diff options
| -rw-r--r-- | .file-metadata | bin | 3714 -> 3666 bytes | |||
| -rw-r--r-- | README-hacking.md | 34 | ||||
| -rwxr-xr-x | configure | 56 | ||||
| -rwxr-xr-x | for-group | 45 | ||||
| -rw-r--r-- | reproduce/config/pipeline/LOCAL.mk.in | 25 | ||||
| -rw-r--r-- | reproduce/src/make/initialize.mk | 8 | ||||
| -rw-r--r-- | reproduce/src/make/top.mk | 33 |
7 files changed, 90 insertions, 111 deletions
diff --git a/.file-metadata b/.file-metadata Binary files differindex 579bf99..11686e3 100644 --- a/.file-metadata +++ b/.file-metadata diff --git a/README-hacking.md b/README-hacking.md index 5fd74b6..ddc9460 100644 --- a/README-hacking.md +++ b/README-hacking.md @@ -339,21 +339,21 @@ Make steps must be called with special conditions which are managed in the `for-group` file. Let's see how this design is implemented. When the `./configure` finishes, -it makes a `Makefile` in the top directory. This Makefile is just a -symbolic link to `reproduce/src/make/top.mk`. Please open and inspect it as -we go along here. The first step (un-commented line) is to import the local -configuration (answers to the questions `./configure` asked you). They are -defined in the configuration-Makefile `reproduce/config/pipeline/LOCAL.mk` -which was also built by `./configure` (based on the `LOCAL.mk.in` -template). +it a `Makefile` will be placed in the top directory. This `Makefile` is +just a symbolic link to `reproduce/src/make/top.mk`. Please open and +inspect it as we go along here. The first step (un-commented line) is to +import the local configuration (answers to the questions `./configure` +asked you). They are defined in the configuration-Makefile +`reproduce/config/pipeline/LOCAL.mk` which was also built by `./configure` +(based on the `LOCAL.mk.in` template). The next non-commented set of lines define the ultimate target of the whole -pipeline (`paper.pdf`). But a sanity check is necessary for situations when -the user is not careful (for example has configured the pipeline for group -access but forgets to run the pipeline with `./for-group`, or the -opposite). Therefore we use a Make conditional to define the `all` target -based on the group permissions being consistent between the initial -configuration and the current run. +pipeline (`paper.pdf`). But to avoid mistakes, a sanity check is necessary +to see if Make is being run with the same group settings as the configure +script (for example when the pipeline is configured for group access using +the `./for-group` script, but Make isn't). Therefore we use a Make +conditional to define the `all` target based on the group permissions being +consistent between the initial configuration and the current run. If there is a problem `all` will not depend on anything and will just print a warning to inform you of the problem. When the group conditions are fine, @@ -621,14 +621,6 @@ advanced in later stages of your work. with. Also check and update this file one last time when you are ready to publish your work (and its reproduction pipeline). - - **`for-group`**: If you will be working on this pipeline with - colleagues, and the build steps involve many files, or are slow, you - need to share the build directory. This script is designed for such - scenarios. So open this file and give the name of the Unix name of - your group to the `thisgroup` variable. You can see the list of groups - you are a member of with the `groups` command. You can ask your system - administrator to define a group with specific members if necessary. - - **Your first commit**: You have already made some small and basic changes in the steps above and you are in the `master` branch. So, you can officially make your first commit in your project's history. But @@ -165,26 +165,27 @@ fi # Make sure the group permissions satisfy the previous configuration (if it # exists and we don't want to re-write it). if [ $rewritepconfig = no ]; then - oldforgroup=$(awk '/FOR-GROUP/ && c==0 {c=1; print $3}' $pconf) - if [ "x$oldforgroup" = xyes ]; then - if [ "x$reproducible_paper_for_group" = x ]; then - echo "-----------------------------" - echo "!!!!!!!! ERROR !!!!!!!!" - echo "-----------------------------" - echo "Previous pipeline was configured for groups." - echo "Either enable re-write, or use './for-group'." - exit 1 - fi - else - if [ "x$reproducible_paper_for_group" = xyes ]; then - echo "-----------------------------" - echo "!!!!!!!! ERROR !!!!!!!!" - echo "-----------------------------" - echo "Previous pipeline was not configured for groups." - echo "Either enable re-write, or don't use './for-group'." - exit 1 - fi - fi + oldgroupname=$(awk '/GROUP-NAME/ {print $3; exit 0}' $pconf) + if [ "x$oldgroupname" = "x$reproducible_paper_group_name" ]; then + just_a_place_holder_to_avoid_not_equal_test=1; + else + echo "-----------------------------" + echo "!!!!!!!! ERROR !!!!!!!!" + echo "-----------------------------" + if [ "x$oldgroupname" = x ]; then + status="NOT configured for groups" + confcommand="./configure" + else + status="configured for '$oldgroupname' group" + confcommand="./for-group $oldgroupname configure" + fi + echo "Previous pipeline was $status!" + echo "Either enable re-write of this configuration file," + echo "or re-run this configuration like this:" + echo + echo " $confcommand"; echo + exit 1 + fi fi @@ -408,19 +409,12 @@ if [ $rewritepconfig = yes ]; then # Make the pipeline configuration's initial comments. create_file_with_notice $pconf - # Fix the group settings. - if [ "x$reproducible_paper_for_group" = xyes ]; then - for_group=yes - else - for_group=no - fi - # Write the values. sed -e's|@bdir[@]|'"$bdir"'|' \ -e's|@indir[@]|'"$indir"'|' \ -e's|@ddir[@]|'"$ddir"'|' \ -e's|@downloader[@]|'"$downloader"'|' \ - -e's|@forgroup[@]|'"$for_group"'|' \ + -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \ $pconf.in >> $pconf else # Read the values from existing configuration file. @@ -783,10 +777,10 @@ ln -s $(pwd)/reproduce/src/make/top.mk Makefile # # The configuration is now complete, we can inform the user on the next # step(s) to take. -if [ x$reproducible_paper_for_group = xyes ]; then - buildcommand="./for-group make -j8" -else +if [ x$reproducible_paper_group_name = x ]; then buildcommand=".local/bin/make -j8" +else + buildcommand="./for-group $reproducible_paper_group_name make -j8" fi cat <<EOF @@ -1,9 +1,9 @@ -#! /bin/sh +#! /bin/bash # # Running examples: # -# $ ./for-group configure -# $ ./for-group make [-jN] +# $ ./for-group group_name configure +# $ ./for-group group_name make [-jN] # # This is a wrapper for the configure and Make steps designed for a group # of users (sharing the same group name) using this pipeline on the same @@ -13,7 +13,7 @@ # (normally done with `.local/bin/make') steps are done with this script, # all the files that are created within the pipeline have these properties: # -# 1) Group owner will be a special group (value of `thisgroup' below). +# 1) Group owner will be the group specified in the command-line. # 2) The permission flags give write access to the group members. # # Original author: @@ -39,13 +39,6 @@ -# Desired group -thisgroup=YOUR-GROUP - - - - - # Script settings # --------------- # Stop the script if there are any errors. @@ -55,14 +48,32 @@ set -e -# Set and run the respective command. +# See if any argument are given at all. if [ "x$1" = x ]; then - echo "$0: an argument is necessary ('configure' or 'make')" + echo "$0: At least two arguments are necessary:" + echo + echo " To configure: $ ./for-group group_name configure" + echo " To build: $ ./for-group group_name make" exit 1 -elif [ "x$1" = xconfigure ]; then script="./configure" -elif [ "x$1" = xmake ]; then script=".local/bin/make $2" +fi + +# Make sure the given group is usable. +testfile=".reproducible-pipeline-group-test" +echo "Just for a test" > $testfile +if chgrp $1 $testfile; then + rm $testfile + if [ "x$2" = xconfigure ]; then script="./configure" + elif [ "x$2" = xmake ]; then script=".local/bin/make $2" + else + echo "$0: a third argument is necessary." + echo "It specifies the action: 'configure' or 'make'" + exit 1 + fi else - echo "$0: argument must be 'configure' or 'make'" + rm $testfile + echo + echo "$0: '$1' is not a usable group name!"; + echo "TIP: you can use the 'groups' command to see your groups." exit 1 fi @@ -76,4 +87,4 @@ fi # We are also exporting a special variable so `./configure' and Make can # prepare for sanity checks and avoid re-doing the whole analysis with a # typo (not using this script properly after configuration). -sg $thisgroup "umask u+r,u+w,g+r,g+w,o-r,o-w,o-x && export reproducible_paper_for_group=yes && $script" +sg $1 "umask u+r,u+w,g+r,g+w,o-r,o-w,o-x && export reproducible_paper_group_name=$1 && $script" diff --git a/reproduce/config/pipeline/LOCAL.mk.in b/reproduce/config/pipeline/LOCAL.mk.in index 846a5b8..41298ee 100644 --- a/reproduce/config/pipeline/LOCAL.mk.in +++ b/reproduce/config/pipeline/LOCAL.mk.in @@ -6,27 +6,4 @@ BDIR = @bdir@ INDIR = @indir@ DEPENDENCIES-DIR = @ddir@ DOWNLOADER = @downloader@ -FOR-GROUP = @forgroup@ - - - - - -# In the top Makefile (which is created after running `./configure' and is -# actually a symbolic link to `reproduce/src/make/top.mk'), we need to -# start by checking if there is no conflict with the running and configured -# group configuration of the pipeline. -good-group-configuration := $(shell \ - if [ "x$(FOR-GROUP)" = xyes ]; then \ - if [ "x$(reproducible_paper_for_group)" = xyes ]; then \ - echo "yes"; \ - else \ - echo "no"; \ - fi; \ - else \ - if [ "x$(reproducible_paper_for_group)" = xyes ]; then \ - echo "no"; \ - else \ - echo "yes"; \ - fi; \ - fi) +GROUP-NAME = @groupname@ diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index da2d012..57565b9 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -68,13 +68,13 @@ gconfdir = reproduce/config/gnuastro # paper is always executed (even if it is present and nothing has # changed). So in terms of over-all efficiency and processing steps, this # doesn't change anything. -ifeq ($(FOR-GROUP),yes) +ifeq (x$(GROUP-NAME),x) +texbdir = $(texdir)/build +final-paper = paper.pdf +else user = $(shell whoami) texbdir = $(texdir)/build-$(user) final-paper = paper-$(user).pdf -else -texbdir = $(texdir)/build -final-paper = paper.pdf endif tikzdir = $(texbdir)/tikz diff --git a/reproduce/src/make/top.mk b/reproduce/src/make/top.mk index 25c4f0b..9b81a58 100644 --- a/reproduce/src/make/top.mk +++ b/reproduce/src/make/top.mk @@ -39,24 +39,29 @@ include reproduce/config/pipeline/LOCAL.mk # configuration settings, necessary for a group building scenario mentioned # next). # +# # Group build # ----------- # # This pipeline can also be configured to have a shared build directory # between multiple users. In this scenario, many users (on a server) can -# have their own/separate version controlled pipeline source of the -# pipeline, but share the same build outputs (in a common directory). This -# will allow a group to work separately, on parallel parts of the analysis. -# It is thus very useful in cases were special storage requirements or CPU -# power is necessary and its not possible/efficient for each user to have a -# fully separate copy of the build directory. +# have their own/separate version controlled pipeline source, but share the +# same build outputs (in a common directory). This will allow a group to +# work separately, on parallel parts of the analysis that don't +# interfere. It is thus very useful in cases were special storage +# requirements or CPU power is necessary and its not possible/efficient for +# each user to have a fully separate copy of the build directory. +# +# Controlling this requires two variables that are available at this stage: +# +# - `GROUP-NAME': from `LOCAL.mk' (which was built by `./configure'). +# - `reproducible_paper_group_name': from the `./for-group' script (if it +# was used to call Make). # -# `FOR-GROUP': from `LOCAL.mk' (which was built by `./configure'). -# `reproducible_paper_for_group': from the `./for-group' script. +# The analysis is only done when both have the same group name. Note that +# when the pipeline isn't being built for a group, both variables will be +# an empty string. # -# The final paper is only built when both have a value of `yes', or when -# `FOR-GROUP' is no and `./for-group' wasn't called (if `./for-group' is -# called before `make', then `reproducible_paper_for_group==yes'). # # Only processing, no LaTeX PDF # ----------------------------- @@ -64,16 +69,16 @@ include reproduce/config/pipeline/LOCAL.mk # If you are just interested in the processing and don't want to build the # PDF, you can skip the creatation of the final PDF by removing the value # of `pdf-build-final' in `reproduce/config/pipeline/pdf-build.mk'. -ifeq ($(good-group-configuration),yes) +ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) all: paper.pdf else all: - @if [ "x$(reproducible_paper_for_group)" = xyes ]; then \ + @if [ "x$(GROUP-NAME)" = x ]; then \ echo "Pipeline is NOT configured for groups, please run"; \ echo " $$ .local/bin/make"; \ else \ echo "Pipeline is configured for groups, please run"; \ - echo " $$ ./for-group make"; \ + echo " $$ ./for-group $(GROUP-NAME) make -j8"; \ fi endif |