diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | Makefile | 9 | ||||
| -rw-r--r-- | README.md | 36 | ||||
| -rwxr-xr-x | configure | 57 | ||||
| -rw-r--r-- | reproduce/config/pipeline/LOCAL.mk.in (renamed from reproduce/config/pipeline/DIRECTORIES.mk.in) | 23 | ||||
| -rw-r--r-- | reproduce/config/pipeline/pdf.mk | 14 | ||||
| -rw-r--r-- | reproduce/config/pipeline/web.mk | 4 | ||||
| -rw-r--r-- | reproduce/src/make/initialize.mk | 68 |
8 files changed, 135 insertions, 78 deletions
@@ -11,4 +11,4 @@ reproduce/build reproduce/BDIR/ tex/pipeline.tex reproduce/SURVEY/ -reproduce/config/pipeline/DIRECTORIES.mk
\ No newline at end of file +reproduce/config/pipeline/LOCAL.mk
\ No newline at end of file @@ -85,16 +85,17 @@ include $(foreach f, initialize download paper, reproduce/src/make/$(f).mk) # try to be remade on every call and `ln' will complain and abort). # # Note that if you don't want the final PDF and just want the processing -# and file outputs, you can remove the value of the `pdf-compile' variable -# in `reproduce/config/pipeline/pdf.mk'. +# and file outputs, you can remove the value of the `BUILD-FINAL-PDF' +# variable in `reproduce/config/LOCAL.mk'. tex/pipeline.tex: $(foreach f, initialize download, $(mtexdir)/$(f).tex) \ | $(bdirsym) # If no PDF is requested, then just exit here. -ifeq ($(pdf-compile),) +ifeq ($(BUILD-FINAL-PDF),) @echo; @echo "Everything is OK until this point, but not building PDF." - @echo "To do so, give a value to the 'pdf-compile' variable." + @echo "To do so, give a value to the 'BUILD-FINAL-PDF' variable." + @echo "It is defined in 'reproduce/config/pipeline/LOCAL.mk'." @echo; @exit 1 endif @@ -179,24 +179,24 @@ been explained here), please let us know to correct it. - **First input dataset**: The user manages the top-level directory of the input data through the variables set in - `reproduce/config/pipeline/DIRECTORIES.mk.in` (the user actually edits - a `DIRECTORIES.mk` file that is created by `configure` from the - `.mk.in` file, but the `.mk` file is not under version control). So - open this file and replace `SURVEY` in the variable name and value - with the name of your input survey or dataset (all in capital - letters), for example if you are working on data from the XDF survey, - replace `SURVEY` with `XDF`. Don't change anything else in the value, - just the the all-caps name. Afterwards, change any occurrence of - `SURVEY` in the whole pipeline with the new name. You can find the - occurrences with a simple command like the ones shown below. We follow - the Make convention here that all `ONLY-CAPITAL` variables are those - directly set by the user and all `small-caps` variables are set by the - pipeline designer. All variables that also depend on this survey have - a `survey` in their name. Hence, also correct all these occurrences to - your new name in small-caps. Of course, ignore those occurrences that - are irrelevant, like those in this file. Note that in the raw version - of this template no target depends on these files, so they are - ignored. Afterwards, set the webpage and correct the filenames in + `reproduce/config/pipeline/LOCAL.mk.in` (the user actually edits a + `LOCAL.mk` file that is created by `configure` from the `.mk.in` file, + but the `.mk` file is not under version control). So open this file + and replace `SURVEY` in the variable name and value with the name of + your input survey or dataset (all in capital letters), for example if + you are working on data from the XDF survey, replace `SURVEY` with + `XDF`. Don't change anything else in the value, just the the all-caps + name. Afterwards, change any occurrence of `SURVEY` in the whole + pipeline with the new name. You can find the occurrences with a simple + command like the ones shown below. We follow the Make convention here + that all `ONLY-CAPITAL` variables are those directly set by the user + and all `small-caps` variables are set by the pipeline designer. All + variables that also depend on this survey have a `survey` in their + name. Hence, also correct all these occurrences to your new name in + small-caps. Of course, ignore those occurrences that are irrelevant, + like those in this file. Note that in the raw version of this template + no target depends on these files, so they are ignored. Afterwards, set + the webpage and correct the filenames in `reproduce/src/make/download.mk` if necessary. ```shell @@ -28,43 +28,37 @@ pdir=reproduce/config/pipeline -# Message to print for editing -function msg { - echo; echo "Top-level reproduction directories are set."; - echo "Please run the following command to start the pipeline:" - echo "(Replace '8' with the number of CPU threads available)" - echo; echo " make -j8" - echo; -} -# If `DIRECTORIES.mk' is already created, then ignore this step. -if [ -f $pdir/DIRECTORIES.mk ]; then + +# If `LOCAL.mk' is already created, then ignore this step. +if [ -f $pdir/LOCAL.mk ]; then echo - echo "$pdir/DIRECTORIES.mk already exists." + echo "$pdir/LOCAL.mk already exists." echo "To change/correct the top-level directories, please remove/edit it manually." echo else # Copy the base file to the desired output file. - if cp $pdir/DIRECTORIES.mk.in $pdir/DIRECTORIES.mk; then + if cp $pdir/LOCAL.mk.in $pdir/LOCAL.mk; then # Tell the user to edit the directories. while [ "$userread" != "y" -a "$userread" != "n" ] do - echo "Top-level directories..." echo - echo "These directories define the input(s) location and the" - echo "directory to host the intermediate/processing files." + echo "------------------------------------" + echo "Reproduction pipeline local configuration" + echo "-----------------------------------------" echo - echo "To help in your ability to read and manage this pipeline," - echo "it is recommended (but not mandatory) to change them to" - echo "a directory outside this reproduction pipeline." + echo "Local settings include things like top-level directories," + echo "or processing steps (e.g., if you want a final PDF output)." echo - echo "More descriptions are provided within the file that is" - echo "opened if you choose to edit the directories." + echo "Pressing 'y' will open the local settings file in an editor" + echo "so you can modify the default values if you want. Each" + echo "variable is also thoroughly described in the comments (lines" + echo "starting with a '#') above it." echo - read -p"Edit the default top-level directories (y/n)? " userread + read -p"Edit the default local configuration (y/n)? " userread done # Only continue if the user wants to edit the top level @@ -72,17 +66,28 @@ else if [ $userread = "y" ]; then # Open a text editor to set the given directories - if emacs $pdir/DIRECTORIES.mk; then msg - elif gedit $pdir/DIRECTORIES.mk; then msg - elif vi $pdir/DIRECTORIES.mk; then msg + if emacs $pdir/LOCAL.mk; then junk=1 + elif gedit $pdir/LOCAL.mk; then junk=1 + elif vi $pdir/LOCAL.mk; then junk=1 else echo echo "No common text editor found on your system." - echo "Please set the values in '$pdir/DIRECTORIES.mk' manually." + echo "Please set the values in '$pdir/LOCAL.mk' manually." echo fi fi + echo + echo "This reproduction pipeline has been configured for this system." + echo "Please run the following command to start the pipeline:" + echo "(Replace '8' with the number of CPU threads available)" + echo + echo " make -j8" + echo + echo + echo "(you can always check/modify the default local settings" + echo " by editing this file: '$pdir/LOCAL.mk')" + echo else - echo; echo "Couldn't create $pdir/DIRECTORIES.mk" + echo; echo "Couldn't create $pdir/LOCAL.mk" fi fi diff --git a/reproduce/config/pipeline/DIRECTORIES.mk.in b/reproduce/config/pipeline/LOCAL.mk.in index 9ebd67b..e5a9f96 100644 --- a/reproduce/config/pipeline/DIRECTORIES.mk.in +++ b/reproduce/config/pipeline/LOCAL.mk.in @@ -22,8 +22,8 @@ -# Input data directories -# ---------------------- +# Input data +# ---------- # # This is where the input data (with the same file-name standard as the # online webpage) are stored. If this directory doesn't exist, or it @@ -48,3 +48,22 @@ SURVEY = reproduce/SURVEY # more easy: the contents of the build directory do not need to be backed # up since they can be reproduced and they can be large. BDIR = reproduce/BDIR + + + + + +# Make the final PDF? +# ------------------- +# +# During the testing a pipeline, it is usually not necessary to build +# the PDF file (which makes a lot of output lines on the command-line +# and can make it hard to find the commands and possible errors (and +# their outputs). Also, in some cases, only the produced results may +# be of interest and not the final PDF, so LaTeX (and its necessary +# packages) may not be installed. +# +# If this variable is given any string, a PDF will be made with +# LaTeX. Otherwise, a notice will just printed that for now, no PDF +# will be created. +BUILD-FINAL-PDF = yes diff --git a/reproduce/config/pipeline/pdf.mk b/reproduce/config/pipeline/pdf.mk deleted file mode 100644 index 51ab933..0000000 --- a/reproduce/config/pipeline/pdf.mk +++ /dev/null @@ -1,14 +0,0 @@ -# Make the final PDF? -# ------------------- -# -# During the testing a pipeline, it is usually not necessary to build -# the PDF file (which makes a lot of output lines on the command-line -# and can make it hard to find the commands and possible errors (and -# their outputs). Also, in some cases, only the produced results may -# be of interest and not the final PDF, so LaTeX (and its necessary -# packages) may not be installed. -# -# If this variable is given any string, a PDF will be made with -# LaTeX. Otherwise, a notice will just printed that for now, no PDF -# will be created. -pdf-compile = yes diff --git a/reproduce/config/pipeline/web.mk b/reproduce/config/pipeline/web.mk index f80b886..5af11a7 100644 --- a/reproduce/config/pipeline/web.mk +++ b/reproduce/config/pipeline/web.mk @@ -1,6 +1,6 @@ # Web server(s) hosting the input data for this pipeline. # # This is the web page containing the files that must be located in the -# `SURVEY' directory of `reproduce/config/pipeline/DIRECTORIES.mk' on the -# local system. +# `SURVEY' directory of `reproduce/config/pipeline/LOCAL.mk' on the local +# system. web-survey = https://some.webpage.com/example/server diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 1478881..927c292 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -33,24 +33,67 @@ # parallel. Also, some programs may not be thread-safe, therefore it will # be necessary to put a lock on them. This pipeline uses the `flock' # program to achieve this. -texdir = $(BDIR)/tex -lockdir = $(BDIR)/locks -bdirsym = reproduce/build -mtexdir = $(texdir)/macros -pconfdir = reproduce/config/pipeline +texdir = $(BDIR)/tex +srcdir = reproduce/src +lockdir = $(BDIR)/locks +bdirsym = reproduce/build +mtexdir = $(texdir)/macros +gconfdir = reproduce/config/gnuastro +pconfdir = reproduce/config/pipeline + + + + + +# Sanity check +# ------------ +# +# We need to make sure that the `./configure' command has already been +# run. The output of `./configure' is the `$(pconfdir)/LOCAL.mk' file and +# this is the non-time-stamp prerequisite of $(BDIR), see below. +# +# There is one problem however: if the user hasn't run `./configure' yet, +# then `BDIR' isn't defined (will just evaluate to blank space). Therefore +# it won't appear in the prerequisites and the pipeline will try to build +# the other directories in the top root directory (`/'). To solve this +# problem, when `BDIR' isn't defined, we'll define it with a place-holder +# name ((only so it won't evaluate to blank space). Note that this +# directory will never be built. +ifeq ($(BDIR),) +configure-run = no +BDIR = reproduce/BDIR +else +configure-run = yes +endif +$(pconfdir)/LOCAL.mk: + @echo + @echo "================================================================" + @echo "For the pipeline's local settings, please run this command first" + @echo "(P.S. this local configuration is only necessary one time)" + @echo + @echo " $$ ./configure" + @echo "================================================================" + @echo + @exit 1 # Make the high-level level directories -# ------------------------------ +# ------------------------------------- # # These are just the top-level directories for all the separate steps. The # directories (or possible sub-directories) for individual steps will be # defined and added within their own Makefiles. -$(BDIR):; mkdir $@; +# +# IMPORTANT NOTE for $(BDIR)'s dependency: it only depends on the existance +# (not the time-stamp) of `$(pconfdir)/LOCAL.mk'. So the user can make any +# changes within that file and if they don't affect the pipeline. For +# example a change of the top $(BDIR) name, while the contents are the same +# as before. $(mtexdir): | $(texdir); mkdir $@ +$(BDIR): | $(pconfdir)/LOCAL.mk; mkdir $@ $(texdir) $(lockdir): | $(BDIR); mkdir $@ @@ -65,11 +108,14 @@ $(texdir) $(lockdir): | $(BDIR); mkdir $@ # included here ensure that the file is always built in every run: for # example the pipeline versions may change within two separate runs, so we # want it to be rebuilt every time. -.PHONY: all clean clean-mmap $(mtexdir)/initialize.tex +.PHONY: all clean distclean clean-mmap $(mtexdir)/initialize.tex +distclean: clean; rm -f $(pconfdir)/LOCAL.mk clean-mmap:; rm -f reproduce/config/gnuastro/mmap* clean: - rm -rf $(BDIR) $(bdirsym) *.pdf *.log *.out *.aux *.auxlock \ - reproduce/config/gnuastro/mmap* +ifeq ($(configure-run),yes) + rm -rf $(BDIR) +endif + rm -f $(bdirsym) $(gconfdir)/mmap* *.pdf *.log *.out *.aux *.auxlock @@ -93,7 +139,7 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\gnuastroversion}{$$v}" >> $@ # Location of the build directory (for LaTeX inputs). - echo "\newcommand{\bdir}{$(BDIR)}" >> $@ + @echo "\newcommand{\bdir}{$(BDIR)}" >> $@ |