From d26535d6665879f77d39e790b4aa9ee0dcb63dcf Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 14 Feb 2018 14:13:36 +0100 Subject: Sanity checks added, local settings now in LOCAL.mk.in The choice of whether or not to make a PDF is now also a local system issue, not a general pipeline issue. So it has been put in the new `LOCAL.mk.in' file which replaces the old `DIRECTORIES.mk.in'. All local settings (things that when changed should not be version-controlled) should be defined in this file. A sanity check was added to find if `./configure' has been run before `make' or not (using the `LOCAL.mk' file which is an output of the configuration step). If `LOCAL.mk' doesn't exist, an error will be printed informing the user that `./configure' needs to be run first. The configure script also provides more clear and hopefully better information on its purpose and what must be done. Since `make clean', it is executed even when `./configure' hasn't been run, it will only delete the build directory and its contents when local configuration has been done. A `distclean' target was also added which will first "clean" the pipeline, then delete the `LOCAL.mk.in' file. To allow rules like `make' to be run even if `BDIR' isn't defined (`./configure' hasn't been run yet), a fake `BDIR' is defined in such cases. --- .gitignore | 2 +- Makefile | 9 ++-- README.md | 36 +++++++-------- configure | 57 +++++++++++++----------- reproduce/config/pipeline/DIRECTORIES.mk.in | 50 --------------------- reproduce/config/pipeline/LOCAL.mk.in | 69 +++++++++++++++++++++++++++++ reproduce/config/pipeline/pdf.mk | 14 ------ reproduce/config/pipeline/web.mk | 4 +- reproduce/src/make/initialize.mk | 68 +++++++++++++++++++++++----- 9 files changed, 183 insertions(+), 126 deletions(-) delete mode 100644 reproduce/config/pipeline/DIRECTORIES.mk.in create mode 100644 reproduce/config/pipeline/LOCAL.mk.in delete mode 100644 reproduce/config/pipeline/pdf.mk diff --git a/.gitignore b/.gitignore index cb41e68..84d739b 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ reproduce/build reproduce/BDIR/ tex/pipeline.tex reproduce/SURVEY/ -reproduce/config/pipeline/DIRECTORIES.mk \ No newline at end of file +reproduce/config/pipeline/LOCAL.mk \ No newline at end of file diff --git a/Makefile b/Makefile index 357a2e0..051332b 100644 --- a/Makefile +++ b/Makefile @@ -85,16 +85,17 @@ include $(foreach f, initialize download paper, reproduce/src/make/$(f).mk) # try to be remade on every call and `ln' will complain and abort). # # Note that if you don't want the final PDF and just want the processing -# and file outputs, you can remove the value of the `pdf-compile' variable -# in `reproduce/config/pipeline/pdf.mk'. +# and file outputs, you can remove the value of the `BUILD-FINAL-PDF' +# variable in `reproduce/config/LOCAL.mk'. tex/pipeline.tex: $(foreach f, initialize download, $(mtexdir)/$(f).tex) \ | $(bdirsym) # If no PDF is requested, then just exit here. -ifeq ($(pdf-compile),) +ifeq ($(BUILD-FINAL-PDF),) @echo; @echo "Everything is OK until this point, but not building PDF." - @echo "To do so, give a value to the 'pdf-compile' variable." + @echo "To do so, give a value to the 'BUILD-FINAL-PDF' variable." + @echo "It is defined in 'reproduce/config/pipeline/LOCAL.mk'." @echo; @exit 1 endif diff --git a/README.md b/README.md index 5f0d3e4..59ecf71 100644 --- a/README.md +++ b/README.md @@ -179,24 +179,24 @@ been explained here), please let us know to correct it. - **First input dataset**: The user manages the top-level directory of the input data through the variables set in - `reproduce/config/pipeline/DIRECTORIES.mk.in` (the user actually edits - a `DIRECTORIES.mk` file that is created by `configure` from the - `.mk.in` file, but the `.mk` file is not under version control). So - open this file and replace `SURVEY` in the variable name and value - with the name of your input survey or dataset (all in capital - letters), for example if you are working on data from the XDF survey, - replace `SURVEY` with `XDF`. Don't change anything else in the value, - just the the all-caps name. Afterwards, change any occurrence of - `SURVEY` in the whole pipeline with the new name. You can find the - occurrences with a simple command like the ones shown below. We follow - the Make convention here that all `ONLY-CAPITAL` variables are those - directly set by the user and all `small-caps` variables are set by the - pipeline designer. All variables that also depend on this survey have - a `survey` in their name. Hence, also correct all these occurrences to - your new name in small-caps. Of course, ignore those occurrences that - are irrelevant, like those in this file. Note that in the raw version - of this template no target depends on these files, so they are - ignored. Afterwards, set the webpage and correct the filenames in + `reproduce/config/pipeline/LOCAL.mk.in` (the user actually edits a + `LOCAL.mk` file that is created by `configure` from the `.mk.in` file, + but the `.mk` file is not under version control). So open this file + and replace `SURVEY` in the variable name and value with the name of + your input survey or dataset (all in capital letters), for example if + you are working on data from the XDF survey, replace `SURVEY` with + `XDF`. Don't change anything else in the value, just the the all-caps + name. Afterwards, change any occurrence of `SURVEY` in the whole + pipeline with the new name. You can find the occurrences with a simple + command like the ones shown below. We follow the Make convention here + that all `ONLY-CAPITAL` variables are those directly set by the user + and all `small-caps` variables are set by the pipeline designer. All + variables that also depend on this survey have a `survey` in their + name. Hence, also correct all these occurrences to your new name in + small-caps. Of course, ignore those occurrences that are irrelevant, + like those in this file. Note that in the raw version of this template + no target depends on these files, so they are ignored. Afterwards, set + the webpage and correct the filenames in `reproduce/src/make/download.mk` if necessary. ```shell diff --git a/configure b/configure index be18c25..8ecb8fa 100755 --- a/configure +++ b/configure @@ -28,43 +28,37 @@ pdir=reproduce/config/pipeline -# Message to print for editing -function msg { - echo; echo "Top-level reproduction directories are set."; - echo "Please run the following command to start the pipeline:" - echo "(Replace '8' with the number of CPU threads available)" - echo; echo " make -j8" - echo; -} -# If `DIRECTORIES.mk' is already created, then ignore this step. -if [ -f $pdir/DIRECTORIES.mk ]; then + +# If `LOCAL.mk' is already created, then ignore this step. +if [ -f $pdir/LOCAL.mk ]; then echo - echo "$pdir/DIRECTORIES.mk already exists." + echo "$pdir/LOCAL.mk already exists." echo "To change/correct the top-level directories, please remove/edit it manually." echo else # Copy the base file to the desired output file. - if cp $pdir/DIRECTORIES.mk.in $pdir/DIRECTORIES.mk; then + if cp $pdir/LOCAL.mk.in $pdir/LOCAL.mk; then # Tell the user to edit the directories. while [ "$userread" != "y" -a "$userread" != "n" ] do - echo "Top-level directories..." echo - echo "These directories define the input(s) location and the" - echo "directory to host the intermediate/processing files." + echo "------------------------------------" + echo "Reproduction pipeline local configuration" + echo "-----------------------------------------" echo - echo "To help in your ability to read and manage this pipeline," - echo "it is recommended (but not mandatory) to change them to" - echo "a directory outside this reproduction pipeline." + echo "Local settings include things like top-level directories," + echo "or processing steps (e.g., if you want a final PDF output)." echo - echo "More descriptions are provided within the file that is" - echo "opened if you choose to edit the directories." + echo "Pressing 'y' will open the local settings file in an editor" + echo "so you can modify the default values if you want. Each" + echo "variable is also thoroughly described in the comments (lines" + echo "starting with a '#') above it." echo - read -p"Edit the default top-level directories (y/n)? " userread + read -p"Edit the default local configuration (y/n)? " userread done # Only continue if the user wants to edit the top level @@ -72,17 +66,28 @@ else if [ $userread = "y" ]; then # Open a text editor to set the given directories - if emacs $pdir/DIRECTORIES.mk; then msg - elif gedit $pdir/DIRECTORIES.mk; then msg - elif vi $pdir/DIRECTORIES.mk; then msg + if emacs $pdir/LOCAL.mk; then junk=1 + elif gedit $pdir/LOCAL.mk; then junk=1 + elif vi $pdir/LOCAL.mk; then junk=1 else echo echo "No common text editor found on your system." - echo "Please set the values in '$pdir/DIRECTORIES.mk' manually." + echo "Please set the values in '$pdir/LOCAL.mk' manually." echo fi fi + echo + echo "This reproduction pipeline has been configured for this system." + echo "Please run the following command to start the pipeline:" + echo "(Replace '8' with the number of CPU threads available)" + echo + echo " make -j8" + echo + echo + echo "(you can always check/modify the default local settings" + echo " by editing this file: '$pdir/LOCAL.mk')" + echo else - echo; echo "Couldn't create $pdir/DIRECTORIES.mk" + echo; echo "Couldn't create $pdir/LOCAL.mk" fi fi diff --git a/reproduce/config/pipeline/DIRECTORIES.mk.in b/reproduce/config/pipeline/DIRECTORIES.mk.in deleted file mode 100644 index 9ebd67b..0000000 --- a/reproduce/config/pipeline/DIRECTORIES.mk.in +++ /dev/null @@ -1,50 +0,0 @@ -# Top-level user specific directories. Note the points below: -# -# - The VALUES to these directories are initially JUST PLACE-HOLDERS! -# Please correct them based on your system. -# -# - The directories don't need to necessarily exist. If they do not exist, -# they will be created and the necessary data will be downloaded into -# them. Ofcourse provided that you have write permissions and an internet -# connection. -# -# - Do not use the tilde expansion `~' or variables for your home -# directory. Please use the full address, for example -# `/home/your-user-name'. -# -# - An ending forward-slash `/' is NOT necessary. In the pipeline, all -# these variables will be followed by a `/', so if you put a `/' at the -# end of the value here, you will see a `//' in the printed outputs -# during the processing. This has no technical problem, but can make -# reading the outputs harder and is thus not recommended. - - - - - -# Input data directories -# ---------------------- -# -# This is where the input data (with the same file-name standard as the -# online webpage) are stored. If this directory doesn't exist, or it -# doesn't contain the files (with the correct file-name formats), it will -# be created and the images will be downloaded. See -# `reproduce/config/pipeline/web.mk', for the URLs containing the expected -# inputs for each survey. -SURVEY = reproduce/SURVEY - - - - - -# Build directory -# --------------- -# -# This is where the intermediate outputs of each step are kept. -# -# Why a separate build directory? So the source and configuration files for -# this reproduction pipeline do not get crowded by all the -# intermediate/derivative files. Also to make synchronization and backups -# more easy: the contents of the build directory do not need to be backed -# up since they can be reproduced and they can be large. -BDIR = reproduce/BDIR diff --git a/reproduce/config/pipeline/LOCAL.mk.in b/reproduce/config/pipeline/LOCAL.mk.in new file mode 100644 index 0000000..e5a9f96 --- /dev/null +++ b/reproduce/config/pipeline/LOCAL.mk.in @@ -0,0 +1,69 @@ +# Top-level user specific directories. Note the points below: +# +# - The VALUES to these directories are initially JUST PLACE-HOLDERS! +# Please correct them based on your system. +# +# - The directories don't need to necessarily exist. If they do not exist, +# they will be created and the necessary data will be downloaded into +# them. Ofcourse provided that you have write permissions and an internet +# connection. +# +# - Do not use the tilde expansion `~' or variables for your home +# directory. Please use the full address, for example +# `/home/your-user-name'. +# +# - An ending forward-slash `/' is NOT necessary. In the pipeline, all +# these variables will be followed by a `/', so if you put a `/' at the +# end of the value here, you will see a `//' in the printed outputs +# during the processing. This has no technical problem, but can make +# reading the outputs harder and is thus not recommended. + + + + + +# Input data +# ---------- +# +# This is where the input data (with the same file-name standard as the +# online webpage) are stored. If this directory doesn't exist, or it +# doesn't contain the files (with the correct file-name formats), it will +# be created and the images will be downloaded. See +# `reproduce/config/pipeline/web.mk', for the URLs containing the expected +# inputs for each survey. +SURVEY = reproduce/SURVEY + + + + + +# Build directory +# --------------- +# +# This is where the intermediate outputs of each step are kept. +# +# Why a separate build directory? So the source and configuration files for +# this reproduction pipeline do not get crowded by all the +# intermediate/derivative files. Also to make synchronization and backups +# more easy: the contents of the build directory do not need to be backed +# up since they can be reproduced and they can be large. +BDIR = reproduce/BDIR + + + + + +# Make the final PDF? +# ------------------- +# +# During the testing a pipeline, it is usually not necessary to build +# the PDF file (which makes a lot of output lines on the command-line +# and can make it hard to find the commands and possible errors (and +# their outputs). Also, in some cases, only the produced results may +# be of interest and not the final PDF, so LaTeX (and its necessary +# packages) may not be installed. +# +# If this variable is given any string, a PDF will be made with +# LaTeX. Otherwise, a notice will just printed that for now, no PDF +# will be created. +BUILD-FINAL-PDF = yes diff --git a/reproduce/config/pipeline/pdf.mk b/reproduce/config/pipeline/pdf.mk deleted file mode 100644 index 51ab933..0000000 --- a/reproduce/config/pipeline/pdf.mk +++ /dev/null @@ -1,14 +0,0 @@ -# Make the final PDF? -# ------------------- -# -# During the testing a pipeline, it is usually not necessary to build -# the PDF file (which makes a lot of output lines on the command-line -# and can make it hard to find the commands and possible errors (and -# their outputs). Also, in some cases, only the produced results may -# be of interest and not the final PDF, so LaTeX (and its necessary -# packages) may not be installed. -# -# If this variable is given any string, a PDF will be made with -# LaTeX. Otherwise, a notice will just printed that for now, no PDF -# will be created. -pdf-compile = yes diff --git a/reproduce/config/pipeline/web.mk b/reproduce/config/pipeline/web.mk index f80b886..5af11a7 100644 --- a/reproduce/config/pipeline/web.mk +++ b/reproduce/config/pipeline/web.mk @@ -1,6 +1,6 @@ # Web server(s) hosting the input data for this pipeline. # # This is the web page containing the files that must be located in the -# `SURVEY' directory of `reproduce/config/pipeline/DIRECTORIES.mk' on the -# local system. +# `SURVEY' directory of `reproduce/config/pipeline/LOCAL.mk' on the local +# system. web-survey = https://some.webpage.com/example/server diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 1478881..927c292 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -33,24 +33,67 @@ # parallel. Also, some programs may not be thread-safe, therefore it will # be necessary to put a lock on them. This pipeline uses the `flock' # program to achieve this. -texdir = $(BDIR)/tex -lockdir = $(BDIR)/locks -bdirsym = reproduce/build -mtexdir = $(texdir)/macros -pconfdir = reproduce/config/pipeline +texdir = $(BDIR)/tex +srcdir = reproduce/src +lockdir = $(BDIR)/locks +bdirsym = reproduce/build +mtexdir = $(texdir)/macros +gconfdir = reproduce/config/gnuastro +pconfdir = reproduce/config/pipeline + + + + + +# Sanity check +# ------------ +# +# We need to make sure that the `./configure' command has already been +# run. The output of `./configure' is the `$(pconfdir)/LOCAL.mk' file and +# this is the non-time-stamp prerequisite of $(BDIR), see below. +# +# There is one problem however: if the user hasn't run `./configure' yet, +# then `BDIR' isn't defined (will just evaluate to blank space). Therefore +# it won't appear in the prerequisites and the pipeline will try to build +# the other directories in the top root directory (`/'). To solve this +# problem, when `BDIR' isn't defined, we'll define it with a place-holder +# name ((only so it won't evaluate to blank space). Note that this +# directory will never be built. +ifeq ($(BDIR),) +configure-run = no +BDIR = reproduce/BDIR +else +configure-run = yes +endif +$(pconfdir)/LOCAL.mk: + @echo + @echo "================================================================" + @echo "For the pipeline's local settings, please run this command first" + @echo "(P.S. this local configuration is only necessary one time)" + @echo + @echo " $$ ./configure" + @echo "================================================================" + @echo + @exit 1 # Make the high-level level directories -# ------------------------------ +# ------------------------------------- # # These are just the top-level directories for all the separate steps. The # directories (or possible sub-directories) for individual steps will be # defined and added within their own Makefiles. -$(BDIR):; mkdir $@; +# +# IMPORTANT NOTE for $(BDIR)'s dependency: it only depends on the existance +# (not the time-stamp) of `$(pconfdir)/LOCAL.mk'. So the user can make any +# changes within that file and if they don't affect the pipeline. For +# example a change of the top $(BDIR) name, while the contents are the same +# as before. $(mtexdir): | $(texdir); mkdir $@ +$(BDIR): | $(pconfdir)/LOCAL.mk; mkdir $@ $(texdir) $(lockdir): | $(BDIR); mkdir $@ @@ -65,11 +108,14 @@ $(texdir) $(lockdir): | $(BDIR); mkdir $@ # included here ensure that the file is always built in every run: for # example the pipeline versions may change within two separate runs, so we # want it to be rebuilt every time. -.PHONY: all clean clean-mmap $(mtexdir)/initialize.tex +.PHONY: all clean distclean clean-mmap $(mtexdir)/initialize.tex +distclean: clean; rm -f $(pconfdir)/LOCAL.mk clean-mmap:; rm -f reproduce/config/gnuastro/mmap* clean: - rm -rf $(BDIR) $(bdirsym) *.pdf *.log *.out *.aux *.auxlock \ - reproduce/config/gnuastro/mmap* +ifeq ($(configure-run),yes) + rm -rf $(BDIR) +endif + rm -f $(bdirsym) $(gconfdir)/mmap* *.pdf *.log *.out *.aux *.auxlock @@ -93,7 +139,7 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\gnuastroversion}{$$v}" >> $@ # Location of the build directory (for LaTeX inputs). - echo "\newcommand{\bdir}{$(BDIR)}" >> $@ + @echo "\newcommand{\bdir}{$(BDIR)}" >> $@ -- cgit v1.2.1