From d26535d6665879f77d39e790b4aa9ee0dcb63dcf Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 14 Feb 2018 14:13:36 +0100 Subject: Sanity checks added, local settings now in LOCAL.mk.in The choice of whether or not to make a PDF is now also a local system issue, not a general pipeline issue. So it has been put in the new `LOCAL.mk.in' file which replaces the old `DIRECTORIES.mk.in'. All local settings (things that when changed should not be version-controlled) should be defined in this file. A sanity check was added to find if `./configure' has been run before `make' or not (using the `LOCAL.mk' file which is an output of the configuration step). If `LOCAL.mk' doesn't exist, an error will be printed informing the user that `./configure' needs to be run first. The configure script also provides more clear and hopefully better information on its purpose and what must be done. Since `make clean', it is executed even when `./configure' hasn't been run, it will only delete the build directory and its contents when local configuration has been done. A `distclean' target was also added which will first "clean" the pipeline, then delete the `LOCAL.mk.in' file. To allow rules like `make' to be run even if `BDIR' isn't defined (`./configure' hasn't been run yet), a fake `BDIR' is defined in such cases. --- reproduce/config/pipeline/DIRECTORIES.mk.in | 50 --------------------- reproduce/config/pipeline/LOCAL.mk.in | 69 +++++++++++++++++++++++++++++ reproduce/config/pipeline/pdf.mk | 14 ------ reproduce/config/pipeline/web.mk | 4 +- reproduce/src/make/initialize.mk | 68 +++++++++++++++++++++++----- 5 files changed, 128 insertions(+), 77 deletions(-) delete mode 100644 reproduce/config/pipeline/DIRECTORIES.mk.in create mode 100644 reproduce/config/pipeline/LOCAL.mk.in delete mode 100644 reproduce/config/pipeline/pdf.mk (limited to 'reproduce') diff --git a/reproduce/config/pipeline/DIRECTORIES.mk.in b/reproduce/config/pipeline/DIRECTORIES.mk.in deleted file mode 100644 index 9ebd67b..0000000 --- a/reproduce/config/pipeline/DIRECTORIES.mk.in +++ /dev/null @@ -1,50 +0,0 @@ -# Top-level user specific directories. Note the points below: -# -# - The VALUES to these directories are initially JUST PLACE-HOLDERS! -# Please correct them based on your system. -# -# - The directories don't need to necessarily exist. If they do not exist, -# they will be created and the necessary data will be downloaded into -# them. Ofcourse provided that you have write permissions and an internet -# connection. -# -# - Do not use the tilde expansion `~' or variables for your home -# directory. Please use the full address, for example -# `/home/your-user-name'. -# -# - An ending forward-slash `/' is NOT necessary. In the pipeline, all -# these variables will be followed by a `/', so if you put a `/' at the -# end of the value here, you will see a `//' in the printed outputs -# during the processing. This has no technical problem, but can make -# reading the outputs harder and is thus not recommended. - - - - - -# Input data directories -# ---------------------- -# -# This is where the input data (with the same file-name standard as the -# online webpage) are stored. If this directory doesn't exist, or it -# doesn't contain the files (with the correct file-name formats), it will -# be created and the images will be downloaded. See -# `reproduce/config/pipeline/web.mk', for the URLs containing the expected -# inputs for each survey. -SURVEY = reproduce/SURVEY - - - - - -# Build directory -# --------------- -# -# This is where the intermediate outputs of each step are kept. -# -# Why a separate build directory? So the source and configuration files for -# this reproduction pipeline do not get crowded by all the -# intermediate/derivative files. Also to make synchronization and backups -# more easy: the contents of the build directory do not need to be backed -# up since they can be reproduced and they can be large. -BDIR = reproduce/BDIR diff --git a/reproduce/config/pipeline/LOCAL.mk.in b/reproduce/config/pipeline/LOCAL.mk.in new file mode 100644 index 0000000..e5a9f96 --- /dev/null +++ b/reproduce/config/pipeline/LOCAL.mk.in @@ -0,0 +1,69 @@ +# Top-level user specific directories. Note the points below: +# +# - The VALUES to these directories are initially JUST PLACE-HOLDERS! +# Please correct them based on your system. +# +# - The directories don't need to necessarily exist. If they do not exist, +# they will be created and the necessary data will be downloaded into +# them. Ofcourse provided that you have write permissions and an internet +# connection. +# +# - Do not use the tilde expansion `~' or variables for your home +# directory. Please use the full address, for example +# `/home/your-user-name'. +# +# - An ending forward-slash `/' is NOT necessary. In the pipeline, all +# these variables will be followed by a `/', so if you put a `/' at the +# end of the value here, you will see a `//' in the printed outputs +# during the processing. This has no technical problem, but can make +# reading the outputs harder and is thus not recommended. + + + + + +# Input data +# ---------- +# +# This is where the input data (with the same file-name standard as the +# online webpage) are stored. If this directory doesn't exist, or it +# doesn't contain the files (with the correct file-name formats), it will +# be created and the images will be downloaded. See +# `reproduce/config/pipeline/web.mk', for the URLs containing the expected +# inputs for each survey. +SURVEY = reproduce/SURVEY + + + + + +# Build directory +# --------------- +# +# This is where the intermediate outputs of each step are kept. +# +# Why a separate build directory? So the source and configuration files for +# this reproduction pipeline do not get crowded by all the +# intermediate/derivative files. Also to make synchronization and backups +# more easy: the contents of the build directory do not need to be backed +# up since they can be reproduced and they can be large. +BDIR = reproduce/BDIR + + + + + +# Make the final PDF? +# ------------------- +# +# During the testing a pipeline, it is usually not necessary to build +# the PDF file (which makes a lot of output lines on the command-line +# and can make it hard to find the commands and possible errors (and +# their outputs). Also, in some cases, only the produced results may +# be of interest and not the final PDF, so LaTeX (and its necessary +# packages) may not be installed. +# +# If this variable is given any string, a PDF will be made with +# LaTeX. Otherwise, a notice will just printed that for now, no PDF +# will be created. +BUILD-FINAL-PDF = yes diff --git a/reproduce/config/pipeline/pdf.mk b/reproduce/config/pipeline/pdf.mk deleted file mode 100644 index 51ab933..0000000 --- a/reproduce/config/pipeline/pdf.mk +++ /dev/null @@ -1,14 +0,0 @@ -# Make the final PDF? -# ------------------- -# -# During the testing a pipeline, it is usually not necessary to build -# the PDF file (which makes a lot of output lines on the command-line -# and can make it hard to find the commands and possible errors (and -# their outputs). Also, in some cases, only the produced results may -# be of interest and not the final PDF, so LaTeX (and its necessary -# packages) may not be installed. -# -# If this variable is given any string, a PDF will be made with -# LaTeX. Otherwise, a notice will just printed that for now, no PDF -# will be created. -pdf-compile = yes diff --git a/reproduce/config/pipeline/web.mk b/reproduce/config/pipeline/web.mk index f80b886..5af11a7 100644 --- a/reproduce/config/pipeline/web.mk +++ b/reproduce/config/pipeline/web.mk @@ -1,6 +1,6 @@ # Web server(s) hosting the input data for this pipeline. # # This is the web page containing the files that must be located in the -# `SURVEY' directory of `reproduce/config/pipeline/DIRECTORIES.mk' on the -# local system. +# `SURVEY' directory of `reproduce/config/pipeline/LOCAL.mk' on the local +# system. web-survey = https://some.webpage.com/example/server diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 1478881..927c292 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -33,24 +33,67 @@ # parallel. Also, some programs may not be thread-safe, therefore it will # be necessary to put a lock on them. This pipeline uses the `flock' # program to achieve this. -texdir = $(BDIR)/tex -lockdir = $(BDIR)/locks -bdirsym = reproduce/build -mtexdir = $(texdir)/macros -pconfdir = reproduce/config/pipeline +texdir = $(BDIR)/tex +srcdir = reproduce/src +lockdir = $(BDIR)/locks +bdirsym = reproduce/build +mtexdir = $(texdir)/macros +gconfdir = reproduce/config/gnuastro +pconfdir = reproduce/config/pipeline + + + + + +# Sanity check +# ------------ +# +# We need to make sure that the `./configure' command has already been +# run. The output of `./configure' is the `$(pconfdir)/LOCAL.mk' file and +# this is the non-time-stamp prerequisite of $(BDIR), see below. +# +# There is one problem however: if the user hasn't run `./configure' yet, +# then `BDIR' isn't defined (will just evaluate to blank space). Therefore +# it won't appear in the prerequisites and the pipeline will try to build +# the other directories in the top root directory (`/'). To solve this +# problem, when `BDIR' isn't defined, we'll define it with a place-holder +# name ((only so it won't evaluate to blank space). Note that this +# directory will never be built. +ifeq ($(BDIR),) +configure-run = no +BDIR = reproduce/BDIR +else +configure-run = yes +endif +$(pconfdir)/LOCAL.mk: + @echo + @echo "================================================================" + @echo "For the pipeline's local settings, please run this command first" + @echo "(P.S. this local configuration is only necessary one time)" + @echo + @echo " $$ ./configure" + @echo "================================================================" + @echo + @exit 1 # Make the high-level level directories -# ------------------------------ +# ------------------------------------- # # These are just the top-level directories for all the separate steps. The # directories (or possible sub-directories) for individual steps will be # defined and added within their own Makefiles. -$(BDIR):; mkdir $@; +# +# IMPORTANT NOTE for $(BDIR)'s dependency: it only depends on the existance +# (not the time-stamp) of `$(pconfdir)/LOCAL.mk'. So the user can make any +# changes within that file and if they don't affect the pipeline. For +# example a change of the top $(BDIR) name, while the contents are the same +# as before. $(mtexdir): | $(texdir); mkdir $@ +$(BDIR): | $(pconfdir)/LOCAL.mk; mkdir $@ $(texdir) $(lockdir): | $(BDIR); mkdir $@ @@ -65,11 +108,14 @@ $(texdir) $(lockdir): | $(BDIR); mkdir $@ # included here ensure that the file is always built in every run: for # example the pipeline versions may change within two separate runs, so we # want it to be rebuilt every time. -.PHONY: all clean clean-mmap $(mtexdir)/initialize.tex +.PHONY: all clean distclean clean-mmap $(mtexdir)/initialize.tex +distclean: clean; rm -f $(pconfdir)/LOCAL.mk clean-mmap:; rm -f reproduce/config/gnuastro/mmap* clean: - rm -rf $(BDIR) $(bdirsym) *.pdf *.log *.out *.aux *.auxlock \ - reproduce/config/gnuastro/mmap* +ifeq ($(configure-run),yes) + rm -rf $(BDIR) +endif + rm -f $(bdirsym) $(gconfdir)/mmap* *.pdf *.log *.out *.aux *.auxlock @@ -93,7 +139,7 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\gnuastroversion}{$$v}" >> $@ # Location of the build directory (for LaTeX inputs). - echo "\newcommand{\bdir}{$(BDIR)}" >> $@ + @echo "\newcommand{\bdir}{$(BDIR)}" >> $@ -- cgit v1.2.1