aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.gitignore2
-rw-r--r--Makefile9
-rw-r--r--README.md36
-rwxr-xr-xconfigure57
-rw-r--r--reproduce/config/pipeline/LOCAL.mk.in (renamed from reproduce/config/pipeline/DIRECTORIES.mk.in)23
-rw-r--r--reproduce/config/pipeline/pdf.mk14
-rw-r--r--reproduce/config/pipeline/web.mk4
-rw-r--r--reproduce/src/make/initialize.mk68
8 files changed, 135 insertions, 78 deletions
diff --git a/.gitignore b/.gitignore
index cb41e68..84d739b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,4 +11,4 @@ reproduce/build
reproduce/BDIR/
tex/pipeline.tex
reproduce/SURVEY/
-reproduce/config/pipeline/DIRECTORIES.mk \ No newline at end of file
+reproduce/config/pipeline/LOCAL.mk \ No newline at end of file
diff --git a/Makefile b/Makefile
index 357a2e0..051332b 100644
--- a/Makefile
+++ b/Makefile
@@ -85,16 +85,17 @@ include $(foreach f, initialize download paper, reproduce/src/make/$(f).mk)
# try to be remade on every call and `ln' will complain and abort).
#
# Note that if you don't want the final PDF and just want the processing
-# and file outputs, you can remove the value of the `pdf-compile' variable
-# in `reproduce/config/pipeline/pdf.mk'.
+# and file outputs, you can remove the value of the `BUILD-FINAL-PDF'
+# variable in `reproduce/config/LOCAL.mk'.
tex/pipeline.tex: $(foreach f, initialize download, $(mtexdir)/$(f).tex) \
| $(bdirsym)
# If no PDF is requested, then just exit here.
-ifeq ($(pdf-compile),)
+ifeq ($(BUILD-FINAL-PDF),)
@echo;
@echo "Everything is OK until this point, but not building PDF."
- @echo "To do so, give a value to the 'pdf-compile' variable."
+ @echo "To do so, give a value to the 'BUILD-FINAL-PDF' variable."
+ @echo "It is defined in 'reproduce/config/pipeline/LOCAL.mk'."
@echo;
@exit 1
endif
diff --git a/README.md b/README.md
index 5f0d3e4..59ecf71 100644
--- a/README.md
+++ b/README.md
@@ -179,24 +179,24 @@ been explained here), please let us know to correct it.
- **First input dataset**: The user manages the top-level directory of the
input data through the variables set in
- `reproduce/config/pipeline/DIRECTORIES.mk.in` (the user actually edits
- a `DIRECTORIES.mk` file that is created by `configure` from the
- `.mk.in` file, but the `.mk` file is not under version control). So
- open this file and replace `SURVEY` in the variable name and value
- with the name of your input survey or dataset (all in capital
- letters), for example if you are working on data from the XDF survey,
- replace `SURVEY` with `XDF`. Don't change anything else in the value,
- just the the all-caps name. Afterwards, change any occurrence of
- `SURVEY` in the whole pipeline with the new name. You can find the
- occurrences with a simple command like the ones shown below. We follow
- the Make convention here that all `ONLY-CAPITAL` variables are those
- directly set by the user and all `small-caps` variables are set by the
- pipeline designer. All variables that also depend on this survey have
- a `survey` in their name. Hence, also correct all these occurrences to
- your new name in small-caps. Of course, ignore those occurrences that
- are irrelevant, like those in this file. Note that in the raw version
- of this template no target depends on these files, so they are
- ignored. Afterwards, set the webpage and correct the filenames in
+ `reproduce/config/pipeline/LOCAL.mk.in` (the user actually edits a
+ `LOCAL.mk` file that is created by `configure` from the `.mk.in` file,
+ but the `.mk` file is not under version control). So open this file
+ and replace `SURVEY` in the variable name and value with the name of
+ your input survey or dataset (all in capital letters), for example if
+ you are working on data from the XDF survey, replace `SURVEY` with
+ `XDF`. Don't change anything else in the value, just the the all-caps
+ name. Afterwards, change any occurrence of `SURVEY` in the whole
+ pipeline with the new name. You can find the occurrences with a simple
+ command like the ones shown below. We follow the Make convention here
+ that all `ONLY-CAPITAL` variables are those directly set by the user
+ and all `small-caps` variables are set by the pipeline designer. All
+ variables that also depend on this survey have a `survey` in their
+ name. Hence, also correct all these occurrences to your new name in
+ small-caps. Of course, ignore those occurrences that are irrelevant,
+ like those in this file. Note that in the raw version of this template
+ no target depends on these files, so they are ignored. Afterwards, set
+ the webpage and correct the filenames in
`reproduce/src/make/download.mk` if necessary.
```shell
diff --git a/configure b/configure
index be18c25..8ecb8fa 100755
--- a/configure
+++ b/configure
@@ -28,43 +28,37 @@
pdir=reproduce/config/pipeline
-# Message to print for editing
-function msg {
- echo; echo "Top-level reproduction directories are set.";
- echo "Please run the following command to start the pipeline:"
- echo "(Replace '8' with the number of CPU threads available)"
- echo; echo " make -j8"
- echo;
-}
-# If `DIRECTORIES.mk' is already created, then ignore this step.
-if [ -f $pdir/DIRECTORIES.mk ]; then
+
+# If `LOCAL.mk' is already created, then ignore this step.
+if [ -f $pdir/LOCAL.mk ]; then
echo
- echo "$pdir/DIRECTORIES.mk already exists."
+ echo "$pdir/LOCAL.mk already exists."
echo "To change/correct the top-level directories, please remove/edit it manually."
echo
else
# Copy the base file to the desired output file.
- if cp $pdir/DIRECTORIES.mk.in $pdir/DIRECTORIES.mk; then
+ if cp $pdir/LOCAL.mk.in $pdir/LOCAL.mk; then
# Tell the user to edit the directories.
while [ "$userread" != "y" -a "$userread" != "n" ]
do
- echo "Top-level directories..."
echo
- echo "These directories define the input(s) location and the"
- echo "directory to host the intermediate/processing files."
+ echo "------------------------------------"
+ echo "Reproduction pipeline local configuration"
+ echo "-----------------------------------------"
echo
- echo "To help in your ability to read and manage this pipeline,"
- echo "it is recommended (but not mandatory) to change them to"
- echo "a directory outside this reproduction pipeline."
+ echo "Local settings include things like top-level directories,"
+ echo "or processing steps (e.g., if you want a final PDF output)."
echo
- echo "More descriptions are provided within the file that is"
- echo "opened if you choose to edit the directories."
+ echo "Pressing 'y' will open the local settings file in an editor"
+ echo "so you can modify the default values if you want. Each"
+ echo "variable is also thoroughly described in the comments (lines"
+ echo "starting with a '#') above it."
echo
- read -p"Edit the default top-level directories (y/n)? " userread
+ read -p"Edit the default local configuration (y/n)? " userread
done
# Only continue if the user wants to edit the top level
@@ -72,17 +66,28 @@ else
if [ $userread = "y" ]; then
# Open a text editor to set the given directories
- if emacs $pdir/DIRECTORIES.mk; then msg
- elif gedit $pdir/DIRECTORIES.mk; then msg
- elif vi $pdir/DIRECTORIES.mk; then msg
+ if emacs $pdir/LOCAL.mk; then junk=1
+ elif gedit $pdir/LOCAL.mk; then junk=1
+ elif vi $pdir/LOCAL.mk; then junk=1
else
echo
echo "No common text editor found on your system."
- echo "Please set the values in '$pdir/DIRECTORIES.mk' manually."
+ echo "Please set the values in '$pdir/LOCAL.mk' manually."
echo
fi
fi
+ echo
+ echo "This reproduction pipeline has been configured for this system."
+ echo "Please run the following command to start the pipeline:"
+ echo "(Replace '8' with the number of CPU threads available)"
+ echo
+ echo " make -j8"
+ echo
+ echo
+ echo "(you can always check/modify the default local settings"
+ echo " by editing this file: '$pdir/LOCAL.mk')"
+ echo
else
- echo; echo "Couldn't create $pdir/DIRECTORIES.mk"
+ echo; echo "Couldn't create $pdir/LOCAL.mk"
fi
fi
diff --git a/reproduce/config/pipeline/DIRECTORIES.mk.in b/reproduce/config/pipeline/LOCAL.mk.in
index 9ebd67b..e5a9f96 100644
--- a/reproduce/config/pipeline/DIRECTORIES.mk.in
+++ b/reproduce/config/pipeline/LOCAL.mk.in
@@ -22,8 +22,8 @@
-# Input data directories
-# ----------------------
+# Input data
+# ----------
#
# This is where the input data (with the same file-name standard as the
# online webpage) are stored. If this directory doesn't exist, or it
@@ -48,3 +48,22 @@ SURVEY = reproduce/SURVEY
# more easy: the contents of the build directory do not need to be backed
# up since they can be reproduced and they can be large.
BDIR = reproduce/BDIR
+
+
+
+
+
+# Make the final PDF?
+# -------------------
+#
+# During the testing a pipeline, it is usually not necessary to build
+# the PDF file (which makes a lot of output lines on the command-line
+# and can make it hard to find the commands and possible errors (and
+# their outputs). Also, in some cases, only the produced results may
+# be of interest and not the final PDF, so LaTeX (and its necessary
+# packages) may not be installed.
+#
+# If this variable is given any string, a PDF will be made with
+# LaTeX. Otherwise, a notice will just printed that for now, no PDF
+# will be created.
+BUILD-FINAL-PDF = yes
diff --git a/reproduce/config/pipeline/pdf.mk b/reproduce/config/pipeline/pdf.mk
deleted file mode 100644
index 51ab933..0000000
--- a/reproduce/config/pipeline/pdf.mk
+++ /dev/null
@@ -1,14 +0,0 @@
-# Make the final PDF?
-# -------------------
-#
-# During the testing a pipeline, it is usually not necessary to build
-# the PDF file (which makes a lot of output lines on the command-line
-# and can make it hard to find the commands and possible errors (and
-# their outputs). Also, in some cases, only the produced results may
-# be of interest and not the final PDF, so LaTeX (and its necessary
-# packages) may not be installed.
-#
-# If this variable is given any string, a PDF will be made with
-# LaTeX. Otherwise, a notice will just printed that for now, no PDF
-# will be created.
-pdf-compile = yes
diff --git a/reproduce/config/pipeline/web.mk b/reproduce/config/pipeline/web.mk
index f80b886..5af11a7 100644
--- a/reproduce/config/pipeline/web.mk
+++ b/reproduce/config/pipeline/web.mk
@@ -1,6 +1,6 @@
# Web server(s) hosting the input data for this pipeline.
#
# This is the web page containing the files that must be located in the
-# `SURVEY' directory of `reproduce/config/pipeline/DIRECTORIES.mk' on the
-# local system.
+# `SURVEY' directory of `reproduce/config/pipeline/LOCAL.mk' on the local
+# system.
web-survey = https://some.webpage.com/example/server
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index 1478881..927c292 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -33,24 +33,67 @@
# parallel. Also, some programs may not be thread-safe, therefore it will
# be necessary to put a lock on them. This pipeline uses the `flock'
# program to achieve this.
-texdir = $(BDIR)/tex
-lockdir = $(BDIR)/locks
-bdirsym = reproduce/build
-mtexdir = $(texdir)/macros
-pconfdir = reproduce/config/pipeline
+texdir = $(BDIR)/tex
+srcdir = reproduce/src
+lockdir = $(BDIR)/locks
+bdirsym = reproduce/build
+mtexdir = $(texdir)/macros
+gconfdir = reproduce/config/gnuastro
+pconfdir = reproduce/config/pipeline
+
+
+
+
+
+# Sanity check
+# ------------
+#
+# We need to make sure that the `./configure' command has already been
+# run. The output of `./configure' is the `$(pconfdir)/LOCAL.mk' file and
+# this is the non-time-stamp prerequisite of $(BDIR), see below.
+#
+# There is one problem however: if the user hasn't run `./configure' yet,
+# then `BDIR' isn't defined (will just evaluate to blank space). Therefore
+# it won't appear in the prerequisites and the pipeline will try to build
+# the other directories in the top root directory (`/'). To solve this
+# problem, when `BDIR' isn't defined, we'll define it with a place-holder
+# name ((only so it won't evaluate to blank space). Note that this
+# directory will never be built.
+ifeq ($(BDIR),)
+configure-run = no
+BDIR = reproduce/BDIR
+else
+configure-run = yes
+endif
+$(pconfdir)/LOCAL.mk:
+ @echo
+ @echo "================================================================"
+ @echo "For the pipeline's local settings, please run this command first"
+ @echo "(P.S. this local configuration is only necessary one time)"
+ @echo
+ @echo " $$ ./configure"
+ @echo "================================================================"
+ @echo
+ @exit 1
# Make the high-level level directories
-# ------------------------------
+# -------------------------------------
#
# These are just the top-level directories for all the separate steps. The
# directories (or possible sub-directories) for individual steps will be
# defined and added within their own Makefiles.
-$(BDIR):; mkdir $@;
+#
+# IMPORTANT NOTE for $(BDIR)'s dependency: it only depends on the existance
+# (not the time-stamp) of `$(pconfdir)/LOCAL.mk'. So the user can make any
+# changes within that file and if they don't affect the pipeline. For
+# example a change of the top $(BDIR) name, while the contents are the same
+# as before.
$(mtexdir): | $(texdir); mkdir $@
+$(BDIR): | $(pconfdir)/LOCAL.mk; mkdir $@
$(texdir) $(lockdir): | $(BDIR); mkdir $@
@@ -65,11 +108,14 @@ $(texdir) $(lockdir): | $(BDIR); mkdir $@
# included here ensure that the file is always built in every run: for
# example the pipeline versions may change within two separate runs, so we
# want it to be rebuilt every time.
-.PHONY: all clean clean-mmap $(mtexdir)/initialize.tex
+.PHONY: all clean distclean clean-mmap $(mtexdir)/initialize.tex
+distclean: clean; rm -f $(pconfdir)/LOCAL.mk
clean-mmap:; rm -f reproduce/config/gnuastro/mmap*
clean:
- rm -rf $(BDIR) $(bdirsym) *.pdf *.log *.out *.aux *.auxlock \
- reproduce/config/gnuastro/mmap*
+ifeq ($(configure-run),yes)
+ rm -rf $(BDIR)
+endif
+ rm -f $(bdirsym) $(gconfdir)/mmap* *.pdf *.log *.out *.aux *.auxlock
@@ -93,7 +139,7 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
echo "\newcommand{\gnuastroversion}{$$v}" >> $@
# Location of the build directory (for LaTeX inputs).
- echo "\newcommand{\bdir}{$(BDIR)}" >> $@
+ @echo "\newcommand{\bdir}{$(BDIR)}" >> $@