First commit to the reproduction pipeline template

Let's start working on this pipeline independently with this first
commit. It is based on my previous experiences, but I had never made a
skeleton of a pipeline before, it was always within a working analysis.

But now that the pipeline has a separate repository for its self, we will
be able to work on it and use it as a base for future work and modify it to
make it even better. Hopefully in time (and with the help of others), it
will grow and become much more robust and useful.

3 files changed, 216 insertions, 0 deletions

diff --git a/reproduce/src/make/download.mk b/reproduce/src/make/download.mk
new file mode 100644
index 0000000..244bd04
--- /dev/null
+++ b/reproduce/src/make/download.mk
@@ -0,0 +1,68 @@
+# Download all the necessary inputs if they are not already present.
+#
+# Since most systems only have one input/connection into the network,
+# downloading is essentially a serial (not parallel) operation. so the
+# recipes in this Makefile all use a single file lock to have one download
+# script running at every instant.
+#
+# Original author:
+#     Your name <your@email.address>
+# Contributing author(s):
+# Copyright (C) YYYY, Your Name.
+#
+# This Makefile is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This Makefile is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details. See <http://www.gnu.org/licenses/>.
+
+
+
+
+
+# Identify the downloader tool
+# ----------------------------
+#
+# If cURL is already present, that will be used, otherwise, we'll use
+# Wget. Since the options specifying the output filename are different
+# between the two, we'll also specify the output option within the
+# `downloader' variable. So it is important to first give the output
+# filename after calling `downloader', then the web address.
+downloader := $(shell if type curl > /dev/null; then downloader="curl -o";  \
+	              else                           downloader="wget -O";  \
+	              fi; echo "$$downloader";                               )
+
+
+
+
+
+# Download SURVEY data
+# --------------------
+#
+# Data from a survey (for example an imaging survey) usually have a special
+# file-name format which should be set here in the `foreach' loop. Note
+# that the `foreach' function needs the backslash (`\') at the end of the
+# line when it is broken into multiple lines.
+all-survey = $(foreach f, $(filters-survey),                                 \
+                          $(SURVEY)/a-special-format-$(f).fits               \
+                          $(SURVEY)/a-possibly-additional-$(f)-format.fits )
+$(SURVEY):; mkdir $@
+$(all-survey): $(SURVEY)/%: | $(SURVEY) $(lockdir)
+	flock $(lockdir)/download -c "$(downloader) $@ $(web-survey)/$*"
+
+
+
+
+
+
+# Final TeX macro
+# ---------------
+#
+# It is very important to mention the address where the data were
+# downloaded in the final report.
+$(mtexdir)/download.tex: $(pconfdir)/web.mk | $(mtexdir)
+	@echo "\\newcommand{\\websurvey}{$(web-survey)}" > $@
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
new file mode 100644
index 0000000..2da5e79
--- /dev/null
+++ b/reproduce/src/make/initialize.mk
@@ -0,0 +1,114 @@
+# Initialize the reproduction pipeline.
+#
+# Original author:
+#     Your name <your@email.address>
+# Contributing author(s):
+# Copyright (C) YYYY, Your Name.
+#
+# This Makefile is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This Makefile is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.
+#
+# A copy of the GNU General Public License is available at
+# <http://www.gnu.org/licenses/>.
+
+
+
+
+
+# High-level directory definitions
+# --------------------------------
+#
+# Basic directories that are used throughout the whole pipeline.
+#
+# Locks are used to make sure that an operation is done in series not in
+# parallel (even if Make is run in parallel with the `-j' option). The most
+# common case is downloads which are better done in series and not in
+# parallel. Also, some programs may not be thread-safe, therefore it will
+# be necessary to put a lock on them. This pipeline uses the `flock'
+# program to achieve this.
+texdir = $(BDIR)/tex
+lockdir = $(BDIR)/locks
+bdirsym = reproduce/build
+mtexdir = $(texdir)/macros
+pconfdir = reproduce/config/pipeline
+
+
+
+
+
+# Make the high-level level directories
+# ------------------------------
+#
+# These are just the top-level directories for all the separate steps. The
+# directories (or possible sub-directories) for individual steps will be
+# defined and added within their own Makefiles.
+$(BDIR):; mkdir $@;
+$(mtexdir): | $(texdir); mkdir $@
+$(texdir) $(lockdir): | $(BDIR); mkdir $@
+
+
+
+
+
+# High-level Makefile management
+# ------------------------------
+#
+# About `.PHONY': these are targets that must be built even if a file with
+# their name exists. Most don't correspond to a file, but those that do are
+# included here ensure that the file is always built in every run: for
+# example the pipeline versions may change within two separate runs, so we
+# want it to be rebuilt every time.
+.PHONY: all clean clean-mmap $(texdir)/versions.tex
+clean-mmap:; rm -f reproduce/config/gnuastro/mmap*
+clean:
+	rm -rf $(BDIR) $(bdirsym) *.pdf *.log *.out *.aux *.auxlock \
+               reproduce/config/gnuastro/mmap*
+
+
+
+
+
+# Pipeline version
+# ----------------
+#
+# The pipeline's version is necessary for the analysis and must be
+# calculated everytime the pipeline is run, so even though this file
+# actually exists, it is also aded as a `.PHONY' target above.
+$(mtexdir)/initialize.tex: | $(mtexdir)
+
+	@v=$$(git describe --dirty --always);                      \
+	echo "\newcommand{\pipelineversion}{$$v}"  > $@
+
+	@v=$$(astnoisechisel --version | awk 'NR==1{print $$NF}'); \
+	echo "\newcommand{\gnuastroversion}{$$v}" >> $@
+
+	echo "\newcommand{\bdir}{$(BDIR)}" >> $@
+
+
+
+
+
+# Symbolic link to build directory
+# --------------------------------
+#
+# Besides $(BDIR), we are also making a symbolic link to it if $(bdirsym)
+# is not empty. In case this symbolic link is not needed, simply remove its
+# value from the definitions above. In that case, it will be read as a
+# blank (non-existant).
+#
+# Note that $(BDIR) might not be an absolute path and this will complicate
+# the symbolic link creation. To be generic, we'll first call `readlink' to
+# make sure we have an absolute address, then we'll make a symbolic link to
+# that.
+ifneq ($(bdirsym),)
+$(bdirsym): | $(BDIR)
+	absbdir=$$(readlink -f $(BDIR));                        \
+	ln -s $$absbdir $(bdirsym)
+endif
diff --git a/reproduce/src/make/paper.mk b/reproduce/src/make/paper.mk
new file mode 100644
index 0000000..0725ec8
--- /dev/null
+++ b/reproduce/src/make/paper.mk
@@ -0,0 +1,34 @@
+# Build the final PDF paper/report.
+#
+# Original author:
+#     Your name <your@email.address>
+# Contributing author(s):
+# Copyright (C) YYYY, Your Name.
+#
+# This script is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This script is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.
+#
+# A copy of the GNU General Public License is available at
+# <http://www.gnu.org/licenses/>.
+
+
+
+
+
+# The final paper
+# ---------------
+#
+# The commands to build the final report. We want the pipeline version to
+# be checked everytime the final PDF is to be built.
+paper.pdf: tex/pipeline.tex paper.tex
+
+        # Make the report.
+	@pdflatex -shell-escape -halt-on-error paper.tex
+	@rm -f *.auxlock *.aux *.out *.log