aboutsummaryrefslogtreecommitdiff
path: root/reproduce/src/make
diff options
context:
space:
mode:
Diffstat (limited to 'reproduce/src/make')
-rw-r--r--reproduce/src/make/Top-Makefile108
-rw-r--r--reproduce/src/make/delete-me.mk12
-rw-r--r--reproduce/src/make/dependencies.mk254
-rw-r--r--reproduce/src/make/initialize.mk84
-rw-r--r--reproduce/src/make/paper.mk22
5 files changed, 407 insertions, 73 deletions
diff --git a/reproduce/src/make/Top-Makefile b/reproduce/src/make/Top-Makefile
new file mode 100644
index 0000000..5d94766
--- /dev/null
+++ b/reproduce/src/make/Top-Makefile
@@ -0,0 +1,108 @@
+# A ONE-LINE DESCRIPTION OF THE WHOLE PIPELINE
+#
+# Original author:
+# Mohammad Akhlaghi <mohammad@akhlaghi.org>
+# Contributing author(s):
+# Your name <your@email.address>
+# Copyright (C) 2018, Your Name.
+#
+# This Makefile is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This Makefile is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# A copy of the GNU General Public License is available at
+# <http://www.gnu.org/licenses/>.
+
+
+
+
+
+# Ultimate target of this pipeline
+# --------------------------------
+#
+# The final paper (in PDF format) is the main target of this whole
+# reproduction pipeline. So as defined in the Make paradigm, we are
+# defining it here.
+#
+# Note that if you don't have LaTeX to build the PDF, or generally are just
+# interested in the processing, you can skip create the final PDF creation
+# with `pdf-build-final' of `reproduce/config/pipeline/pdf-build.mk'.
+all: paper.pdf
+
+
+
+
+
+# Include specific Makefiles
+# --------------------------
+#
+# To keep things clean, managable and readable, each set of operations is
+# (and must be) classified (modularized) by context into separate
+# Makefiles: the more the better. They are included in this top-level
+# Makefile through the command below.
+#
+# To further help in readability, it is best to avoid including Makefiles
+# within any other Makefile. So in short, it is best that the `foreach'
+# loop below contains all the `reproduce/src/make/*.mk' files.
+#
+# IMPORTANT NOTE: order matters in the inclusion of the processing
+# Makefiles. As the pipeline grows, some Makefiles will probably define
+# variables/dependencies that others need. Therefore unlike the
+# `reproduce/config/pipeline/*.mk' Makefiles which only define low-level
+# variables (not dependent on other variables and contain no rules), the
+# high-level processing Makefiles are included through the `foreach' loop
+# below by explicitly requesting them in a specific order here.
+include reproduce/config/pipeline/*.mk
+include $(foreach f, initialize \
+ download \
+ delete-me \
+ paper \
+ , reproduce/src/make/$(f).mk)
+
+
+
+
+
+# LaTeX macros for paper
+# ----------------------
+#
+# The final report's PDF (final target of this reproduction pipeline) takes
+# variable strings from the pipeline. Those variables are defined as LaTeX
+# macros in `tex/pipeline.tex'. This file is thus the interface between the
+# pipeline scripts and the final PDF.
+#
+# Each of the pipeline steps will save their macros into their own `.tex'
+# file in the `$(mtexdir)' directory. Those individual macros are the
+# pre-requisite to `tex/pipeline.txt'. `tex/pipeline.tex' is thus a
+# high-level output and is defined in this top-most Makefile (and not
+# `reproduce/src/make/paper.mk'). This enables a clear demonstration of the
+# top-level dependencies clearly.
+#
+# Note that if you don't want the final PDF and just want the processing
+# and file outputs, you can remove the value of the `pdf-build-final'
+# variable in `reproduce/config/pdf-build.mk'.
+tex/pipeline.tex: $(foreach f, initialize \
+ download \
+ delete-me \
+ , $(mtexdir)/$(f).tex)
+
+ # If no PDF is requested, then just exit here.
+ifeq ($(pdf-build-final),)
+ @echo
+ @echo
+ @echo "-----"
+ @echo "Everything is OK until this point, but not building PDF."
+ @echo "To do so, give a value to the 'pdf-build-final' variable."
+ @echo "It is defined in 'reproduce/config/pipeline/pdf-build.mk'."
+ @echo
+ @exit 1
+endif
+
+ # Merge all the TeX macros that are prepared for building the PDF.
+ @cat $(mtexdir)/*.tex > $@
diff --git a/reproduce/src/make/delete-me.mk b/reproduce/src/make/delete-me.mk
index de72873..67f0440 100644
--- a/reproduce/src/make/delete-me.mk
+++ b/reproduce/src/make/delete-me.mk
@@ -63,10 +63,10 @@ $(mtexdir)/delete-me.tex: $(dm)
# Here, we are first using AWK to find the minimum and maximum
# values, then using it again to read each separately to use in the
# macro definition.
- mm=$$(awk 'BEGIN{min=99999; max=-min} \
- {if($$2>max) max=$$2; if($$2<min) min=$$2;} \
- END{print min, max}' $(dm)); \
- v=$$(echo "$$mm" | awk '{printf "%.3f", $$1}'); \
- echo "\newcommand{\deletememin}{$$v}" >> $@; \
- v=$$(echo "$$mm" | awk '{printf "%.3f", $$2}'); \
+ mm=$$(awk 'BEGIN{min=99999; max=-min}
+ {if($$2>max) max=$$2; if($$2<min) min=$$2;}
+ END{print min, max}' $(dm));
+ v=$$(echo "$$mm" | awk '{printf "%.3f", $$1}');
+ echo "\newcommand{\deletememin}{$$v}" >> $@;
+ v=$$(echo "$$mm" | awk '{printf "%.3f", $$2}');
echo "\newcommand{\deletememax}{$$v}" >> $@
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
new file mode 100644
index 0000000..0fb5a34
--- /dev/null
+++ b/reproduce/src/make/dependencies.mk
@@ -0,0 +1,254 @@
+# Build the reproduction pipeline dependencies (programs and libraries).
+#
+# ------------------------------------------------------------------------
+# !!!!! IMPORTANT NOTES !!!!!
+#
+# This Makefile will be run by the initial `./configure' script. It is not
+# included into the reproduction pipe after that.
+#
+# This Makefile also builds GNU Bash and GNU Make. Therefore this is the
+# only Makefile in the reproduction pipeline where you MUST NOT assume that
+# GNU Bash or GNU Make are to be used.
+#
+# ------------------------------------------------------------------------
+#
+# Original author:
+# Mohammad Akhlaghi <mohammad@akhlaghi.org>
+# Contributing author(s):
+# Your name <your@email.address>
+# Copyright (C) 2018, Your Name.
+#
+# This Makefile is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This Makefile is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# A copy of the GNU General Public License is available at
+# <http://www.gnu.org/licenses/>.
+
+
+
+# Top level environment
+include reproduce/config/pipeline/LOCAL.mk
+include reproduce/config/pipeline/dependency-versions.mk
+
+ddir = $(BDIR)/dependencies
+tdir = $(BDIR)/dependencies/tarballs
+idir = $(BDIR)/dependencies/installed
+ibdir = $(BDIR)/dependencies/installed/bin
+ildir = $(BDIR)/dependencies/installed/lib
+
+# Define the top-level programs to build (installed in `.local/bin', so for
+# Coreutils, only one of its executables is enough).
+top-level-programs = ls gawk gs grep libtool sed astnoisechisel
+all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
+
+# This Makefile will be called to also build Bash locally. So when we don't
+# have it yet, we'll have to use the system's bash.
+ifeq ($(USE_LOCAL_BASH),yes)
+SHELL := $(ibdir)/bash
+else
+SHELL := /bin/sh
+endif
+
+# Other basic environment settings.
+.ONESHELL:
+.SHELLFLAGS = -ec
+PATH := $(ibdir):$(PATH)
+LDFLAGS := -L$(ildir) $(LDFLAGS)
+CPPFLAGS := -I$(idir)/include $(CPPFLAGS)
+LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH)
+
+
+
+
+
+# Tarballs
+# --------
+#
+# All the necessary tarballs are defined and prepared with this rule.
+tarballs = $(foreach t, bash-$(bash-version).tar.gz \
+ cfitsio$(cfitsio-version).tar.gz \
+ coreutils-$(coreutils-version).tar.xz \
+ gawk-$(gawk-version).tar.gz \
+ ghostscript-$(ghostscript-version).tar.gz \
+ gnuastro-$(gnuastro-version).tar.gz \
+ grep-$(grep-version).tar.xz \
+ gsl-$(gsl-version).tar.gz \
+ jpegsrc.$(libjpeg-version).tar.gz \
+ libtool-$(libtool-version).tar.gz \
+ libgit2-$(libgit2-version).tar.gz \
+ sed-$(sed-version).tar.xz \
+ make-$(make-version).tar.gz \
+ wcslib-$(wcslib-version).tar.bz2 \
+ , $(tdir)/$(t) )
+$(tarballs): $(tdir)/%:
+ if [ -f $(DEPENDENCIES-DIR)/$* ]; then
+ cp $(DEPENDENCIES-DIR)/$* $@
+ else
+ # Remove all numbers, `-' and `.' from the tarball name so we can
+ # search more easily only with the program name.
+ n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \
+ | awk '{print $$1}' )
+
+ # Set the top download link of the requested tarball.
+ if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash
+ elif [ $$n = cfitsio ]; then w=WWWWWWWWWWWWWWWW
+ elif [ $$n = coreutils ]; then w=WWWWWWWWWWWWWWWW
+ elif [ $$n = gawk ]; then w=WWWWWWWWWWWWWWWW
+ elif [ $$n = ghostscript ]; then w=WWWWWWWWWWWWWWWW
+ elif [ $$n = gnuastro ]; then w=http://akhlaghi.org
+ elif [ $$n = grep ]; then w=WWWWWWWWWWWWWWWW
+ elif [ $$n = gsl ]; then w=WWWWWWWWWWWWWWWW
+ elif [ $$n = jpegsrc ]; then w=WWWWWWWWWWWWWWWW
+ elif [ $$n = libtool ]; then w=WWWWWWWWWWWWWWWW
+ elif [ $$n = libgit ]; then w=WWWWWWWWWWWWWWWW
+ elif [ $$n = sed ]; then w=WWWWWWWWWWWWWWWW
+ elif [ $$n = make ]; then w=http://akhlaghi.org
+ elif [ $$n = wcslib ]; then w=WWWWWWWWWWWWWWWW
+ else
+ echo; echo; echo;
+ echo "'$$n' not recognized as a dependency name to download."
+ echo; echo; echo;
+ exit 1
+ fi
+
+ # Download the requested tarball.
+ $(DOWNLOADER) $@ $$w/$*
+ fi
+
+
+
+
+
+# Customized build
+# ----------------
+#
+# Programs that need some customization on their build.
+# For CFITSIO we'll need to intervene manually to remove the check on
+# libcurl (which can be real trouble in this controlled environment).
+$(ildir)/libcfitsio.a: $(ibdir)/ls \
+ $(tdir)/cfitsio$(cfitsio-version).tar.gz
+ # Same as before
+ cd $(ddir)
+ tar xf $(tdir)/cfitsio$(cfitsio-version).tar.gz
+ cd cfitsio
+
+ # Remove the part that checks for the CURL library, so it assumes
+ # that the CURL library wasn't found.
+ awk 'NR<4785 || NR>4847' configure > new_configure
+ mv new_configure configure
+ chmod +x configure
+
+ # Do the standard configuring and building
+ ./configure CFLAGS=--static --disable-shared --prefix=$(idir)
+ make; make install;
+ cd ..; rm -rf cfitsio
+
+
+# Why not shared: Gnuastro's configure can't link with it in static mode.
+$(ildir)/libgit2.a: $(tdir)/libgit2-$(libgit2-version).tar.gz
+ cd $(ddir)
+ tar xf $(tdir)/libgit2-$(libgit2-version).tar.gz
+ cd libgit2-$(libgit2-version)
+ mkdir build
+ cd build
+ export CFLAGS="--static $$CFLAGS"
+ cmake .. -DUSE_SSH=OFF -DUSE_OPENSSL=OFF -DBUILD_SHARED_LIBS=OFF \
+ -DBUILD_CLAR=OFF -DTHREADSAFE=ON
+ cmake --build .
+ cmake .. -DCMAKE_INSTALL_PREFIX=$(idir)
+ cmake --build . --target install
+ cd ../..
+ rm -rf libgit2-$(libgit2-version)
+
+
+
+
+
+# GNU Build system programs
+# -------------------------
+#
+# Programs that use the basic GNU build system.
+gbuild = cd $(ddir); tar xf $(tdir)/$(1); cd $(2); \
+ if [ $(3)x = staticx ]; then \
+ opts="CFLAGS=--static --disable-shared"; \
+ fi; \
+ ./configure $$opts $(4) --prefix=$(idir); make $(5); \
+ check="$(6)"; if [ x"$$check" != x ]; then $$check; fi; \
+ make install; cd ..; rm -rf $(2)
+
+$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz
+ $(call gbuild,$(subst $(tdir),,$<), bash-$(bash-version), static)
+
+
+# Unfortunately GNU Make needs dynamic linking in two instances: when
+# loading objects (dynamically linked libraries), or when using the
+# `getpwnam' function (for tilde expansion). The first can be disabled with
+# `--disable-load', but unfortunately I don't know any way to fix the
+# second. So, we'll have to build it dynamically for now.
+$(ibdir)/make: $(tdir)/make-$(make-version).tar.gz
+ $(call gbuild,$(subst $(tdir),,$<), make-$(make-version))
+
+
+$(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz
+ $(call gbuild,$(subst $(tdir),,$<), coreutils-$(coreutils-version), \
+ static)
+
+
+$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.gz \
+ $(ibdir)/ls
+ $(call gbuild,$(subst $(tdir),,$<), gawk-$(gawk-version), static)
+
+
+$(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \
+ $(ibdir)/ls
+ $(call gbuild,$(subst $(tdir),,$<), sed-$(sed-version), static)
+
+
+$(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \
+ $(ibdir)/ls
+ $(call gbuild,$(subst $(tdir),,$<), grep-$(grep-version), static)
+
+
+$(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.gz \
+ $(ibdir)/ls
+ $(call gbuild,$(subst $(tdir),,$<), libtool-$(libtool-version), static)
+
+
+$(ildir)/libgsl.a: $(tdir)/gsl-$(gsl-version).tar.gz \
+ $(ibdir)/ls
+ $(call gbuild,$(subst $(tdir),,$<), gsl-$(gsl-version), static)
+
+
+$(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
+ $(ildir)/libcfitsio.a
+ $(call gbuild,$(subst $(tdir),,$<), wcslib-$(wcslib-version), , \
+ LIBS="-pthread -lcurl -lm" --without-pgplot \
+ --disable-fortran)
+
+
+$(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz \
+ $(ibdir)/ls
+ $(call gbuild,$(subst $(tdir),,$<), ghostscript-$(ghostscript-version))
+
+
+$(ildir)/libjpeg.a: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz
+ $(call gbuild,$(subst $(tdir),,$<), jpeg-9b, static)
+
+
+$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.gz \
+ $(ildir)/libgsl.a \
+ $(ildir)/libcfitsio.a \
+ $(ildir)/libwcs.a \
+ $(ibdir)/gs \
+ $(ildir)/libjpeg.a \
+ $(ildir)/libgit2.a \
+
+ $(call gbuild,$(subst $(tdir),,$<), gnuastro-$(gnuastro-version), \
+ static, , -j8, make check -j8)
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index f615e22..165db78 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -47,36 +47,24 @@ pconfdir = reproduce/config/pipeline
-# Sanity check
-# ------------
+# High level environment
+# ----------------------
#
-# We need to make sure that the `./configure' command has already been
-# run. The output of `./configure' is the `$(pconfdir)/LOCAL.mk' file and
-# this is the non-time-stamp prerequisite of $(BDIR), see below.
+# We want the full recipe to be executed in one call to the shell. Also we
+# want Make to run the specific version of Bash that we have installed
+# during `./configure' time.
#
-# There is one problem however: if the user hasn't run `./configure' yet,
-# then `BDIR' isn't defined (will just evaluate to blank space). Therefore
-# it won't appear in the prerequisites and the pipeline will try to build
-# the other directories in the top root directory (`/'). To solve this
-# problem, when `BDIR' isn't defined, we'll define it with a place-holder
-# name (only so it won't evaluate to blank space). Note that this
-# directory will never be built.
-ifeq ($(BDIR),)
-configure-run = no
-BDIR = reproduce/BDIR
-else
-configure-run = yes
-endif
-$(pconfdir)/LOCAL.mk:
- @echo
- @echo "================================================================"
- @echo "For the pipeline's local settings, please run this command first"
- @echo "(P.S. this local configuration is only necessary one time)"
- @echo
- @echo " $$ ./configure"
- @echo "================================================================"
- @echo
- @exit 1
+# Regarding the directories, this pipeline builds its major dependencies
+# itself and doesn't use the local system's default tools. With these
+# environment variables, we are setting it to prefer the software we have
+# build here.
+.ONESHELL:
+.SHELLFLAGS = -ec
+SHELL := .local/bin/bash
+PATH := .local/bin:$(PATH)
+LDFLAGS := -L.local/lib $(LDFLAGS)
+CPPFLAGS := -I.local/include $(CPPFLAGS)
+LD_LIBRARY_PATH := .local/lib:$(LD_LIBRARY_PATH)
@@ -103,52 +91,32 @@ $(pconfdir)/LOCAL.mk:
# are looking for in this pipeline.
.SUFFIXES:
$(tikzdir): | $(texbdir); mkdir $@
-$(BDIR): | $(pconfdir)/LOCAL.mk; mkdir $@
$(texdir) $(lockdir): | $(BDIR); mkdir $@
$(mtexdir) $(texbdir): | $(texdir); mkdir $@
-# Symbolic link to build directory
-# --------------------------------
-#
-# Besides $(BDIR), we are also making a symbolic link to it for easy
-# access. Recall that it is recommended that the actual build directory be
-# in a completely separate part of the file system (a place that may easily
-# be completely deleted).
-#
-# Note that $(BDIR) might not be an absolute path and this will complicate
-# the symbolic link creation. To be generic, we'll first call `readlink' to
-# make sure we have an absolute address, then we'll make a symbolic link to
-# that.
-reproduce/build: | $(BDIR)
- absbdir=$$(readlink -f $(BDIR)); \
- ln -s $$absbdir $@
-
-
-
-
# High-level Makefile management
# ------------------------------
#
# About `.PHONY': these are targets that must be built even if a file with
-# their name exists. Most don't correspond to a file, but those that do are
-# included here ensure that the file is always built in every run: for
-# example the pipeline versions may change within two separate runs, so we
-# want it to be rebuilt every time.
+# their name exists.
+#
+# Only `$(mtexdir)/initialize.tex' corresponds to a file. This is because
+# we want to ensure that the file is always built in every run: it contains
+# the pipeline version which may change between two separate runs, even
+# when no file actually differs.
.PHONY: all clean distclean clean-mmap $(mtexdir)/initialize.tex
-distclean: clean; rm -f $(pconfdir)/LOCAL.mk
# --------- Delete for no Gnuastro ---------
clean-mmap:; rm -f reproduce/config/gnuastro/mmap*
# ------------------------------------------
clean: clean-mmap
-ifeq ($(configure-run),yes)
rm -rf $(BDIR)
-endif
rm -f reproduce/build *.pdf *.log *.out *.aux *.auxlock
-
+distclean: clean
+ rm -f Makefile $(pconfdir)/LOCAL.mk .gnuastro
@@ -163,12 +131,12 @@ endif
$(mtexdir)/initialize.tex: | $(mtexdir)
# Version of the pipeline.
- @v=$$(git describe --dirty --always); \
+ @v=$$(git describe --dirty --always);
echo "\newcommand{\pipelineversion}{$$v}" > $@
# --------- Delete for no Gnuastro ---------
# Version of Gnuastro.
- @v=$$(astnoisechisel --version | awk 'NR==1{print $$NF}'); \
+ @v=$$(astnoisechisel --version | awk 'NR==1{print $$NF}');
echo "\newcommand{\gnuastroversion}{$$v}" >> $@
# ------------------------------------------
diff --git a/reproduce/src/make/paper.mk b/reproduce/src/make/paper.mk
index 844f157..79d7722 100644
--- a/reproduce/src/make/paper.mk
+++ b/reproduce/src/make/paper.mk
@@ -40,10 +40,10 @@ $(texbdir)/paper.bbl: tex/references.tex \
# We'll run LaTeX first to generate the `.bcf' file (necessary for
# `biber') and then run `biber' to generate the `.bbl' file.
- p=$$(pwd); \
- export TEXINPUTS=$$p:$$TEXINPUTS; \
- cd $(texbdir); \
- pdflatex -shell-escape -halt-on-error $$p/paper.tex; \
+ p=$$(pwd);
+ export TEXINPUTS=$$p:$$TEXINPUTS;
+ cd $(texbdir);
+ pdflatex -shell-escape -halt-on-error $$p/paper.tex;
biber paper
@@ -61,9 +61,13 @@ $(texbdir)/paper.bbl: tex/references.tex \
paper.pdf: tex/pipeline.tex paper.tex $(texbdir)/paper.bbl \
| $(tikzdir) $(texbdir)
- # Make the report.
- p=$$(pwd); \
- export TEXINPUTS=$$p:$$TEXINPUTS; \
- cd $(texbdir); \
- pdflatex -shell-escape -halt-on-error $$p/paper.tex
+ # Go into the top TeX build directory and make the paper.
+ p=$$(pwd)
+ export TEXINPUTS=$$p:$$TEXINPUTS
+ cd $(texbdir)
+ pdflatex -shell-escape -halt-on-error $$p/paper.tex
+
+ # Come back to the top pipeline directory and copy the built PDF
+ # file here.
+ cd $$p
cp $(texbdir)/$@ $@