diff options
Diffstat (limited to 'reproduce/src')
| -rw-r--r-- | reproduce/src/make/Top-Makefile | 108 | ||||
| -rw-r--r-- | reproduce/src/make/delete-me.mk | 12 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies.mk | 254 | ||||
| -rw-r--r-- | reproduce/src/make/initialize.mk | 84 | ||||
| -rw-r--r-- | reproduce/src/make/paper.mk | 22 |
5 files changed, 407 insertions, 73 deletions
diff --git a/reproduce/src/make/Top-Makefile b/reproduce/src/make/Top-Makefile new file mode 100644 index 0000000..5d94766 --- /dev/null +++ b/reproduce/src/make/Top-Makefile @@ -0,0 +1,108 @@ +# A ONE-LINE DESCRIPTION OF THE WHOLE PIPELINE +# +# Original author: +# Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Contributing author(s): +# Your name <your@email.address> +# Copyright (C) 2018, Your Name. +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + + + + + +# Ultimate target of this pipeline +# -------------------------------- +# +# The final paper (in PDF format) is the main target of this whole +# reproduction pipeline. So as defined in the Make paradigm, we are +# defining it here. +# +# Note that if you don't have LaTeX to build the PDF, or generally are just +# interested in the processing, you can skip create the final PDF creation +# with `pdf-build-final' of `reproduce/config/pipeline/pdf-build.mk'. +all: paper.pdf + + + + + +# Include specific Makefiles +# -------------------------- +# +# To keep things clean, managable and readable, each set of operations is +# (and must be) classified (modularized) by context into separate +# Makefiles: the more the better. They are included in this top-level +# Makefile through the command below. +# +# To further help in readability, it is best to avoid including Makefiles +# within any other Makefile. So in short, it is best that the `foreach' +# loop below contains all the `reproduce/src/make/*.mk' files. +# +# IMPORTANT NOTE: order matters in the inclusion of the processing +# Makefiles. As the pipeline grows, some Makefiles will probably define +# variables/dependencies that others need. Therefore unlike the +# `reproduce/config/pipeline/*.mk' Makefiles which only define low-level +# variables (not dependent on other variables and contain no rules), the +# high-level processing Makefiles are included through the `foreach' loop +# below by explicitly requesting them in a specific order here. +include reproduce/config/pipeline/*.mk +include $(foreach f, initialize \ + download \ + delete-me \ + paper \ + , reproduce/src/make/$(f).mk) + + + + + +# LaTeX macros for paper +# ---------------------- +# +# The final report's PDF (final target of this reproduction pipeline) takes +# variable strings from the pipeline. Those variables are defined as LaTeX +# macros in `tex/pipeline.tex'. This file is thus the interface between the +# pipeline scripts and the final PDF. +# +# Each of the pipeline steps will save their macros into their own `.tex' +# file in the `$(mtexdir)' directory. Those individual macros are the +# pre-requisite to `tex/pipeline.txt'. `tex/pipeline.tex' is thus a +# high-level output and is defined in this top-most Makefile (and not +# `reproduce/src/make/paper.mk'). This enables a clear demonstration of the +# top-level dependencies clearly. +# +# Note that if you don't want the final PDF and just want the processing +# and file outputs, you can remove the value of the `pdf-build-final' +# variable in `reproduce/config/pdf-build.mk'. +tex/pipeline.tex: $(foreach f, initialize \ + download \ + delete-me \ + , $(mtexdir)/$(f).tex) + + # If no PDF is requested, then just exit here. +ifeq ($(pdf-build-final),) + @echo + @echo + @echo "-----" + @echo "Everything is OK until this point, but not building PDF." + @echo "To do so, give a value to the 'pdf-build-final' variable." + @echo "It is defined in 'reproduce/config/pipeline/pdf-build.mk'." + @echo + @exit 1 +endif + + # Merge all the TeX macros that are prepared for building the PDF. + @cat $(mtexdir)/*.tex > $@ diff --git a/reproduce/src/make/delete-me.mk b/reproduce/src/make/delete-me.mk index de72873..67f0440 100644 --- a/reproduce/src/make/delete-me.mk +++ b/reproduce/src/make/delete-me.mk @@ -63,10 +63,10 @@ $(mtexdir)/delete-me.tex: $(dm) # Here, we are first using AWK to find the minimum and maximum # values, then using it again to read each separately to use in the # macro definition. - mm=$$(awk 'BEGIN{min=99999; max=-min} \ - {if($$2>max) max=$$2; if($$2<min) min=$$2;} \ - END{print min, max}' $(dm)); \ - v=$$(echo "$$mm" | awk '{printf "%.3f", $$1}'); \ - echo "\newcommand{\deletememin}{$$v}" >> $@; \ - v=$$(echo "$$mm" | awk '{printf "%.3f", $$2}'); \ + mm=$$(awk 'BEGIN{min=99999; max=-min} + {if($$2>max) max=$$2; if($$2<min) min=$$2;} + END{print min, max}' $(dm)); + v=$$(echo "$$mm" | awk '{printf "%.3f", $$1}'); + echo "\newcommand{\deletememin}{$$v}" >> $@; + v=$$(echo "$$mm" | awk '{printf "%.3f", $$2}'); echo "\newcommand{\deletememax}{$$v}" >> $@ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk new file mode 100644 index 0000000..0fb5a34 --- /dev/null +++ b/reproduce/src/make/dependencies.mk @@ -0,0 +1,254 @@ +# Build the reproduction pipeline dependencies (programs and libraries). +# +# ------------------------------------------------------------------------ +# !!!!! IMPORTANT NOTES !!!!! +# +# This Makefile will be run by the initial `./configure' script. It is not +# included into the reproduction pipe after that. +# +# This Makefile also builds GNU Bash and GNU Make. Therefore this is the +# only Makefile in the reproduction pipeline where you MUST NOT assume that +# GNU Bash or GNU Make are to be used. +# +# ------------------------------------------------------------------------ +# +# Original author: +# Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Contributing author(s): +# Your name <your@email.address> +# Copyright (C) 2018, Your Name. +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + + + +# Top level environment +include reproduce/config/pipeline/LOCAL.mk +include reproduce/config/pipeline/dependency-versions.mk + +ddir = $(BDIR)/dependencies +tdir = $(BDIR)/dependencies/tarballs +idir = $(BDIR)/dependencies/installed +ibdir = $(BDIR)/dependencies/installed/bin +ildir = $(BDIR)/dependencies/installed/lib + +# Define the top-level programs to build (installed in `.local/bin', so for +# Coreutils, only one of its executables is enough). +top-level-programs = ls gawk gs grep libtool sed astnoisechisel +all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) + +# This Makefile will be called to also build Bash locally. So when we don't +# have it yet, we'll have to use the system's bash. +ifeq ($(USE_LOCAL_BASH),yes) +SHELL := $(ibdir)/bash +else +SHELL := /bin/sh +endif + +# Other basic environment settings. +.ONESHELL: +.SHELLFLAGS = -ec +PATH := $(ibdir):$(PATH) +LDFLAGS := -L$(ildir) $(LDFLAGS) +CPPFLAGS := -I$(idir)/include $(CPPFLAGS) +LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) + + + + + +# Tarballs +# -------- +# +# All the necessary tarballs are defined and prepared with this rule. +tarballs = $(foreach t, bash-$(bash-version).tar.gz \ + cfitsio$(cfitsio-version).tar.gz \ + coreutils-$(coreutils-version).tar.xz \ + gawk-$(gawk-version).tar.gz \ + ghostscript-$(ghostscript-version).tar.gz \ + gnuastro-$(gnuastro-version).tar.gz \ + grep-$(grep-version).tar.xz \ + gsl-$(gsl-version).tar.gz \ + jpegsrc.$(libjpeg-version).tar.gz \ + libtool-$(libtool-version).tar.gz \ + libgit2-$(libgit2-version).tar.gz \ + sed-$(sed-version).tar.xz \ + make-$(make-version).tar.gz \ + wcslib-$(wcslib-version).tar.bz2 \ + , $(tdir)/$(t) ) +$(tarballs): $(tdir)/%: + if [ -f $(DEPENDENCIES-DIR)/$* ]; then + cp $(DEPENDENCIES-DIR)/$* $@ + else + # Remove all numbers, `-' and `.' from the tarball name so we can + # search more easily only with the program name. + n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ + | awk '{print $$1}' ) + + # Set the top download link of the requested tarball. + if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash + elif [ $$n = cfitsio ]; then w=WWWWWWWWWWWWWWWW + elif [ $$n = coreutils ]; then w=WWWWWWWWWWWWWWWW + elif [ $$n = gawk ]; then w=WWWWWWWWWWWWWWWW + elif [ $$n = ghostscript ]; then w=WWWWWWWWWWWWWWWW + elif [ $$n = gnuastro ]; then w=http://akhlaghi.org + elif [ $$n = grep ]; then w=WWWWWWWWWWWWWWWW + elif [ $$n = gsl ]; then w=WWWWWWWWWWWWWWWW + elif [ $$n = jpegsrc ]; then w=WWWWWWWWWWWWWWWW + elif [ $$n = libtool ]; then w=WWWWWWWWWWWWWWWW + elif [ $$n = libgit ]; then w=WWWWWWWWWWWWWWWW + elif [ $$n = sed ]; then w=WWWWWWWWWWWWWWWW + elif [ $$n = make ]; then w=http://akhlaghi.org + elif [ $$n = wcslib ]; then w=WWWWWWWWWWWWWWWW + else + echo; echo; echo; + echo "'$$n' not recognized as a dependency name to download." + echo; echo; echo; + exit 1 + fi + + # Download the requested tarball. + $(DOWNLOADER) $@ $$w/$* + fi + + + + + +# Customized build +# ---------------- +# +# Programs that need some customization on their build. +# For CFITSIO we'll need to intervene manually to remove the check on +# libcurl (which can be real trouble in this controlled environment). +$(ildir)/libcfitsio.a: $(ibdir)/ls \ + $(tdir)/cfitsio$(cfitsio-version).tar.gz + # Same as before + cd $(ddir) + tar xf $(tdir)/cfitsio$(cfitsio-version).tar.gz + cd cfitsio + + # Remove the part that checks for the CURL library, so it assumes + # that the CURL library wasn't found. + awk 'NR<4785 || NR>4847' configure > new_configure + mv new_configure configure + chmod +x configure + + # Do the standard configuring and building + ./configure CFLAGS=--static --disable-shared --prefix=$(idir) + make; make install; + cd ..; rm -rf cfitsio + + +# Why not shared: Gnuastro's configure can't link with it in static mode. +$(ildir)/libgit2.a: $(tdir)/libgit2-$(libgit2-version).tar.gz + cd $(ddir) + tar xf $(tdir)/libgit2-$(libgit2-version).tar.gz + cd libgit2-$(libgit2-version) + mkdir build + cd build + export CFLAGS="--static $$CFLAGS" + cmake .. -DUSE_SSH=OFF -DUSE_OPENSSL=OFF -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_CLAR=OFF -DTHREADSAFE=ON + cmake --build . + cmake .. -DCMAKE_INSTALL_PREFIX=$(idir) + cmake --build . --target install + cd ../.. + rm -rf libgit2-$(libgit2-version) + + + + + +# GNU Build system programs +# ------------------------- +# +# Programs that use the basic GNU build system. +gbuild = cd $(ddir); tar xf $(tdir)/$(1); cd $(2); \ + if [ $(3)x = staticx ]; then \ + opts="CFLAGS=--static --disable-shared"; \ + fi; \ + ./configure $$opts $(4) --prefix=$(idir); make $(5); \ + check="$(6)"; if [ x"$$check" != x ]; then $$check; fi; \ + make install; cd ..; rm -rf $(2) + +$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz + $(call gbuild,$(subst $(tdir),,$<), bash-$(bash-version), static) + + +# Unfortunately GNU Make needs dynamic linking in two instances: when +# loading objects (dynamically linked libraries), or when using the +# `getpwnam' function (for tilde expansion). The first can be disabled with +# `--disable-load', but unfortunately I don't know any way to fix the +# second. So, we'll have to build it dynamically for now. +$(ibdir)/make: $(tdir)/make-$(make-version).tar.gz + $(call gbuild,$(subst $(tdir),,$<), make-$(make-version)) + + +$(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz + $(call gbuild,$(subst $(tdir),,$<), coreutils-$(coreutils-version), \ + static) + + +$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.gz \ + $(ibdir)/ls + $(call gbuild,$(subst $(tdir),,$<), gawk-$(gawk-version), static) + + +$(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \ + $(ibdir)/ls + $(call gbuild,$(subst $(tdir),,$<), sed-$(sed-version), static) + + +$(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \ + $(ibdir)/ls + $(call gbuild,$(subst $(tdir),,$<), grep-$(grep-version), static) + + +$(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.gz \ + $(ibdir)/ls + $(call gbuild,$(subst $(tdir),,$<), libtool-$(libtool-version), static) + + +$(ildir)/libgsl.a: $(tdir)/gsl-$(gsl-version).tar.gz \ + $(ibdir)/ls + $(call gbuild,$(subst $(tdir),,$<), gsl-$(gsl-version), static) + + +$(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ + $(ildir)/libcfitsio.a + $(call gbuild,$(subst $(tdir),,$<), wcslib-$(wcslib-version), , \ + LIBS="-pthread -lcurl -lm" --without-pgplot \ + --disable-fortran) + + +$(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz \ + $(ibdir)/ls + $(call gbuild,$(subst $(tdir),,$<), ghostscript-$(ghostscript-version)) + + +$(ildir)/libjpeg.a: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz + $(call gbuild,$(subst $(tdir),,$<), jpeg-9b, static) + + +$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.gz \ + $(ildir)/libgsl.a \ + $(ildir)/libcfitsio.a \ + $(ildir)/libwcs.a \ + $(ibdir)/gs \ + $(ildir)/libjpeg.a \ + $(ildir)/libgit2.a \ + + $(call gbuild,$(subst $(tdir),,$<), gnuastro-$(gnuastro-version), \ + static, , -j8, make check -j8) diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index f615e22..165db78 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -47,36 +47,24 @@ pconfdir = reproduce/config/pipeline -# Sanity check -# ------------ +# High level environment +# ---------------------- # -# We need to make sure that the `./configure' command has already been -# run. The output of `./configure' is the `$(pconfdir)/LOCAL.mk' file and -# this is the non-time-stamp prerequisite of $(BDIR), see below. +# We want the full recipe to be executed in one call to the shell. Also we +# want Make to run the specific version of Bash that we have installed +# during `./configure' time. # -# There is one problem however: if the user hasn't run `./configure' yet, -# then `BDIR' isn't defined (will just evaluate to blank space). Therefore -# it won't appear in the prerequisites and the pipeline will try to build -# the other directories in the top root directory (`/'). To solve this -# problem, when `BDIR' isn't defined, we'll define it with a place-holder -# name (only so it won't evaluate to blank space). Note that this -# directory will never be built. -ifeq ($(BDIR),) -configure-run = no -BDIR = reproduce/BDIR -else -configure-run = yes -endif -$(pconfdir)/LOCAL.mk: - @echo - @echo "================================================================" - @echo "For the pipeline's local settings, please run this command first" - @echo "(P.S. this local configuration is only necessary one time)" - @echo - @echo " $$ ./configure" - @echo "================================================================" - @echo - @exit 1 +# Regarding the directories, this pipeline builds its major dependencies +# itself and doesn't use the local system's default tools. With these +# environment variables, we are setting it to prefer the software we have +# build here. +.ONESHELL: +.SHELLFLAGS = -ec +SHELL := .local/bin/bash +PATH := .local/bin:$(PATH) +LDFLAGS := -L.local/lib $(LDFLAGS) +CPPFLAGS := -I.local/include $(CPPFLAGS) +LD_LIBRARY_PATH := .local/lib:$(LD_LIBRARY_PATH) @@ -103,52 +91,32 @@ $(pconfdir)/LOCAL.mk: # are looking for in this pipeline. .SUFFIXES: $(tikzdir): | $(texbdir); mkdir $@ -$(BDIR): | $(pconfdir)/LOCAL.mk; mkdir $@ $(texdir) $(lockdir): | $(BDIR); mkdir $@ $(mtexdir) $(texbdir): | $(texdir); mkdir $@ -# Symbolic link to build directory -# -------------------------------- -# -# Besides $(BDIR), we are also making a symbolic link to it for easy -# access. Recall that it is recommended that the actual build directory be -# in a completely separate part of the file system (a place that may easily -# be completely deleted). -# -# Note that $(BDIR) might not be an absolute path and this will complicate -# the symbolic link creation. To be generic, we'll first call `readlink' to -# make sure we have an absolute address, then we'll make a symbolic link to -# that. -reproduce/build: | $(BDIR) - absbdir=$$(readlink -f $(BDIR)); \ - ln -s $$absbdir $@ - - - - # High-level Makefile management # ------------------------------ # # About `.PHONY': these are targets that must be built even if a file with -# their name exists. Most don't correspond to a file, but those that do are -# included here ensure that the file is always built in every run: for -# example the pipeline versions may change within two separate runs, so we -# want it to be rebuilt every time. +# their name exists. +# +# Only `$(mtexdir)/initialize.tex' corresponds to a file. This is because +# we want to ensure that the file is always built in every run: it contains +# the pipeline version which may change between two separate runs, even +# when no file actually differs. .PHONY: all clean distclean clean-mmap $(mtexdir)/initialize.tex -distclean: clean; rm -f $(pconfdir)/LOCAL.mk # --------- Delete for no Gnuastro --------- clean-mmap:; rm -f reproduce/config/gnuastro/mmap* # ------------------------------------------ clean: clean-mmap -ifeq ($(configure-run),yes) rm -rf $(BDIR) -endif rm -f reproduce/build *.pdf *.log *.out *.aux *.auxlock - +distclean: clean + rm -f Makefile $(pconfdir)/LOCAL.mk .gnuastro @@ -163,12 +131,12 @@ endif $(mtexdir)/initialize.tex: | $(mtexdir) # Version of the pipeline. - @v=$$(git describe --dirty --always); \ + @v=$$(git describe --dirty --always); echo "\newcommand{\pipelineversion}{$$v}" > $@ # --------- Delete for no Gnuastro --------- # Version of Gnuastro. - @v=$$(astnoisechisel --version | awk 'NR==1{print $$NF}'); \ + @v=$$(astnoisechisel --version | awk 'NR==1{print $$NF}'); echo "\newcommand{\gnuastroversion}{$$v}" >> $@ # ------------------------------------------ diff --git a/reproduce/src/make/paper.mk b/reproduce/src/make/paper.mk index 844f157..79d7722 100644 --- a/reproduce/src/make/paper.mk +++ b/reproduce/src/make/paper.mk @@ -40,10 +40,10 @@ $(texbdir)/paper.bbl: tex/references.tex \ # We'll run LaTeX first to generate the `.bcf' file (necessary for # `biber') and then run `biber' to generate the `.bbl' file. - p=$$(pwd); \ - export TEXINPUTS=$$p:$$TEXINPUTS; \ - cd $(texbdir); \ - pdflatex -shell-escape -halt-on-error $$p/paper.tex; \ + p=$$(pwd); + export TEXINPUTS=$$p:$$TEXINPUTS; + cd $(texbdir); + pdflatex -shell-escape -halt-on-error $$p/paper.tex; biber paper @@ -61,9 +61,13 @@ $(texbdir)/paper.bbl: tex/references.tex \ paper.pdf: tex/pipeline.tex paper.tex $(texbdir)/paper.bbl \ | $(tikzdir) $(texbdir) - # Make the report. - p=$$(pwd); \ - export TEXINPUTS=$$p:$$TEXINPUTS; \ - cd $(texbdir); \ - pdflatex -shell-escape -halt-on-error $$p/paper.tex + # Go into the top TeX build directory and make the paper. + p=$$(pwd) + export TEXINPUTS=$$p:$$TEXINPUTS + cd $(texbdir) + pdflatex -shell-escape -halt-on-error $$p/paper.tex + + # Come back to the top pipeline directory and copy the built PDF + # file here. + cd $$p cp $(texbdir)/$@ $@ |