Libcurl, Git, CMake, TIFF, Zlib also built at configure time

During the configuration step several new programs that were necessary for
a more complete controlled environment are now also downloaded and built
statically.

5 files changed, 188 insertions, 126 deletions

diff --git a/configure b/configure
index d9c2ef6..72d7e1c 100755
--- a/configure
+++ b/configure
@@ -76,25 +76,29 @@ echo
 # Identify the downloader tool
 # ----------------------------
 #
-# If cURL is already present, that will be used, otherwise, we'll use
-# Wget. Since the options specifying the output filename are different
-# between the two, we'll also specify the output option within the
-# `downloader' variable. So it is important to first give the output
-# filename after calling `DOWNLOADER' within the Makefiles, and finish the
-# command with the web address.
+# We use Wget for the downloading. As of November 12th, cURL couldn't
+# download Ghostscript's source from its standard address, but Wget
+# could. Until November 12th, we would check for cURL or Wget, so we
+# defined this check system here (explained in the next paragraph of this
+# comment). For now its redundant, but if the fix for cURL is found later,
+# we can add it back (or add other downloaders). So we'll keep it.
+#
+# Since the options specifying the output filename are different between
+# the two, we'll also specify the output option within the `downloader'
+# variable. So it is important to first give the output filename after
+# calling `DOWNLOADER' within the Makefiles, and finish the command with
+# the web address.
 print_downloader_notice=1
-if type curl > /dev/null; then
-    downloader="curl -o"
-elif type wget > /dev/null; then
+if type wget > /dev/null; then
     downloader="wget -O";
 else
     echo "======="
     echo "Warning"
     echo "======="
-    echo "Couldn't find any of the 'curl' or 'wget' programs. They are used"
-    echo "for downloading necessary programs and data if they aren't already"
-    echo "present in the specified directories. Therefore the pipeline will"
-    echo "crash if the necessary files are not already present on the system."
+    echo "Couldn't find GNU Wget. It is used for downloading necessary "
+    echo "programs and data if they aren't already present in the specified "
+    echo "directories. Therefore the pipeline will crash if the necessary "
+    echo "files are not already present on the system."
     echo "======="
     echo
     downloader="no-downloader-found"
@@ -256,7 +260,7 @@ echo
 
 # Notice on build Make and Bash, build top directories
 # ----------------------------------------------------
-tsec=10
+tsec=0
 echo;
 echo;
 echo "----------------";
@@ -268,7 +272,7 @@ echo
 sleep $tsec
 export USE_LOCAL_BASH=no
 bindir=$bdir/dependencies/installed/bin
-make -j2 -f reproduce/src/make/dependencies.mk $bindir/bash $bindir/make
+make -f reproduce/src/make/dependencies.mk $bindir/bash $bindir/make -j2
 
 
 
@@ -313,7 +317,7 @@ echo "Please run the following command to start."
 echo
 echo "(Replace '8' with the number of CPU threads)"
 echo
-echo "    ./local/bin/make -j8"
+echo "    .local/bin/make -j8"
 echo
 echo "To change the configuration later, please re-run './configure',"
 echo "DO NOT manually edit the relevant files."
diff --git a/reproduce/config/pipeline/LOCAL.mk.in b/reproduce/config/pipeline/LOCAL.mk.in
index 02f8b11..b74d9f6 100644
--- a/reproduce/config/pipeline/LOCAL.mk.in
+++ b/reproduce/config/pipeline/LOCAL.mk.in
@@ -22,21 +22,6 @@
 
 
 
-# (OPTIONAL) Dependencies directory
-# ---------------------------------
-#
-# To ensure an identical build environment, the pipeline will use its own
-# build of the programs it needs. Therefore the tarball of the relevant
-# programs are necessary for this pipeline. If a tarball isn't present in
-# the specified directory, it will be downloaded by the pipeline. Therefore
-# an internet connection will be mandatory.
-#
-# Important note: Keep atleast one blank space before and after `='.
-DEPENDENCIES-DIR = /optional/path/to/directory/containing/tarballs
-
-
-
-
 # Build directory
 # ---------------
 #
@@ -55,8 +40,24 @@ BDIR = /path/of/directory/for/building
 
 
 
-# Memory mapping minimum size
-# ---------------------------
+# (OPTIONAL) Dependencies directory
+# ---------------------------------
+#
+# To ensure an identical build environment, the pipeline will use its own
+# build of the programs it needs. Therefore the tarball of the relevant
+# programs are necessary for this pipeline. If a tarball isn't present in
+# the specified directory, it will be downloaded by the pipeline. Therefore
+# an internet connection will be mandatory.
+#
+# Important note: Keep atleast one blank space before and after `='.
+DEPENDENCIES-DIR = /optional/path/to/directory/containing/tarballs
+
+
+
+
+
+# (OPTIONAL) Memory mapping minimum size
+# --------------------------------------
 #
 # Some programs (for example Gnuastro) can deal with cases where the local
 # system doesn't have enough memory (RAM) to keep large files there. For
@@ -75,8 +76,8 @@ MINMAPSIZE = 1000000000
 
 
 
-# Downloader program
-# ------------------
+# (OPTIONAL) Downloader program
+# -----------------------------
 #
 # The downloder program (and its output option name, for example `wget -O'
 # or `curl -o') that will be used if any of the necessary datasets aren't
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index 3d9d8b3..dac56b1 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -1,15 +1,20 @@
 # Versions of the various dependnecies
 bash-version        = 4.4.18
 cfitsio-version     = 3450
+cmake-version       = 3.12.4
 coreutils-version   = 8.30
+curl-version        = 7.62.0
 gawk-version        = 4.2.1
 ghostscript-version = 9.25
-gnuastro-version    = 0.7.58-e72a
+git-version         = 2.19.1
+gnuastro-version    = 0.7.60-4c9eb
 grep-version        = 3.1
 gsl-version         = 2.5
 libjpeg-version     = v9b
 libgit2-version     = 0.26.0
+libtiff-version     = 4.0.10
 libtool-version     = 2.4.6
 make-version        = 4.2.90
 sed-version         = 4.5
 wcslib-version      = 6.2
+zlib-version        = 1.2.11
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index 671d6a5..f4d2e88 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -45,7 +45,7 @@ ildir = $(BDIR)/dependencies/installed/lib
 
 # Define the top-level programs to build (installed in `.local/bin', so for
 # Coreutils, only one of its executables is enough).
-top-level-programs = ls gawk gs grep libtool sed astnoisechisel
+top-level-programs = ls gawk gs grep libtool sed git astnoisechisel
 all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
 
 # This Makefile will be called to also build Bash locally. So when we don't
@@ -74,18 +74,23 @@ LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH)
 # All the necessary tarballs are defined and prepared with this rule.
 tarballs = $(foreach t, bash-$(bash-version).tar.gz                 \
 	                cfitsio$(cfitsio-version).tar.gz            \
+                        cmake-$(cmake-version).tar.gz               \
                         coreutils-$(coreutils-version).tar.xz       \
+                        curl-$(curl-version).tar.gz                 \
 	                gawk-$(gawk-version).tar.gz                 \
 	                ghostscript-$(ghostscript-version).tar.gz   \
-	                gnuastro-$(gnuastro-version).tar.gz         \
+	                git-$(git-version).tar.xz                   \
+	                gnuastro-$(gnuastro-version).tar.lz         \
 	                grep-$(grep-version).tar.xz                 \
 	                gsl-$(gsl-version).tar.gz                   \
 	                jpegsrc.$(libjpeg-version).tar.gz           \
+                        tiff-$(libtiff-version).tar.gz              \
 	                libtool-$(libtool-version).tar.gz           \
                         libgit2-$(libgit2-version).tar.gz           \
 	                sed-$(sed-version).tar.xz                   \
 	                make-$(make-version).tar.gz                 \
 	                wcslib-$(wcslib-version).tar.bz2            \
+                        zlib-$(zlib-version).tar.gz                 \
                       , $(tdir)/$(t) )
 $(tarballs): $(tdir)/%:
 	if [ -f $(DEPENDENCIES-DIR)/$* ]; then
@@ -97,23 +102,28 @@ $(tarballs): $(tdir)/%:
 	               | awk '{print $$1}' )
 
           # Set the top download link of the requested tarball.
-	  prefix=""
+	  mergenames=1
 	  if   [ $$n = bash        ]; then w=http://ftp.gnu.org/gnu/bash
 	  elif [ $$n = cfitsio     ]; then w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c
+	  elif [ $$n = cmake       ]; then w=https://cmake.org/files/v3.12
 	  elif [ $$n = coreutils   ]; then w=http://ftp.gnu.org/gnu/coreutils
+	  elif [ $$n = curl        ]; then w=https://curl.haxx.se/download
 	  elif [ $$n = gawk        ]; then w=http://ftp.gnu.org/gnu/gawk
 	  elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs925
-	  elif [ $$n = gnuastro    ]; then w=http://akhlaghi.org
+	  elif [ $$n = git         ]; then w=https://mirrors.edge.kernel.org/pub/software/scm/git
+	  elif [ $$n = gnuastro    ]; then w=http://akhlaghi.org/src
 	  elif [ $$n = grep        ]; then w=http://ftp.gnu.org/gnu/grep
 	  elif [ $$n = gsl         ]; then w=http://ftp.gnu.org/gnu/gsl
 	  elif [ $$n = jpegsrc     ]; then w=http://ijg.org/files
 	  elif [ $$n = libtool     ]; then w=ftp://ftp.gnu.org/gnu/libtool
 	  elif [ $$n = libgit      ]; then
-            w=https://github.com/libgit2/libgit2/archive/v0.27.7.tar.gz
-	    prefix=v
+	    mergenames=0
+	    w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
+	  elif [ $$n = make        ]; then w=http://akhlaghi.org/src
 	  elif [ $$n = sed         ]; then w=http://ftp.gnu.org/gnu/sed
-	  elif [ $$n = make        ]; then w=http://akhlaghi.org
+	  elif [ $$n = tiff        ]; then w=https://download.osgeo.org/libtiff
 	  elif [ $$n = wcslib      ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib
+	  elif [ $$n = zlib        ]; then w=https://www.zlib.net
 	  else
 	    echo; echo; echo;
 	    echo "'$$n' not recognized as a dependency name to download."
@@ -121,74 +131,111 @@ $(tarballs): $(tdir)/%:
 	    exit 1
 	  fi
 
-          # Download the requested tarball.
-	  $(DOWNLOADER) $@ $$w/$$prefix$*
+          # Download the requested tarball. Note that some packages may not
+          # follow our naming convention (where the package name is merged
+          # with its version number). In such cases, `w' will be the full
+          # address, not just the top directory address. But since we are
+          # storing all the tarballs in one directory, we want it to have
+          # the same naming convention, so we'll download it to a temporary
+          # name, then rename that.
+	  if [ $$mergenames = 1 ]; then  tarballurl=$$w/"$*"
+	  else                           tarballurl=$$w
+	  fi
+	  echo "Downloading $$tarballurl"
+	  if [ $$mergenames = 1 ]; then
+	    $(DOWNLOADER) $@ $$tarballurl
+	  else
+	    $(DOWNLOADER) $@_tmp $$tarballurl
+	    mv $@_tmp $@
+	  fi
 	fi
 
 
 
 
 
-# Customized build
-# ----------------
-#
-# Programs that need some customization on their build.
-# For CFITSIO we'll need to intervene manually to remove the check on
-# libcurl (which can be real trouble in this controlled environment).
-$(ildir)/libcfitsio.a: $(ibdir)/ls                              \
-                          $(tdir)/cfitsio$(cfitsio-version).tar.gz
-        # Same as before
-	cd $(ddir)
-	tar xf $(tdir)/cfitsio$(cfitsio-version).tar.gz
-	cd cfitsio
-
-        # Remove the part that checks for the CURL library, so it assumes
-        # that the CURL library wasn't found.
-	awk 'NR<4785 || NR>4847' configure > new_configure
-	mv new_configure configure
-	chmod +x configure
-
-        # Do the standard configuring and building
-	./configure CFLAGS=--static --disable-shared --prefix=$(idir)
-	make; make install;
-	cd ..; rm -rf cfitsio
-
-
-# Why not shared: Gnuastro's configure can't link with it in static mode.
-$(ildir)/libgit2.a: $(tdir)/libgit2-$(libgit2-version).tar.gz
-	cd $(ddir)
-	tar xf $(tdir)/libgit2-$(libgit2-version).tar.gz
-	cd libgit2-$(libgit2-version)
-	mkdir build
-	cd build
-	export CFLAGS="--static $$CFLAGS"
-	cmake .. -DUSE_SSH=OFF -DUSE_OPENSSL=OFF -DBUILD_SHARED_LIBS=OFF \
-                 -DBUILD_CLAR=OFF -DTHREADSAFE=ON
-	cmake --build .
-	cmake .. -DCMAKE_INSTALL_PREFIX=$(idir)
-	cmake --build . --target install
-	cd ../..
-	rm -rf libgit2-$(libgit2-version)
-
-
-
-
-
-# GNU Build system programs
-# -------------------------
-#
-# Programs that use the basic GNU build system.
-gbuild = cd $(ddir); tar xf $(tdir)/$(1); cd $(2);               \
-         if [ $(3)x = staticx ]; then                            \
-         opts="CFLAGS=--static --disable-shared";                \
-         fi;                                                     \
-         ./configure $$opts $(4) --prefix=$(idir); make $(5);    \
-         check="$(6)"; if [ x"$$check" != x ]; then $$check; fi; \
+# Build system rules
+# ------------------
+gbuild = cd $(ddir); rm -rf $(2); tar xf $(tdir)/$(1); cd $(2);      \
+         if [ $(3)x = staticx ]; then                                \
+         opts="CFLAGS=--static --disable-shared";                    \
+         fi;                                                         \
+         ./configure $$opts $(4) --prefix=$(idir); make $(5);        \
+         check="$(6)"; if [ x"$$check" != x ]; then $$check; fi;     \
          make install; cd ..; rm -rf $(2)
 
+
+cbuild = cd $(ddir); rm -rf $(2); tar xf $(tdir)/$(1); cd $(2);      \
+	 rm -rf my-build; mkdir my-build; cd my-build; opts="";      \
+	 if [ $(3)x = staticx ]; then                                \
+	   export CFLAGS="--static $$CFLAGS";                        \
+	   opts="-DBUILD_SHARED_LIBS=OFF";                           \
+	 fi;                                                         \
+	 cmake .. $$opts $(4);                                       \
+	 cmake --build .;                                            \
+	 cmake .. -DCMAKE_INSTALL_PREFIX=$(idir);                    \
+	 cmake --build . --target install; cd ../..; rm -rf $(2)
+
+
+
+
+
+# Libraries
+# ---------
+$(ildir)/libcfitsio.a: $(tdir)/cfitsio$(cfitsio-version).tar.gz         \
+                       $(ildir)/libcurl.a                               \
+                       $(ibdir)/ls
+	$(call gbuild,$(subst $(tdir),,$<), gsl-$(gsl-version), static, \
+                      --enable-sse2 --enable-reentrant)
+
+$(ildir)/libcurl.a: $(tdir)/curl-$(curl-version).tar.gz \
+                    $(ildir)/libz.a                     \
+                    $(ibdir)/ls
+	$(call gbuild,$(subst $(tdir),,$<), curl-$(curl-version), static, \
+                      --without-brotli)
+
+$(ildir)/libgit2.a: $(tdir)/libgit2-$(libgit2-version).tar.gz             \
+                    $(ildir)/libcurl.a                                    \
+                    $(ibdir)/cmake
+	$(call cbuild,$(subst $(tdir),,$<), libgit2-$(libgit2-version),   \
+	              static, -DUSE_SSH=OFF -DUSE_OPENSSL=OFF             \
+                      -DBUILD_CLAR=OFF -DTHREADSAFE=ON, )
+
+$(ildir)/libgsl.a: $(tdir)/gsl-$(gsl-version).tar.gz \
+                   $(ibdir)/ls
+	$(call gbuild,$(subst $(tdir),,$<), gsl-$(gsl-version), static)
+
+$(ildir)/libjpeg.a: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz
+	$(call gbuild,$(subst $(tdir),,$<), jpeg-9b, static)
+
+$(ildir)/libtiff.a: $(tdir)/tiff-$(libtiff-version).tar.gz \
+                   $(ibdir)/ls
+	$(call gbuild,$(subst $(tdir),,$<), tiff-$(libtiff-version), static)
+
+$(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
+	           $(ildir)/libcfitsio.a
+	$(call gbuild,$(subst $(tdir),,$<), wcslib-$(wcslib-version),     \
+                      static, LIBS="-pthread -lcurl -lm" --without-pgplot \
+                         --disable-fortran)
+
+# Zlib's `./configure' doesn't use Autoconf's configure script, it just
+# accepts a direct `--static' option.
+$(ildir)/libz.a: $(tdir)/zlib-$(zlib-version).tar.gz
+	$(call gbuild,$(subst $(tdir),,$<), zlib-$(zlib-version), , \
+                      --static)
+
+
+
+
+
+# Programs
+# --------
 $(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz
 	$(call gbuild,$(subst $(tdir),,$<), bash-$(bash-version), static)
 
+$(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
+                $(ibdir)/ls
+	$(call cbuild,$(subst $(tdir),,$<), cmake-$(cmake-version))
 
 # Unfortunately GNU Make needs dynamic linking in two instances: when
 # loading objects (dynamically linked libraries), or when using the
@@ -198,59 +245,41 @@ $(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz
 $(ibdir)/make: $(tdir)/make-$(make-version).tar.gz
 	$(call gbuild,$(subst $(tdir),,$<), make-$(make-version))
 
-
 $(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz
 	$(call gbuild,$(subst $(tdir),,$<), coreutils-$(coreutils-version), \
                       static)
 
-
 $(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.gz \
                $(ibdir)/ls
 	$(call gbuild,$(subst $(tdir),,$<), gawk-$(gawk-version), static)
 
-
 $(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \
               $(ibdir)/ls
 	$(call gbuild,$(subst $(tdir),,$<), sed-$(sed-version), static)
 
-
 $(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \
                $(ibdir)/ls
 	$(call gbuild,$(subst $(tdir),,$<), grep-$(grep-version), static)
 
-
 $(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.gz \
                   $(ibdir)/ls
 	$(call gbuild,$(subst $(tdir),,$<), libtool-$(libtool-version), static)
 
-
-$(ildir)/libgsl.a: $(tdir)/gsl-$(gsl-version).tar.gz \
-                   $(ibdir)/ls
-	$(call gbuild,$(subst $(tdir),,$<), gsl-$(gsl-version), static)
-
-
-$(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
-	           $(ildir)/libcfitsio.a
-	$(call gbuild,$(subst $(tdir),,$<), wcslib-$(wcslib-version), , \
-                      LIBS="-pthread -lcurl -lm" --without-pgplot       \
-                         --disable-fortran)
-
-
 $(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz \
              $(ibdir)/ls
 	$(call gbuild,$(subst $(tdir),,$<), ghostscript-$(ghostscript-version))
 
+$(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \
+             $(ibdir)/ls
+	$(call gbuild,$(subst $(tdir),,$<), git-$(git-version), static)
 
-$(ildir)/libjpeg.a: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz
-	$(call gbuild,$(subst $(tdir),,$<), jpeg-9b, static)
-
-
-$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.gz \
+$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \
                          $(ildir)/libgsl.a                           \
                          $(ildir)/libcfitsio.a                       \
                          $(ildir)/libwcs.a                           \
                          $(ibdir)/gs                                 \
                          $(ildir)/libjpeg.a                          \
+                         $(ildir)/libtiff.a                          \
                          $(ildir)/libgit2.a                          \
 
 	$(call gbuild,$(subst $(tdir),,$<), gnuastro-$(gnuastro-version), \
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index 165db78..5fd87c9 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -47,6 +47,16 @@ pconfdir    = reproduce/config/pipeline
 
 
 
+# System's environment
+# --------------------
+#
+# Before defining the local sub-environment here, we'll need to save the
+# system's environment for some scenarios (for example after `clean'ing the
+# built programs).
+sys-rm := $(shell which rm)
+
+
+
 # High level environment
 # ----------------------
 #
@@ -113,10 +123,23 @@ $(mtexdir) $(texbdir): | $(texdir); mkdir $@
 clean-mmap:; rm -f reproduce/config/gnuastro/mmap*
 # ------------------------------------------
 clean: clean-mmap
-	rm -rf $(BDIR)
-	rm -f reproduce/build *.pdf *.log *.out *.aux *.auxlock
+        # Delete the top-level PDF file.
+	rm -f *.pdf
+
+        # Delete all the built outputs except the dependency
+        # programs. We'll use Bash's extended options builtin (`shopt') to
+        # enable "extended glob" (for listing of files). It allows extended
+        # features like ignoring the listing of a file with `!()' that we
+        # are using afterwards.
+	shopt -s extglob
+	rm -rf $(BDIR)/!(dependencies)
 distclean: clean
-	rm -f Makefile $(pconfdir)/LOCAL.mk .gnuastro
+        # We'll be deleting the built environent programs and just need the
+        # `rm' program. So for this recipe, we'll use the host system's
+        # `rm', not our own.
+	$(sys-rm) -rf $(BDIR) reproduce/build
+	$(sys-rm) -f Makefile $(pconfdir)/LOCAL.mk .gnuastro .local
+