Binutils and other compressors also included in pipeline

To have better control over the build, GNU Binutils, Bzip2, GNU Gzip, and
XZ Utils have also been added to the pipeline. Some other minor cleanups
and fixes were also implemented throughout the process.

5 files changed, 151 insertions, 68 deletions

diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index fdad202..9f8d99a 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -2,6 +2,8 @@
 
 # Programs
 bash-version        = 4.4.18
+binutils-version    = 2.31.1
+bzip2-version       = 1.0.6
 cmake-version       = 3.12.4
 coreutils-version   = 8.30
 gawk-version        = 4.2.1
@@ -9,11 +11,13 @@ ghostscript-version = 9.25
 git-version         = 2.19.1
 gnuastro-version    = 0.7.62-13b0
 grep-version        = 3.1
+gzip-version        = 1.9.10-051e
 libtool-version     = 2.4.6
 lzip-version        = 1.20
 make-version        = 4.2.90
 sed-version         = 4.5
 tar-version         = 1.30
+xz-version          = 5.2.4
 
 # Libraries
 cfitsio-version     = 3.45
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index 8fbfeaf..be6e5ed 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -1,5 +1,5 @@
-# Build the VERY BASIC reproduction pipeline dependencies (programs and
-# libraries).
+# Build the VERY BASIC reproduction pipeline dependencies before everything
+# else using minimum Make and Shell.
 #
 # ------------------------------------------------------------------------
 #                      !!!!! IMPORTANT NOTES !!!!!
@@ -7,9 +7,11 @@
 # This Makefile will be run by the initial `./configure' script. It is not
 # included into the reproduction pipe after that.
 #
-# This Makefile builds very low-level and basic tools like GNU Bash and GNU
-# Make. Therefore this is the only Makefile in the reproduction pipeline
-# where you MUST NOT assume that modern GNU Bash or GNU Make are used.
+# This Makefile builds very low-level and basic tools like GNU Tar, and
+# various compression programs, GNU Bash, and GNU Make. Therefore this is
+# the only Makefile in the reproduction pipeline where you MUST NOT assume
+# that modern GNU Bash or GNU Make are used. After this Makefile, other
+# Makefiles can safely assume the fixed version of all these software.
 #
 # This Makefile is a very simplified version of `dependencies.mk' in the
 # same directory. See there for more comments.
@@ -48,8 +50,11 @@ ibdir = $(BDIR)/dependencies/installed/bin
 ildir = $(BDIR)/dependencies/installed/lib
 
 # As we build more programs, we want to use our own pipeline's built
-# programs, not the systems.
-PATH := $(ibdir):$(PATH)
+# programs and libraries, not the host's.
+PATH            := $(ibdir):$(PATH)
+LDFLAGS         := -L$(ildir) $(LDFLAGS)
+CPPFLAGS        := -I$(idir)/include $(CPPFLAGS)
+LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH)
 
 top-level-programs = bash
 all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
@@ -65,10 +70,30 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
 # is not recognized by some versions of Make (even older GNU Makes). So
 # we'll have to make sure the recipe doesn't break into multiple shell
 # calls (so we can preserve the variables).
+#
+# Software hosted at akhlaghi.org/src: As of our latest check (November
+# 2018) their major release tarballs either crash or don't build on some
+# systems (for example Make or Gzip), or they don't exist (for example
+# Bzip2).
+#
+# In the first case, we used their Git repo and bootstrapped them (just
+# like Gnuastro) and built the most recent tarball off of that. In the case
+# of Bzip2: its webpage has expired and doesn't host the data any more. It
+# is available on the link below (archive.org):
+#
+# https://web.archive.org/web/20180624184806/http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz
+#
+# However, downloading from this link is slow (because its just a link). So
+# its easier to just keep a with the others.
 tarballs = $(foreach t, bash-$(bash-version).tar.gz                         \
+                        binutils-$(binutils-version).tar.lz                 \
+                        bzip2-$(bzip2-version).tar.gz                       \
+	                gzip-$(gzip-version).tar.lz                         \
                         lzip-$(lzip-version).tar.gz                         \
-	                make-$(make-version).tar.gz                         \
+	                make-$(make-version).tar.lz                         \
 	                tar-$(tar-version).tar.gz                           \
+                        xz-$(xz-version).tar.gz                             \
+                        zlib-$(zlib-version).tar.gz                         \
                       , $(tdir)/$(t) )
 $(tarballs): $(tdir)/%:
 	if [ -f $(DEPENDENCIES-DIR)/$* ]; then                              \
@@ -79,10 +104,15 @@ $(tarballs): $(tdir)/%:
 	               | awk '{print $$1}' );                               \
 	                                                                    \
 	  mergenames=1;                                                     \
-	  if   [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash;          \
-	  elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \
-	  elif [ $$n = make ]; then w=http://akhlaghi.org/src;              \
-	  elif [ $$n = tar  ]; then w=http://ftp.gnu.org/gnu/tar;           \
+	  if   [ $$n = bash     ]; then w=http://ftp.gnu.org/gnu/bash;      \
+	  elif [ $$n = binutils ]; then w=http://ftp.gnu.org/gnu/binutils;  \
+	  elif [ $$n = bzip     ]; then w=http://akhlaghi.org/src;          \
+	  elif [ $$n = gzip     ]; then w=http://akhlaghi.org/src;          \
+	  elif [ $$n = lzip     ]; then w=http://download.savannah.gnu.org/releases/lzip; \
+	  elif [ $$n = make     ]; then w=http://akhlaghi.org/src;          \
+	  elif [ $$n = tar      ]; then w=http://ftp.gnu.org/gnu/tar;       \
+	  elif [ $$n = xz       ]; then w=http://tukaani.org/xz;            \
+	  elif [ $$n = zlib     ]; then w=http://www.zlib.net               \
 	  else                                                              \
 	    echo; echo; echo;                                               \
 	    echo "'$$n' not a dependency name (for downloading)."           \
@@ -101,27 +131,65 @@ $(tarballs): $(tdir)/%:
 
 
 
-# Basic programs (sorted alphabetically), see prerequisites for which one
-# will be built first.
-$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
-	       $(ibdir)/make
-	$(call gbuild,$(subst $(tdir)/,,$<), bash-$(bash-version), , \
-               --enable-static-link)
-
+# GNU Lzip: For a static build, the `-static' flag should be given to
+# LDFLAGS on the command-line (not from the environment).
 $(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz
-	$(call gbuild,$(subst $(tdir)/,,$<), lzip-$(lzip-version), static)
+	$(call gbuild,$(subst $(tdir)/,,$<), lzip-$(lzip-version), , \
+                      LDFLAGS="-static")
+
+
+# GNU Gzip.
+$(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.lz  \
+	       $(ibdir)/lzip
+	$(call gbuild,$(subst $(tdir)/,,$<), gzip-$(gzip-version), static)
+
+
+# Zlib: its `./configure' doesn't use Autoconf's configure script, it just
+# accepts a direct `--static' option.
+$(ildir)/libz.a: $(tdir)/zlib-$(zlib-version).tar.gz
+	$(call gbuild,$(subst $(tdir)/,,$<), zlib-$(zlib-version), ,   \
+                      --static)
 
-# Unfortunately GNU Make needs dynamic linking in two instances: when
+# XZ Utils
+$(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz
+	$(call gbuild,$(subst $(tdir)/,,$<), xz-$(xz-version), static)
+
+# Bzip2: Bzip2 doesn't have a configure script.
+$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
+	 tdir=bzip2-$(bzip2-version);                                  \
+	 cd $(ddir) && rm -rf $$tdir && tar xf $< && cd $$tdir &&      \
+	 make LDFLAGS=-static && make install PREFIX=$(idir) &&        \
+	 cd .. && rm -rf $$tdir
+
+
+# GNU Binutils:
+$(ibdir)/nm: $(tdir)/binutils-$(binutils-version).tar.lz \
+	     $(ibdir)/lzip                               \
+             $(ildir)/libz.a
+	$(call gbuild,$(subst $(tdir)/,,$<), binutils-$(binutils-version), \
+                      static)
+
+# GNU Tar: When built statically, tar gives a segmentation fault on
+# unpacking Bash. So we'll build it dynamically.
+$(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
+	      $(ibdir)/bzip2                    \
+	      $(ibdir)/gzip                     \
+	      $(ibdir)/lzip                     \
+	      $(ibdir)/xz                       \
+              $(ibdir)/nm
+	$(call gbuild,$(subst $(tdir)/,,$<), tar-$(tar-version))
+
+# GNU Make: Unfortunately it needs dynamic linking in two instances: when
 # loading objects (dynamically linked libraries), or when using the
 # `getpwnam' function (for tilde expansion). The first can be disabled with
 # `--disable-load', but unfortunately I don't know any way to fix the
 # second. So, we'll have to build it dynamically for now.
-$(ibdir)/make: $(tdir)/make-$(make-version).tar.gz \
+$(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \
                $(ibdir)/tar
-	$(call gbuild,$(subst $(tdir)/,,$<), make-$(make-version), , , ,)
+	$(call gbuild,$(subst $(tdir)/,,$<), make-$(make-version))
 
-# When built statically, tar gives a segmentation fault on unpacking
-# Bash. So we'll build it dynamically.
-$(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
-              $(ibdir)/lzip
-	$(call gbuild,$(subst $(tdir)/,,$<), tar-$(tar-version))
+# GNU Bash
+$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
+	       $(ibdir)/make
+	$(call gbuild,$(subst $(tdir)/,,$<), bash-$(bash-version), , \
+               --enable-static-link)
diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk
index a2e4a89..31e7759 100644
--- a/reproduce/src/make/dependencies-build-rules.mk
+++ b/reproduce/src/make/dependencies-build-rules.mk
@@ -48,15 +48,15 @@
 #  4: Extra configuration options.
 #  5: Extra options/arguments to pass to Make.
 #  6: Step to run between `make' and `make install': usually `make check'.
-gbuild = if [ $(3)x = staticx ]; then export LDFLAGS="$$LDFLAGS -static"; fi;\
-	 check="$(6)";                                                       \
-	 if [ x"$$check" = x ]; then check="echo Skipping-check"; fi;        \
-	 cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) &&      \
-	 ./configure $(4) --prefix=$(idir) &&                                \
-	 make $(5) &&                                                        \
-	 $$check &&                                                          \
-	 make install&&                                                      \
-	 cd ..&& rm -rf $(2)
+gbuild = if [ $(3)x = staticx ]; then export LDFLAGS="$$LDFLAGS -static"; fi; \
+	 check="$(6)";                                                        \
+	 if [ x"$$check" = x ]; then check="echo Skipping-check"; fi;         \
+	 cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) &&       \
+	 ./configure $(4) --prefix=$(idir) &&                                 \
+	 make $(5) &&                                                         \
+	 $$check &&                                                           \
+	 make install&&                                                       \
+	 cd .. && rm -rf $(2)
 
 
 
@@ -64,15 +64,16 @@ gbuild = if [ $(3)x = staticx ]; then export LDFLAGS="$$LDFLAGS -static"; fi;\
 
 # CMake
 # -----
-cbuild = if [ $(3)x = staticx ]; then                                        \
-	   export LDFLAGS="$$LDFLAGS -static";                               \
-	   opts="-DBUILD_SHARED_LIBS=OFF";                                   \
-	 fi;                                                                 \
-	 cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) &&      \
-	 rm -rf my-build && mkdir my-build && cd my-build &&                 \
-	 cmake .. $$opts $(4) &&                                             \
-	 cmake --build . &&                                                  \
-	 cmake .. -DCMAKE_INSTALL_PREFIX=$(idir) &&                          \
-	 cmake --build . --target install &&                                 \
-	 cd ../.. &&                                                         \
+cbuild = if [ $(3)x = staticx ]; then                                         \
+	   export LDFLAGS="$$LDFLAGS -static";                                \
+	   opts="-DBUILD_SHARED_LIBS=OFF";                                    \
+	 fi;                                                                  \
+	 cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) &&       \
+	 rm -rf pipeline-build && mkdir pipeline-build &&                     \
+	 cd pipeline-build &&                                                 \
+	 cmake .. $$opts $(4) &&                                              \
+	 cmake --build . &&                                                   \
+	 cmake .. -DCMAKE_INSTALL_PREFIX=$(idir) &&                           \
+	 cmake --build . --target install &&                                  \
+	 cd ../.. &&                                                          \
 	 rm -rf $(2)
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index 45ceadc..f24fafc 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -72,7 +72,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz             \
                         cmake-$(cmake-version).tar.gz                 \
                         coreutils-$(coreutils-version).tar.xz         \
                         curl-$(curl-version).tar.gz                   \
-	                gawk-$(gawk-version).tar.gz                   \
+	                gawk-$(gawk-version).tar.lz                   \
 	                ghostscript-$(ghostscript-version).tar.gz     \
 	                git-$(git-version).tar.xz                     \
 	                gnuastro-$(gnuastro-version).tar.lz           \
@@ -80,11 +80,10 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz             \
 	                gsl-$(gsl-version).tar.gz                     \
 	                jpegsrc.$(libjpeg-version).tar.gz             \
                         tiff-$(libtiff-version).tar.gz                \
-	                libtool-$(libtool-version).tar.gz             \
+	                libtool-$(libtool-version).tar.xz             \
                         libgit2-$(libgit2-version).tar.gz             \
 	                sed-$(sed-version).tar.xz                     \
 	                wcslib-$(wcslib-version).tar.bz2              \
-                        zlib-$(zlib-version).tar.gz                   \
                       , $(tdir)/$(t) )
 $(tarballs): $(tdir)/%:
 	if [ -f $(DEPENDENCIES-DIR)/$* ]; then
@@ -123,7 +122,6 @@ $(tarballs): $(tdir)/%:
 	  elif [ $$n = sed         ]; then w=http://ftp.gnu.org/gnu/sed
 	  elif [ $$n = tiff        ]; then w=https://download.osgeo.org/libtiff
 	  elif [ $$n = wcslib      ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib
-	  elif [ $$n = zlib        ]; then w=https://www.zlib.net
 	  else
 	    echo; echo; echo;
 	    echo "'$$n' not recognized as a dependency name to download."
@@ -152,15 +150,15 @@ $(tarballs): $(tdir)/%:
 # Libraries
 # ---------
 $(ildir)/libcfitsio.a: $(tdir)/cfitsio-$(cfitsio-version).tar.gz           \
-                       $(ildir)/libcurl.a                                  \
+                       $(ibdir)/curl                                       \
                        $(ibdir)/ls
 	$(call gbuild,$(subst $(tdir)/,,$<), cfitsio, static,              \
                       --enable-sse2 --enable-reentrant)
 
 
 $(ildir)/libgit2.a: $(tdir)/libgit2-$(libgit2-version).tar.gz              \
-                    $(ildir)/libcurl.a                                     \
-                    $(ibdir)/cmake
+                    $(ibdir)/cmake                                         \
+                    $(ibdir)/curl
 	$(call cbuild,$(subst $(tdir)/,,$<), libgit2-$(libgit2-version),   \
 	              static, -DUSE_SSH=OFF -DUSE_OPENSSL=OFF              \
 	              -DBUILD_CLAR=OFF -DTHREADSAFE=ON)
@@ -179,15 +177,13 @@ $(ildir)/libtiff.a: $(tdir)/tiff-$(libtiff-version).tar.gz                 \
 
 $(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2                \
 	           $(ildir)/libcfitsio.a
-	$(call gbuild,$(subst $(tdir)/,,$<), wcslib-$(wcslib-version),     \
-                      static, LIBS="-pthread -lcurl -lm" --without-pgplot  \
-                         --disable-fortran)
-
-# Zlib's `./configure' doesn't use Autoconf's configure script, it just
-# accepts a direct `--static' option.
-$(ildir)/libz.a: $(tdir)/zlib-$(zlib-version).tar.gz
-	$(call gbuild,$(subst $(tdir)/,,$<), zlib-$(zlib-version), ,       \
-                      --static)
+        # Unfortunately WCSLIB forces the building of shared libraries. So
+        # we'll allow it to finish, then remove the shared libraries
+        # afterwards.
+	$(call gbuild,$(subst $(tdir)/,,$<), wcslib-$(wcslib-version), ,   \
+                      LIBS="-pthread -lcurl -lm" --without-pgplot          \
+                      --disable-fortran)
+	rm -f $(ildir)/libwcs.so*
 
 
 
@@ -209,7 +205,7 @@ $(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz
 	$(call gbuild,$(subst $(tdir)/,,$<), coreutils-$(coreutils-version), \
                       static)
 
-$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.gz \
+$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \
                $(ibdir)/ls
 	$(call gbuild,$(subst $(tdir)/,,$<), gawk-$(gawk-version), static)
 
@@ -221,7 +217,7 @@ $(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \
                $(ibdir)/ls
 	$(call gbuild,$(subst $(tdir)/,,$<), grep-$(grep-version), static)
 
-$(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.gz \
+$(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.xz \
                   $(ibdir)/ls
 	$(call gbuild,$(subst $(tdir)/,,$<), libtool-$(libtool-version), static)
 
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index 1acec9f..5c0aac5 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -152,7 +152,7 @@ pvcheck = prog="$(strip $(1))";                                          \
 	  ver="$(strip $(2))";                                           \
 	  name="$(strip $(3))";                                          \
 	  macro="$(strip $(4))";                                         \
-	  v=$$($$prog --version | awk '/'$$ver'/{print "y"}');           \
+	  v=$$($$prog --version | awk '/'$$ver'/{print "y"; exit 0}');   \
 	  if [ x$$v != xy ]; then                                        \
 	    echo; echo "PIPELINE ERROR: Not running $$name $$ver"; echo; \
 	    exit 1;                                                      \
@@ -164,7 +164,7 @@ lvcheck = idir=$(BDIR)/dependencies/installed/include;                   \
 	  ver="$(strip $(2))";                                           \
 	  name="$(strip $(3))";                                          \
 	  macro="$(strip $(4))";                                         \
-	  v=$$(awk '$$1=="\#define" && /'$$ver'/ {print "y"}' $$f);      \
+	  v=$$(awk '$$1=="\#define" && /'$$ver'/{print "y";exit 0}' $$f);\
 	  if [ x$$v != xy ]; then                                        \
 	    echo; echo "PIPELINE ERROR: Not linking with $$name $$ver";  \
 	    echo; exit 1;                                                \
@@ -190,6 +190,7 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
 
         # Versions of programs (same order as 'dependency-versions.mk').
 	$(call pvcheck, bash, $(bash-version), GNU Bash, bashversion)
+	$(call pvcheck, nm, $(binutils-version), GNU Binutils, binutilsversion)
 	$(call pvcheck, cmake, $(cmake-version), CMake, cmakeversion)
 	$(call pvcheck, curl, $(curl-version), cURL, curlversion)
 	$(call pvcheck, ls, $(coreutils-version), GNU Coreutils,       \
@@ -207,6 +208,19 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
 	$(call pvcheck, make, $(make-version), GNU Make, makeversion)
 	$(call pvcheck, sed, $(sed-version), GNU SED, sedversion)
 	$(call pvcheck, tar, $(tar-version), GNU Tar, tarversion)
+	$(call pvcheck, xz, $(xz-version), XZ Utils, xzversion)
+
+        # Bzip2 prints its version in standard error, not standard output!
+	echo "here0"
+	echo "" | bzip2 --version &> $@_bzip2_ver;
+	v=$$(awk 'NR==1 && /'$(bzip2-version)'/{print "y"; exit 0}'        \
+	         $@_bzip2_ver);                                            \
+	if [ x$$v != xy ]; then                                            \
+	  echo; echo "PIPELINE ERROR: Not running Bzip2 $(bzip2-version)"; \
+	  echo; exit 1;                                                    \
+	fi;                                                                \
+	echo "\newcommand{\\bziptwoversion}{$(bzip2-version)}" >> $@
+
 
         # Versions of libraries.
 	$(call lvcheck, fitsio.h, $(cfitsio-version), CFITSIO, cfitsioversion)