High-level dependencies build without system's PATH

The high-level dependencies are now built without having access to the
system's PATH. To do this, all the necessary software that we aren't
building ourselves are now brought into the installed `bin/' directory
using a symbolic link to the corresponding software on the host. To do
this, it was also necessary to increase the number of basic/low-level
packages that we are building, and add several more (Diffutils and
Findutils).

With this process in place, we now have a list of the exact software
packages that we are not building our selves, enabling easy building of all
such dependencies in the future.

5 files changed, 129 insertions, 66 deletions

diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index e5b4bbb..9c6f18f 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -5,6 +5,8 @@ bash-version        = 4.4.18
 bzip2-version       = 1.0.6
 cmake-version       = 3.12.4
 coreutils-version   = 8.30
+diffutils-version   = 3.6
+findutils-version   = 4.6.0.199-e3fc
 flock-version       = 0.2.3
 gawk-version        = 4.2.1
 ghostscript-version = 9.26
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index 17bd73f..a3d679e 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -49,6 +49,10 @@ idir  = $(BDIR)/dependencies/installed
 ibdir = $(BDIR)/dependencies/installed/bin
 ildir = $(BDIR)/dependencies/installed/lib
 
+# We'll need the system's PATH for making links to low-level programs we
+# won't be building ourselves.
+syspath         := $(PATH)
+
 # As we build more programs, we want to use our own pipeline's built
 # programs and libraries, not the host's.
 PATH            := $(ibdir):$(PATH)
@@ -56,7 +60,7 @@ LDFLAGS         := -L$(ildir) $(LDFLAGS)
 CPPFLAGS        := -I$(idir)/include $(CPPFLAGS)
 LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH)
 
-top-level-programs = bash which ls
+top-level-programs = bash which ls sed gawk grep diff find
 all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
 
 
@@ -88,9 +92,14 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
 tarballs = $(foreach t, bash-$(bash-version).tar.gz                         \
                         bzip2-$(bzip2-version).tar.gz                       \
                         coreutils-$(coreutils-version).tar.xz               \
-	                gzip-$(gzip-version).tar.gz                         \
+                        diffutils-$(diffutils-version).tar.xz               \
+                        findutils-$(findutils-version).tar.lz               \
+                        gawk-$(gawk-version).tar.lz                         \
+                        grep-$(grep-version).tar.xz                         \
+                        gzip-$(gzip-version).tar.gz                         \
                         lzip-$(lzip-version).tar.gz                         \
-	                make-$(make-version).tar.lz                         \
+                        make-$(make-version).tar.lz                         \
+                        sed-$(sed-version).tar.xz                           \
 	                tar-$(tar-version).tar.gz                           \
                         which-$(which-version).tar.gz                       \
                         xz-$(xz-version).tar.gz                             \
@@ -107,9 +116,14 @@ $(tarballs): $(tdir)/%:
 	  if   [ $$n = bash      ]; then w=http://ftp.gnu.org/gnu/bash;     \
 	  elif [ $$n = bzip      ]; then w=http://akhlaghi.org/src;         \
 	  elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils;\
+	  elif [ $$n = diffutils ]; then w=http://ftp.gnu.org/gnu/diffutils;\
+	  elif [ $$n = findutils ]; then w=http://akhlaghi.org/src;         \
+	  elif [ $$n = gawk      ]; then w=http://ftp.gnu.org/gnu/gawk;     \
+	  elif [ $$n = grep      ]; then w=http://ftp.gnu.org/gnu/grep;     \
 	  elif [ $$n = gzip      ]; then w=http://akhlaghi.org/src;         \
 	  elif [ $$n = lzip      ]; then w=http://download.savannah.gnu.org/releases/lzip; \
 	  elif [ $$n = make      ]; then w=http://akhlaghi.org/src;         \
+	  elif [ $$n = sed       ]; then w=http://ftp.gnu.org/gnu/sed;      \
 	  elif [ $$n = tar       ]; then w=http://ftp.gnu.org/gnu/tar;      \
 	  elif [ $$n = which     ]; then w=http://ftp.gnu.org/gnu/which;    \
 	  elif [ $$n = xz        ]; then w=http://tukaani.org/xz;           \
@@ -131,37 +145,75 @@ $(tarballs): $(tdir)/%:
 
 
 
-# GNU Gzip.
-$(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz
-	$(call gbuild, $<, gzip-$(gzip-version), static)
+# Low-level (not built) programs
+# ------------------------------
+#
+# For the time being, we aren't building a local C compiler, but we'll use
+# any C compiler that the system already has and just make a symbolic link
+# to it.
+makelink = a=$$(which $(1) 2> /dev/null); \
+	   if [ x$$a != x ]; then ln -s $$a $(ibdir)/$(1); fi
+$(ibdir):; mkdir $@
+$(ibdir)/low-level: | $(ibdir)
+        # We aren't building these low-levels tools yet ourselves. We'll
+        # thus just use what the host operating system has available.
+	PATH=$(syspath)
 
+        # The Assembler
+	$(call makelink,as)
 
+        # The compiler
+	$(call makelink,clang)
+	$(call makelink,gcc)
+	$(call makelink,g++)
+	$(call makelink,cc)
 
+        # The linker
+	$(call makelink,ar)
+	$(call makelink,ld)
 
+        # GNU Gettext (translate messages)
+	$(call makelink,msgfmt)
 
-# GNU Lzip: For a static build, the `-static' flag should be given to
-# LDFLAGS on the command-line (not from the environment).
-$(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz
-ifeq ($(static_build),yes)
-	$(call gbuild, $<, lzip-$(lzip-version), , LDFLAGS="-static")
-else
-	$(call gbuild, $<, lzip-$(lzip-version))
-endif
+        # GNU M4 (for managing building macros)
+	$(call makelink,m4)
 
+        # Needed by TeXLive specifically
+	$(call makelink,perl)
+	$(call makelink,wget)
 
+	echo "Low-level program links are setup" > $@
 
 
 
-# XZ Utils
-$(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz
-	$(call gbuild, $<, xz-$(xz-version), static)
 
 
+# Compression programs
+# --------------------
+#
+# The first set of programs to be built are those that we need to unpack
+# the source code tarballs of each program. First, we'll build the
+# necessary programs, then we'll build GNU Tar.
+$(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz \
+	       $(ibdir)/low-level
+	$(call gbuild, $<, gzip-$(gzip-version), static)
 
+# GNU Lzip: For a static build, the `-static' flag should be given to
+# LDFLAGS on the command-line (not from the environment).
+$(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz \
+               $(ibdir)/low-level
+ifeq ($(static_build),yes)
+	$(call gbuild, $<, lzip-$(lzip-version), , LDFLAGS="-static")
+else
+	$(call gbuild, $<, lzip-$(lzip-version))
+endif
 
+$(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz \
+             $(ibdir)/low-level
+	$(call gbuild, $<, xz-$(xz-version), static)
 
-# Bzip2: Bzip2 doesn't have a configure script.
-$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
+$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz \
+	        $(ibdir)/low-level
 	 tdir=bzip2-$(bzip2-version);                                  \
 	 if [ $(static_build) = yes ]; then                            \
 	   makecommand="make LDFLAGS=-static";                         \
@@ -172,10 +224,6 @@ $(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
 	 $$makecommand && make install PREFIX=$(idir) &&               \
 	 cd .. && rm -rf $$tdir
 
-
-
-
-
 # GNU Tar: When built statically, tar gives a segmentation fault on
 # unpacking Bash. So we'll build it dynamically.
 $(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
@@ -189,9 +237,15 @@ $(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
 
 
 
-# GNU Make: Unfortunately it needs dynamic linking in two instances: when
-# loading objects (dynamically linked libraries), or when using the
-# `getpwnam' function (for tilde expansion). The first can be disabled with
+# GNU Make
+# --------
+#
+# GNU Make is the second layer that we'll need to build the basic
+# dependencies.
+#
+# Unfortunately it needs dynamic linking in two instances: when loading
+# objects (dynamically linked libraries), or when using the `getpwnam'
+# function (for tilde expansion). The first can be disabled with
 # `--disable-load', but unfortunately I don't know any way to fix the
 # second. So, we'll have to build it dynamically for now.
 $(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \
@@ -202,7 +256,33 @@ $(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \
 
 
 
-# GNU Which:
+# Basic command-line programs necessary in build process of the
+# higher-level dependencies: Note that during the building of those
+# programs, there is no access to the system's PATH.
+$(ibdir)/diff: $(tdir)/diffutils-$(diffutils-version).tar.xz \
+               $(ibdir)/make
+	$(call gbuild, $<, diffutils-$(diffutils-version), static)
+
+$(ibdir)/find: $(tdir)/findutils-$(findutils-version).tar.lz \
+               $(ibdir)/make
+	$(call gbuild, $<, findutils-$(findutils-version), static)
+
+$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \
+               $(ibdir)/make
+	$(call gbuild, $<, gawk-$(gawk-version), static)
+
+$(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \
+               $(ibdir)/make
+	$(call gbuild, $<, grep-$(grep-version), static)
+
+$(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz \
+             $(ibdir)/make
+	$(call gbuild, $<, coreutils-$(coreutils-version), static)
+
+$(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \
+              $(ibdir)/make
+	$(call gbuild, $<, sed-$(sed-version), static)
+
 $(ibdir)/which: $(tdir)/which-$(which-version).tar.gz \
 	        $(ibdir)/make
 	$(call gbuild, $<, which-$(which-version), static)
@@ -234,12 +314,3 @@ endif
         # before making the link, we'll see if the file actually exists
         # there.
 	if [ -f $@ ]; then ln -s $@ $(ibdir)/sh; fi
-
-
-
-
-
-# GNU Coreutils
-$(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz \
-             $(ibdir)/make
-	$(call gbuild, $<, coreutils-$(coreutils-version), static)
diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk
index 25e2444..7b3a419 100644
--- a/reproduce/src/make/dependencies-build-rules.mk
+++ b/reproduce/src/make/dependencies-build-rules.mk
@@ -53,6 +53,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then          \
 	 fi;                                                                  \
 	 check="$(6)";                                                        \
 	 if [ x"$$check" = x ]; then check="echo Skipping-check"; fi;         \
+	 export SHELL=$(ibdir)/bash;                                          \
 	 cd $(ddir) && rm -rf $(2) && tar xf $(1) && cd $(2) &&               \
 	 ./configure $(4) --prefix=$(idir) &&                                 \
 	 make $(5) &&                                                         \
@@ -70,6 +71,7 @@ cbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then          \
 	   export LDFLAGS="$$LDFLAGS -static";                                \
 	   opts="-DBUILD_SHARED_LIBS=OFF";                                    \
 	 fi;                                                                  \
+	 export SHELL=$(ibdir)/bash;                                          \
 	 cd $(ddir) && rm -rf $(2) && tar xf $(1) && cd $(2) &&               \
 	 rm -rf pipeline-build && mkdir pipeline-build &&                     \
 	 cd pipeline-build &&                                                 \
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index a784883..c795b34 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -43,7 +43,7 @@ ildir  = $(BDIR)/dependencies/installed/lib
 ilidir = $(BDIR)/dependencies/installed/lib/built
 
 # Define the top-level programs to build (installed in `.local/bin').
-top-level-programs = gawk gs grep sed git flock astnoisechisel texlive-ready
+top-level-programs = gs git flock astnoisechisel texlive-ready
 all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
 
 # Other basic environment settings: We are only including the host
@@ -52,11 +52,11 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
 # using our internally built libraries.
 .ONESHELL:
 .SHELLFLAGS     := -ec
-SHELL           := $(ibdir)/bash
-PATH            := $(ibdir):$(PATH)
+PATH            := $(ibdir)
+LD_LIBRARY_PATH := $(ildir)
 LDFLAGS         := -L$(ildir)
+SHELL           := $(ibdir)/bash
 CPPFLAGS        := -I$(idir)/include
-LD_LIBRARY_PATH := $(ildir)
 
 
 
@@ -76,17 +76,14 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz             \
                         cmake-$(cmake-version).tar.gz                 \
                         curl-$(curl-version).tar.gz                   \
 	                flock-$(flock-version).tar.xz                 \
-	                gawk-$(gawk-version).tar.lz                   \
 	                ghostscript-$(ghostscript-version).tar.gz     \
 	                git-$(git-version).tar.xz                     \
 	                gnuastro-$(gnuastro-version).tar.lz           \
-	                grep-$(grep-version).tar.xz                   \
 	                gsl-$(gsl-version).tar.gz                     \
 	                jpegsrc.$(libjpeg-version).tar.gz             \
                         tiff-$(libtiff-version).tar.gz                \
                         libtool-$(libtool-version).tar.xz             \
                         libgit2-$(libgit2-version).tar.gz             \
-	                sed-$(sed-version).tar.xz                     \
 	                wcslib-$(wcslib-version).tar.bz2              \
                         zlib-$(zlib-version).tar.gz                   \
                       , $(tdir)/$(t) )
@@ -113,18 +110,15 @@ $(tarballs): $(tdir)/%:
 	  elif [ $$n = cmake       ]; then w=https://cmake.org/files/v3.12
 	  elif [ $$n = curl        ]; then w=https://curl.haxx.se/download
 	  elif [ $$n = flock       ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version)
-	  elif [ $$n = gawk        ]; then w=http://ftp.gnu.org/gnu/gawk
 	  elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926
 	  elif [ $$n = git         ]; then w=https://mirrors.edge.kernel.org/pub/software/scm/git
 	  elif [ $$n = gnuastro    ]; then w=http://akhlaghi.org/src
-	  elif [ $$n = grep        ]; then w=http://ftp.gnu.org/gnu/grep
 	  elif [ $$n = gsl         ]; then w=http://ftp.gnu.org/gnu/gsl
 	  elif [ $$n = jpegsrc     ]; then w=http://ijg.org/files
 	  elif [ $$n = libtool     ]; then w=ftp://ftp.gnu.org/gnu/libtool
 	  elif [ $$n = libgit      ]; then
 	    mergenames=0
 	    w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
-	  elif [ $$n = sed         ]; then w=http://ftp.gnu.org/gnu/sed
 	  elif [ $$n = tiff        ]; then w=https://download.osgeo.org/libtiff
 	  elif [ $$n = wcslib      ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib
 	  elif [ $$n = zlib        ]; then w=http://www.zlib.net
@@ -231,15 +225,6 @@ $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz                           \
                $(ilidir)/zlib
 	$(call gbuild, $<, curl-$(curl-version), static, --without-brotli)
 
-$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz
-	$(call gbuild, $<, gawk-$(gawk-version), static)
-
-$(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz
-	$(call gbuild, $<, sed-$(sed-version), static)
-
-$(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz
-	$(call gbuild, $<, grep-$(grep-version), static)
-
 $(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.xz
 	$(call gbuild, $<, libtool-$(libtool-version), static)
 
@@ -251,7 +236,8 @@ $(ibdir)/flock: $(tdir)/flock-$(flock-version).tar.xz
 
 $(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \
               $(ilidir)/zlib
-	$(call gbuild, $<, git-$(git-version), static)
+	$(call gbuild, $<, git-$(git-version), static, \
+                       --without-tcltk --with-shell=$(ibdir)/bash)
 
 $(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \
                          $(ibdir)/gs                                 \
@@ -281,10 +267,9 @@ endif
 $(ibdir)/texlive-ready-tlmgr: reproduce/config/pipeline/texlive.conf
 
         # To work with TeX live installation, we'll need the internet.
-	if ping -c1 ctan.org; then
-          # Download the TeX Live installation tarball.
-	  $(DOWNLOADER) $(tdir)/install-tl-unx.tar.gz \
-	     http://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz
+	if $(DOWNLOADER) $(tdir)/install-tl-unx.tar.gz                         \
+	   http://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz; \
+          then
 
           # Unpack, enter the directory, and install based on the given
           # configuration (prerequisite of this rule).
@@ -321,9 +306,8 @@ $(ibdir)/texlive-ready: reproduce/config/pipeline/dependency-texlive.mk \
 
         # To work with TeX live installation, we'll need the internet.
 	res=$(cat $(ibdir)/texlive-ready-tlmgr)
-	if ping -c1 ctan.org                                            \
-	   && [ -f $(ibdir)/texlive-ready-tlmgr ]                       \
-           && [ x$$res != x"NOT" ]; then
+	if [ -f $(ibdir)/texlive-ready-tlmgr ]; then
+
           # The current directory is necessary later.
 	  topdir=$$(pwd)
 
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index 41a5e05..61f28cf 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -191,18 +191,22 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
 	@echo "\newcommand{\bdir}{$(BDIR)}"       >> $@
 
         # Versions of programs (same order as 'dependency-versions.mk').
+	$(call pvcheck, awk, $(gawk-version), GNU AWK, gawkversion)
 	$(call pvcheck, bash, $(bash-version), GNU Bash, bashversion)
 	$(call pvcheck, cmake, $(cmake-version), CMake, cmakeversion)
 	$(call pvcheck, curl, $(curl-version), cURL, curlversion)
-	$(call pvcheck, ls, $(coreutils-version), GNU Coreutils,       \
-	                coreutilsversion)
-	$(call pvcheck, awk, $(gawk-version), GNU AWK, gawkversion)
+	$(call pvcheck, diff, $(diffutils-version), GNU Diffutils,     \
+	                diffutilsversion)
+	$(call pvcheck, find, $(findutils-version), GNU Findutils,       \
+	                findutilsversion)
 	$(call pvcheck, gs, $(ghostscript-version), GPL Ghostscript,   \
 	                    ghostscriptversion)
 	$(call pvcheck, git, $(git-version), Git, gitversion)
 	$(call pvcheck, grep, $(grep-version), GNU Grep, grepversion)
 	$(call pvcheck, libtool, $(libtool-version), GNU Libtool,      \
 	                libtoolversion)
+	$(call pvcheck, ls, $(coreutils-version), GNU Coreutils,       \
+	                coreutilsversion)
 	$(call pvcheck, lzip, $(lzip-version), Lzip, lzipversion)
 	$(call pvcheck, make, $(make-version), GNU Make, makeversion)
 	$(call pvcheck, sed, $(sed-version), GNU SED, sedversion)