From 5bb72e98b312d22191b13f3821f258449e548516 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 26 Nov 2018 18:09:35 +0000 Subject: High-level dependencies build without system's PATH The high-level dependencies are now built without having access to the system's PATH. To do this, all the necessary software that we aren't building ourselves are now brought into the installed `bin/' directory using a symbolic link to the corresponding software on the host. To do this, it was also necessary to increase the number of basic/low-level packages that we are building, and add several more (Diffutils and Findutils). With this process in place, we now have a list of the exact software packages that we are not building our selves, enabling easy building of all such dependencies in the future. --- paper.tex | 42 +++---- reproduce/config/pipeline/dependency-versions.mk | 2 + reproduce/src/make/dependencies-basic.mk | 143 +++++++++++++++++------ reproduce/src/make/dependencies-build-rules.mk | 2 + reproduce/src/make/dependencies.mk | 38 ++---- reproduce/src/make/initialize.mk | 10 +- 6 files changed, 151 insertions(+), 86 deletions(-) diff --git a/paper.tex b/paper.tex index 97077d2..6aa30bd 100644 --- a/paper.tex +++ b/paper.tex @@ -204,26 +204,28 @@ SUNDIAL ITN, and from the Spanish Ministry of Economy and Competitiveness The following free software tools were also critical component of this research (in alphabetical order): Bzip2 \bziptwoversion, CFITSIO \cfitsioversion, CMake \cmakeversion, cURL \curlversion, Discoteq flock -\flockversion, Git \gitversion, GNU Bash \bashversion, GNU Coreutils -\coreutilsversion, GNU AWK \gawkversion, GNU Grep \grepversion, GNU Libtool -\libtoolversion, GNU Make \makeversion, GNU Sed \sedversion, GNU Scientific -Library (GSL) \gslversion, GNU Tar \tarversion, GNU Which \whichversion, -Lzip \lzipversion, GPL Ghostscript \ghostscriptversion, Libgit2 -\libgitwoversion, Libtiff \libtiffversion, WCSLIB \wcslibversion, XZ Utils -\xzversion, and ZLib \zlibversion. The final paper was produced with \TeX{} -Live \texliveversion, using the following packages: \TeX{} \textexversion, -EC \texecversion, NewTX \texnewtxversion, Fontaxes \texfontaxesversion, -Keyval, \texxkeyvalversion, Etoolbox \texetoolboxversion, Xcolor -\texxcolorversion, Setspace \texsetspaceversion, Caption -\texcaptionversion, Footmisc \texfootmiscversion, Datetime -\texdatetimeversion, Fmtcount \texfmtcountversion, Titlesec -\textitlesecversion, Preprint \texpreprintversion, Ulem \texulemversion, -Bib\LaTeX{} \texbiblatexversion, Biber \texbiberversion, Logreq -\texlogreqversion, PGF/TiKZ \texpgfversion, PGFPlots \texpgfplotsversion, -FP \texfpversion, Courier \texcourierversion, \TeX-gyre \textexgyreversion, -TXFonts \textxfontsversion, Times \textimesversion. We are very grateful to -all their creators for freely providing this necessary infrastructure. This -research would not be possible without them. +\flockversion, Git \gitversion, GNU AWK \gawkversion, GNU Bash +\bashversion, GNU Coreutils \coreutilsversion, GNU Diffutils +\diffutilsversion, GNU Findutils \findutilsversion, GNU Grep \grepversion, +GNU Libtool \libtoolversion, GNU Make \makeversion, GNU Sed \sedversion, +GNU Scientific Library (GSL) \gslversion, GNU Tar \tarversion, GNU Which +\whichversion, Lzip \lzipversion, GPL Ghostscript \ghostscriptversion, +Libgit2 \libgitwoversion, Libtiff \libtiffversion, WCSLIB \wcslibversion, +XZ Utils \xzversion, and ZLib \zlibversion. The final paper was produced +with \TeX{} Live \texliveversion, using the following packages: \TeX{} +\textexversion, EC \texecversion, NewTX \texnewtxversion, Fontaxes +\texfontaxesversion, Keyval, \texxkeyvalversion, Etoolbox +\texetoolboxversion, Xcolor \texxcolorversion, Setspace +\texsetspaceversion, Caption \texcaptionversion, Footmisc +\texfootmiscversion, Datetime \texdatetimeversion, Fmtcount +\texfmtcountversion, Titlesec \textitlesecversion, Preprint +\texpreprintversion, Ulem \texulemversion, Bib\LaTeX{} \texbiblatexversion, +Biber \texbiberversion, Logreq \texlogreqversion, PGF/TiKZ \texpgfversion, +PGFPlots \texpgfplotsversion, FP \texfpversion, Courier \texcourierversion, +\TeX-gyre \textexgyreversion, TXFonts \textxfontsversion, Times +\textimesversion. We are very grateful to all their creators for freely +providing this necessary infrastructure. This research would not be +possible without them. %% Tell BibLaTeX to put the bibliography list here. \printbibliography diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index e5b4bbb..9c6f18f 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -5,6 +5,8 @@ bash-version = 4.4.18 bzip2-version = 1.0.6 cmake-version = 3.12.4 coreutils-version = 8.30 +diffutils-version = 3.6 +findutils-version = 4.6.0.199-e3fc flock-version = 0.2.3 gawk-version = 4.2.1 ghostscript-version = 9.26 diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 17bd73f..a3d679e 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -49,6 +49,10 @@ idir = $(BDIR)/dependencies/installed ibdir = $(BDIR)/dependencies/installed/bin ildir = $(BDIR)/dependencies/installed/lib +# We'll need the system's PATH for making links to low-level programs we +# won't be building ourselves. +syspath := $(PATH) + # As we build more programs, we want to use our own pipeline's built # programs and libraries, not the host's. PATH := $(ibdir):$(PATH) @@ -56,7 +60,7 @@ LDFLAGS := -L$(ildir) $(LDFLAGS) CPPFLAGS := -I$(idir)/include $(CPPFLAGS) LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) -top-level-programs = bash which ls +top-level-programs = bash which ls sed gawk grep diff find all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) @@ -88,9 +92,14 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) tarballs = $(foreach t, bash-$(bash-version).tar.gz \ bzip2-$(bzip2-version).tar.gz \ coreutils-$(coreutils-version).tar.xz \ - gzip-$(gzip-version).tar.gz \ + diffutils-$(diffutils-version).tar.xz \ + findutils-$(findutils-version).tar.lz \ + gawk-$(gawk-version).tar.lz \ + grep-$(grep-version).tar.xz \ + gzip-$(gzip-version).tar.gz \ lzip-$(lzip-version).tar.gz \ - make-$(make-version).tar.lz \ + make-$(make-version).tar.lz \ + sed-$(sed-version).tar.xz \ tar-$(tar-version).tar.gz \ which-$(which-version).tar.gz \ xz-$(xz-version).tar.gz \ @@ -107,9 +116,14 @@ $(tarballs): $(tdir)/%: if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash; \ elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \ elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils;\ + elif [ $$n = diffutils ]; then w=http://ftp.gnu.org/gnu/diffutils;\ + elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk; \ + elif [ $$n = grep ]; then w=http://ftp.gnu.org/gnu/grep; \ elif [ $$n = gzip ]; then w=http://akhlaghi.org/src; \ elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \ elif [ $$n = make ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed; \ elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \ elif [ $$n = which ]; then w=http://ftp.gnu.org/gnu/which; \ elif [ $$n = xz ]; then w=http://tukaani.org/xz; \ @@ -131,37 +145,75 @@ $(tarballs): $(tdir)/%: -# GNU Gzip. -$(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz - $(call gbuild, $<, gzip-$(gzip-version), static) +# Low-level (not built) programs +# ------------------------------ +# +# For the time being, we aren't building a local C compiler, but we'll use +# any C compiler that the system already has and just make a symbolic link +# to it. +makelink = a=$$(which $(1) 2> /dev/null); \ + if [ x$$a != x ]; then ln -s $$a $(ibdir)/$(1); fi +$(ibdir):; mkdir $@ +$(ibdir)/low-level: | $(ibdir) + # We aren't building these low-levels tools yet ourselves. We'll + # thus just use what the host operating system has available. + PATH=$(syspath) + # The Assembler + $(call makelink,as) + # The compiler + $(call makelink,clang) + $(call makelink,gcc) + $(call makelink,g++) + $(call makelink,cc) + # The linker + $(call makelink,ar) + $(call makelink,ld) + # GNU Gettext (translate messages) + $(call makelink,msgfmt) -# GNU Lzip: For a static build, the `-static' flag should be given to -# LDFLAGS on the command-line (not from the environment). -$(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz -ifeq ($(static_build),yes) - $(call gbuild, $<, lzip-$(lzip-version), , LDFLAGS="-static") -else - $(call gbuild, $<, lzip-$(lzip-version)) -endif + # GNU M4 (for managing building macros) + $(call makelink,m4) + # Needed by TeXLive specifically + $(call makelink,perl) + $(call makelink,wget) + echo "Low-level program links are setup" > $@ -# XZ Utils -$(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz - $(call gbuild, $<, xz-$(xz-version), static) +# Compression programs +# -------------------- +# +# The first set of programs to be built are those that we need to unpack +# the source code tarballs of each program. First, we'll build the +# necessary programs, then we'll build GNU Tar. +$(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz \ + $(ibdir)/low-level + $(call gbuild, $<, gzip-$(gzip-version), static) +# GNU Lzip: For a static build, the `-static' flag should be given to +# LDFLAGS on the command-line (not from the environment). +$(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz \ + $(ibdir)/low-level +ifeq ($(static_build),yes) + $(call gbuild, $<, lzip-$(lzip-version), , LDFLAGS="-static") +else + $(call gbuild, $<, lzip-$(lzip-version)) +endif +$(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz \ + $(ibdir)/low-level + $(call gbuild, $<, xz-$(xz-version), static) -# Bzip2: Bzip2 doesn't have a configure script. -$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz +$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz \ + $(ibdir)/low-level tdir=bzip2-$(bzip2-version); \ if [ $(static_build) = yes ]; then \ makecommand="make LDFLAGS=-static"; \ @@ -172,10 +224,6 @@ $(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz $$makecommand && make install PREFIX=$(idir) && \ cd .. && rm -rf $$tdir - - - - # GNU Tar: When built statically, tar gives a segmentation fault on # unpacking Bash. So we'll build it dynamically. $(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \ @@ -189,9 +237,15 @@ $(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \ -# GNU Make: Unfortunately it needs dynamic linking in two instances: when -# loading objects (dynamically linked libraries), or when using the -# `getpwnam' function (for tilde expansion). The first can be disabled with +# GNU Make +# -------- +# +# GNU Make is the second layer that we'll need to build the basic +# dependencies. +# +# Unfortunately it needs dynamic linking in two instances: when loading +# objects (dynamically linked libraries), or when using the `getpwnam' +# function (for tilde expansion). The first can be disabled with # `--disable-load', but unfortunately I don't know any way to fix the # second. So, we'll have to build it dynamically for now. $(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \ @@ -202,7 +256,33 @@ $(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \ -# GNU Which: +# Basic command-line programs necessary in build process of the +# higher-level dependencies: Note that during the building of those +# programs, there is no access to the system's PATH. +$(ibdir)/diff: $(tdir)/diffutils-$(diffutils-version).tar.xz \ + $(ibdir)/make + $(call gbuild, $<, diffutils-$(diffutils-version), static) + +$(ibdir)/find: $(tdir)/findutils-$(findutils-version).tar.lz \ + $(ibdir)/make + $(call gbuild, $<, findutils-$(findutils-version), static) + +$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \ + $(ibdir)/make + $(call gbuild, $<, gawk-$(gawk-version), static) + +$(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \ + $(ibdir)/make + $(call gbuild, $<, grep-$(grep-version), static) + +$(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz \ + $(ibdir)/make + $(call gbuild, $<, coreutils-$(coreutils-version), static) + +$(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \ + $(ibdir)/make + $(call gbuild, $<, sed-$(sed-version), static) + $(ibdir)/which: $(tdir)/which-$(which-version).tar.gz \ $(ibdir)/make $(call gbuild, $<, which-$(which-version), static) @@ -234,12 +314,3 @@ endif # before making the link, we'll see if the file actually exists # there. if [ -f $@ ]; then ln -s $@ $(ibdir)/sh; fi - - - - - -# GNU Coreutils -$(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz \ - $(ibdir)/make - $(call gbuild, $<, coreutils-$(coreutils-version), static) diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk index 25e2444..7b3a419 100644 --- a/reproduce/src/make/dependencies-build-rules.mk +++ b/reproduce/src/make/dependencies-build-rules.mk @@ -53,6 +53,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then \ fi; \ check="$(6)"; \ if [ x"$$check" = x ]; then check="echo Skipping-check"; fi; \ + export SHELL=$(ibdir)/bash; \ cd $(ddir) && rm -rf $(2) && tar xf $(1) && cd $(2) && \ ./configure $(4) --prefix=$(idir) && \ make $(5) && \ @@ -70,6 +71,7 @@ cbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then \ export LDFLAGS="$$LDFLAGS -static"; \ opts="-DBUILD_SHARED_LIBS=OFF"; \ fi; \ + export SHELL=$(ibdir)/bash; \ cd $(ddir) && rm -rf $(2) && tar xf $(1) && cd $(2) && \ rm -rf pipeline-build && mkdir pipeline-build && \ cd pipeline-build && \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index a784883..c795b34 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,7 +43,7 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). -top-level-programs = gawk gs grep sed git flock astnoisechisel texlive-ready +top-level-programs = gs git flock astnoisechisel texlive-ready all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # Other basic environment settings: We are only including the host @@ -52,11 +52,11 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # using our internally built libraries. .ONESHELL: .SHELLFLAGS := -ec -SHELL := $(ibdir)/bash -PATH := $(ibdir):$(PATH) +PATH := $(ibdir) +LD_LIBRARY_PATH := $(ildir) LDFLAGS := -L$(ildir) +SHELL := $(ibdir)/bash CPPFLAGS := -I$(idir)/include -LD_LIBRARY_PATH := $(ildir) @@ -76,17 +76,14 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ flock-$(flock-version).tar.xz \ - gawk-$(gawk-version).tar.lz \ ghostscript-$(ghostscript-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ - grep-$(grep-version).tar.xz \ gsl-$(gsl-version).tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ - sed-$(sed-version).tar.xz \ wcslib-$(wcslib-version).tar.bz2 \ zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) @@ -113,18 +110,15 @@ $(tarballs): $(tdir)/%: elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) - elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=https://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://akhlaghi.org/src - elif [ $$n = grep ]; then w=http://ftp.gnu.org/gnu/grep elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl elif [ $$n = jpegsrc ]; then w=http://ijg.org/files elif [ $$n = libtool ]; then w=ftp://ftp.gnu.org/gnu/libtool elif [ $$n = libgit ]; then mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz - elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib elif [ $$n = zlib ]; then w=http://www.zlib.net @@ -231,15 +225,6 @@ $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz \ $(ilidir)/zlib $(call gbuild, $<, curl-$(curl-version), static, --without-brotli) -$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz - $(call gbuild, $<, gawk-$(gawk-version), static) - -$(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz - $(call gbuild, $<, sed-$(sed-version), static) - -$(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz - $(call gbuild, $<, grep-$(grep-version), static) - $(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.xz $(call gbuild, $<, libtool-$(libtool-version), static) @@ -251,7 +236,8 @@ $(ibdir)/flock: $(tdir)/flock-$(flock-version).tar.xz $(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \ $(ilidir)/zlib - $(call gbuild, $<, git-$(git-version), static) + $(call gbuild, $<, git-$(git-version), static, \ + --without-tcltk --with-shell=$(ibdir)/bash) $(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \ $(ibdir)/gs \ @@ -281,10 +267,9 @@ endif $(ibdir)/texlive-ready-tlmgr: reproduce/config/pipeline/texlive.conf # To work with TeX live installation, we'll need the internet. - if ping -c1 ctan.org; then - # Download the TeX Live installation tarball. - $(DOWNLOADER) $(tdir)/install-tl-unx.tar.gz \ - http://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz + if $(DOWNLOADER) $(tdir)/install-tl-unx.tar.gz \ + http://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz; \ + then # Unpack, enter the directory, and install based on the given # configuration (prerequisite of this rule). @@ -321,9 +306,8 @@ $(ibdir)/texlive-ready: reproduce/config/pipeline/dependency-texlive.mk \ # To work with TeX live installation, we'll need the internet. res=$(cat $(ibdir)/texlive-ready-tlmgr) - if ping -c1 ctan.org \ - && [ -f $(ibdir)/texlive-ready-tlmgr ] \ - && [ x$$res != x"NOT" ]; then + if [ -f $(ibdir)/texlive-ready-tlmgr ]; then + # The current directory is necessary later. topdir=$$(pwd) diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 41a5e05..61f28cf 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -191,18 +191,22 @@ $(mtexdir)/initialize.tex: | $(mtexdir) @echo "\newcommand{\bdir}{$(BDIR)}" >> $@ # Versions of programs (same order as 'dependency-versions.mk'). + $(call pvcheck, awk, $(gawk-version), GNU AWK, gawkversion) $(call pvcheck, bash, $(bash-version), GNU Bash, bashversion) $(call pvcheck, cmake, $(cmake-version), CMake, cmakeversion) $(call pvcheck, curl, $(curl-version), cURL, curlversion) - $(call pvcheck, ls, $(coreutils-version), GNU Coreutils, \ - coreutilsversion) - $(call pvcheck, awk, $(gawk-version), GNU AWK, gawkversion) + $(call pvcheck, diff, $(diffutils-version), GNU Diffutils, \ + diffutilsversion) + $(call pvcheck, find, $(findutils-version), GNU Findutils, \ + findutilsversion) $(call pvcheck, gs, $(ghostscript-version), GPL Ghostscript, \ ghostscriptversion) $(call pvcheck, git, $(git-version), Git, gitversion) $(call pvcheck, grep, $(grep-version), GNU Grep, grepversion) $(call pvcheck, libtool, $(libtool-version), GNU Libtool, \ libtoolversion) + $(call pvcheck, ls, $(coreutils-version), GNU Coreutils, \ + coreutilsversion) $(call pvcheck, lzip, $(lzip-version), Lzip, lzipversion) $(call pvcheck, make, $(make-version), GNU Make, makeversion) $(call pvcheck, sed, $(sed-version), GNU SED, sedversion) -- cgit v1.2.1