diff options
| -rwxr-xr-x | configure | 2 | ||||
| -rw-r--r-- | paper.tex | 23 | ||||
| -rw-r--r-- | reproduce/config/pipeline/dependency-versions.mk | 4 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies-basic.mk | 124 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies-build-rules.mk | 41 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies.mk | 32 | ||||
| -rw-r--r-- | reproduce/src/make/initialize.mk | 18 |
7 files changed, 164 insertions, 80 deletions
@@ -456,7 +456,7 @@ echo echo "Please run the following command to start." echo "(Replace '8' with the number of CPU threads)" echo -echo " .local/bin/make -j8" +echo " ./.local/bin/make -j8" echo echo "To change the configuration later, please re-run './configure'," echo "DO NOT manually edit the relevant files." @@ -177,17 +177,18 @@ SUNDIAL ITN, and from the Spanish Ministry of Economy and Competitiveness (MINECO) under grant number AYA2016-76219-P. The following free software tools were also critical component of this -research (in alphabetical order): {\small CFITSIO} \cfitsioversion, CMake -\cmakeversion, c{\small URL} \curlversion, Git \gitversion, \gnu{Bash} -\bashversion, \gnu{Coreutils} \coreutilsversion, \gnu{{\small AWK}} -\gawkversion, \gnu{Grep} \grepversion, \gnu{Libtool} \libtoolversion, -\gnu{Make} \makeversion, \gnu{Sed} \sedversion, \gnu{Scientific Library} -({\small GSL}) \gslversion, \gnu{Tar} \tarversion, Lzip \lzipversion, - {\small GPL} Ghostscript \ghostscriptversion, Libgit2 \libgitwoversion, - Libtiff \libtiffversion, {{\small WCSLIB}} \wcslibversion, and ZLib - \zlibversion. We are very grateful to all their creators for freely - providing this necessary infrastructure. This research would not be - possible without them. +research (in alphabetical order): Bzip2 \bziptwoversion, {\small CFITSIO} +\cfitsioversion, CMake \cmakeversion, c{\small URL} \curlversion, Git +\gitversion, \gnu{Bash} \bashversion, \gnu{Binutils} \binutilsversion, +\gnu{Coreutils} \coreutilsversion, \gnu{{\small AWK}} \gawkversion, +\gnu{Grep} \grepversion, \gnu{Libtool} \libtoolversion, \gnu{Make} +\makeversion, \gnu{Sed} \sedversion, \gnu{Scientific Library} ({\small + GSL}) \gslversion, \gnu{Tar} \tarversion, Lzip \lzipversion, {\small GPL} +Ghostscript \ghostscriptversion, Libgit2 \libgitwoversion, Libtiff +\libtiffversion, {{\small WCSLIB}} \wcslibversion, {\small XZ} Utils +\xzversion, and ZLib \zlibversion. We are very grateful to all their +creators for freely providing this necessary infrastructure. This research +would not be possible without them. %% Tell BibLaTeX to put the bibliography list here. \printbibliography diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index fdad202..9f8d99a 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -2,6 +2,8 @@ # Programs bash-version = 4.4.18 +binutils-version = 2.31.1 +bzip2-version = 1.0.6 cmake-version = 3.12.4 coreutils-version = 8.30 gawk-version = 4.2.1 @@ -9,11 +11,13 @@ ghostscript-version = 9.25 git-version = 2.19.1 gnuastro-version = 0.7.62-13b0 grep-version = 3.1 +gzip-version = 1.9.10-051e libtool-version = 2.4.6 lzip-version = 1.20 make-version = 4.2.90 sed-version = 4.5 tar-version = 1.30 +xz-version = 5.2.4 # Libraries cfitsio-version = 3.45 diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 8fbfeaf..be6e5ed 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -1,5 +1,5 @@ -# Build the VERY BASIC reproduction pipeline dependencies (programs and -# libraries). +# Build the VERY BASIC reproduction pipeline dependencies before everything +# else using minimum Make and Shell. # # ------------------------------------------------------------------------ # !!!!! IMPORTANT NOTES !!!!! @@ -7,9 +7,11 @@ # This Makefile will be run by the initial `./configure' script. It is not # included into the reproduction pipe after that. # -# This Makefile builds very low-level and basic tools like GNU Bash and GNU -# Make. Therefore this is the only Makefile in the reproduction pipeline -# where you MUST NOT assume that modern GNU Bash or GNU Make are used. +# This Makefile builds very low-level and basic tools like GNU Tar, and +# various compression programs, GNU Bash, and GNU Make. Therefore this is +# the only Makefile in the reproduction pipeline where you MUST NOT assume +# that modern GNU Bash or GNU Make are used. After this Makefile, other +# Makefiles can safely assume the fixed version of all these software. # # This Makefile is a very simplified version of `dependencies.mk' in the # same directory. See there for more comments. @@ -48,8 +50,11 @@ ibdir = $(BDIR)/dependencies/installed/bin ildir = $(BDIR)/dependencies/installed/lib # As we build more programs, we want to use our own pipeline's built -# programs, not the systems. -PATH := $(ibdir):$(PATH) +# programs and libraries, not the host's. +PATH := $(ibdir):$(PATH) +LDFLAGS := -L$(ildir) $(LDFLAGS) +CPPFLAGS := -I$(idir)/include $(CPPFLAGS) +LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) top-level-programs = bash all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) @@ -65,10 +70,30 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # is not recognized by some versions of Make (even older GNU Makes). So # we'll have to make sure the recipe doesn't break into multiple shell # calls (so we can preserve the variables). +# +# Software hosted at akhlaghi.org/src: As of our latest check (November +# 2018) their major release tarballs either crash or don't build on some +# systems (for example Make or Gzip), or they don't exist (for example +# Bzip2). +# +# In the first case, we used their Git repo and bootstrapped them (just +# like Gnuastro) and built the most recent tarball off of that. In the case +# of Bzip2: its webpage has expired and doesn't host the data any more. It +# is available on the link below (archive.org): +# +# https://web.archive.org/web/20180624184806/http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz +# +# However, downloading from this link is slow (because its just a link). So +# its easier to just keep a with the others. tarballs = $(foreach t, bash-$(bash-version).tar.gz \ + binutils-$(binutils-version).tar.lz \ + bzip2-$(bzip2-version).tar.gz \ + gzip-$(gzip-version).tar.lz \ lzip-$(lzip-version).tar.gz \ - make-$(make-version).tar.gz \ + make-$(make-version).tar.lz \ tar-$(tar-version).tar.gz \ + xz-$(xz-version).tar.gz \ + zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then \ @@ -79,10 +104,15 @@ $(tarballs): $(tdir)/%: | awk '{print $$1}' ); \ \ mergenames=1; \ - if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash; \ - elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \ - elif [ $$n = make ]; then w=http://akhlaghi.org/src; \ - elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \ + if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash; \ + elif [ $$n = binutils ]; then w=http://ftp.gnu.org/gnu/binutils; \ + elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = gzip ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \ + elif [ $$n = make ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \ + elif [ $$n = xz ]; then w=http://tukaani.org/xz; \ + elif [ $$n = zlib ]; then w=http://www.zlib.net \ else \ echo; echo; echo; \ echo "'$$n' not a dependency name (for downloading)." \ @@ -101,27 +131,65 @@ $(tarballs): $(tdir)/%: -# Basic programs (sorted alphabetically), see prerequisites for which one -# will be built first. -$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ - $(ibdir)/make - $(call gbuild,$(subst $(tdir)/,,$<), bash-$(bash-version), , \ - --enable-static-link) - +# GNU Lzip: For a static build, the `-static' flag should be given to +# LDFLAGS on the command-line (not from the environment). $(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz - $(call gbuild,$(subst $(tdir)/,,$<), lzip-$(lzip-version), static) + $(call gbuild,$(subst $(tdir)/,,$<), lzip-$(lzip-version), , \ + LDFLAGS="-static") + + +# GNU Gzip. +$(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.lz \ + $(ibdir)/lzip + $(call gbuild,$(subst $(tdir)/,,$<), gzip-$(gzip-version), static) + + +# Zlib: its `./configure' doesn't use Autoconf's configure script, it just +# accepts a direct `--static' option. +$(ildir)/libz.a: $(tdir)/zlib-$(zlib-version).tar.gz + $(call gbuild,$(subst $(tdir)/,,$<), zlib-$(zlib-version), , \ + --static) -# Unfortunately GNU Make needs dynamic linking in two instances: when +# XZ Utils +$(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz + $(call gbuild,$(subst $(tdir)/,,$<), xz-$(xz-version), static) + +# Bzip2: Bzip2 doesn't have a configure script. +$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz + tdir=bzip2-$(bzip2-version); \ + cd $(ddir) && rm -rf $$tdir && tar xf $< && cd $$tdir && \ + make LDFLAGS=-static && make install PREFIX=$(idir) && \ + cd .. && rm -rf $$tdir + + +# GNU Binutils: +$(ibdir)/nm: $(tdir)/binutils-$(binutils-version).tar.lz \ + $(ibdir)/lzip \ + $(ildir)/libz.a + $(call gbuild,$(subst $(tdir)/,,$<), binutils-$(binutils-version), \ + static) + +# GNU Tar: When built statically, tar gives a segmentation fault on +# unpacking Bash. So we'll build it dynamically. +$(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \ + $(ibdir)/bzip2 \ + $(ibdir)/gzip \ + $(ibdir)/lzip \ + $(ibdir)/xz \ + $(ibdir)/nm + $(call gbuild,$(subst $(tdir)/,,$<), tar-$(tar-version)) + +# GNU Make: Unfortunately it needs dynamic linking in two instances: when # loading objects (dynamically linked libraries), or when using the # `getpwnam' function (for tilde expansion). The first can be disabled with # `--disable-load', but unfortunately I don't know any way to fix the # second. So, we'll have to build it dynamically for now. -$(ibdir)/make: $(tdir)/make-$(make-version).tar.gz \ +$(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \ $(ibdir)/tar - $(call gbuild,$(subst $(tdir)/,,$<), make-$(make-version), , , ,) + $(call gbuild,$(subst $(tdir)/,,$<), make-$(make-version)) -# When built statically, tar gives a segmentation fault on unpacking -# Bash. So we'll build it dynamically. -$(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \ - $(ibdir)/lzip - $(call gbuild,$(subst $(tdir)/,,$<), tar-$(tar-version)) +# GNU Bash +$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ + $(ibdir)/make + $(call gbuild,$(subst $(tdir)/,,$<), bash-$(bash-version), , \ + --enable-static-link) diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk index a2e4a89..31e7759 100644 --- a/reproduce/src/make/dependencies-build-rules.mk +++ b/reproduce/src/make/dependencies-build-rules.mk @@ -48,15 +48,15 @@ # 4: Extra configuration options. # 5: Extra options/arguments to pass to Make. # 6: Step to run between `make' and `make install': usually `make check'. -gbuild = if [ $(3)x = staticx ]; then export LDFLAGS="$$LDFLAGS -static"; fi;\ - check="$(6)"; \ - if [ x"$$check" = x ]; then check="echo Skipping-check"; fi; \ - cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) && \ - ./configure $(4) --prefix=$(idir) && \ - make $(5) && \ - $$check && \ - make install&& \ - cd ..&& rm -rf $(2) +gbuild = if [ $(3)x = staticx ]; then export LDFLAGS="$$LDFLAGS -static"; fi; \ + check="$(6)"; \ + if [ x"$$check" = x ]; then check="echo Skipping-check"; fi; \ + cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) && \ + ./configure $(4) --prefix=$(idir) && \ + make $(5) && \ + $$check && \ + make install&& \ + cd .. && rm -rf $(2) @@ -64,15 +64,16 @@ gbuild = if [ $(3)x = staticx ]; then export LDFLAGS="$$LDFLAGS -static"; fi;\ # CMake # ----- -cbuild = if [ $(3)x = staticx ]; then \ - export LDFLAGS="$$LDFLAGS -static"; \ - opts="-DBUILD_SHARED_LIBS=OFF"; \ - fi; \ - cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) && \ - rm -rf my-build && mkdir my-build && cd my-build && \ - cmake .. $$opts $(4) && \ - cmake --build . && \ - cmake .. -DCMAKE_INSTALL_PREFIX=$(idir) && \ - cmake --build . --target install && \ - cd ../.. && \ +cbuild = if [ $(3)x = staticx ]; then \ + export LDFLAGS="$$LDFLAGS -static"; \ + opts="-DBUILD_SHARED_LIBS=OFF"; \ + fi; \ + cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) && \ + rm -rf pipeline-build && mkdir pipeline-build && \ + cd pipeline-build && \ + cmake .. $$opts $(4) && \ + cmake --build . && \ + cmake .. -DCMAKE_INSTALL_PREFIX=$(idir) && \ + cmake --build . --target install && \ + cd ../.. && \ rm -rf $(2) diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 45ceadc..f24fafc 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -72,7 +72,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ coreutils-$(coreutils-version).tar.xz \ curl-$(curl-version).tar.gz \ - gawk-$(gawk-version).tar.gz \ + gawk-$(gawk-version).tar.lz \ ghostscript-$(ghostscript-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ @@ -80,11 +80,10 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ gsl-$(gsl-version).tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ - libtool-$(libtool-version).tar.gz \ + libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ sed-$(sed-version).tar.xz \ wcslib-$(wcslib-version).tar.bz2 \ - zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then @@ -123,7 +122,6 @@ $(tarballs): $(tdir)/%: elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib - elif [ $$n = zlib ]; then w=https://www.zlib.net else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." @@ -152,15 +150,15 @@ $(tarballs): $(tdir)/%: # Libraries # --------- $(ildir)/libcfitsio.a: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ - $(ildir)/libcurl.a \ + $(ibdir)/curl \ $(ibdir)/ls $(call gbuild,$(subst $(tdir)/,,$<), cfitsio, static, \ --enable-sse2 --enable-reentrant) $(ildir)/libgit2.a: $(tdir)/libgit2-$(libgit2-version).tar.gz \ - $(ildir)/libcurl.a \ - $(ibdir)/cmake + $(ibdir)/cmake \ + $(ibdir)/curl $(call cbuild,$(subst $(tdir)/,,$<), libgit2-$(libgit2-version), \ static, -DUSE_SSH=OFF -DUSE_OPENSSL=OFF \ -DBUILD_CLAR=OFF -DTHREADSAFE=ON) @@ -179,15 +177,13 @@ $(ildir)/libtiff.a: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ $(ildir)/libcfitsio.a - $(call gbuild,$(subst $(tdir)/,,$<), wcslib-$(wcslib-version), \ - static, LIBS="-pthread -lcurl -lm" --without-pgplot \ - --disable-fortran) - -# Zlib's `./configure' doesn't use Autoconf's configure script, it just -# accepts a direct `--static' option. -$(ildir)/libz.a: $(tdir)/zlib-$(zlib-version).tar.gz - $(call gbuild,$(subst $(tdir)/,,$<), zlib-$(zlib-version), , \ - --static) + # Unfortunately WCSLIB forces the building of shared libraries. So + # we'll allow it to finish, then remove the shared libraries + # afterwards. + $(call gbuild,$(subst $(tdir)/,,$<), wcslib-$(wcslib-version), , \ + LIBS="-pthread -lcurl -lm" --without-pgplot \ + --disable-fortran) + rm -f $(ildir)/libwcs.so* @@ -209,7 +205,7 @@ $(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz $(call gbuild,$(subst $(tdir)/,,$<), coreutils-$(coreutils-version), \ static) -$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.gz \ +$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \ $(ibdir)/ls $(call gbuild,$(subst $(tdir)/,,$<), gawk-$(gawk-version), static) @@ -221,7 +217,7 @@ $(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \ $(ibdir)/ls $(call gbuild,$(subst $(tdir)/,,$<), grep-$(grep-version), static) -$(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.gz \ +$(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.xz \ $(ibdir)/ls $(call gbuild,$(subst $(tdir)/,,$<), libtool-$(libtool-version), static) diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 1acec9f..5c0aac5 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -152,7 +152,7 @@ pvcheck = prog="$(strip $(1))"; \ ver="$(strip $(2))"; \ name="$(strip $(3))"; \ macro="$(strip $(4))"; \ - v=$$($$prog --version | awk '/'$$ver'/{print "y"}'); \ + v=$$($$prog --version | awk '/'$$ver'/{print "y"; exit 0}'); \ if [ x$$v != xy ]; then \ echo; echo "PIPELINE ERROR: Not running $$name $$ver"; echo; \ exit 1; \ @@ -164,7 +164,7 @@ lvcheck = idir=$(BDIR)/dependencies/installed/include; \ ver="$(strip $(2))"; \ name="$(strip $(3))"; \ macro="$(strip $(4))"; \ - v=$$(awk '$$1=="\#define" && /'$$ver'/ {print "y"}' $$f); \ + v=$$(awk '$$1=="\#define" && /'$$ver'/{print "y";exit 0}' $$f);\ if [ x$$v != xy ]; then \ echo; echo "PIPELINE ERROR: Not linking with $$name $$ver"; \ echo; exit 1; \ @@ -190,6 +190,7 @@ $(mtexdir)/initialize.tex: | $(mtexdir) # Versions of programs (same order as 'dependency-versions.mk'). $(call pvcheck, bash, $(bash-version), GNU Bash, bashversion) + $(call pvcheck, nm, $(binutils-version), GNU Binutils, binutilsversion) $(call pvcheck, cmake, $(cmake-version), CMake, cmakeversion) $(call pvcheck, curl, $(curl-version), cURL, curlversion) $(call pvcheck, ls, $(coreutils-version), GNU Coreutils, \ @@ -207,6 +208,19 @@ $(mtexdir)/initialize.tex: | $(mtexdir) $(call pvcheck, make, $(make-version), GNU Make, makeversion) $(call pvcheck, sed, $(sed-version), GNU SED, sedversion) $(call pvcheck, tar, $(tar-version), GNU Tar, tarversion) + $(call pvcheck, xz, $(xz-version), XZ Utils, xzversion) + + # Bzip2 prints its version in standard error, not standard output! + echo "here0" + echo "" | bzip2 --version &> $@_bzip2_ver; + v=$$(awk 'NR==1 && /'$(bzip2-version)'/{print "y"; exit 0}' \ + $@_bzip2_ver); \ + if [ x$$v != xy ]; then \ + echo; echo "PIPELINE ERROR: Not running Bzip2 $(bzip2-version)"; \ + echo; exit 1; \ + fi; \ + echo "\newcommand{\\bziptwoversion}{$(bzip2-version)}" >> $@ + # Versions of libraries. $(call lvcheck, fitsio.h, $(cfitsio-version), CFITSIO, cfitsioversion) |