aboutsummaryrefslogtreecommitdiff
path: root/reproduce
diff options
context:
space:
mode:
Diffstat (limited to 'reproduce')
-rw-r--r--reproduce/config/pipeline/dependency-versions.mk4
-rw-r--r--reproduce/src/make/dependencies-basic.mk124
-rw-r--r--reproduce/src/make/dependencies-build-rules.mk41
-rw-r--r--reproduce/src/make/dependencies.mk32
-rw-r--r--reproduce/src/make/initialize.mk18
5 files changed, 151 insertions, 68 deletions
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index fdad202..9f8d99a 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -2,6 +2,8 @@
# Programs
bash-version = 4.4.18
+binutils-version = 2.31.1
+bzip2-version = 1.0.6
cmake-version = 3.12.4
coreutils-version = 8.30
gawk-version = 4.2.1
@@ -9,11 +11,13 @@ ghostscript-version = 9.25
git-version = 2.19.1
gnuastro-version = 0.7.62-13b0
grep-version = 3.1
+gzip-version = 1.9.10-051e
libtool-version = 2.4.6
lzip-version = 1.20
make-version = 4.2.90
sed-version = 4.5
tar-version = 1.30
+xz-version = 5.2.4
# Libraries
cfitsio-version = 3.45
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index 8fbfeaf..be6e5ed 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -1,5 +1,5 @@
-# Build the VERY BASIC reproduction pipeline dependencies (programs and
-# libraries).
+# Build the VERY BASIC reproduction pipeline dependencies before everything
+# else using minimum Make and Shell.
#
# ------------------------------------------------------------------------
# !!!!! IMPORTANT NOTES !!!!!
@@ -7,9 +7,11 @@
# This Makefile will be run by the initial `./configure' script. It is not
# included into the reproduction pipe after that.
#
-# This Makefile builds very low-level and basic tools like GNU Bash and GNU
-# Make. Therefore this is the only Makefile in the reproduction pipeline
-# where you MUST NOT assume that modern GNU Bash or GNU Make are used.
+# This Makefile builds very low-level and basic tools like GNU Tar, and
+# various compression programs, GNU Bash, and GNU Make. Therefore this is
+# the only Makefile in the reproduction pipeline where you MUST NOT assume
+# that modern GNU Bash or GNU Make are used. After this Makefile, other
+# Makefiles can safely assume the fixed version of all these software.
#
# This Makefile is a very simplified version of `dependencies.mk' in the
# same directory. See there for more comments.
@@ -48,8 +50,11 @@ ibdir = $(BDIR)/dependencies/installed/bin
ildir = $(BDIR)/dependencies/installed/lib
# As we build more programs, we want to use our own pipeline's built
-# programs, not the systems.
-PATH := $(ibdir):$(PATH)
+# programs and libraries, not the host's.
+PATH := $(ibdir):$(PATH)
+LDFLAGS := -L$(ildir) $(LDFLAGS)
+CPPFLAGS := -I$(idir)/include $(CPPFLAGS)
+LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH)
top-level-programs = bash
all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
@@ -65,10 +70,30 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
# is not recognized by some versions of Make (even older GNU Makes). So
# we'll have to make sure the recipe doesn't break into multiple shell
# calls (so we can preserve the variables).
+#
+# Software hosted at akhlaghi.org/src: As of our latest check (November
+# 2018) their major release tarballs either crash or don't build on some
+# systems (for example Make or Gzip), or they don't exist (for example
+# Bzip2).
+#
+# In the first case, we used their Git repo and bootstrapped them (just
+# like Gnuastro) and built the most recent tarball off of that. In the case
+# of Bzip2: its webpage has expired and doesn't host the data any more. It
+# is available on the link below (archive.org):
+#
+# https://web.archive.org/web/20180624184806/http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz
+#
+# However, downloading from this link is slow (because its just a link). So
+# its easier to just keep a with the others.
tarballs = $(foreach t, bash-$(bash-version).tar.gz \
+ binutils-$(binutils-version).tar.lz \
+ bzip2-$(bzip2-version).tar.gz \
+ gzip-$(gzip-version).tar.lz \
lzip-$(lzip-version).tar.gz \
- make-$(make-version).tar.gz \
+ make-$(make-version).tar.lz \
tar-$(tar-version).tar.gz \
+ xz-$(xz-version).tar.gz \
+ zlib-$(zlib-version).tar.gz \
, $(tdir)/$(t) )
$(tarballs): $(tdir)/%:
if [ -f $(DEPENDENCIES-DIR)/$* ]; then \
@@ -79,10 +104,15 @@ $(tarballs): $(tdir)/%:
| awk '{print $$1}' ); \
\
mergenames=1; \
- if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash; \
- elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \
- elif [ $$n = make ]; then w=http://akhlaghi.org/src; \
- elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \
+ if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash; \
+ elif [ $$n = binutils ]; then w=http://ftp.gnu.org/gnu/binutils; \
+ elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \
+ elif [ $$n = gzip ]; then w=http://akhlaghi.org/src; \
+ elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \
+ elif [ $$n = make ]; then w=http://akhlaghi.org/src; \
+ elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \
+ elif [ $$n = xz ]; then w=http://tukaani.org/xz; \
+ elif [ $$n = zlib ]; then w=http://www.zlib.net \
else \
echo; echo; echo; \
echo "'$$n' not a dependency name (for downloading)." \
@@ -101,27 +131,65 @@ $(tarballs): $(tdir)/%:
-# Basic programs (sorted alphabetically), see prerequisites for which one
-# will be built first.
-$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
- $(ibdir)/make
- $(call gbuild,$(subst $(tdir)/,,$<), bash-$(bash-version), , \
- --enable-static-link)
-
+# GNU Lzip: For a static build, the `-static' flag should be given to
+# LDFLAGS on the command-line (not from the environment).
$(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz
- $(call gbuild,$(subst $(tdir)/,,$<), lzip-$(lzip-version), static)
+ $(call gbuild,$(subst $(tdir)/,,$<), lzip-$(lzip-version), , \
+ LDFLAGS="-static")
+
+
+# GNU Gzip.
+$(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.lz \
+ $(ibdir)/lzip
+ $(call gbuild,$(subst $(tdir)/,,$<), gzip-$(gzip-version), static)
+
+
+# Zlib: its `./configure' doesn't use Autoconf's configure script, it just
+# accepts a direct `--static' option.
+$(ildir)/libz.a: $(tdir)/zlib-$(zlib-version).tar.gz
+ $(call gbuild,$(subst $(tdir)/,,$<), zlib-$(zlib-version), , \
+ --static)
-# Unfortunately GNU Make needs dynamic linking in two instances: when
+# XZ Utils
+$(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz
+ $(call gbuild,$(subst $(tdir)/,,$<), xz-$(xz-version), static)
+
+# Bzip2: Bzip2 doesn't have a configure script.
+$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
+ tdir=bzip2-$(bzip2-version); \
+ cd $(ddir) && rm -rf $$tdir && tar xf $< && cd $$tdir && \
+ make LDFLAGS=-static && make install PREFIX=$(idir) && \
+ cd .. && rm -rf $$tdir
+
+
+# GNU Binutils:
+$(ibdir)/nm: $(tdir)/binutils-$(binutils-version).tar.lz \
+ $(ibdir)/lzip \
+ $(ildir)/libz.a
+ $(call gbuild,$(subst $(tdir)/,,$<), binutils-$(binutils-version), \
+ static)
+
+# GNU Tar: When built statically, tar gives a segmentation fault on
+# unpacking Bash. So we'll build it dynamically.
+$(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
+ $(ibdir)/bzip2 \
+ $(ibdir)/gzip \
+ $(ibdir)/lzip \
+ $(ibdir)/xz \
+ $(ibdir)/nm
+ $(call gbuild,$(subst $(tdir)/,,$<), tar-$(tar-version))
+
+# GNU Make: Unfortunately it needs dynamic linking in two instances: when
# loading objects (dynamically linked libraries), or when using the
# `getpwnam' function (for tilde expansion). The first can be disabled with
# `--disable-load', but unfortunately I don't know any way to fix the
# second. So, we'll have to build it dynamically for now.
-$(ibdir)/make: $(tdir)/make-$(make-version).tar.gz \
+$(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \
$(ibdir)/tar
- $(call gbuild,$(subst $(tdir)/,,$<), make-$(make-version), , , ,)
+ $(call gbuild,$(subst $(tdir)/,,$<), make-$(make-version))
-# When built statically, tar gives a segmentation fault on unpacking
-# Bash. So we'll build it dynamically.
-$(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
- $(ibdir)/lzip
- $(call gbuild,$(subst $(tdir)/,,$<), tar-$(tar-version))
+# GNU Bash
+$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
+ $(ibdir)/make
+ $(call gbuild,$(subst $(tdir)/,,$<), bash-$(bash-version), , \
+ --enable-static-link)
diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk
index a2e4a89..31e7759 100644
--- a/reproduce/src/make/dependencies-build-rules.mk
+++ b/reproduce/src/make/dependencies-build-rules.mk
@@ -48,15 +48,15 @@
# 4: Extra configuration options.
# 5: Extra options/arguments to pass to Make.
# 6: Step to run between `make' and `make install': usually `make check'.
-gbuild = if [ $(3)x = staticx ]; then export LDFLAGS="$$LDFLAGS -static"; fi;\
- check="$(6)"; \
- if [ x"$$check" = x ]; then check="echo Skipping-check"; fi; \
- cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) && \
- ./configure $(4) --prefix=$(idir) && \
- make $(5) && \
- $$check && \
- make install&& \
- cd ..&& rm -rf $(2)
+gbuild = if [ $(3)x = staticx ]; then export LDFLAGS="$$LDFLAGS -static"; fi; \
+ check="$(6)"; \
+ if [ x"$$check" = x ]; then check="echo Skipping-check"; fi; \
+ cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) && \
+ ./configure $(4) --prefix=$(idir) && \
+ make $(5) && \
+ $$check && \
+ make install&& \
+ cd .. && rm -rf $(2)
@@ -64,15 +64,16 @@ gbuild = if [ $(3)x = staticx ]; then export LDFLAGS="$$LDFLAGS -static"; fi;\
# CMake
# -----
-cbuild = if [ $(3)x = staticx ]; then \
- export LDFLAGS="$$LDFLAGS -static"; \
- opts="-DBUILD_SHARED_LIBS=OFF"; \
- fi; \
- cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) && \
- rm -rf my-build && mkdir my-build && cd my-build && \
- cmake .. $$opts $(4) && \
- cmake --build . && \
- cmake .. -DCMAKE_INSTALL_PREFIX=$(idir) && \
- cmake --build . --target install && \
- cd ../.. && \
+cbuild = if [ $(3)x = staticx ]; then \
+ export LDFLAGS="$$LDFLAGS -static"; \
+ opts="-DBUILD_SHARED_LIBS=OFF"; \
+ fi; \
+ cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) && \
+ rm -rf pipeline-build && mkdir pipeline-build && \
+ cd pipeline-build && \
+ cmake .. $$opts $(4) && \
+ cmake --build . && \
+ cmake .. -DCMAKE_INSTALL_PREFIX=$(idir) && \
+ cmake --build . --target install && \
+ cd ../.. && \
rm -rf $(2)
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index 45ceadc..f24fafc 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -72,7 +72,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
cmake-$(cmake-version).tar.gz \
coreutils-$(coreutils-version).tar.xz \
curl-$(curl-version).tar.gz \
- gawk-$(gawk-version).tar.gz \
+ gawk-$(gawk-version).tar.lz \
ghostscript-$(ghostscript-version).tar.gz \
git-$(git-version).tar.xz \
gnuastro-$(gnuastro-version).tar.lz \
@@ -80,11 +80,10 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
gsl-$(gsl-version).tar.gz \
jpegsrc.$(libjpeg-version).tar.gz \
tiff-$(libtiff-version).tar.gz \
- libtool-$(libtool-version).tar.gz \
+ libtool-$(libtool-version).tar.xz \
libgit2-$(libgit2-version).tar.gz \
sed-$(sed-version).tar.xz \
wcslib-$(wcslib-version).tar.bz2 \
- zlib-$(zlib-version).tar.gz \
, $(tdir)/$(t) )
$(tarballs): $(tdir)/%:
if [ -f $(DEPENDENCIES-DIR)/$* ]; then
@@ -123,7 +122,6 @@ $(tarballs): $(tdir)/%:
elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed
elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff
elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib
- elif [ $$n = zlib ]; then w=https://www.zlib.net
else
echo; echo; echo;
echo "'$$n' not recognized as a dependency name to download."
@@ -152,15 +150,15 @@ $(tarballs): $(tdir)/%:
# Libraries
# ---------
$(ildir)/libcfitsio.a: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \
- $(ildir)/libcurl.a \
+ $(ibdir)/curl \
$(ibdir)/ls
$(call gbuild,$(subst $(tdir)/,,$<), cfitsio, static, \
--enable-sse2 --enable-reentrant)
$(ildir)/libgit2.a: $(tdir)/libgit2-$(libgit2-version).tar.gz \
- $(ildir)/libcurl.a \
- $(ibdir)/cmake
+ $(ibdir)/cmake \
+ $(ibdir)/curl
$(call cbuild,$(subst $(tdir)/,,$<), libgit2-$(libgit2-version), \
static, -DUSE_SSH=OFF -DUSE_OPENSSL=OFF \
-DBUILD_CLAR=OFF -DTHREADSAFE=ON)
@@ -179,15 +177,13 @@ $(ildir)/libtiff.a: $(tdir)/tiff-$(libtiff-version).tar.gz \
$(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
$(ildir)/libcfitsio.a
- $(call gbuild,$(subst $(tdir)/,,$<), wcslib-$(wcslib-version), \
- static, LIBS="-pthread -lcurl -lm" --without-pgplot \
- --disable-fortran)
-
-# Zlib's `./configure' doesn't use Autoconf's configure script, it just
-# accepts a direct `--static' option.
-$(ildir)/libz.a: $(tdir)/zlib-$(zlib-version).tar.gz
- $(call gbuild,$(subst $(tdir)/,,$<), zlib-$(zlib-version), , \
- --static)
+ # Unfortunately WCSLIB forces the building of shared libraries. So
+ # we'll allow it to finish, then remove the shared libraries
+ # afterwards.
+ $(call gbuild,$(subst $(tdir)/,,$<), wcslib-$(wcslib-version), , \
+ LIBS="-pthread -lcurl -lm" --without-pgplot \
+ --disable-fortran)
+ rm -f $(ildir)/libwcs.so*
@@ -209,7 +205,7 @@ $(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz
$(call gbuild,$(subst $(tdir)/,,$<), coreutils-$(coreutils-version), \
static)
-$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.gz \
+$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \
$(ibdir)/ls
$(call gbuild,$(subst $(tdir)/,,$<), gawk-$(gawk-version), static)
@@ -221,7 +217,7 @@ $(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \
$(ibdir)/ls
$(call gbuild,$(subst $(tdir)/,,$<), grep-$(grep-version), static)
-$(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.gz \
+$(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.xz \
$(ibdir)/ls
$(call gbuild,$(subst $(tdir)/,,$<), libtool-$(libtool-version), static)
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index 1acec9f..5c0aac5 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -152,7 +152,7 @@ pvcheck = prog="$(strip $(1))"; \
ver="$(strip $(2))"; \
name="$(strip $(3))"; \
macro="$(strip $(4))"; \
- v=$$($$prog --version | awk '/'$$ver'/{print "y"}'); \
+ v=$$($$prog --version | awk '/'$$ver'/{print "y"; exit 0}'); \
if [ x$$v != xy ]; then \
echo; echo "PIPELINE ERROR: Not running $$name $$ver"; echo; \
exit 1; \
@@ -164,7 +164,7 @@ lvcheck = idir=$(BDIR)/dependencies/installed/include; \
ver="$(strip $(2))"; \
name="$(strip $(3))"; \
macro="$(strip $(4))"; \
- v=$$(awk '$$1=="\#define" && /'$$ver'/ {print "y"}' $$f); \
+ v=$$(awk '$$1=="\#define" && /'$$ver'/{print "y";exit 0}' $$f);\
if [ x$$v != xy ]; then \
echo; echo "PIPELINE ERROR: Not linking with $$name $$ver"; \
echo; exit 1; \
@@ -190,6 +190,7 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
# Versions of programs (same order as 'dependency-versions.mk').
$(call pvcheck, bash, $(bash-version), GNU Bash, bashversion)
+ $(call pvcheck, nm, $(binutils-version), GNU Binutils, binutilsversion)
$(call pvcheck, cmake, $(cmake-version), CMake, cmakeversion)
$(call pvcheck, curl, $(curl-version), cURL, curlversion)
$(call pvcheck, ls, $(coreutils-version), GNU Coreutils, \
@@ -207,6 +208,19 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
$(call pvcheck, make, $(make-version), GNU Make, makeversion)
$(call pvcheck, sed, $(sed-version), GNU SED, sedversion)
$(call pvcheck, tar, $(tar-version), GNU Tar, tarversion)
+ $(call pvcheck, xz, $(xz-version), XZ Utils, xzversion)
+
+ # Bzip2 prints its version in standard error, not standard output!
+ echo "here0"
+ echo "" | bzip2 --version &> $@_bzip2_ver;
+ v=$$(awk 'NR==1 && /'$(bzip2-version)'/{print "y"; exit 0}' \
+ $@_bzip2_ver); \
+ if [ x$$v != xy ]; then \
+ echo; echo "PIPELINE ERROR: Not running Bzip2 $(bzip2-version)"; \
+ echo; exit 1; \
+ fi; \
+ echo "\newcommand{\\bziptwoversion}{$(bzip2-version)}" >> $@
+
# Versions of libraries.
$(call lvcheck, fitsio.h, $(cfitsio-version), CFITSIO, cfitsioversion)