From 39372eedc72c4f608fc6bd9df10e67dfb8462bf1 Mon Sep 17 00:00:00 2001
From: Mohammad Akhlaghi <mohammad@akhlaghi.org>
Date: Fri, 5 Apr 2019 16:54:41 +0100
Subject: Software acknowledgement section is automatically generated
Until now, management of the software names and versions in the paper was
done manually (a macro had to be defined in `initialize.mk', then used in
`paper.tex', so they had to be manually set in two places). Managing this
was not easy.
To fix this, with this commit, each software building rule's target is a
text file that contains its human-readable name and its version. In the
end, the configure script sorts them by their name and writes them into a
LaTeX input file that we can easily import as a file into the main paper.
---
reproduce/src/make/dependencies-basic.mk | 223 +++++++++++++++++-------------
reproduce/src/make/dependencies-python.mk | 183 ++++++++++++++----------
reproduce/src/make/dependencies.mk | 204 ++++++++++++++-------------
reproduce/src/make/initialize.mk | 2 +-
reproduce/src/make/paper.mk | 3 +-
5 files changed, 346 insertions(+), 269 deletions(-)
(limited to 'reproduce/src')
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index 1573f39..d4830ca 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -49,7 +49,8 @@ tdir = $(BDIR)/dependencies/tarballs
idir = $(BDIR)/dependencies/installed
ibdir = $(BDIR)/dependencies/installed/bin
ildir = $(BDIR)/dependencies/installed/lib
-ilidir = $(BDIR)/dependencies/installed/lib/built
+ibidir = $(BDIR)/dependencies/installed/version-info/bin
+ilidir = $(BDIR)/dependencies/installed/version-info/lib
# We'll need the system's PATH for making links to low-level programs we
# won't be building ourselves.
@@ -67,7 +68,7 @@ export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH)
# Define the programs that don't depend on any other.
top-level-programs = low-level-links wget gcc
-all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
+all: $(foreach p, $(top-level-programs), $(ibidir)/$(p))
@@ -126,7 +127,7 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \
xz-$(xz-version).tar.gz \
zlib-$(zlib-version).tar.gz \
, $(tdir)/$(t) )
-$(tarballs): $(tdir)/%: $(lockdir)
+$(tarballs): $(tdir)/%: | $(lockdir)
if [ -f $(DEPENDENCIES-DIR)/$* ]; then \
cp $(DEPENDENCIES-DIR)/$* $@; \
else \
@@ -213,7 +214,7 @@ makelink = origpath="$$PATH"; \
if [ x$$a != x ]; then $$c $$a $(ibdir)/$(1); fi; \
export PATH="$$origpath"
$(ibdir) $(ildir):; mkdir $@
-$(ibdir)/low-level-links: | $(ibdir) $(ildir)
+$(ibidir)/low-level-links: | $(ibdir) $(ildir)
# The Assembler
$(call makelink,as)
@@ -260,7 +261,9 @@ $(ibdir)/low-level-links: | $(ibdir) $(ildir)
fi; \
done
- echo "Low-level symbolic links are setup" > $@
+ # We want this to be empty (so it doesn't interefere with the other
+ # files in `ibidir'.
+ touch $@
@@ -277,22 +280,26 @@ $(ibdir)/low-level-links: | $(ibdir) $(ildir)
# The first set of programs to be built are those that we need to unpack
# the source code tarballs of each program. First, we'll build the
# necessary programs, then we'll build GNU Tar.
-$(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz
- $(call gbuild, $<, gzip-$(gzip-version), static, , V=1)
+$(ibidir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz
+ $(call gbuild, $<, gzip-$(gzip-version), static, , V=1) \
+ && echo "GNU Gzip $(gzip-version)" > $@
# GNU Lzip: For a static build, the `-static' flag should be given to
# LDFLAGS on the command-line (not from the environment).
-$(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz
ifeq ($(static_build),yes)
- $(call gbuild, $<, lzip-$(lzip-version), , LDFLAGS="-static")
+lzipconf="LDFLAGS=-static"
else
- $(call gbuild, $<, lzip-$(lzip-version))
+lzipconf=
endif
+$(ibidir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz
+ $(call gbuild, $<, lzip-$(lzip-version), , $(lzipconf)) \
+ && echo "Lzip $(lzip-version)" > $@
-$(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz
- $(call gbuild, $<, xz-$(xz-version), static)
+$(ibidir)/xz: $(tdir)/xz-$(xz-version).tar.gz
+ $(call gbuild, $<, xz-$(xz-version), static) \
+ && echo "XZ Utils $(xz-version)" > $@
-$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
+$(ibidir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
# Bzip2 doesn't have a `./configure' script, and its Makefile
# doesn't build a shared library. So we can't use the `gbuild'
# function here and we need to take some extra steps (inspired
@@ -326,20 +333,22 @@ $(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
&& cd .. \
&& rm -rf $$tdir \
&& cd $(ildir) \
- && ln -fs libbz2.so.1.0 libbz2.so
+ && ln -fs libbz2.so.1.0 libbz2.so \
+ && echo "Bzip2 $(bzip2-version)" > $@
# GNU Tar: When built statically, tar gives a segmentation fault on
# unpacking Bash. So we'll build it dynamically.
-$(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
- $(ibdir)/bzip2 \
- $(ibdir)/lzip \
- $(ibdir)/gzip \
- $(ibdir)/xz
+$(ibidir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
+ $(ibidir)/bzip2 \
+ $(ibidir)/lzip \
+ $(ibidir)/gzip \
+ $(ibidir)/xz
# Since all later programs depend on Tar, the pipeline will be
# stuck here, only making Tar. So its more efficient to built it on
# multiple threads (when the user's Make doesn't pass down the
# number of threads).
- $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads))
+ $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads)) \
+ && echo "GNU Tar $(tar-version)" > $@
@@ -361,13 +370,14 @@ $(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
# function (for tilde expansion). The first can be disabled with
# `--disable-load', but unfortunately I don't know any way to fix the
# second. So, we'll have to build it dynamically for now.
-$(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \
- $(ibdir)/tar
+$(ibidir)/make: $(tdir)/make-$(make-version).tar.lz \
+ $(ibidir)/tar
# See Tar's comments for the `-j' option.
- $(call gbuild, $<, make-$(make-version), , , -j$(numthreads))
+ $(call gbuild, $<, make-$(make-version), , , -j$(numthreads)) \
+ && echo "GNU Make $(make-version)" > $@
$(ilidir)/ncurses: $(tdir)/ncurses-$(ncurses-version).tar.gz \
- $(ibdir)/make | $(ilidir)
+ $(ibidir)/make
# Delete the library that will be installed (so we can make sure
# the build process completed afterwards and reset the links).
@@ -447,7 +457,7 @@ $(ilidir)/ncurses: $(tdir)/ncurses-$(ncurses-version).tar.gz \
ln -fs $(ildir)/pkgconfig/ncursesw.pc pkgconfig/cursesw.pc; \
\
ln -fs $(idir)/include/ncursesw $(idir)/include/ncurses; \
- echo "GNU ncurses is built and ready" > $@; \
+ echo "GNU NCURSES $(ncurses-version)" > $@; \
else \
exit 1; \
fi
@@ -456,12 +466,13 @@ $(ilidir)/readline: $(tdir)/readline-$(readline-version).tar.gz \
$(ilidir)/ncurses
$(call gbuild, $<, readline-$(readline-version), static, \
--with-curses --disable-install-examples, \
- SHLIB_LIBS="-lncursesw" ) && \
- echo "GNU Readline is built and ready" > $@
+ SHLIB_LIBS="-lncursesw" ) \
+ && echo "GNU Readline $(readline-version)" > $@
-$(ibdir)/patchelf: $(tdir)/patchelf-$(patchelf-version).tar.gz \
- $(ibdir)/make
- $(call gbuild, $<, patchelf-$(patchelf-version), static)
+$(ibidir)/patchelf: $(tdir)/patchelf-$(patchelf-version).tar.gz \
+ $(ibidir)/make
+ $(call gbuild, $<, patchelf-$(patchelf-version), static) \
+ && echo "PatchELF $(patchelf-version)" > $@
# IMPORTANT: Even though we have enabled `rpath', Bash doesn't write the
@@ -475,14 +486,14 @@ $(ibdir)/patchelf: $(tdir)/patchelf-$(patchelf-version).tar.gz \
ifeq ($(on_mac_os),yes)
needpatchelf =
else
-needpatchelf = $(ibdir)/patchelf
+needpatchelf = $(ibidir)/patchelf
endif
-$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
- $(ilidir)/readline \
- $(needpatchelf)
+$(ibidir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
+ $(ilidir)/readline \
+ $(needpatchelf)
- # Delete any possibly existing output
- rm -f $@
+ # Delete the (possibly) existing Bash executable.
+ rm -f $(ibdir)/bash
# Build Bash. Note that we aren't building Bash with
# `--with-installed-readline'. This is because (as described above)
@@ -522,7 +533,11 @@ $(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
# Just to be sure that the installation step above went well,
# before making the link, we'll see if the file actually exists
# there.
- if [ -f $@ ]; then ln -fs $@ $(ibdir)/sh; else exit 1; fi
+ if [ -f $(ibdir)/bash ]; then \
+ ln -fs $(ibdir)/bash $(ibdir)/sh; \
+ echo "GNU Bash $(bash-version)" > $@; \
+ else \
+ echo "GNU Bash not built!"; exit 1; fi
@@ -537,10 +552,10 @@ $(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
# Note for a static-only build: Zlib's `./configure' doesn't use Autoconf's
# configure script, it just accepts a direct `--static' option.
$(idir)/etc:; mkdir $@
-$(ilidir): | $(ildir); mkdir $@
$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \
- $(ibdir)/bash | $(ilidir)
- $(call gbuild, $<, zlib-$(zlib-version)) && echo "Zlib is built" > $@
+ $(ibidir)/bash
+ $(call gbuild, $<, zlib-$(zlib-version)) \
+ && echo "Zlib $(zlib-version)" > $@
# OpenSSL: Some programs/libraries later need dynamic linking. So we'll
# build libssl (and libcrypto) dynamically also.
@@ -584,7 +599,7 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \
else \
patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \
fi; \
- echo "OpenSSL is built and ready" > $@; \
+ echo "OpenSSL $(openssl-version)" > $@; \
fi
# GNU Wget
@@ -600,9 +615,9 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \
# building as part of this pipeline. So to avoid too much dependency on the
# host system (especially a crash when these libraries are updated on the
# host), they are disabled here.
-$(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \
- $(ibdir)/pkg-config \
- $(ilidir)/openssl
+$(ibidir)/wget: $(tdir)/wget-$(wget-version).tar.lz \
+ $(ibidir)/pkg-config \
+ $(ilidir)/openssl
libs="-pthread"; \
if [ x$(needs_ldl) = xyes ]; then libs="$$libs -ldl"; fi; \
$(call gbuild, $<, wget-$(wget-version), , \
@@ -616,7 +631,8 @@ $(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \
--without-libidn \
--disable-pcre2 \
--disable-pcre \
- --disable-iri )
+ --disable-iri ) \
+ && echo "GNU Wget $(wget-version)" > $@
@@ -625,19 +641,22 @@ $(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \
# Basic command-line programs necessary in build process of the
# higher-level dependencies: Note that during the building of those
# programs, there is no access to the system's PATH.
-$(ibdir)/diff: $(tdir)/diffutils-$(diffutils-version).tar.xz \
- $(ibdir)/bash
- $(call gbuild, $<, diffutils-$(diffutils-version), static)
-
-$(ibdir)/find: $(tdir)/findutils-$(findutils-version).tar.lz \
- $(ibdir)/bash
- $(call gbuild, $<, findutils-$(findutils-version), static)
-
-$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \
- $(ibdir)/bash
+$(ibidir)/diffutils: $(tdir)/diffutils-$(diffutils-version).tar.xz \
+ $(ibidir)/bash
+ $(call gbuild, $<, diffutils-$(diffutils-version), static, , V=1) \
+ && echo "GNU Diffutils $(diffutils-version)" > $@
+
+$(ibidir)/findutils: $(tdir)/findutils-$(findutils-version).tar.lz \
+ $(ibidir)/bash
+ $(call gbuild, $<, findutils-$(findutils-version), static, , V=1) \
+ && echo "GNU Findutils $(findutils-version)" > $@
+
+$(ibidir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \
+ $(ibidir)/bash
# Build the main program.
$(call gbuild, $<, gawk-$(gawk-version), static, \
- --with-readline=$(idir));
+ --with-readline=$(idir)) \
+ && echo "GNU AWK $(gawk-version)" > $@
# Since AWK doesn't include RPATH by default, we'll have to
# manually include it using the `patchelf' program. Just note that
@@ -650,31 +669,36 @@ $(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \
fi; \
fi
-$(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \
- $(ibdir)/bash
- $(call gbuild, $<, grep-$(grep-version), static)
+$(ibidir)/grep: $(tdir)/grep-$(grep-version).tar.xz \
+ $(ibidir)/bash
+ $(call gbuild, $<, grep-$(grep-version), static) \
+ && echo "GNU Grep $(grep-version)" > $@
-$(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz \
- $(ilidir)/openssl
+$(ibidir)/coreutils: $(tdir)/coreutils-$(coreutils-version).tar.xz \
+ $(ilidir)/openssl
# Coreutils will use the hashing features of OpenSSL's `libcrypto'.
# See Tar's comments for the `-j' option.
$(call gbuild, $<, coreutils-$(coreutils-version), static, \
LDFLAGS="$(LDFLAGS)" CPPFLAGS="$(CPPFLAGS)" \
--enable-rpath --disable-silent-rules --with-openssl, \
- -j$(numthreads))
+ -j$(numthreads)) \
+ && echo "GNU Coreutils $(coreutils-version)" > $@
-$(ibdir)/pkg-config: $(tdir)/pkg-config-$(pkgconfig-version).tar.gz \
- $(ibdir)/bash
+$(ibidir)/pkg-config: $(tdir)/pkg-config-$(pkgconfig-version).tar.gz \
+ $(ibidir)/bash
$(call gbuild, $<, pkg-config-$(pkgconfig-version), static, \
- --with-internal-glib --with-pc-path=$(ildir)/pkgconfig)
+ --with-internal-glib --with-pc-path=$(ildir)/pkgconfig) \
+ && echo "pkg-config $(pkgconfig-version)" > $@
-$(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \
- $(ibdir)/bash
- $(call gbuild, $<, sed-$(sed-version), static)
+$(ibidir)/sed: $(tdir)/sed-$(sed-version).tar.xz \
+ $(ibidir)/bash
+ $(call gbuild, $<, sed-$(sed-version), static) \
+ && echo "GNU Sed $(sed-version)" > $@
-$(ibdir)/which: $(tdir)/which-$(which-version).tar.gz \
- $(ibdir)/bash
- $(call gbuild, $<, which-$(which-version), static)
+$(ibidir)/which: $(tdir)/which-$(which-version).tar.gz \
+ $(ibidir)/bash
+ $(call gbuild, $<, which-$(which-version), static) \
+ && echo "GNU Which $(which-version)" > $@
@@ -685,33 +709,34 @@ $(ibdir)/which: $(tdir)/which-$(which-version).tar.gz \
-# (CURRENTLY IGNORED) GCC prerequisites
-# -------------------------------------
+# GCC prerequisites
+# -----------------
$(ilidir)/gmp: $(tdir)/gmp-$(gmp-version).tar.lz \
- $(ibdir)/bash | $(ilidir)
+ $(ibidir)/bash
$(call gbuild, $<, gmp-$(gmp-version), static, , , make check) \
- && echo "GNU multiple precision arithmetic library is built" > $@
+ && echo "GNU Multiple Precision Arithmetic Library $(gmp-version)" > $@
$(ilidir)/mpfr: $(tdir)/mpfr-$(mpfr-version).tar.xz \
$(ilidir)/gmp
$(call gbuild, $<, mpfr-$(mpfr-version), static, , , make check) \
- && echo "GNU MPFR library is built" > $@
+ && echo "GNU Multiple Precision Floating-Point Reliably $(mpfr-version)" > $@
$(ilidir)/mpc: $(tdir)/mpc-$(mpc-version).tar.gz \
$(ilidir)/mpfr
$(call gbuild, $<, mpc-$(mpc-version), static, , , make check) \
- && echo "GNU MPC library is built" > $@
+ && echo "GNU Multiple Precision Complex library" > $@
$(ilidir)/isl: $(tdir)/isl-$(isl-version).tar.bz2 \
$(ilidir)/gmp
$(call gbuild, $<, isl-$(isl-version), static) \
- && echo "GCC's ISL library is built" > $@
+ && echo "GNU Integer Set Library $(isl-version)" > $@
# Binutils' linker `ld' is apparently only good for GNU/Linux systems and
# other OSs have their own. So for now we aren't actually building
# Binutils (`ld' isn't a prerequisite of GCC).
-$(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz
- $(call gbuild, $<, binutils-$(binutils-version), static)
+$(ibidir)/binutils: $(tdir)/binutils-$(binutils-version).tar.lz
+ $(call gbuild, $<, binutils-$(binutils-version), static) \
+ && echo "GNU Binutils $(binutils-version)" > $@
@@ -742,22 +767,22 @@ $(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz
#
# We are currently having problems installing GCC on macOS, so for the time
# being, if the pipeline is being run on a macOS, we'll just set a link.
-#ifeq ($(on_mac_os),yes)
-#gcc-prerequisites =
-#else
+ifeq ($(on_mac_os),yes)
+gcc-prerequisites =
+else
gcc-prerequisites = $(tdir)/gcc-$(gcc-version).tar.xz \
$(ilidir)/isl \
$(ilidir)/mpc
-#endif
-$(ibdir)/gcc: $(gcc-prerequisites) \
- $(ibdir)/ls \
- $(ibdir)/sed \
- $(ibdir)/gawk \
- $(ibdir)/grep \
- $(ibdir)/diff \
- $(ibdir)/find \
- $(ibdir)/bash \
- $(ibdir)/which
+endif
+$(ibidir)/gcc: $(gcc-prerequisites) \
+ $(ibidir)/sed \
+ $(ibidir)/gawk \
+ $(ibidir)/grep \
+ $(ibidir)/bash \
+ $(ibidir)/which \
+ $(ibidir)/findutils \
+ $(ibidir)/diffutils \
+ $(ibidir)/coreutils
# On a macOS, we (currently!) won't build GCC because of some
# errors we are still trying to find. So, we'll just make a
@@ -769,14 +794,15 @@ $(ibdir)/gcc: $(gcc-prerequisites) \
# in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic
# link to '$(idir)/lib'.
-# SO FAR WE HAVEN'T BEEN ABLE TO GET A CONSISTENT BUILD OF GCC ON MAC
-# (SOMETIMES IT CRASHES IN libiberty with g++) AND SOMETIMES IT FINISHES,
-# SO, MORE TESTS ARE NEEDED ON MAC AND WE'LL USE THE HOST'S COMPILER UNTIL
-# THEN.
+ # SO FAR WE HAVEN'T BEEN ABLE TO GET A CONSISTENT BUILD OF GCC ON
+ # MAC (SOMETIMES IT CRASHES IN libiberty with g++) AND SOMETIMES IT
+ # FINISHES, SO, MORE TESTS ARE NEEDED ON MAC AND WE'LL USE THE
+ # HOST'S COMPILER UNTIL THEN.
if [ "x$(on_mac_os)" = xyes ]; then \
$(call makelink,g++); \
$(call makelink,gfortran); \
$(call makelink,gcc,copy); \
+ echo "" > $@; \
else \
rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\
rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \
@@ -819,5 +845,6 @@ $(ibdir)/gcc: $(gcc-prerequisites) \
patchelf --set-rpath $(ildir) $$f; \
fi; \
done; \
- fi; \
+ fi \
+ && echo "GNU Compiler Collection (GCC) $(gcc-version)" > $@; \
fi
diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk
index 6711d17..034afed 100644
--- a/reproduce/src/make/dependencies-python.mk
+++ b/reproduce/src/make/dependencies-python.mk
@@ -193,47 +193,16 @@ $(pytarballs): $(tdir)/%:
else tarballurl=$$h
fi
- # If the download fails, Wget will write the error message in the
- # target file, so Make will think that its done! To avoid this
- # problem, we'll rename the output.
- echo "Downloading $$tarballurl"
- if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then
- rm -f $@
- echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1
- fi
+ # Download using the script specially defined for this job.
+ touch $(lockdir)/download
+ $(downloadwrapper) "$$downloader" $(lockdir)/download \
+ $$tarballurl $@
fi
-# Install without pip
-# --------------------
-#
-# To build Python packages with direct access to a `setup.py' (if no direct
-# access to `setup.py' is needed, pip can be used).
-# Arguments of this function are the numbers
-# 1) Unpack command
-# 2) Package name
-# 3) Unpacked directory name after unpacking the tarball
-# 4) site.cfg file (optional)
-pybuild = cd $(ddir); rm -rf $(3); \
- if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \
- cd $(3); \
- if [ "x$(4)" != x ]; then \
- sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \
- -e 's|@INCDIR[@]|'"$(idir)/include"'|' \
- $(4) > site.cfg; \
- fi; \
- python3 setup.py build \
- && python3 setup.py install \
- && cd .. && rm -rf $(3) \
- && echo "done!" > $@
-
-
-
-
-
# Necessary programs and libraries
# --------------------------------
#
@@ -241,10 +210,10 @@ pybuild = cd $(ddir); rm -rf $(3); \
# certain programs (like Python itself), or libraries for the modules.
$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz
$(call gbuild, $<, libffi-$(libffi-version)) \
- echo "libffi is built" > $@
+ echo "Libffi $(libffi-version)" > $@
-$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \
- $(ilidir)/libffi
+$(ibidir)/python3: $(tdir)/python-$(python-version).tar.gz \
+ $(ilidir)/libffi
# On Mac systems, the build complains about `clang' specific
# features, so we can't use our own GCC build here.
if [ x$(on_mac_os) = xyes ]; then \
@@ -258,7 +227,39 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \
&& v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \
{printf "%d.%d\n", $$1, $$2}') \
&& ln -s $(ildir)/python$$v $(ildir)/python \
- && rm -rf $(ipydir) && mkdir $(ipydir)
+ && rm -rf $(ipydir) \
+ && mkdir $(ipydir) \
+ && echo "Python $(python-version)" > $@
+
+
+
+
+
+# Non-PiP Python module installation
+# ----------------------------------
+#
+# To build Python packages with direct access to a `setup.py' (if no direct
+# access to `setup.py' is needed, pip can be used).
+# Arguments of this function are the numbers
+# 1) Unpack command
+# 2) Package name
+# 3) Unpacked directory name after unpacking the tarball
+# 4) site.cfg file (optional)
+# 5) Official software name.(for paper).
+pybuild = cd $(ddir); rm -rf $(3); \
+ if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \
+ cd $(3); \
+ if [ "x$(4)" != x ]; then \
+ sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \
+ -e 's|@INCDIR[@]|'"$(idir)/include"'|' \
+ $(4) > site.cfg; \
+ fi; \
+ python3 setup.py build \
+ && python3 setup.py install \
+ && cd .. \
+ && rm -rf $(3) \
+ && echo "$(5)" > $@
+
@@ -269,7 +270,8 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \
# All the necessary Python modules go here.
$(ipydir)/asn1crypto: $(tdir)/asn1crypto-$(asn1crypto-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, asn1crypto-$(asn1crypto-version))
+ $(call pybuild, tar xf, $<, asn1crypto-$(asn1crypto-version), , \
+ Asn1crypto $(asn1crypto-version))
$(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \
$(ipydir)/astropy \
@@ -278,74 +280,89 @@ $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \
$(ipydir)/keyring \
$(ipydir)/numpy \
$(ipydir)/requests
- $(call pybuild, tar xf, $<, astroquery-$(astroquery-version))
+ $(call pybuild, tar xf, $<, astroquery-$(astroquery-version), ,\
+ Astroquery $(astroquery-version))
$(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \
$(ipydir)/h5py \
$(ipydir)/numpy \
$(ipydir)/scipy
- $(call pybuild, tar xf, $<, astropy-$(astropy-version))
+ $(call pybuild, tar xf, $<, astropy-$(astropy-version), ,\
+ "Astropy $(astropy-version) \citep{astropy2013,astropy2018}")
$(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \
$(ipydir)/soupsieve
- $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version))
+ $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version), ,\
+ BeautifulSoup $(beautifulsoup4-version))
$(ipydir)/certifi: $(tdir)/certifi-$(certifi-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, certifi-$(certifi-version))
+ $(call pybuild, tar xf, $<, certifi-$(certifi-version), ,\
+ Certifi $(certifi-version))
$(ipydir)/cffi: $(tdir)/cffi-$(cffi-version).tar.gz \
$(ilidir)/libffi \
$(ipydir)/pycparser
- $(call pybuild, tar xf, $<, cffi-$(cffi-version))
+ $(call pybuild, tar xf, $<, cffi-$(cffi-version), ,\
+ cffi $(cffi-version))
$(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, chardet-$(chardet-version))
+ $(call pybuild, tar xf, $<, chardet-$(chardet-version), ,\
+ Chardet $(chardet-version))
$(ipydir)/cryptography: $(tdir)/cryptography-$(cryptography-version).tar.gz \
$(ipydir)/asn1crypto \
$(ipydir)/cffi
- $(call pybuild, tar xf, $<, cryptography-$(cryptography-version))
+ $(call pybuild, tar xf, $<, cryptography-$(cryptography-version), ,\
+ Cryptography $(cryptography-version))
$(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \
$(ipydir)/six
- $(call pybuild, tar xf, $<, cycler-$(cycler-version))
+ $(call pybuild, tar xf, $<, cycler-$(cycler-version), ,\
+ Cycler $(cycler-version))
$(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version))
+ $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version), ,\
+ EntryPoints $(entrypoints-version))
$(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \
$(ipydir)/setuptools \
$(ilidir)/hdf5
# $(ipydir)/mpi4py # AFTER its problem is fixed.
#export HDF5_MPI=ON; # AFTER its problem is fixed.
- export HDF5_DIR=$(ildir); \
- $(call pybuild, tar xf, $<, h5py-$(h5py-version))
+ export HDF5_DIR=$(ildir); \
+ $(call pybuild, tar xf, $<, h5py-$(h5py-version), ,\
+ h5py $(h5py-version))
$(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \
$(ipydir)/six \
$(ipydir)/webencodings
- $(call pybuild, tar xf, $<, html5lib-$(html5lib-version))
+ $(call pybuild, tar xf, $<, html5lib-$(html5lib-version), ,\
+ HTML5lib $(html5lib-version))
$(ipydir)/idna: $(tdir)/idna-$(idna-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, idna-$(idna-version))
+ $(call pybuild, tar xf, $<, idna-$(idna-version), ,\
+ idna $(idna-version))
$(ipydir)/jeepney: $(tdir)/jeepney-$(jeepney-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, jeepney-$(jeepney-version))
+ $(call pybuild, tar xf, $<, jeepney-$(jeepney-version), ,\
+ Jeepney $(jeepney-version))
$(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \
$(ipydir)/entrypoints \
$(ipydir)/secretstorage \
$(ipydir)/setuptools_scm
- $(call pybuild, tar xf, $<, keyring-$(keyring-version))
+ $(call pybuild, tar xf, $<, keyring-$(keyring-version), ,\
+ Keyring $(keyring-version))
$(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version))
+ $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version), ,\
+ Kiwisolver $(kiwisolver-version))
$(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
$(ipydir)/cycler \
@@ -354,7 +371,8 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
$(ipydir)/numpy \
$(ipydir)/pyparsing \
$(ipydir)/python-dateutil
- $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version))
+ $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version), ,\
+ Matplotlib $(matplotlib-version))
# Currently mpi4py doesn't build because of some conflict with OpenMPI:
#
@@ -368,7 +386,8 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
$(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \
$(ipydir)/setuptools \
$(ilidir)/openmpi
- $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version))
+ $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version), ,\
+ mpi4py $(mpi4py-version))
$(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
$(ipydir)/setuptools \
@@ -380,24 +399,29 @@ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
export LDFLAGS="$(LDFLAGS) -shared"; \
fi; \
conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
- $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf)
+ $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf, \
+ Numpy $(numpy-version))
-$(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \
- $(ipydir)/setuptools
- $(call pybuild, tar xf, $<, pip-$(pip-version))
+$(ibidir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \
+ $(ipydir)/setuptools
+ $(call pybuild, tar xf, $<, pip-$(pip-version), ,\
+ PiP $(pip-version))
$(ipydir)/pycparser: $(tdir)/pycparser-$(pycparser-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, pycparser-$(pycparser-version))
+ $(call pybuild, tar xf, $<, pycparser-$(pycparser-version), ,\
+ pycparser $(pycparser-version))
$(ipydir)/pyparsing: $(tdir)/pyparsing-$(pyparsing-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version))
+ $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version), ,\
+ PyParsing $(pyparsing-version))
$(ipydir)/python-dateutil: $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz \
$(ipydir)/setuptools_scm \
$(ipydir)/six
- $(call pybuild, tar xf, $<, python-dateutil-$(python-dateutil-version))
+ $(call pybuild, tar xf, $<, python-dateutil-$(python-dateutil-version), ,\
+ python-dateutil $(python-dateutil-version))
$(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \
$(ipydir)/certifi \
@@ -405,7 +429,8 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \
$(ipydir)/idna \
$(ipydir)/numpy \
$(ipydir)/urllib3
- $(call pybuild, tar xf, $<, requests-$(requests-version))
+ $(call pybuild, tar xf, $<, requests-$(requests-version), ,\
+ Requests $(requests-version))
$(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \
$(ipydir)/numpy
@@ -415,33 +440,41 @@ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \
export LDFLAGS="$(LDFLAGS) -shared"; \
fi; \
conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
- $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf)
+ $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf,\
+ Scipy $(scipy-version))
$(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \
$(ipydir)/cryptography \
$(ipydir)/jeepney
- $(call pybuild, tar xf, $<, SecretStorage-$(secretstorage-version))
+ $(call pybuild, tar xf, $<, SecretStorage-$(secretstorage-version), ,\
+ SecretStorage $(secretstorage-version))
$(ipydir)/setuptools: $(tdir)/setuptools-$(setuptools-version).zip \
- $(ibdir)/python3
- $(call pybuild, unzip, $<, setuptools-$(setuptools-version))
+ $(ibidir)/python3
+ $(call pybuild, unzip, $<, setuptools-$(setuptools-version), ,\
+ Setuptools $(setuptools-version))
$(ipydir)/setuptools_scm: $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version))
+ $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version), ,\
+ Setuptools-scm $(setuptools_scm-version))
$(ipydir)/six: $(tdir)/six-$(six-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, six-$(six-version))
+ $(call pybuild, tar xf, $<, six-$(six-version), ,\
+ Six $(six-version))
$(ipydir)/soupsieve: $(tdir)/soupsieve-$(soupsieve-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version))
+ $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version), ,\
+ SoupSieve $(soupsieve-version))
$(ipydir)/urllib3: $(tdir)/urllib3-$(urllib3-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, urllib3-$(urllib3-version))
+ $(call pybuild, tar xf, $<, urllib3-$(urllib3-version), ,\
+ Urllib3 $(urllib3-version))
$(ipydir)/webencodings: $(tdir)/webencodings-$(webencodings-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, webencodings-$(webencodings-version))
+ $(call pybuild, tar xf, $<, webencodings-$(webencodings-version), ,\
+ Webencodings $(webencodings-version))
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index fdbfd7f..780a4f0 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -35,14 +35,16 @@ include reproduce/src/make/dependencies-build-rules.mk
include reproduce/config/pipeline/dependency-texlive.mk
include reproduce/config/pipeline/dependency-versions.mk
-
-ddir = $(BDIR)/dependencies
-tdir = $(BDIR)/dependencies/tarballs
-idir = $(BDIR)/dependencies/installed
-ibdir = $(BDIR)/dependencies/installed/bin
-ildir = $(BDIR)/dependencies/installed/lib
-ilidir = $(BDIR)/dependencies/installed/lib/built
-ipydir = $(BDIR)/dependencies/installed/lib/built/python
+lockdir = $(BDIR)/locks
+ddir = $(BDIR)/dependencies
+tdir = $(BDIR)/dependencies/tarballs
+idir = $(BDIR)/dependencies/installed
+ibdir = $(BDIR)/dependencies/installed/bin
+ildir = $(BDIR)/dependencies/installed/lib
+ibidir = $(BDIR)/dependencies/installed/version-info/bin
+ilidir = $(BDIR)/dependencies/installed/version-info/lib
+itidir = $(BDIR)/dependencies/installed/version-info/tex
+ipydir = $(BDIR)/dependencies/installed/version-info/python
# Define the top-level programs to build (installed in `.local/bin').
#
@@ -51,12 +53,12 @@ ipydir = $(BDIR)/dependencies/installed/lib/built/python
# successfully on Mac (only static) and GNU/Linux (shared and static). But,
# since it takes a few hours to build, it is not currently a target.
top-level-libraries = # atlas
-top-level-programs = astnoisechisel flock metastore unzip zip
+top-level-programs = astnoisechisel metastore unzip zip
top-level-python = astroquery matplotlib
-all: $(ddir)/texlive-versions.tex \
- $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \
- $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \
- $(foreach p, $(top-level-python), $(ipydir)/$(p))
+all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \
+ $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \
+ $(foreach p, $(top-level-python), $(ipydir)/$(p)) \
+ $(itidir)/texlive
# Other basic environment settings: We are only including the host
# operating system's PATH environment variable (after our own!) for the
@@ -85,6 +87,15 @@ export LD_LIBRARY_PATH := $(ildir):$(il64dir)
export LDFLAGS := $(rpath_command) -L$(ildir)
+# We want the download to happen on a single thread. So we need to define a
+# lock, and call a special script we have written for this job. These are
+# placed here because we want them both in the `dependencies.mk' and
+# `dependencies-python.mk'.
+$(lockdir): | $(BDIR); mkdir $@
+downloader="wget --no-use-server-timestamps -O";
+downloadwrapper = ./reproduce/src/bash/download-multi-try
+
+
# Python packages
include reproduce/src/make/dependencies-python.mk
@@ -125,7 +136,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
wcslib-$(wcslib-version).tar.bz2 \
zip-$(zip-version).tar.gz \
, $(tdir)/$(t) )
-$(tarballs): $(tdir)/%:
+$(tarballs): $(tdir)/%: | $(lockdir)
if [ -f $(DEPENDENCIES-DIR)/$* ]; then
cp $(DEPENDENCIES-DIR)/$* $@
else
@@ -203,14 +214,10 @@ $(tarballs): $(tdir)/%:
else tarballurl=$$w
fi
- # If the download fails, Wget will write the error message in the
- # target file, so Make will think that its done! To avoid this
- # problem, we'll rename the output.
- echo "Downloading $$tarballurl"
- if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then
- rm -f $@
- echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1
- fi
+ # Download using the script specially defined for this job.
+ touch $(lockdir)/download
+ $(downloadwrapper) "$$downloader" $(lockdir)/download \
+ $$tarballurl $@
fi
@@ -232,7 +239,7 @@ $(tarballs): $(tdir)/%:
# simple plain text file in it with the basic library name (an no prefix)
# and create/write into it when the library is successfully built.
$(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \
- $(ibdir)/curl
+ $(ibidir)/curl
# CFITSIO hard-codes the absolute address of cURL's `curl-config'
# program (which gives the necessary header and linking
@@ -252,22 +259,22 @@ $(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \
$(call gbuild, $$customtar, cfitsio, static, \
--enable-sse2 --enable-reentrant) \
&& rm $$customtar \
- && echo "CFITSIO is built" > $@
+ && echo "CFITSIO $(cfitsio-version)" > $@
$(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz
$(call gbuild, $<, gsl-$(gsl-version), static) \
- && echo "GNU Scientific Library is built" > $@
+ && echo "GNU Scientific Library $(gsl-version)" > $@
$(ilidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz
$(call gbuild, $<, fftw-$(fftw-version), static, \
--enable-shared) \
- && echo "FFTW is built" > $@
+ && echo "FFTW $(fftw-version)" > $@
# Freetype is necessary to install matplotlib
$(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \
$(ilidir)/libpng
$(call gbuild, $<, freetype-$(freetype-version), static) \
- && echo "freetype is built" > $@
+ && echo "FreeType $(freetype-version)" > $@
$(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \
$(ilidir)/openmpi
@@ -276,28 +283,29 @@ $(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \
$(call gbuild, $<, hdf5-$(hdf5-version), static, \
--enable-parallel \
--enable-fortran, V=1) \
- && echo "HDF5 library is built" > $@
+ && echo "HDF5 library $(hdf5-version)" > $@
$(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz
$(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \
- && echo "libbsd is built" > $@
+ && echo "Libbsd $(libbsd-version)" > $@
$(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz
- $(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@
+ $(call gbuild, $<, jpeg-9b, static) \
+ && echo "Libjpeg $(libjpeg-version)" > $@
$(ilidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz
$(call gbuild, $<, libpng-$(libpng-version), static) \
- && echo "Libpng is built" > $@
+ && echo "Libpng $(libpng-version)" > $@
$(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \
$(ilidir)/libjpeg
$(call gbuild, $<, tiff-$(libtiff-version), static, \
--disable-webp --disable-zstd) \
- && echo "Libtiff is built" > $@
+ && echo "Libtiff $(libtiff-version)" > $@
$(ilidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz
$(call gbuild, $<, openmpi-$(openmpi-version), static, , V=1) \
- && echo "OpenMPI is built" > $@
+ && echo "OpenMPI $(openmpi-version)" > $@
$(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
$(tdir)/lapack-$(lapack-version).tar.gz
@@ -375,7 +383,9 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
# We'll check the full installation with the static library (not
# currently building shared library on Mac.
- if [ -f $(ildir)/libatlas.a ]; then echo "Atlas is built" > $@; fi
+ if [ -f $(ildir)/libatlas.a ]; then \
+ echo "ATLAS $(atlas-version)" > $@; \
+ fi
$(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz
if [ x$(on_mac_os) = xyes ]; then \
@@ -388,7 +398,7 @@ $(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz
&& make PREFIX=$(idir) install \
&& cd .. \
&& rm -rf OpenBLAS-$(openblas-version) \
- && echo "Libtiff is built" > $@
+ && echo "OpenBLAS $(openblas-version)" > $@
@@ -411,8 +421,8 @@ $(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz
# problem we have been having so far with Mac systems:
# https://libgit2.org/docs/guides/build-and-link
$(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \
- $(ibdir)/cmake \
- $(ibdir)/curl
+ $(ibidir)/cmake \
+ $(ibidir)/curl
# Build and install the library.
$(call cbuild, $<, libgit2-$(libgit2-version), static, \
-DUSE_SSH=OFF -DBUILD_CLAR=OFF \
@@ -425,7 +435,7 @@ $(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \
fi
# Write the target file.
- echo "Libgit2 is built" > $@
+ echo "Libgit2 $(libgit2-version)" > $@
$(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
$(ilidir)/cfitsio
@@ -443,7 +453,7 @@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
fi
# Write the target file.
- echo "WCSLIB is built" > $@
+ echo "WCSLIB $(wcslib-version)" > $@
@@ -453,8 +463,8 @@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
# --------
#
# CMake can be built with its custom `./bootstrap' script.
-$(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
- $(ibdir)/curl
+$(ibidir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
+ $(ibidir)/curl
# After searching in `bootstrap', I couldn't find `LIBS', only
# `LDFLAGS'. So the extra libraries are being added to `LDFLAGS',
# not `LIBS'.
@@ -465,13 +475,16 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
export CC=clang; \
export CXX=clang++; \
fi; \
- cd $(ddir) && rm -rf cmake-$(cmake-version) && \
- tar xf $< && cd cmake-$(cmake-version) && \
- ./bootstrap --prefix=$(idir) --system-curl --system-zlib \
- --system-bzip2 --system-liblzma --no-qt-gui && \
- make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 && \
- make install && \
- cd ..&& rm -rf cmake-$(cmake-version)
+ cd $(ddir) \
+ && rm -rf cmake-$(cmake-version) \
+ && tar xf $< && cd cmake-$(cmake-version) \
+ && ./bootstrap --prefix=$(idir) --system-curl --system-zlib\
+ --system-bzip2 --system-liblzma --no-qt-gui \
+ && make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 \
+ && make install \
+ && cd .. \
+ && rm -rf cmake-$(cmake-version) \
+ && echo "CMake $(cmake-version)" > $@
# cURL (and its library, which is needed by several programs here) can
# optionally link with many different network-related libraries on the host
@@ -481,7 +494,7 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
# that if it does link with them, the pipeline will crash when the library
# is updated/changed by the host, and the whole purpose of this pipeline is
# avoid dependency on the host as much as possible.
-$(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz
+$(ibidir)/curl: $(tdir)/curl-$(curl-version).tar.gz
$(call gbuild, $<, curl-$(curl-version), , \
LIBS="-pthread" \
--with-zlib=$(ildir) \
@@ -498,22 +511,26 @@ $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz
--without-axtls \
--disable-ldaps \
--disable-ldap \
- --without-nss, V=1)
+ --without-nss, V=1) \
+ && echo "cURL $(curl-version)" > $@
# On Mac OS, libtool does different things, so to avoid confusion, we'll
# prefix GNU's libtool executables with `glibtool'.
-$(ibdir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz
+$(ibidir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz
$(call gbuild, $<, libtool-$(libtool-version), static, \
- --program-prefix=g)
+ --program-prefix=g) \
+ && echo "GNU Libtool $(libtool-version)" > $@
-$(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz
- $(call gbuild, $<, ghostscript-$(ghostscript-version))
+$(ibidir)/ghostscript: $(tdir)/ghostscript-$(ghostscript-version).tar.gz
+ $(call gbuild, $<, ghostscript-$(ghostscript-version)) \
+ && echo "GPL Ghostscript $(ghostscript-version)" > $@
-$(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \
- $(ibdir)/curl
+$(ibidir)/git: $(tdir)/git-$(git-version).tar.xz \
+ $(ibidir)/curl
$(call gbuild, $<, git-$(git-version), static, \
--without-tcltk --with-shell=$(ibdir)/bash, \
- V=1)
+ V=1) \
+ && echo "Git $(git-version)" > $@
# Metastore is used (through a Git hook) to restore the source modification
# dates of files after a Git checkout. Another Git hook saves all file
@@ -533,9 +550,9 @@ needlibbsd =
else
needlibbsd = $(ilidir)/libbsd
endif
-$(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
- $(needlibbsd) \
- $(ibdir)/git
+$(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
+ $(needlibbsd) \
+ $(ibidir)/git
# The build command below will change the current directory of this
# build, so we'll fix its value here.
@@ -559,7 +576,7 @@ $(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
user=$$(whoami)
group=$$(groups | awk '{print $$1}')
cd $$current_dir
- if [ -f $@ ]; then
+ if [ -f $(ibdir)/metastore ]; then
for f in pre-commit post-checkout; do
sed -e's|@USER[@]|'$$user'|g' \
-e's|@GROUP[@]|'$$group'|g' \
@@ -567,6 +584,7 @@ $(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
-e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \
reproduce/src/bash/git-$$f > .git/hooks/$$f
chmod +x .git/hooks/$$f
+ echo "Metastore (forked) $(metastore-version)" > $@
done
else
echo; echo; echo;
@@ -584,36 +602,39 @@ $(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
# build. Also, Ghostscript and GSL are relatively large packages. So when
# building in parallel, its better to have these packages start building
# early.
-$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \
- $(ilidir)/libgit2 \
- $(ibdir)/gs \
- $(ilidir)/gsl \
- $(ibdir)/glibtool \
- $(ilidir)/libjpeg \
- $(ilidir)/libtiff \
- $(ilidir)/wcslib
+$(ibidir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \
+ $(ilidir)/gsl \
+ $(ilidir)/wcslib \
+ $(ilidir)/libjpeg \
+ $(ilidir)/libtiff \
+ $(ilidir)/libgit2 \
+ $(ibidir)/glibtool \
+ $(ibidir)/ghostscript
ifeq ($(static_build),yes)
staticopts="--enable-static=yes --enable-shared=no";
endif
$(call gbuild, $<, gnuastro-$(gnuastro-version), static, \
$$staticopts, -j$(numthreads), \
- make check -j$(numthreads))
+ make check -j$(numthreads)) \
+ && echo "GNU Astronomy Utilities $(gnuastro-version)" > $@
-$(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz
+$(ibidir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz
v=$$(echo $(unzip-version) | sed -e's/\.//')
- $(call gbuild, $<, unzip$$v, static,, \
- -f unix/Makefile generic_gcc \
- CFLAGS="-DBIG_MEM -DMMAP",,pwd, \
- -f unix/Makefile \
- BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 )
-
-$(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz
+ $(call gbuild, $<, unzip$$v, static,, \
+ -f unix/Makefile generic_gcc \
+ CFLAGS="-DBIG_MEM -DMMAP",,pwd, \
+ -f unix/Makefile \
+ BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \
+ && echo "Unzip $(unzip-version)" > $@
+
+$(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz
v=$$(echo $(zip-version) | sed -e's/\.//')
- $(call gbuild, $<, zip$$v, static,, \
- -f unix/Makefile generic_gcc \
- CFLAGS="-DBIG_MEM -DMMAP",,pwd, \
- -f unix/Makefile \
- BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 )
+ $(call gbuild, $<, zip$$v, static,, \
+ -f unix/Makefile generic_gcc \
+ CFLAGS="-DBIG_MEM -DMMAP",,pwd, \
+ -f unix/Makefile \
+ BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \
+ && echo "Zip $(zip-version)" > $@
@@ -626,8 +647,8 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz
# of the final PDF). So we'll make a simple ASCII file called
# `texlive-ready-tlmgr' and use its contents to mark if we can use it or
# not.
-$(ibdir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \
- reproduce/config/pipeline/texlive.conf
+$(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \
+ reproduce/config/pipeline/texlive.conf
# Unpack, enter the directory, and install based on the given
# configuration (prerequisite of this rule).
@@ -668,11 +689,11 @@ $(ibdir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \
# To keep things modular and simple, we'll break up the installation of TeX
# Live itself (only very basic TeX and LaTeX) and the installation of its
# necessary packages into two packages.
-$(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \
- $(ibdir)/texlive-ready-tlmgr
+$(itidir)/texlive: reproduce/config/pipeline/dependency-texlive.mk \
+ $(itidir)/texlive-ready-tlmgr
# To work with TeX live installation, we'll need the internet.
- @res=$$(cat $(ibdir)/texlive-ready-tlmgr)
+ @res=$$(cat $(itidir)/texlive-ready-tlmgr)
if [ x"$$res" = x"NOT!" ]; then
echo "" > $@
else
@@ -688,7 +709,6 @@ $(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \
#
# We are putting a notice, because if there is no internet,
# `tlmgr' just hangs waiting.
- echo; echo; echo "Downloading necessary TeX packages..."; echo;
tlmgr install $(texlive-packages)
# Make a symbolic link of all the TeX Live executables in the bin
@@ -698,15 +718,13 @@ $(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \
# Get all the necessary versions.
texlive=$$(pdflatex --version | awk 'NR==1' | sed 's/.*(\(.*\))/\1/' \
| awk '{print $$NF}');
- echo "\newcommand{\\texliveversion}{$$texlive}" > $@
- # LaTeX Package versions.
+ # Package names and versions.
tlmgr info $(texlive-packages) --only-installed | awk \
'$$1=="package:" {version=0; \
if($$NF=="tex-gyre") name="texgyre"; \
else name=$$NF} \
$$1=="cat-version:" {version=$$NF} \
$$1=="cat-date:" {if(version==0) version=$$2; \
- printf("\\newcommand{\\tex%sversion}{%s}\n", \
- name, version)}' >> $@
+ printf("%s %s\n", name, version)}' >> $@
fi
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index 3ae580a..3c272d8 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -347,4 +347,4 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
# Version of the pipeline and build directory (for LaTeX inputs).
@v=$$(git describe --dirty --always);
- echo "\newcommand{\pipelineversion}{$$v}" > $@
+ echo "\newcommand{\pipelineversion}{$$v}" > $@
diff --git a/reproduce/src/make/paper.mk b/reproduce/src/make/paper.mk
index af9a70e..d148c08 100644
--- a/reproduce/src/make/paper.mk
+++ b/reproduce/src/make/paper.mk
@@ -54,8 +54,7 @@ $(mtexdir)/pipeline.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s
if [ ! -e tex/pipeline ]; then ln -s $(texdir) tex/pipeline; fi
# Put a LaTeX input command for all the necessary macro files.
- echo "\input{tex/pipeline/macros/dependency-versions.tex}" \
- > $(mtexdir)/pipeline.tex
+ rm -f $(mtexdir)/pipeline.tex
for t in $(subst paper,,$(makesrc)); do
echo "\input{tex/pipeline/macros/$$t.tex}" >> $(mtexdir)/pipeline.tex
done
--
cgit v1.2.1