aboutsummaryrefslogtreecommitdiff
path: root/reproduce
diff options
context:
space:
mode:
authorMohammad Akhlaghi <mohammad@akhlaghi.org>2019-04-05 16:54:41 +0100
committerMohammad Akhlaghi <mohammad@akhlaghi.org>2019-04-05 16:54:41 +0100
commit39372eedc72c4f608fc6bd9df10e67dfb8462bf1 (patch)
tree9fb8c79c84a734fe6f712bb3dcfe7a447e24aa82 /reproduce
parentac439cf62255ab38da940eb7bba0ccc00fc835f2 (diff)
Software acknowledgement section is automatically generated
Until now, management of the software names and versions in the paper was done manually (a macro had to be defined in `initialize.mk', then used in `paper.tex', so they had to be manually set in two places). Managing this was not easy. To fix this, with this commit, each software building rule's target is a text file that contains its human-readable name and its version. In the end, the configure script sorts them by their name and writes them into a LaTeX input file that we can easily import as a file into the main paper.
Diffstat (limited to 'reproduce')
-rw-r--r--reproduce/src/make/dependencies-basic.mk223
-rw-r--r--reproduce/src/make/dependencies-python.mk183
-rw-r--r--reproduce/src/make/dependencies.mk204
-rw-r--r--reproduce/src/make/initialize.mk2
-rw-r--r--reproduce/src/make/paper.mk3
5 files changed, 346 insertions, 269 deletions
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index 1573f39..d4830ca 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -49,7 +49,8 @@ tdir = $(BDIR)/dependencies/tarballs
idir = $(BDIR)/dependencies/installed
ibdir = $(BDIR)/dependencies/installed/bin
ildir = $(BDIR)/dependencies/installed/lib
-ilidir = $(BDIR)/dependencies/installed/lib/built
+ibidir = $(BDIR)/dependencies/installed/version-info/bin
+ilidir = $(BDIR)/dependencies/installed/version-info/lib
# We'll need the system's PATH for making links to low-level programs we
# won't be building ourselves.
@@ -67,7 +68,7 @@ export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH)
# Define the programs that don't depend on any other.
top-level-programs = low-level-links wget gcc
-all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
+all: $(foreach p, $(top-level-programs), $(ibidir)/$(p))
@@ -126,7 +127,7 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \
xz-$(xz-version).tar.gz \
zlib-$(zlib-version).tar.gz \
, $(tdir)/$(t) )
-$(tarballs): $(tdir)/%: $(lockdir)
+$(tarballs): $(tdir)/%: | $(lockdir)
if [ -f $(DEPENDENCIES-DIR)/$* ]; then \
cp $(DEPENDENCIES-DIR)/$* $@; \
else \
@@ -213,7 +214,7 @@ makelink = origpath="$$PATH"; \
if [ x$$a != x ]; then $$c $$a $(ibdir)/$(1); fi; \
export PATH="$$origpath"
$(ibdir) $(ildir):; mkdir $@
-$(ibdir)/low-level-links: | $(ibdir) $(ildir)
+$(ibidir)/low-level-links: | $(ibdir) $(ildir)
# The Assembler
$(call makelink,as)
@@ -260,7 +261,9 @@ $(ibdir)/low-level-links: | $(ibdir) $(ildir)
fi; \
done
- echo "Low-level symbolic links are setup" > $@
+ # We want this to be empty (so it doesn't interefere with the other
+ # files in `ibidir'.
+ touch $@
@@ -277,22 +280,26 @@ $(ibdir)/low-level-links: | $(ibdir) $(ildir)
# The first set of programs to be built are those that we need to unpack
# the source code tarballs of each program. First, we'll build the
# necessary programs, then we'll build GNU Tar.
-$(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz
- $(call gbuild, $<, gzip-$(gzip-version), static, , V=1)
+$(ibidir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz
+ $(call gbuild, $<, gzip-$(gzip-version), static, , V=1) \
+ && echo "GNU Gzip $(gzip-version)" > $@
# GNU Lzip: For a static build, the `-static' flag should be given to
# LDFLAGS on the command-line (not from the environment).
-$(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz
ifeq ($(static_build),yes)
- $(call gbuild, $<, lzip-$(lzip-version), , LDFLAGS="-static")
+lzipconf="LDFLAGS=-static"
else
- $(call gbuild, $<, lzip-$(lzip-version))
+lzipconf=
endif
+$(ibidir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz
+ $(call gbuild, $<, lzip-$(lzip-version), , $(lzipconf)) \
+ && echo "Lzip $(lzip-version)" > $@
-$(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz
- $(call gbuild, $<, xz-$(xz-version), static)
+$(ibidir)/xz: $(tdir)/xz-$(xz-version).tar.gz
+ $(call gbuild, $<, xz-$(xz-version), static) \
+ && echo "XZ Utils $(xz-version)" > $@
-$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
+$(ibidir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
# Bzip2 doesn't have a `./configure' script, and its Makefile
# doesn't build a shared library. So we can't use the `gbuild'
# function here and we need to take some extra steps (inspired
@@ -326,20 +333,22 @@ $(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
&& cd .. \
&& rm -rf $$tdir \
&& cd $(ildir) \
- && ln -fs libbz2.so.1.0 libbz2.so
+ && ln -fs libbz2.so.1.0 libbz2.so \
+ && echo "Bzip2 $(bzip2-version)" > $@
# GNU Tar: When built statically, tar gives a segmentation fault on
# unpacking Bash. So we'll build it dynamically.
-$(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
- $(ibdir)/bzip2 \
- $(ibdir)/lzip \
- $(ibdir)/gzip \
- $(ibdir)/xz
+$(ibidir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
+ $(ibidir)/bzip2 \
+ $(ibidir)/lzip \
+ $(ibidir)/gzip \
+ $(ibidir)/xz
# Since all later programs depend on Tar, the pipeline will be
# stuck here, only making Tar. So its more efficient to built it on
# multiple threads (when the user's Make doesn't pass down the
# number of threads).
- $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads))
+ $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads)) \
+ && echo "GNU Tar $(tar-version)" > $@
@@ -361,13 +370,14 @@ $(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
# function (for tilde expansion). The first can be disabled with
# `--disable-load', but unfortunately I don't know any way to fix the
# second. So, we'll have to build it dynamically for now.
-$(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \
- $(ibdir)/tar
+$(ibidir)/make: $(tdir)/make-$(make-version).tar.lz \
+ $(ibidir)/tar
# See Tar's comments for the `-j' option.
- $(call gbuild, $<, make-$(make-version), , , -j$(numthreads))
+ $(call gbuild, $<, make-$(make-version), , , -j$(numthreads)) \
+ && echo "GNU Make $(make-version)" > $@
$(ilidir)/ncurses: $(tdir)/ncurses-$(ncurses-version).tar.gz \
- $(ibdir)/make | $(ilidir)
+ $(ibidir)/make
# Delete the library that will be installed (so we can make sure
# the build process completed afterwards and reset the links).
@@ -447,7 +457,7 @@ $(ilidir)/ncurses: $(tdir)/ncurses-$(ncurses-version).tar.gz \
ln -fs $(ildir)/pkgconfig/ncursesw.pc pkgconfig/cursesw.pc; \
\
ln -fs $(idir)/include/ncursesw $(idir)/include/ncurses; \
- echo "GNU ncurses is built and ready" > $@; \
+ echo "GNU NCURSES $(ncurses-version)" > $@; \
else \
exit 1; \
fi
@@ -456,12 +466,13 @@ $(ilidir)/readline: $(tdir)/readline-$(readline-version).tar.gz \
$(ilidir)/ncurses
$(call gbuild, $<, readline-$(readline-version), static, \
--with-curses --disable-install-examples, \
- SHLIB_LIBS="-lncursesw" ) && \
- echo "GNU Readline is built and ready" > $@
+ SHLIB_LIBS="-lncursesw" ) \
+ && echo "GNU Readline $(readline-version)" > $@
-$(ibdir)/patchelf: $(tdir)/patchelf-$(patchelf-version).tar.gz \
- $(ibdir)/make
- $(call gbuild, $<, patchelf-$(patchelf-version), static)
+$(ibidir)/patchelf: $(tdir)/patchelf-$(patchelf-version).tar.gz \
+ $(ibidir)/make
+ $(call gbuild, $<, patchelf-$(patchelf-version), static) \
+ && echo "PatchELF $(patchelf-version)" > $@
# IMPORTANT: Even though we have enabled `rpath', Bash doesn't write the
@@ -475,14 +486,14 @@ $(ibdir)/patchelf: $(tdir)/patchelf-$(patchelf-version).tar.gz \
ifeq ($(on_mac_os),yes)
needpatchelf =
else
-needpatchelf = $(ibdir)/patchelf
+needpatchelf = $(ibidir)/patchelf
endif
-$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
- $(ilidir)/readline \
- $(needpatchelf)
+$(ibidir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
+ $(ilidir)/readline \
+ $(needpatchelf)
- # Delete any possibly existing output
- rm -f $@
+ # Delete the (possibly) existing Bash executable.
+ rm -f $(ibdir)/bash
# Build Bash. Note that we aren't building Bash with
# `--with-installed-readline'. This is because (as described above)
@@ -522,7 +533,11 @@ $(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
# Just to be sure that the installation step above went well,
# before making the link, we'll see if the file actually exists
# there.
- if [ -f $@ ]; then ln -fs $@ $(ibdir)/sh; else exit 1; fi
+ if [ -f $(ibdir)/bash ]; then \
+ ln -fs $(ibdir)/bash $(ibdir)/sh; \
+ echo "GNU Bash $(bash-version)" > $@; \
+ else \
+ echo "GNU Bash not built!"; exit 1; fi
@@ -537,10 +552,10 @@ $(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
# Note for a static-only build: Zlib's `./configure' doesn't use Autoconf's
# configure script, it just accepts a direct `--static' option.
$(idir)/etc:; mkdir $@
-$(ilidir): | $(ildir); mkdir $@
$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \
- $(ibdir)/bash | $(ilidir)
- $(call gbuild, $<, zlib-$(zlib-version)) && echo "Zlib is built" > $@
+ $(ibidir)/bash
+ $(call gbuild, $<, zlib-$(zlib-version)) \
+ && echo "Zlib $(zlib-version)" > $@
# OpenSSL: Some programs/libraries later need dynamic linking. So we'll
# build libssl (and libcrypto) dynamically also.
@@ -584,7 +599,7 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \
else \
patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \
fi; \
- echo "OpenSSL is built and ready" > $@; \
+ echo "OpenSSL $(openssl-version)" > $@; \
fi
# GNU Wget
@@ -600,9 +615,9 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \
# building as part of this pipeline. So to avoid too much dependency on the
# host system (especially a crash when these libraries are updated on the
# host), they are disabled here.
-$(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \
- $(ibdir)/pkg-config \
- $(ilidir)/openssl
+$(ibidir)/wget: $(tdir)/wget-$(wget-version).tar.lz \
+ $(ibidir)/pkg-config \
+ $(ilidir)/openssl
libs="-pthread"; \
if [ x$(needs_ldl) = xyes ]; then libs="$$libs -ldl"; fi; \
$(call gbuild, $<, wget-$(wget-version), , \
@@ -616,7 +631,8 @@ $(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \
--without-libidn \
--disable-pcre2 \
--disable-pcre \
- --disable-iri )
+ --disable-iri ) \
+ && echo "GNU Wget $(wget-version)" > $@
@@ -625,19 +641,22 @@ $(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \
# Basic command-line programs necessary in build process of the
# higher-level dependencies: Note that during the building of those
# programs, there is no access to the system's PATH.
-$(ibdir)/diff: $(tdir)/diffutils-$(diffutils-version).tar.xz \
- $(ibdir)/bash
- $(call gbuild, $<, diffutils-$(diffutils-version), static)
-
-$(ibdir)/find: $(tdir)/findutils-$(findutils-version).tar.lz \
- $(ibdir)/bash
- $(call gbuild, $<, findutils-$(findutils-version), static)
-
-$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \
- $(ibdir)/bash
+$(ibidir)/diffutils: $(tdir)/diffutils-$(diffutils-version).tar.xz \
+ $(ibidir)/bash
+ $(call gbuild, $<, diffutils-$(diffutils-version), static, , V=1) \
+ && echo "GNU Diffutils $(diffutils-version)" > $@
+
+$(ibidir)/findutils: $(tdir)/findutils-$(findutils-version).tar.lz \
+ $(ibidir)/bash
+ $(call gbuild, $<, findutils-$(findutils-version), static, , V=1) \
+ && echo "GNU Findutils $(findutils-version)" > $@
+
+$(ibidir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \
+ $(ibidir)/bash
# Build the main program.
$(call gbuild, $<, gawk-$(gawk-version), static, \
- --with-readline=$(idir));
+ --with-readline=$(idir)) \
+ && echo "GNU AWK $(gawk-version)" > $@
# Since AWK doesn't include RPATH by default, we'll have to
# manually include it using the `patchelf' program. Just note that
@@ -650,31 +669,36 @@ $(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \
fi; \
fi
-$(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \
- $(ibdir)/bash
- $(call gbuild, $<, grep-$(grep-version), static)
+$(ibidir)/grep: $(tdir)/grep-$(grep-version).tar.xz \
+ $(ibidir)/bash
+ $(call gbuild, $<, grep-$(grep-version), static) \
+ && echo "GNU Grep $(grep-version)" > $@
-$(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz \
- $(ilidir)/openssl
+$(ibidir)/coreutils: $(tdir)/coreutils-$(coreutils-version).tar.xz \
+ $(ilidir)/openssl
# Coreutils will use the hashing features of OpenSSL's `libcrypto'.
# See Tar's comments for the `-j' option.
$(call gbuild, $<, coreutils-$(coreutils-version), static, \
LDFLAGS="$(LDFLAGS)" CPPFLAGS="$(CPPFLAGS)" \
--enable-rpath --disable-silent-rules --with-openssl, \
- -j$(numthreads))
+ -j$(numthreads)) \
+ && echo "GNU Coreutils $(coreutils-version)" > $@
-$(ibdir)/pkg-config: $(tdir)/pkg-config-$(pkgconfig-version).tar.gz \
- $(ibdir)/bash
+$(ibidir)/pkg-config: $(tdir)/pkg-config-$(pkgconfig-version).tar.gz \
+ $(ibidir)/bash
$(call gbuild, $<, pkg-config-$(pkgconfig-version), static, \
- --with-internal-glib --with-pc-path=$(ildir)/pkgconfig)
+ --with-internal-glib --with-pc-path=$(ildir)/pkgconfig) \
+ && echo "pkg-config $(pkgconfig-version)" > $@
-$(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \
- $(ibdir)/bash
- $(call gbuild, $<, sed-$(sed-version), static)
+$(ibidir)/sed: $(tdir)/sed-$(sed-version).tar.xz \
+ $(ibidir)/bash
+ $(call gbuild, $<, sed-$(sed-version), static) \
+ && echo "GNU Sed $(sed-version)" > $@
-$(ibdir)/which: $(tdir)/which-$(which-version).tar.gz \
- $(ibdir)/bash
- $(call gbuild, $<, which-$(which-version), static)
+$(ibidir)/which: $(tdir)/which-$(which-version).tar.gz \
+ $(ibidir)/bash
+ $(call gbuild, $<, which-$(which-version), static) \
+ && echo "GNU Which $(which-version)" > $@
@@ -685,33 +709,34 @@ $(ibdir)/which: $(tdir)/which-$(which-version).tar.gz \
-# (CURRENTLY IGNORED) GCC prerequisites
-# -------------------------------------
+# GCC prerequisites
+# -----------------
$(ilidir)/gmp: $(tdir)/gmp-$(gmp-version).tar.lz \
- $(ibdir)/bash | $(ilidir)
+ $(ibidir)/bash
$(call gbuild, $<, gmp-$(gmp-version), static, , , make check) \
- && echo "GNU multiple precision arithmetic library is built" > $@
+ && echo "GNU Multiple Precision Arithmetic Library $(gmp-version)" > $@
$(ilidir)/mpfr: $(tdir)/mpfr-$(mpfr-version).tar.xz \
$(ilidir)/gmp
$(call gbuild, $<, mpfr-$(mpfr-version), static, , , make check) \
- && echo "GNU MPFR library is built" > $@
+ && echo "GNU Multiple Precision Floating-Point Reliably $(mpfr-version)" > $@
$(ilidir)/mpc: $(tdir)/mpc-$(mpc-version).tar.gz \
$(ilidir)/mpfr
$(call gbuild, $<, mpc-$(mpc-version), static, , , make check) \
- && echo "GNU MPC library is built" > $@
+ && echo "GNU Multiple Precision Complex library" > $@
$(ilidir)/isl: $(tdir)/isl-$(isl-version).tar.bz2 \
$(ilidir)/gmp
$(call gbuild, $<, isl-$(isl-version), static) \
- && echo "GCC's ISL library is built" > $@
+ && echo "GNU Integer Set Library $(isl-version)" > $@
# Binutils' linker `ld' is apparently only good for GNU/Linux systems and
# other OSs have their own. So for now we aren't actually building
# Binutils (`ld' isn't a prerequisite of GCC).
-$(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz
- $(call gbuild, $<, binutils-$(binutils-version), static)
+$(ibidir)/binutils: $(tdir)/binutils-$(binutils-version).tar.lz
+ $(call gbuild, $<, binutils-$(binutils-version), static) \
+ && echo "GNU Binutils $(binutils-version)" > $@
@@ -742,22 +767,22 @@ $(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz
#
# We are currently having problems installing GCC on macOS, so for the time
# being, if the pipeline is being run on a macOS, we'll just set a link.
-#ifeq ($(on_mac_os),yes)
-#gcc-prerequisites =
-#else
+ifeq ($(on_mac_os),yes)
+gcc-prerequisites =
+else
gcc-prerequisites = $(tdir)/gcc-$(gcc-version).tar.xz \
$(ilidir)/isl \
$(ilidir)/mpc
-#endif
-$(ibdir)/gcc: $(gcc-prerequisites) \
- $(ibdir)/ls \
- $(ibdir)/sed \
- $(ibdir)/gawk \
- $(ibdir)/grep \
- $(ibdir)/diff \
- $(ibdir)/find \
- $(ibdir)/bash \
- $(ibdir)/which
+endif
+$(ibidir)/gcc: $(gcc-prerequisites) \
+ $(ibidir)/sed \
+ $(ibidir)/gawk \
+ $(ibidir)/grep \
+ $(ibidir)/bash \
+ $(ibidir)/which \
+ $(ibidir)/findutils \
+ $(ibidir)/diffutils \
+ $(ibidir)/coreutils
# On a macOS, we (currently!) won't build GCC because of some
# errors we are still trying to find. So, we'll just make a
@@ -769,14 +794,15 @@ $(ibdir)/gcc: $(gcc-prerequisites) \
# in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic
# link to '$(idir)/lib'.
-# SO FAR WE HAVEN'T BEEN ABLE TO GET A CONSISTENT BUILD OF GCC ON MAC
-# (SOMETIMES IT CRASHES IN libiberty with g++) AND SOMETIMES IT FINISHES,
-# SO, MORE TESTS ARE NEEDED ON MAC AND WE'LL USE THE HOST'S COMPILER UNTIL
-# THEN.
+ # SO FAR WE HAVEN'T BEEN ABLE TO GET A CONSISTENT BUILD OF GCC ON
+ # MAC (SOMETIMES IT CRASHES IN libiberty with g++) AND SOMETIMES IT
+ # FINISHES, SO, MORE TESTS ARE NEEDED ON MAC AND WE'LL USE THE
+ # HOST'S COMPILER UNTIL THEN.
if [ "x$(on_mac_os)" = xyes ]; then \
$(call makelink,g++); \
$(call makelink,gfortran); \
$(call makelink,gcc,copy); \
+ echo "" > $@; \
else \
rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\
rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \
@@ -819,5 +845,6 @@ $(ibdir)/gcc: $(gcc-prerequisites) \
patchelf --set-rpath $(ildir) $$f; \
fi; \
done; \
- fi; \
+ fi \
+ && echo "GNU Compiler Collection (GCC) $(gcc-version)" > $@; \
fi
diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk
index 6711d17..034afed 100644
--- a/reproduce/src/make/dependencies-python.mk
+++ b/reproduce/src/make/dependencies-python.mk
@@ -193,47 +193,16 @@ $(pytarballs): $(tdir)/%:
else tarballurl=$$h
fi
- # If the download fails, Wget will write the error message in the
- # target file, so Make will think that its done! To avoid this
- # problem, we'll rename the output.
- echo "Downloading $$tarballurl"
- if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then
- rm -f $@
- echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1
- fi
+ # Download using the script specially defined for this job.
+ touch $(lockdir)/download
+ $(downloadwrapper) "$$downloader" $(lockdir)/download \
+ $$tarballurl $@
fi
-# Install without pip
-# --------------------
-#
-# To build Python packages with direct access to a `setup.py' (if no direct
-# access to `setup.py' is needed, pip can be used).
-# Arguments of this function are the numbers
-# 1) Unpack command
-# 2) Package name
-# 3) Unpacked directory name after unpacking the tarball
-# 4) site.cfg file (optional)
-pybuild = cd $(ddir); rm -rf $(3); \
- if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \
- cd $(3); \
- if [ "x$(4)" != x ]; then \
- sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \
- -e 's|@INCDIR[@]|'"$(idir)/include"'|' \
- $(4) > site.cfg; \
- fi; \
- python3 setup.py build \
- && python3 setup.py install \
- && cd .. && rm -rf $(3) \
- && echo "done!" > $@
-
-
-
-
-
# Necessary programs and libraries
# --------------------------------
#
@@ -241,10 +210,10 @@ pybuild = cd $(ddir); rm -rf $(3); \
# certain programs (like Python itself), or libraries for the modules.
$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz
$(call gbuild, $<, libffi-$(libffi-version)) \
- echo "libffi is built" > $@
+ echo "Libffi $(libffi-version)" > $@
-$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \
- $(ilidir)/libffi
+$(ibidir)/python3: $(tdir)/python-$(python-version).tar.gz \
+ $(ilidir)/libffi
# On Mac systems, the build complains about `clang' specific
# features, so we can't use our own GCC build here.
if [ x$(on_mac_os) = xyes ]; then \
@@ -258,7 +227,39 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \
&& v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \
{printf "%d.%d\n", $$1, $$2}') \
&& ln -s $(ildir)/python$$v $(ildir)/python \
- && rm -rf $(ipydir) && mkdir $(ipydir)
+ && rm -rf $(ipydir) \
+ && mkdir $(ipydir) \
+ && echo "Python $(python-version)" > $@
+
+
+
+
+
+# Non-PiP Python module installation
+# ----------------------------------
+#
+# To build Python packages with direct access to a `setup.py' (if no direct
+# access to `setup.py' is needed, pip can be used).
+# Arguments of this function are the numbers
+# 1) Unpack command
+# 2) Package name
+# 3) Unpacked directory name after unpacking the tarball
+# 4) site.cfg file (optional)
+# 5) Official software name.(for paper).
+pybuild = cd $(ddir); rm -rf $(3); \
+ if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \
+ cd $(3); \
+ if [ "x$(4)" != x ]; then \
+ sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \
+ -e 's|@INCDIR[@]|'"$(idir)/include"'|' \
+ $(4) > site.cfg; \
+ fi; \
+ python3 setup.py build \
+ && python3 setup.py install \
+ && cd .. \
+ && rm -rf $(3) \
+ && echo "$(5)" > $@
+
@@ -269,7 +270,8 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \
# All the necessary Python modules go here.
$(ipydir)/asn1crypto: $(tdir)/asn1crypto-$(asn1crypto-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, asn1crypto-$(asn1crypto-version))
+ $(call pybuild, tar xf, $<, asn1crypto-$(asn1crypto-version), , \
+ Asn1crypto $(asn1crypto-version))
$(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \
$(ipydir)/astropy \
@@ -278,74 +280,89 @@ $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \
$(ipydir)/keyring \
$(ipydir)/numpy \
$(ipydir)/requests
- $(call pybuild, tar xf, $<, astroquery-$(astroquery-version))
+ $(call pybuild, tar xf, $<, astroquery-$(astroquery-version), ,\
+ Astroquery $(astroquery-version))
$(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \
$(ipydir)/h5py \
$(ipydir)/numpy \
$(ipydir)/scipy
- $(call pybuild, tar xf, $<, astropy-$(astropy-version))
+ $(call pybuild, tar xf, $<, astropy-$(astropy-version), ,\
+ "Astropy $(astropy-version) \citep{astropy2013,astropy2018}")
$(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \
$(ipydir)/soupsieve
- $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version))
+ $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version), ,\
+ BeautifulSoup $(beautifulsoup4-version))
$(ipydir)/certifi: $(tdir)/certifi-$(certifi-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, certifi-$(certifi-version))
+ $(call pybuild, tar xf, $<, certifi-$(certifi-version), ,\
+ Certifi $(certifi-version))
$(ipydir)/cffi: $(tdir)/cffi-$(cffi-version).tar.gz \
$(ilidir)/libffi \
$(ipydir)/pycparser
- $(call pybuild, tar xf, $<, cffi-$(cffi-version))
+ $(call pybuild, tar xf, $<, cffi-$(cffi-version), ,\
+ cffi $(cffi-version))
$(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, chardet-$(chardet-version))
+ $(call pybuild, tar xf, $<, chardet-$(chardet-version), ,\
+ Chardet $(chardet-version))
$(ipydir)/cryptography: $(tdir)/cryptography-$(cryptography-version).tar.gz \
$(ipydir)/asn1crypto \
$(ipydir)/cffi
- $(call pybuild, tar xf, $<, cryptography-$(cryptography-version))
+ $(call pybuild, tar xf, $<, cryptography-$(cryptography-version), ,\
+ Cryptography $(cryptography-version))
$(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \
$(ipydir)/six
- $(call pybuild, tar xf, $<, cycler-$(cycler-version))
+ $(call pybuild, tar xf, $<, cycler-$(cycler-version), ,\
+ Cycler $(cycler-version))
$(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version))
+ $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version), ,\
+ EntryPoints $(entrypoints-version))
$(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \
$(ipydir)/setuptools \
$(ilidir)/hdf5
# $(ipydir)/mpi4py # AFTER its problem is fixed.
#export HDF5_MPI=ON; # AFTER its problem is fixed.
- export HDF5_DIR=$(ildir); \
- $(call pybuild, tar xf, $<, h5py-$(h5py-version))
+ export HDF5_DIR=$(ildir); \
+ $(call pybuild, tar xf, $<, h5py-$(h5py-version), ,\
+ h5py $(h5py-version))
$(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \
$(ipydir)/six \
$(ipydir)/webencodings
- $(call pybuild, tar xf, $<, html5lib-$(html5lib-version))
+ $(call pybuild, tar xf, $<, html5lib-$(html5lib-version), ,\
+ HTML5lib $(html5lib-version))
$(ipydir)/idna: $(tdir)/idna-$(idna-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, idna-$(idna-version))
+ $(call pybuild, tar xf, $<, idna-$(idna-version), ,\
+ idna $(idna-version))
$(ipydir)/jeepney: $(tdir)/jeepney-$(jeepney-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, jeepney-$(jeepney-version))
+ $(call pybuild, tar xf, $<, jeepney-$(jeepney-version), ,\
+ Jeepney $(jeepney-version))
$(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \
$(ipydir)/entrypoints \
$(ipydir)/secretstorage \
$(ipydir)/setuptools_scm
- $(call pybuild, tar xf, $<, keyring-$(keyring-version))
+ $(call pybuild, tar xf, $<, keyring-$(keyring-version), ,\
+ Keyring $(keyring-version))
$(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version))
+ $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version), ,\
+ Kiwisolver $(kiwisolver-version))
$(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
$(ipydir)/cycler \
@@ -354,7 +371,8 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
$(ipydir)/numpy \
$(ipydir)/pyparsing \
$(ipydir)/python-dateutil
- $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version))
+ $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version), ,\
+ Matplotlib $(matplotlib-version))
# Currently mpi4py doesn't build because of some conflict with OpenMPI:
#
@@ -368,7 +386,8 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
$(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \
$(ipydir)/setuptools \
$(ilidir)/openmpi
- $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version))
+ $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version), ,\
+ mpi4py $(mpi4py-version))
$(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
$(ipydir)/setuptools \
@@ -380,24 +399,29 @@ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
export LDFLAGS="$(LDFLAGS) -shared"; \
fi; \
conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
- $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf)
+ $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf, \
+ Numpy $(numpy-version))
-$(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \
- $(ipydir)/setuptools
- $(call pybuild, tar xf, $<, pip-$(pip-version))
+$(ibidir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \
+ $(ipydir)/setuptools
+ $(call pybuild, tar xf, $<, pip-$(pip-version), ,\
+ PiP $(pip-version))
$(ipydir)/pycparser: $(tdir)/pycparser-$(pycparser-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, pycparser-$(pycparser-version))
+ $(call pybuild, tar xf, $<, pycparser-$(pycparser-version), ,\
+ pycparser $(pycparser-version))
$(ipydir)/pyparsing: $(tdir)/pyparsing-$(pyparsing-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version))
+ $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version), ,\
+ PyParsing $(pyparsing-version))
$(ipydir)/python-dateutil: $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz \
$(ipydir)/setuptools_scm \
$(ipydir)/six
- $(call pybuild, tar xf, $<, python-dateutil-$(python-dateutil-version))
+ $(call pybuild, tar xf, $<, python-dateutil-$(python-dateutil-version), ,\
+ python-dateutil $(python-dateutil-version))
$(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \
$(ipydir)/certifi \
@@ -405,7 +429,8 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \
$(ipydir)/idna \
$(ipydir)/numpy \
$(ipydir)/urllib3
- $(call pybuild, tar xf, $<, requests-$(requests-version))
+ $(call pybuild, tar xf, $<, requests-$(requests-version), ,\
+ Requests $(requests-version))
$(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \
$(ipydir)/numpy
@@ -415,33 +440,41 @@ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \
export LDFLAGS="$(LDFLAGS) -shared"; \
fi; \
conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
- $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf)
+ $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf,\
+ Scipy $(scipy-version))
$(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \
$(ipydir)/cryptography \
$(ipydir)/jeepney
- $(call pybuild, tar xf, $<, SecretStorage-$(secretstorage-version))
+ $(call pybuild, tar xf, $<, SecretStorage-$(secretstorage-version), ,\
+ SecretStorage $(secretstorage-version))
$(ipydir)/setuptools: $(tdir)/setuptools-$(setuptools-version).zip \
- $(ibdir)/python3
- $(call pybuild, unzip, $<, setuptools-$(setuptools-version))
+ $(ibidir)/python3
+ $(call pybuild, unzip, $<, setuptools-$(setuptools-version), ,\
+ Setuptools $(setuptools-version))
$(ipydir)/setuptools_scm: $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version))
+ $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version), ,\
+ Setuptools-scm $(setuptools_scm-version))
$(ipydir)/six: $(tdir)/six-$(six-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, six-$(six-version))
+ $(call pybuild, tar xf, $<, six-$(six-version), ,\
+ Six $(six-version))
$(ipydir)/soupsieve: $(tdir)/soupsieve-$(soupsieve-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version))
+ $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version), ,\
+ SoupSieve $(soupsieve-version))
$(ipydir)/urllib3: $(tdir)/urllib3-$(urllib3-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, urllib3-$(urllib3-version))
+ $(call pybuild, tar xf, $<, urllib3-$(urllib3-version), ,\
+ Urllib3 $(urllib3-version))
$(ipydir)/webencodings: $(tdir)/webencodings-$(webencodings-version).tar.gz \
$(ipydir)/setuptools
- $(call pybuild, tar xf, $<, webencodings-$(webencodings-version))
+ $(call pybuild, tar xf, $<, webencodings-$(webencodings-version), ,\
+ Webencodings $(webencodings-version))
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index fdbfd7f..780a4f0 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -35,14 +35,16 @@ include reproduce/src/make/dependencies-build-rules.mk
include reproduce/config/pipeline/dependency-texlive.mk
include reproduce/config/pipeline/dependency-versions.mk
-
-ddir = $(BDIR)/dependencies
-tdir = $(BDIR)/dependencies/tarballs
-idir = $(BDIR)/dependencies/installed
-ibdir = $(BDIR)/dependencies/installed/bin
-ildir = $(BDIR)/dependencies/installed/lib
-ilidir = $(BDIR)/dependencies/installed/lib/built
-ipydir = $(BDIR)/dependencies/installed/lib/built/python
+lockdir = $(BDIR)/locks
+ddir = $(BDIR)/dependencies
+tdir = $(BDIR)/dependencies/tarballs
+idir = $(BDIR)/dependencies/installed
+ibdir = $(BDIR)/dependencies/installed/bin
+ildir = $(BDIR)/dependencies/installed/lib
+ibidir = $(BDIR)/dependencies/installed/version-info/bin
+ilidir = $(BDIR)/dependencies/installed/version-info/lib
+itidir = $(BDIR)/dependencies/installed/version-info/tex
+ipydir = $(BDIR)/dependencies/installed/version-info/python
# Define the top-level programs to build (installed in `.local/bin').
#
@@ -51,12 +53,12 @@ ipydir = $(BDIR)/dependencies/installed/lib/built/python
# successfully on Mac (only static) and GNU/Linux (shared and static). But,
# since it takes a few hours to build, it is not currently a target.
top-level-libraries = # atlas
-top-level-programs = astnoisechisel flock metastore unzip zip
+top-level-programs = astnoisechisel metastore unzip zip
top-level-python = astroquery matplotlib
-all: $(ddir)/texlive-versions.tex \
- $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \
- $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \
- $(foreach p, $(top-level-python), $(ipydir)/$(p))
+all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \
+ $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \
+ $(foreach p, $(top-level-python), $(ipydir)/$(p)) \
+ $(itidir)/texlive
# Other basic environment settings: We are only including the host
# operating system's PATH environment variable (after our own!) for the
@@ -85,6 +87,15 @@ export LD_LIBRARY_PATH := $(ildir):$(il64dir)
export LDFLAGS := $(rpath_command) -L$(ildir)
+# We want the download to happen on a single thread. So we need to define a
+# lock, and call a special script we have written for this job. These are
+# placed here because we want them both in the `dependencies.mk' and
+# `dependencies-python.mk'.
+$(lockdir): | $(BDIR); mkdir $@
+downloader="wget --no-use-server-timestamps -O";
+downloadwrapper = ./reproduce/src/bash/download-multi-try
+
+
# Python packages
include reproduce/src/make/dependencies-python.mk
@@ -125,7 +136,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
wcslib-$(wcslib-version).tar.bz2 \
zip-$(zip-version).tar.gz \
, $(tdir)/$(t) )
-$(tarballs): $(tdir)/%:
+$(tarballs): $(tdir)/%: | $(lockdir)
if [ -f $(DEPENDENCIES-DIR)/$* ]; then
cp $(DEPENDENCIES-DIR)/$* $@
else
@@ -203,14 +214,10 @@ $(tarballs): $(tdir)/%:
else tarballurl=$$w
fi
- # If the download fails, Wget will write the error message in the
- # target file, so Make will think that its done! To avoid this
- # problem, we'll rename the output.
- echo "Downloading $$tarballurl"
- if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then
- rm -f $@
- echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1
- fi
+ # Download using the script specially defined for this job.
+ touch $(lockdir)/download
+ $(downloadwrapper) "$$downloader" $(lockdir)/download \
+ $$tarballurl $@
fi
@@ -232,7 +239,7 @@ $(tarballs): $(tdir)/%:
# simple plain text file in it with the basic library name (an no prefix)
# and create/write into it when the library is successfully built.
$(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \
- $(ibdir)/curl
+ $(ibidir)/curl
# CFITSIO hard-codes the absolute address of cURL's `curl-config'
# program (which gives the necessary header and linking
@@ -252,22 +259,22 @@ $(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \
$(call gbuild, $$customtar, cfitsio, static, \
--enable-sse2 --enable-reentrant) \
&& rm $$customtar \
- && echo "CFITSIO is built" > $@
+ && echo "CFITSIO $(cfitsio-version)" > $@
$(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz
$(call gbuild, $<, gsl-$(gsl-version), static) \
- && echo "GNU Scientific Library is built" > $@
+ && echo "GNU Scientific Library $(gsl-version)" > $@
$(ilidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz
$(call gbuild, $<, fftw-$(fftw-version), static, \
--enable-shared) \
- && echo "FFTW is built" > $@
+ && echo "FFTW $(fftw-version)" > $@
# Freetype is necessary to install matplotlib
$(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \
$(ilidir)/libpng
$(call gbuild, $<, freetype-$(freetype-version), static) \
- && echo "freetype is built" > $@
+ && echo "FreeType $(freetype-version)" > $@
$(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \
$(ilidir)/openmpi
@@ -276,28 +283,29 @@ $(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \
$(call gbuild, $<, hdf5-$(hdf5-version), static, \
--enable-parallel \
--enable-fortran, V=1) \
- && echo "HDF5 library is built" > $@
+ && echo "HDF5 library $(hdf5-version)" > $@
$(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz
$(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \
- && echo "libbsd is built" > $@
+ && echo "Libbsd $(libbsd-version)" > $@
$(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz
- $(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@
+ $(call gbuild, $<, jpeg-9b, static) \
+ && echo "Libjpeg $(libjpeg-version)" > $@
$(ilidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz
$(call gbuild, $<, libpng-$(libpng-version), static) \
- && echo "Libpng is built" > $@
+ && echo "Libpng $(libpng-version)" > $@
$(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \
$(ilidir)/libjpeg
$(call gbuild, $<, tiff-$(libtiff-version), static, \
--disable-webp --disable-zstd) \
- && echo "Libtiff is built" > $@
+ && echo "Libtiff $(libtiff-version)" > $@
$(ilidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz
$(call gbuild, $<, openmpi-$(openmpi-version), static, , V=1) \
- && echo "OpenMPI is built" > $@
+ && echo "OpenMPI $(openmpi-version)" > $@
$(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
$(tdir)/lapack-$(lapack-version).tar.gz
@@ -375,7 +383,9 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
# We'll check the full installation with the static library (not
# currently building shared library on Mac.
- if [ -f $(ildir)/libatlas.a ]; then echo "Atlas is built" > $@; fi
+ if [ -f $(ildir)/libatlas.a ]; then \
+ echo "ATLAS $(atlas-version)" > $@; \
+ fi
$(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz
if [ x$(on_mac_os) = xyes ]; then \
@@ -388,7 +398,7 @@ $(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz
&& make PREFIX=$(idir) install \
&& cd .. \
&& rm -rf OpenBLAS-$(openblas-version) \
- && echo "Libtiff is built" > $@
+ && echo "OpenBLAS $(openblas-version)" > $@
@@ -411,8 +421,8 @@ $(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz
# problem we have been having so far with Mac systems:
# https://libgit2.org/docs/guides/build-and-link
$(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \
- $(ibdir)/cmake \
- $(ibdir)/curl
+ $(ibidir)/cmake \
+ $(ibidir)/curl
# Build and install the library.
$(call cbuild, $<, libgit2-$(libgit2-version), static, \
-DUSE_SSH=OFF -DBUILD_CLAR=OFF \
@@ -425,7 +435,7 @@ $(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \
fi
# Write the target file.
- echo "Libgit2 is built" > $@
+ echo "Libgit2 $(libgit2-version)" > $@
$(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
$(ilidir)/cfitsio
@@ -443,7 +453,7 @@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
fi
# Write the target file.
- echo "WCSLIB is built" > $@
+ echo "WCSLIB $(wcslib-version)" > $@
@@ -453,8 +463,8 @@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
# --------
#
# CMake can be built with its custom `./bootstrap' script.
-$(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
- $(ibdir)/curl
+$(ibidir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
+ $(ibidir)/curl
# After searching in `bootstrap', I couldn't find `LIBS', only
# `LDFLAGS'. So the extra libraries are being added to `LDFLAGS',
# not `LIBS'.
@@ -465,13 +475,16 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
export CC=clang; \
export CXX=clang++; \
fi; \
- cd $(ddir) && rm -rf cmake-$(cmake-version) && \
- tar xf $< && cd cmake-$(cmake-version) && \
- ./bootstrap --prefix=$(idir) --system-curl --system-zlib \
- --system-bzip2 --system-liblzma --no-qt-gui && \
- make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 && \
- make install && \
- cd ..&& rm -rf cmake-$(cmake-version)
+ cd $(ddir) \
+ && rm -rf cmake-$(cmake-version) \
+ && tar xf $< && cd cmake-$(cmake-version) \
+ && ./bootstrap --prefix=$(idir) --system-curl --system-zlib\
+ --system-bzip2 --system-liblzma --no-qt-gui \
+ && make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 \
+ && make install \
+ && cd .. \
+ && rm -rf cmake-$(cmake-version) \
+ && echo "CMake $(cmake-version)" > $@
# cURL (and its library, which is needed by several programs here) can
# optionally link with many different network-related libraries on the host
@@ -481,7 +494,7 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
# that if it does link with them, the pipeline will crash when the library
# is updated/changed by the host, and the whole purpose of this pipeline is
# avoid dependency on the host as much as possible.
-$(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz
+$(ibidir)/curl: $(tdir)/curl-$(curl-version).tar.gz
$(call gbuild, $<, curl-$(curl-version), , \
LIBS="-pthread" \
--with-zlib=$(ildir) \
@@ -498,22 +511,26 @@ $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz
--without-axtls \
--disable-ldaps \
--disable-ldap \
- --without-nss, V=1)
+ --without-nss, V=1) \
+ && echo "cURL $(curl-version)" > $@
# On Mac OS, libtool does different things, so to avoid confusion, we'll
# prefix GNU's libtool executables with `glibtool'.
-$(ibdir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz
+$(ibidir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz
$(call gbuild, $<, libtool-$(libtool-version), static, \
- --program-prefix=g)
+ --program-prefix=g) \
+ && echo "GNU Libtool $(libtool-version)" > $@
-$(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz
- $(call gbuild, $<, ghostscript-$(ghostscript-version))
+$(ibidir)/ghostscript: $(tdir)/ghostscript-$(ghostscript-version).tar.gz
+ $(call gbuild, $<, ghostscript-$(ghostscript-version)) \
+ && echo "GPL Ghostscript $(ghostscript-version)" > $@
-$(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \
- $(ibdir)/curl
+$(ibidir)/git: $(tdir)/git-$(git-version).tar.xz \
+ $(ibidir)/curl
$(call gbuild, $<, git-$(git-version), static, \
--without-tcltk --with-shell=$(ibdir)/bash, \
- V=1)
+ V=1) \
+ && echo "Git $(git-version)" > $@
# Metastore is used (through a Git hook) to restore the source modification
# dates of files after a Git checkout. Another Git hook saves all file
@@ -533,9 +550,9 @@ needlibbsd =
else
needlibbsd = $(ilidir)/libbsd
endif
-$(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
- $(needlibbsd) \
- $(ibdir)/git
+$(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
+ $(needlibbsd) \
+ $(ibidir)/git
# The build command below will change the current directory of this
# build, so we'll fix its value here.
@@ -559,7 +576,7 @@ $(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
user=$$(whoami)
group=$$(groups | awk '{print $$1}')
cd $$current_dir
- if [ -f $@ ]; then
+ if [ -f $(ibdir)/metastore ]; then
for f in pre-commit post-checkout; do
sed -e's|@USER[@]|'$$user'|g' \
-e's|@GROUP[@]|'$$group'|g' \
@@ -567,6 +584,7 @@ $(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
-e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \
reproduce/src/bash/git-$$f > .git/hooks/$$f
chmod +x .git/hooks/$$f
+ echo "Metastore (forked) $(metastore-version)" > $@
done
else
echo; echo; echo;
@@ -584,36 +602,39 @@ $(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
# build. Also, Ghostscript and GSL are relatively large packages. So when
# building in parallel, its better to have these packages start building
# early.
-$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \
- $(ilidir)/libgit2 \
- $(ibdir)/gs \
- $(ilidir)/gsl \
- $(ibdir)/glibtool \
- $(ilidir)/libjpeg \
- $(ilidir)/libtiff \
- $(ilidir)/wcslib
+$(ibidir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \
+ $(ilidir)/gsl \
+ $(ilidir)/wcslib \
+ $(ilidir)/libjpeg \
+ $(ilidir)/libtiff \
+ $(ilidir)/libgit2 \
+ $(ibidir)/glibtool \
+ $(ibidir)/ghostscript
ifeq ($(static_build),yes)
staticopts="--enable-static=yes --enable-shared=no";
endif
$(call gbuild, $<, gnuastro-$(gnuastro-version), static, \
$$staticopts, -j$(numthreads), \
- make check -j$(numthreads))
+ make check -j$(numthreads)) \
+ && echo "GNU Astronomy Utilities $(gnuastro-version)" > $@
-$(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz
+$(ibidir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz
v=$$(echo $(unzip-version) | sed -e's/\.//')
- $(call gbuild, $<, unzip$$v, static,, \
- -f unix/Makefile generic_gcc \
- CFLAGS="-DBIG_MEM -DMMAP",,pwd, \
- -f unix/Makefile \
- BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 )
-
-$(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz
+ $(call gbuild, $<, unzip$$v, static,, \
+ -f unix/Makefile generic_gcc \
+ CFLAGS="-DBIG_MEM -DMMAP",,pwd, \
+ -f unix/Makefile \
+ BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \
+ && echo "Unzip $(unzip-version)" > $@
+
+$(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz
v=$$(echo $(zip-version) | sed -e's/\.//')
- $(call gbuild, $<, zip$$v, static,, \
- -f unix/Makefile generic_gcc \
- CFLAGS="-DBIG_MEM -DMMAP",,pwd, \
- -f unix/Makefile \
- BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 )
+ $(call gbuild, $<, zip$$v, static,, \
+ -f unix/Makefile generic_gcc \
+ CFLAGS="-DBIG_MEM -DMMAP",,pwd, \
+ -f unix/Makefile \
+ BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \
+ && echo "Zip $(zip-version)" > $@
@@ -626,8 +647,8 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz
# of the final PDF). So we'll make a simple ASCII file called
# `texlive-ready-tlmgr' and use its contents to mark if we can use it or
# not.
-$(ibdir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \
- reproduce/config/pipeline/texlive.conf
+$(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \
+ reproduce/config/pipeline/texlive.conf
# Unpack, enter the directory, and install based on the given
# configuration (prerequisite of this rule).
@@ -668,11 +689,11 @@ $(ibdir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \
# To keep things modular and simple, we'll break up the installation of TeX
# Live itself (only very basic TeX and LaTeX) and the installation of its
# necessary packages into two packages.
-$(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \
- $(ibdir)/texlive-ready-tlmgr
+$(itidir)/texlive: reproduce/config/pipeline/dependency-texlive.mk \
+ $(itidir)/texlive-ready-tlmgr
# To work with TeX live installation, we'll need the internet.
- @res=$$(cat $(ibdir)/texlive-ready-tlmgr)
+ @res=$$(cat $(itidir)/texlive-ready-tlmgr)
if [ x"$$res" = x"NOT!" ]; then
echo "" > $@
else
@@ -688,7 +709,6 @@ $(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \
#
# We are putting a notice, because if there is no internet,
# `tlmgr' just hangs waiting.
- echo; echo; echo "Downloading necessary TeX packages..."; echo;
tlmgr install $(texlive-packages)
# Make a symbolic link of all the TeX Live executables in the bin
@@ -698,15 +718,13 @@ $(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \
# Get all the necessary versions.
texlive=$$(pdflatex --version | awk 'NR==1' | sed 's/.*(\(.*\))/\1/' \
| awk '{print $$NF}');
- echo "\newcommand{\\texliveversion}{$$texlive}" > $@
- # LaTeX Package versions.
+ # Package names and versions.
tlmgr info $(texlive-packages) --only-installed | awk \
'$$1=="package:" {version=0; \
if($$NF=="tex-gyre") name="texgyre"; \
else name=$$NF} \
$$1=="cat-version:" {version=$$NF} \
$$1=="cat-date:" {if(version==0) version=$$2; \
- printf("\\newcommand{\\tex%sversion}{%s}\n", \
- name, version)}' >> $@
+ printf("%s %s\n", name, version)}' >> $@
fi
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index 3ae580a..3c272d8 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -347,4 +347,4 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
# Version of the pipeline and build directory (for LaTeX inputs).
@v=$$(git describe --dirty --always);
- echo "\newcommand{\pipelineversion}{$$v}" > $@
+ echo "\newcommand{\pipelineversion}{$$v}" > $@
diff --git a/reproduce/src/make/paper.mk b/reproduce/src/make/paper.mk
index af9a70e..d148c08 100644
--- a/reproduce/src/make/paper.mk
+++ b/reproduce/src/make/paper.mk
@@ -54,8 +54,7 @@ $(mtexdir)/pipeline.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s
if [ ! -e tex/pipeline ]; then ln -s $(texdir) tex/pipeline; fi
# Put a LaTeX input command for all the necessary macro files.
- echo "\input{tex/pipeline/macros/dependency-versions.tex}" \
- > $(mtexdir)/pipeline.tex
+ rm -f $(mtexdir)/pipeline.tex
for t in $(subst paper,,$(makesrc)); do
echo "\input{tex/pipeline/macros/$$t.tex}" >> $(mtexdir)/pipeline.tex
done