aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.file-metadatabin3923 -> 4007 bytes
-rwxr-xr-xconfigure18
-rw-r--r--reproduce/config/pipeline/dependency-versions.mk38
-rw-r--r--reproduce/src/bash/git-post-checkout9
-rw-r--r--reproduce/src/bash/git-pre-commit11
-rw-r--r--reproduce/src/make/dependencies-basic.mk122
-rw-r--r--reproduce/src/make/dependencies-build-rules.mk17
-rw-r--r--reproduce/src/make/dependencies-python.mk365
-rw-r--r--reproduce/src/make/dependencies.mk61
-rw-r--r--reproduce/src/make/initialize.mk13
10 files changed, 542 insertions, 112 deletions
diff --git a/.file-metadata b/.file-metadata
index a8b09db..d58756a 100644
--- a/.file-metadata
+++ b/.file-metadata
Binary files differ
diff --git a/configure b/configure
index 268ee4c..c8e2b63 100755
--- a/configure
+++ b/configure
@@ -718,6 +718,24 @@ numthreads=$($instdir/bin/nproc)
+# Python dependencies
+# -------------------
+#
+# Python has its own installation program. To help in managing the
+# dependencies we make the installation of packages as a separate
+# Makefile.
+./.local/bin/make -f reproduce/src/make/dependencies-python.mk \
+ rpath_command=$rpath_command \
+ static_build=$static_build \
+ on_mac_os=$on_mac_os \
+ numthreads=$numthreads \
+ -j$numthreads
+
+
+
+
+
+
# Make sure TeX Live installed successfully
# -----------------------------------------
#
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index c704e27..9104852 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -3,14 +3,14 @@
# Programs
bash-version = 5.0
binutils-version = 2.31.1
-bzip2-version = 1.0.6
cmake-version = 3.12.4
coreutils-version = 8.30
diffutils-version = 3.7
findutils-version = 4.6.0.199-e3fc
flock-version = 0.2.3
+freetype-version = 2.9
gawk-version = 4.2.1
-gcc-version = 8.2.0
+gcc-version = 8.3.0
ghostscript-version = 9.26
git-version = 2.20.1
gmp-version = 6.1.2
@@ -18,8 +18,9 @@ gnuastro-version = 0.8
grep-version = 3.3
gzip-version = 1.10
isl-version = 0.18
-libtool-version = 2.4.6
libbsd-version = 0.9.1
+libpng-version = 1.6.36
+libtool-version = 2.4.6
lzip-version = 1.20
make-version = 4.2.90
metastore-version = 1.1.2-23-fa9170b
@@ -52,15 +53,38 @@ zlib-version = 1.2.11
#
# IMPORTANT: Fix url in `reproduce/src/make/dependencies.mk'
# if changing the version
+astroquery-version = 0.3.9
astropy-version = 3.1.1
+beautifulsoup4-version = 4.7.1
+certifi-version = 2018.11.29
+chardet-version = 3.0.4
+cycler-version = 0.10.0
+entrypoints-version = 0.3
+html5lib-version = 1.0.1
+idna-version = 2.8
+kiwisolver-version = 1.0.1
+keyring-version = 18.0.0
+matplotlib-version = 3.0.2
numpy-version = 1.16.1
+pip-version = 19.0.2
+pyparsing-version = 2.3.1
+python-dateutil-version = 2.8.0
+requests-version = 2.21.0
+scipy-version = 1.2.1
+setuptools-version = 40.8.0
+setuptools_scm-version = 3.2.0
+six-version = 1.12.0
+soupsieve-version = 1.8
+urllib3-version = 1.24.1
+virtualenv-version = 16.4.0
+webencodings-version = 0.5.1
# Special libraries
# -----------------
#
-# The shared library name of the following libraries is explicity mentioned
-# the software build Makefiles (`reproduce/src/make/dependencies*.mk'). If
-# you change their version, also please change the explicit shared library
-# names also.
+# When updating the version of these libraries, please look into the build
+# rule first: In one way or another, the version string becomes necessary
+# during their build and must be accounted for.
+bzip2-version = 1.0.6
libgit2-version = 0.26.0
wcslib-version = 6.2
diff --git a/reproduce/src/bash/git-post-checkout b/reproduce/src/bash/git-post-checkout
index e2a0fd0..153b277 100644
--- a/reproduce/src/bash/git-post-checkout
+++ b/reproduce/src/bash/git-post-checkout
@@ -9,8 +9,13 @@
# special characters for the installation location of meta-store so our own
# installation is found by Git.
-# File containig the metadata.
+# File containig the metadata and metastore executable.
MSFILE=".file-metadata"
+MSBIN=@BINDIR@/metastore
+
+# If metastore is not installed, then ignore this script (exit with a
+# status of 0).
+if [ ! -f $MSBIN ]; then exit 0; fi
# Delete all temporary files
find @TOP_PROJECT_DIR@/ -name "*~" -type f -delete
@@ -32,7 +37,7 @@ fi
# Run metastore.
exit_on_fail \
- @BINDIR@/metastore -a -m -e -E -q -O @USER@ -G @GROUP@ -f "$MSFILE"
+ $MSBIN -a -m -e -E -q -O @USER@ -G @GROUP@ -f "$MSFILE"
# Return with a success code (0).
exit 0
diff --git a/reproduce/src/bash/git-pre-commit b/reproduce/src/bash/git-pre-commit
index ca28757..0fa6c52 100644
--- a/reproduce/src/bash/git-pre-commit
+++ b/reproduce/src/bash/git-pre-commit
@@ -20,8 +20,13 @@
# git reset HEAD -- .metadata
# git checkout HEAD -- .metadata
-# File containig the metadata.
+# File containig the metadata and metastore executable.
MSFILE=".file-metadata"
+MSBIN=@BINDIR@/metastore
+
+# If metastore is not installed, then ignore this script (exit with a
+# status of 0).
+if [ ! -f $MSBIN ]; then exit 0; fi
# Function to help in reporting a crash.
exit_on_fail() {
@@ -34,12 +39,12 @@ exit_on_fail() {
# Run metastore.
exit_on_fail \
- @BINDIR@/metastore -O @USER@ -G @GROUP@ -s -f "$MSFILE"
+ $MSBIN -O @USER@ -G @GROUP@ -s -f "$MSFILE"
# If it's first metastore commit, store again to include $MSFILE in $MSFILE.
if ! git-ls-tree --name-only HEAD 2>/dev/null | grep -Fqx "$MSFILE"; then
exit_on_fail \
- @BINDIR@/metastore -O @USER@ -G @GROUP@ -s -f "$MSFILE"
+ $MSBIN -O @USER@ -G @GROUP@ -s -f "$MSFILE"
fi
# Check if the metadata file exists.
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index 4f5362f..cb3a7f2 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -65,8 +65,7 @@ export CPPFLAGS := -I$(idir)/include $(CPPFLAGS)
export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH)
# Define the programs that don't depend on any other.
-top-level-programs = low-level-links ls sed gawk grep diff find \
- wget which
+top-level-programs = low-level-links wget gcc
all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
@@ -141,7 +140,7 @@ $(tarballs): $(tdir)/%:
elif [ $$n = diffutils ]; then w=http://ftpmirror.gnu.org/gnu/diffutils;\
elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \
elif [ $$n = gawk ]; then w=http://ftpmirror.gnu.org/gnu/gawk; \
- elif [ $$n = gcc ]; then w=http://ftpmirror.gnu.org/gcc/gcc-$(gcc-version); \
+ elif [ $$n = gcc ]; then w=http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version); \
elif [ $$n = gmp ]; then w=https://gmplib.org/download/gmp; \
elif [ $$n = grep ]; then w=http://ftpmirror.gnu.org/gnu/grep; \
elif [ $$n = gzip ]; then w=http://ftpmirror.gnu.org/gnu/gzip; \
@@ -210,14 +209,14 @@ makelink = export PATH=$$(echo $(syspath)| tr : '\n' |grep -v ccache \
if [ x$$a != x ]; then ln -s $$a $(ibdir)/$(1); fi
$(ibdir) $(ildir):; mkdir $@
$(ibdir)/low-level-links: | $(ibdir) $(ildir)
+
# The Assembler
$(call makelink,as)
- # The compiler
+ # Compiler (Cmake needs the clang compiler which we aren't building
+ # yet in the pipeline).
$(call makelink,clang)
- $(call makelink,gcc)
- $(call makelink,g++)
- $(call makelink,cc)
+ $(call makelink,clang++)
# The linker
$(call makelink,ar)
@@ -227,7 +226,9 @@ $(ibdir)/low-level-links: | $(ibdir) $(ildir)
$(call makelink,ranlib)
# Mac OS specific
+ $(call makelink,sysctl)
$(call makelink,sw_vers)
+ $(call makelink,dsymutil)
$(call makelink,install_name_tool)
# On Mac OS, libtool is different compared to GNU Libtool. The
@@ -287,15 +288,34 @@ $(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz
$(call gbuild, $<, xz-$(xz-version), static)
$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
- tdir=bzip2-$(bzip2-version); \
- if [ $(static_build) = yes ]; then \
- makecommand="make LDFLAGS=-static"; \
- else \
- makecommand="make"; \
- fi; \
- cd $(ddir) && rm -rf $$tdir && tar xf $< && cd $$tdir && \
- $$makecommand && make install PREFIX=$(idir) && \
- cd .. && rm -rf $$tdir
+ # Bzip2 doesn't have a `./configure' script, and its Makefile
+ # doesn't build a shared library. So we can't use the `gbuild'
+ # function here and we need to take some extra steps (inspired
+ # from the "Linux from Scratch" guide for Bzip2):
+ # 1) The `sed' call is for relative installed symbolic links.
+ # 2) The special Makefile-libbz2_so builds the shared library.
+ #
+ # NOTE: the major version number appears in the final symbolic
+ # link.
+ tdir=bzip2-$(bzip2-version); \
+ if [ $(static_build) = yes ]; then \
+ makecommand="make LDFLAGS=-static"; \
+ makeshared="echo no-shared"; \
+ else \
+ makecommand="make"; \
+ makeshared="make -f Makefile-libbz2_so"; \
+ fi; \
+ cd $(ddir) && rm -rf $$tdir && tar xf $< && cd $$tdir \
+ && sed -i 's@\(ln -s -f \)$$(PREFIX)/bin/@\1@' Makefile \
+ && $$makeshared \
+ && cp -a libbz2* $(ildir)/ \
+ && make clean \
+ && $$makecommand \
+ && make install PREFIX=$(idir) \
+ && cd .. \
+ && rm -rf $$tdir \
+ && cd $(ildir) \
+ && ln -fs libbz2.so.1.0 libbz2.so
# GNU Tar: When built statically, tar gives a segmentation fault on
# unpacking Bash. So we'll build it dynamically.
@@ -698,6 +718,9 @@ $(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz
# We want to build GCC after building all the basic tools that are often
# used in a configure script to enable GCC's configure script to work as
# smoothly/robustly as possible.
+#
+# Including Objective C and Objective C++ is necessary for installing
+# `matplotlib'.
$(ibdir)/gcc: $(tdir)/gcc-$(gcc-version).tar.xz \
$(ibdir)/ls \
$(ibdir)/sed \
@@ -710,38 +733,39 @@ $(ibdir)/gcc: $(tdir)/gcc-$(gcc-version).tar.xz \
$(ibdir)/bash \
$(ibdir)/which
+ # Clean up (possibly existing) gcc installation
+ rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*
+ rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc* \
+ $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*
+
# Un-pack all the necessary tools in the top building directory
- cd $(ddir); \
- rm -rf gcc-build gcc-$(gcc-version); \
- tar xf $< && \
- mkdir $(ddir)/gcc-build && \
- cd $(ddir)/gcc-build && \
- ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \
- --prefix=$(idir) \
- --with-mpc=$(idir) \
- --with-mpfr=$(idir) \
- --with-gmp=$(idir) \
- --with-isl=$(idir) \
- --with-build-time-tools=$(idir) \
- --enable-shared \
- --disable-multilib \
- --disable-multiarch \
- --enable-threads=posix \
- --enable-libmpx \
- --with-local-prefix=$(idir) \
- --enable-linker-build-id \
- --with-gnu-as \
- --with-gnu-ld \
- --enable-lto \
- --with-linker-hash-style=gnu \
- --enable-languages=c,c++,fortran\
- --disable-libada \
- --disable-nls \
- --enable-default-pie \
- --enable-default-ssp \
- --enable-cet=auto \
- --enable-decimal-float && \
- make SHELL=$(ibdir)/bash -j$$(nproc) && \
- make SHELL=$(ibdir)/bash install && \
- cd .. && \
+ cd $(ddir); \
+ rm -rf gcc-build gcc-$(gcc-version); \
+ tar xf $< && \
+ mkdir $(ddir)/gcc-build && \
+ cd $(ddir)/gcc-build && \
+ ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \
+ --prefix=$(idir) \
+ --with-mpc=$(idir) \
+ --with-mpfr=$(idir) \
+ --with-gmp=$(idir) \
+ --with-isl=$(idir) \
+ --with-build-time-tools=$(idir) \
+ --enable-shared \
+ --disable-multilib \
+ --disable-multiarch \
+ --enable-threads=posix \
+ --with-local-prefix=$(idir) \
+ --enable-linker-build-id \
+ --enable-lto \
+ --enable-languages=c,c++,fortran,objc,obj-c++ \
+ --disable-libada \
+ --disable-nls \
+ --enable-default-pie \
+ --enable-default-ssp \
+ --enable-cet=auto \
+ --enable-decimal-float && \
+ make SHELL=$(ibdir)/bash -j$$(nproc) && \
+ make SHELL=$(ibdir)/bash install && \
+ cd .. && \
rm -rf gcc-build gcc-$(gcc-version)
diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk
index 2247518..5216488 100644
--- a/reproduce/src/make/dependencies-build-rules.mk
+++ b/reproduce/src/make/dependencies-build-rules.mk
@@ -102,23 +102,6 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then \
-
-# Python
-# ------
-#
-# To build Python packages with direct access to a `setup.py' (if no direct
-# access to `setup.py' is needed, pip can be used)
-pybuild = cd $(ddir); rm -rf $(2); \
- if ! tar xf $(1); then echo; echo "Tar error"; exit 1; fi; \
- cd $(2); \
- python3 setup.py build && \
- python3 setup.py install && \
- cd .. && rm -rf $(2)
-
-
-
-
-
# CMake
# -----
#
diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk
new file mode 100644
index 0000000..1e4f611
--- /dev/null
+++ b/reproduce/src/make/dependencies-python.mk
@@ -0,0 +1,365 @@
+# Build the reproduction pipeline Python dependencies.
+#
+# ------------------------------------------------------------------------
+# !!!!! IMPORTANT NOTES !!!!!
+#
+# This Makefile will be run by the initial `./configure' script. It is not
+# included into the reproduction pipe after that.
+#
+# ------------------------------------------------------------------------
+#
+# Original author:
+# Raul Infante-Sainz <infantesainz@gmail.com>
+# Contributing author(s):
+# Mohammad Akhlaghi <mohammad@akhlaghi.org>
+# Copyright (C) 2019, Your Name.
+#
+# This Makefile is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This Makefile is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# A copy of the GNU General Public License is available at
+# <http://www.gnu.org/licenses/>.
+
+
+
+# Top level environment
+include reproduce/config/pipeline/LOCAL.mk
+include reproduce/src/make/dependencies-build-rules.mk
+include reproduce/config/pipeline/dependency-versions.mk
+
+ddir = $(BDIR)/dependencies
+tdir = $(BDIR)/dependencies/tarballs
+idir = $(BDIR)/dependencies/installed
+ibdir = $(BDIR)/dependencies/installed/bin
+ildir = $(BDIR)/dependencies/installed/lib
+ilidir = $(BDIR)/dependencies/installed/lib/built
+ipydir = $(BDIR)/dependencies/installed/lib/built/python
+
+# Define the top-level programs to build (installed in `.local/bin').
+top-level-python = astroquery matplotlib #scipy
+all: $(foreach p, $(top-level-python), $(ipydir)/$(p))
+
+# Other basic environment settings: We are only including the host
+# operating system's PATH environment variable (after our own!) for the
+# compiler and linker. For the library binaries and headers, we are only
+# using our internally built libraries.
+#
+# To investigate:
+#
+# 1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all
+# the parameters defined bellow as `NAME=VALUE' statements before
+# calling Bash. This will enable us to completely ignore the user's
+# native environment.
+#
+# 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the
+# user's environment.
+.ONESHELL:
+.SHELLFLAGS := --noprofile --norc -ec
+export CCACHE_DISABLE := 1
+export PATH := $(ibdir)
+export LD_RUN_PATH := $(ildir)
+export LD_LIBRARY_PATH := $(ildir)
+export SHELL := $(ibdir)/bash
+export CPPFLAGS := -I$(idir)/include
+export PKG_CONFIG_PATH := $(ildir)/pkgconfig
+export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig
+export LDFLAGS := $(rpath_command) -L$(ildir)
+
+
+
+
+
+# Tarballs
+# --------
+#
+# All the necessary tarballs are defined and prepared with this rule.
+#
+# Note that we want the tarballs to follow the convention of NAME-VERSION
+# before the `tar.XX' prefix. For those programs that don't follow this
+# convention, but include the name/version in their tarball names with
+# another format, we'll do the modification before the download so the
+# downloaded file has our desired format.
+tarballs = $(foreach t, astroquery-$(astroquery-version).tar.gz \
+ astropy-$(astropy-version).tar.gz \
+ beautifulsoup4-$(beautifulsoup4-version).tar.gz \
+ certifi-$(certifi-version).tar.gz \
+ chardet-$(chardet-version).tar.gz \
+ cycler-$(cycler-version).tar.gz \
+ entrypoints-$(entrypoints-version).tar.gz \
+ html5lib-$(html5lib-version).tar.gz \
+ idna-$(idna-version).tar.gz \
+ kiwisolver-$(kiwisolver-version).tar.gz \
+ keyring-$(keyring-version).tar.gz \
+ matplotlib-$(matplotlib-version).tar.gz \
+ numpy-$(numpy-version).zip \
+ pip-$(pip-version).tar.gz \
+ python-$(python-version).tar.gz \
+ python-dateutil-$(python-dateutil-version).tar.gz \
+ pyparsing-$(pyparsing-version).tar.gz \
+ requests-$(requests-version).tar.gz \
+ scipy-$(scipy-version).tar.gz \
+ setuptools-$(setuptools-version).zip \
+ setuptools_scm-$(setuptools_scm-version).tar.gz \
+ six-$(six-version).tar.gz \
+ soupsieve-$(soupsieve-version).tar.gz \
+ urllib3-$(urllib3-version).tar.gz \
+ webencodings-$(webencodings-version).tar.gz \
+ virtualenv-$(virtualenv-version).tar.gz \
+ , $(tdir)/$(t) )
+topurl=https://files.pythonhosted.org/packages
+$(tarballs): $(tdir)/%:
+ if [ -f $(DEPENDENCIES-DIR)/$* ]; then
+ cp $(DEPENDENCIES-DIR)/$* $@
+ else
+
+ # Convenience variable
+ # --------------------
+ #
+ # `n' is just for convenience and to avoid having to repeat the
+ # package tarball name in the conditional to find its URL.
+ #
+ # For some packages (for example `python-dateutil', or those with
+ # a number or dash in their name), we need special consideration
+ # because the tokenization above will produce `python' as the
+ # first string.
+ if [ $* = python-dateutil-$(python-dateutil-version).tar.gz ]; then
+ n=dateutil
+ # elif [ $* = strange-tarball5name-version.tar.gz ]; then
+ # n=strange5-name
+ else
+ # Remove all numbers, `-' and `.' from the tarball name so we can
+ # search more easily only with the program name.
+ n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \
+ | awk '{print $$1}')
+ fi
+
+ # Set the top download link of the requested tarball.
+ mergenames=1
+ if [ $$n = python ]; then
+ mergenames=0
+ h=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz
+ elif [ $$n = astroquery ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396
+ elif [ $$n = astropy ]; then h=eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352
+ elif [ $$n = beautifulsoup ]; then h=80/f2/f6aca7f1b209bb9a7ef069d68813b091c8c3620642b568dac4eb0e507748
+ elif [ $$n = certifi ]; then h=55/54/3ce77783acba5979ce16674fc98b1920d00b01d337cfaaf5db22543505ed
+ elif [ $$n = chardet ]; then h=fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d
+ elif [ $$n = cycler ]; then h=c2/4b/137dea450d6e1e3d474e1d873cd1d4f7d3beed7e0dc973b06e8e10d32488
+ elif [ $$n = entrypoints ]; then h=b4/ef/063484f1f9ba3081e920ec9972c96664e2edb9fdc3d8669b0e3b8fc0ad7c
+ elif [ $$n = html ]; then h=85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f
+ elif [ $$n = idna ]; then h=ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7
+ elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2
+ elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864
+ elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f
+ elif [ $$n = numpy ]; then h=2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c
+ elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af
+ elif [ $$n = pyparsing ]; then h=b9/b8/6b32b3e84014148dcd60dd05795e35c2e7f4b72f918616c61fdce83d27fc
+ elif [ $$n = dateutil ]; then h=ad/99/5b2e99737edeb28c71bcbec5b5dda19d0d9ef3ca3e92e3e925e7c0bb364c
+ elif [ $$n = requests ]; then h=52/2c/514e4ac25da2b08ca5a464c50463682126385c4272c18193876e91f4bc38
+ elif [ $$n = scipy ]; then h=a9/b4/5598a706697d1e2929eaf7fe68898ef4bea76e4950b9efbe1ef396b8813a
+ elif [ $$n = setuptools ]; then h=c2/f7/c7b501b783e5a74cf1768bc174ee4fb0a8a6ee5af6afa92274ff964703e0
+ elif [ $$n = setuptools_scm ]; then h=54/85/514ba3ca2a022bddd68819f187ae826986051d130ec5b972076e4f58a9f3
+ elif [ $$n = six ]; then h=dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca
+ elif [ $$n = soupsieve ]; then h=0c/52/e9088bb9b96e2d39fc3b33fcda5b4fde9d71473536ac660a1ca9a0958a2f
+ elif [ $$n = urllib ]; then h=b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1
+ elif [ $$n = virtualenv ]; then h=51/aa/c395a6e6eaaedfa5a04723b6446a1df783b16cca6fec66e671cede514688
+ elif [ $$n = webencodings ]; then h=0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47
+# elif [ $$n = strange5-name ]; then h=XXXXX
+ else
+ echo; echo; echo;
+ echo "'$$n' not recognized as a dependency name to download."
+ echo; echo; echo;
+ exit 1
+ fi
+
+ # Download the requested tarball. Note that some packages may not
+ # follow our naming convention (where the package name is merged
+ # with its version number). In such cases, `w' will be the full
+ # address, not just the top directory address. But since we are
+ # storing all the tarballs in one directory, we want it to have
+ # the same naming convention, so we'll download it to a temporary
+ # name, then rename that.
+ if [ $$mergenames = 1 ]; then tarballurl=$(topurl)/$$h/"$*"
+ else tarballurl=$$h
+ fi
+
+ # If the download fails, Wget will write the error message in the
+ # target file, so Make will think that its done! To avoid this
+ # problem, we'll rename the output.
+ echo "Downloading $$tarballurl"
+ if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then
+ rm -f $@
+ echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1
+ fi
+ fi
+
+
+
+
+
+# Install without pip
+# --------------------
+#
+# To build Python packages with direct access to a `setup.py' (if no direct
+# access to `setup.py' is needed, pip can be used)
+# Arguments of this function are the numbers
+# 1) Unpack command
+# 2) Package name
+# 3) Unpacked directory name after unpacking the tarball
+pybuild = cd $(ddir); rm -rf $(3); \
+ if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \
+ cd $(3); \
+ python3 setup.py build && \
+ python3 setup.py install && \
+ cd .. && rm -rf $(3) && \
+ echo "done!" > $@
+
+
+
+
+
+# Python installation
+# -------------------
+#
+$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz
+ # On Mac systems, the build complains about `clang' specific
+ # features, so we can't use our own GCC build here.
+# if [ x$(on_mac_os) = xyes ]; then \
+# export CC=clang; \
+# export CXX=clang++; \
+# fi; \
+
+ $(call gbuild, $<, Python-$(python-version)) \
+ && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \
+ {printf "%d.%d\n", $$1, $$2}') \
+ && ln -s $(ildir)/python$$v $(ildir)/python \
+ && rm -rf $(ipydir) && mkdir $(ipydir)
+
+
+
+
+# Python packages
+# ---------------
+#
+$(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \
+ $(ipydir)/astropy \
+ $(ipydir)/beautifulsoup4 \
+ $(ipydir)/html5lib \
+ $(ipydir)/keyring \
+ $(ipydir)/numpy \
+ $(ipydir)/requests
+ $(call pybuild, tar xf, $<, astroquery-$(astroquery-version))
+
+$(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \
+ $(ipydir)/numpy
+ $(call pybuild, tar xf, $<, astropy-$(astropy-version))
+
+$(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \
+ $(ipydir)/soupsieve
+ $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version))
+
+$(ipydir)/certifi: $(tdir)/certifi-$(certifi-version).tar.gz \
+ $(ibdir)/python3
+ $(call pybuild, tar xf, $<, certifi-$(certifi-version))
+
+$(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \
+ $(ibdir)/python3
+ $(call pybuild, tar xf, $<, chardet-$(chardet-version))
+
+$(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \
+ $(ipydir)/six
+ $(call pybuild, tar xf, $<, cycler-$(cycler-version))
+
+$(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \
+ $(ibdir)/python3
+ $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version))
+
+$(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \
+ $(ipydir)/six \
+ $(ipydir)/webencodings
+ $(call pybuild, tar xf, $<, html5lib-$(html5lib-version))
+
+$(ipydir)/idna: $(tdir)/idna-$(idna-version).tar.gz \
+ $(ibdir)/python3
+ $(call pybuild, tar xf, $<, idna-$(idna-version))
+
+$(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \
+ $(ipydir)/entrypoints \
+ $(ipydir)/setuptools_scm
+ $(call pybuild, tar xf, $<, keyring-$(keyring-version))
+
+$(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \
+ $(ipydir)/setuptools
+ $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version))
+
+
+$(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
+ $(ipydir)/cycler \
+ $(ilidir)/freetype \
+ $(ipydir)/numpy \
+ $(ipydir)/pyparsing \
+ $(ipydir)/python-dateutil \
+ $(ipydir)/kiwisolver
+ $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version))
+
+$(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
+ $(ibdir)/python3
+ export BLAS=None; \
+ export ATLAS=None; \
+ export LAPACK=None; \
+ $(call pybuild, unzip, $<, numpy-$(numpy-version))
+
+$(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \
+ $(ibdir)/python3
+ $(call pybuild, tar xf, $<, pip-$(pip-version))
+
+$(ipydir)/pyparsing: $(tdir)/pyparsing-$(pyparsing-version).tar.gz \
+ $(ibdir)/python3
+ $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version))
+
+$(ipydir)/python-dateutil: $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz \
+ $(ipydir)/six
+ $(call pybuild, tar xf, $<, python-dateutil-$(python-dateutil-version))
+
+$(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \
+ $(ipydir)/certifi \
+ $(ipydir)/chardet \
+ $(ipydir)/idna \
+ $(ipydir)/numpy \
+ $(ipydir)/urllib3
+ $(call pybuild, tar xf, $<, requests-$(requests-version))
+
+$(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \
+ $(ipydir)/numpy
+ $(call pybuild, tar xf, $<, scipy-$(scipy-version))
+
+$(ipydir)/setuptools: $(tdir)/setuptools-$(setuptools-version).zip \
+ $(ibdir)/python3
+ $(call pybuild, unzip, $<, setuptools-$(setuptools-version))
+
+$(ipydir)/setuptools_scm: $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.gz \
+ $(ipydir)/setuptools
+ $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version))
+
+$(ipydir)/six: $(tdir)/six-$(six-version).tar.gz \
+ $(ibdir)/python3
+ $(call pybuild, tar xf, $<, six-$(six-version))
+
+$(ipydir)/soupsieve: $(tdir)/soupsieve-$(soupsieve-version).tar.gz \
+ $(ibdir)/python3
+ $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version))
+
+$(ipydir)/urllib3: $(tdir)/urllib3-$(urllib3-version).tar.gz \
+ $(ibdir)/python3
+ $(call pybuild, tar xf, $<, urllib3-$(urllib3-version))
+
+$(ipydir)/webencodings: $(tdir)/webencodings-$(webencodings-version).tar.gz \
+ $(ibdir)/python3
+ $(call pybuild, tar xf, $<, webencodings-$(webencodings-version))
+
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index 92febb1..deb9f39 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -43,11 +43,11 @@ ildir = $(BDIR)/dependencies/installed/lib
ilidir = $(BDIR)/dependencies/installed/lib/built
# Define the top-level programs to build (installed in `.local/bin').
-top-level-python = astropy
-top-level-programs = astnoisechisel metastore flock unzip zip
+top-level-programs = astnoisechisel flock metastore unzip zip
+top-level-libraries = freetype
all: $(ddir)/texlive-versions.tex \
- $(foreach p, $(top-level-python), $(ilidir)/$(p)) \
- $(foreach p, $(top-level-programs), $(ibdir)/$(p))
+ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \
+ $(foreach p, $(top-level-libraries), $(ilidir)/$(p))
# Other basic environment settings: We are only including the host
# operating system's PATH environment variable (after our own!) for the
@@ -89,11 +89,11 @@ export LDFLAGS := $(rpath_command) -L$(ildir)
# convention, but include the name/version in their tarball names with
# another format, we'll do the modification before the download so the
# downloaded file has our desired format.
-tarballs = $(foreach t, astropy-$(astropy-version).tar.gz \
- cfitsio-$(cfitsio-version).tar.gz \
+tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
cmake-$(cmake-version).tar.gz \
curl-$(curl-version).tar.gz \
flock-$(flock-version).tar.xz \
+ freetype-$(freetype-version).tar.gz \
ghostscript-$(ghostscript-version).tar.gz \
git-$(git-version).tar.xz \
gnuastro-$(gnuastro-version).tar.lz \
@@ -101,11 +101,10 @@ tarballs = $(foreach t, astropy-$(astropy-version).tar.gz \
install-tl-unx.tar.gz \
jpegsrc.$(libjpeg-version).tar.gz \
libbsd-$(libbsd-version).tar.xz \
+ libpng-$(libpng-version).tar.xz \
libtool-$(libtool-version).tar.xz \
libgit2-$(libgit2-version).tar.gz \
metastore-$(metastore-version).tar.gz \
- numpy-$(numpy-version).zip \
- python-$(python-version).tar.gz \
unzip-$(unzip-version).tar.gz \
tiff-$(libtiff-version).tar.gz \
wcslib-$(wcslib-version).tar.bz2 \
@@ -122,8 +121,7 @@ $(tarballs): $(tdir)/%:
# Set the top download link of the requested tarball.
mergenames=1
- if [ $$n = astropy ]; then w=https://files.pythonhosted.org/packages/eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352
- elif [ $$n = cfitsio ]; then
+ if [ $$n = cfitsio ]; then
mergenames=0
v=$$(echo $(cfitsio-version) | sed -e's/\.//' \
| awk '{l=length($$1); \
@@ -135,6 +133,7 @@ $(tarballs): $(tdir)/%:
elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12
elif [ $$n = curl ]; then w=https://curl.haxx.se/download
elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version)
+ elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype
elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926
elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git
elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro
@@ -142,15 +141,12 @@ $(tarballs): $(tdir)/%:
elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet
elif [ $$n = jpegsrc ]; then w=http://ijg.org/files
elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases
+ elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng
elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool
elif [ $$n = libgit ]; then
mergenames=0
w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
elif [ $$n = metastore ]; then w=http://akhlaghi.org/src
- elif [ $$n = numpy ]; then w=https://files.pythonhosted.org/packages/2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c
- elif [ $$n = python ]; then
- mergenames=0
- w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz
elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff
elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src
mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//')
@@ -232,6 +228,12 @@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz
$(call gbuild, $<, gsl-$(gsl-version), static) \
&& echo "GNU Scientific Library is built" > $@
+# Freetype is necessary to install matplotlib
+$(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \
+ $(ilidir)/libpng
+ $(call gbuild, $<, freetype-$(freetype-version), static) \
+ && echo "freetype is built" > $@
+
$(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz
$(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \
&& echo "libbsd is built" > $@
@@ -239,6 +241,10 @@ $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz
$(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz
$(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@
+$(ilidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz
+ $(call gbuild, $<, libpng-$(libpng-version), static) \
+ && echo "Libpng is built" > $@
+
$(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \
$(ilidir)/libjpeg
$(call gbuild, $<, tiff-$(libtiff-version), static, \
@@ -314,6 +320,13 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
# After searching in `bootstrap', I couldn't find `LIBS', only
# `LDFLAGS'. So the extra libraries are being added to `LDFLAGS',
# not `LIBS'.
+ #
+ # On Mac systems, the build complains about `clang' specific
+ # features, so we can't use our own GCC build here.
+ if [ x$(on_mac_os) = xyes ]; then \
+ export CC=clang; \
+ export CXX=clang++; \
+ fi; \
cd $(ddir) && rm -rf cmake-$(cmake-version) && \
tar xf $< && cd cmake-$(cmake-version) && \
./bootstrap --prefix=$(idir) --system-curl --system-zlib \
@@ -456,12 +469,6 @@ endif
$$staticopts, -j$(numthreads), \
make check -j$(numthreads))
-$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz
- $(call gbuild, $<, Python-$(python-version)) \
- && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \
- {printf "%d.%d\n", $$1, $$2}') \
- && ln -s $(ildir)/python$$v $(ildir)/python
-
$(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz
v=$$(echo $(unzip-version) | sed -e's/\.//')
$(call gbuild, $<, unzip$$v, static,, \
@@ -482,20 +489,6 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz
-# Python packages
-# ---------------
-$(ilidir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
- $(ibdir)/python3
- pip3 install $< --verbose && echo "numpy is built" > $@
-
-$(ilidir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \
- $(ilidir)/numpy
- pip3 install $< --verbose && echo "astropy is built" > $@
-
-
-
-
-
# Since we want to avoid complicating the PATH, we are putting a symbolic
# link of all the TeX Live executables in $(ibdir). But symbolic links are
# hard to track for Make (as a target). Also, TeX in general is optional
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index 71cdbb8..638338b 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -116,6 +116,19 @@ export LDFLAGS := -L$(installdir)/lib
export SHELL := $(installdir)/bin/bash
export CPPFLAGS := -I$(installdir)/include
+# Python enviroment
+# So far we have found several other Python-related environment
+# variables which might interfere. So we are just removing all
+# of their values within the pipeline.
+export PYTHONPATH := $(installdir)/lib/python/site-packages
+export PYTHONPATH3 := $(PYTHONPATH)
+export _LMFILES_ :=
+export PYTHONPATH2 :=
+export LOADEDMODULES :=
+export MPI_PYTHON_SITEARCH :=
+export MPI_PYTHON2_SITEARCH :=
+export MPI_PYTHON3_SITEARCH :=
+