Imported recent work on building Python within the pipeline

Raul Infante-Sainz added the building of Python (along with the Numpy and
Astropy packages) into the pipeline. That work is now being merged into the
main pipeline branch.

There was only this small problem that needed to be fixed: the Python
tarball's name after unpacking is actually `Python-X.X.X' (with a captial
P), not `python-X.X.X'. This has been corrected with this merge.

4 files changed, 125 insertions, 66 deletions

diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index 41a4cf9..f0774bc 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -1,50 +1,59 @@
 # Versions of the various dependnecies
 
 # Programs
-bash-version        = 5.0
-binutils-version    = 2.31.1
-bzip2-version       = 1.0.6
-cmake-version       = 3.12.4
-coreutils-version   = 8.30
-diffutils-version   = 3.7
-findutils-version   = 4.6.0.199-e3fc
-flock-version       = 0.2.3
-gawk-version        = 4.2.1
-gcc-version         = 8.2.0
-ghostscript-version = 9.26
-git-version         = 2.20.1
-gmp-version         = 6.1.2
-gnuastro-version    = 0.8
-grep-version        = 3.3
-gzip-version        = 1.10
-isl-version         = 0.18
-libtool-version     = 2.4.6
-libbsd-version      = 0.9.1
-lzip-version        = 1.20
-make-version        = 4.2.90
-metastore-version   = 1.1.2-23-fa9170b
-mpfr-version        = 4.0.1
-mpc-version         = 1.1.0
-ncurses-version     = 6.1
-openssl-version     = 1.1.1a
-patchelf-version    = 0.9
-pkgconfig-version   = 0.29.2
-readline-version    = 8.0
-sed-version         = 4.7
-tar-version         = 1.31
-unzip-version       = 6.0
-wget-version        = 1.20.1
-which-version       = 2.21
-xz-version          = 5.2.4
-zip-version         = 3.0
+bash-version               = 5.0
+binutils-version           = 2.31.1
+bzip2-version              = 1.0.6
+cmake-version              = 3.12.4
+coreutils-version          = 8.30
+diffutils-version          = 3.7
+findutils-version          = 4.6.0.199-e3fc
+flock-version              = 0.2.3
+gawk-version               = 4.2.1
+gcc-version                = 8.2.0
+ghostscript-version        = 9.26
+git-version                = 2.20.1
+gmp-version                = 6.1.2
+gnuastro-version           = 0.8
+grep-version               = 3.3
+gzip-version               = 1.10
+isl-version                = 0.18
+libtool-version            = 2.4.6
+libbsd-version             = 0.9.1
+lzip-version               = 1.20
+make-version               = 4.2.90
+metastore-version          = 1.1.2-23-fa9170b
+mpfr-version               = 4.0.1
+mpc-version                = 1.1.0
+ncurses-version            = 6.1
+openssl-version            = 1.1.1a
+patchelf-version           = 0.9
+pkgconfig-version          = 0.29.2
+python-version             = 3.6.8
+readline-version           = 8.0
+sed-version                = 4.7
+tar-version                = 1.31
+unzip-version              = 6.0
+wget-version               = 1.20.1
+which-version              = 2.21
+xz-version                 = 5.2.4
+zip-version                = 3.0
 
 # Libraries
-cfitsio-version     = 3.45
-curl-version        = 7.63.0
-gsl-version         = 2.5
-libjpeg-version     = v9b
-libtiff-version     = 4.0.10
-zlib-version        = 1.2.11
+cfitsio-version            = 3.45
+curl-version               = 7.63.0
+gsl-version                = 2.5
+libjpeg-version            = v9b
+libtiff-version            = 4.0.10
+zlib-version               = 1.2.11
+
+# Python packages
+# ---------------
+#
+# IMPORTANT: Fix url in `reproduce/src/make/dependencies.mk'
+# if changing the version
+astropy-version            = 3.1.1
+numpy-version              = 1.16.1
 
 # Special libraries
 # -----------------
@@ -53,5 +62,5 @@ zlib-version        = 1.2.11
 # the software build Makefiles (`reproduce/src/make/dependencies*.mk'). If
 # you change their version, also please change the explicit shared library
 # names also.
-libgit2-version     = 0.26.0
-wcslib-version      = 6.2
+libgit2-version            = 0.26.0
+wcslib-version             = 6.2
diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk
index 0c7262e..2247518 100644
--- a/reproduce/src/make/dependencies-build-rules.mk
+++ b/reproduce/src/make/dependencies-build-rules.mk
@@ -103,6 +103,22 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then        \
 
 
 
+# Python
+# ------
+#
+# To build Python packages with direct access to a `setup.py' (if no direct
+# access to `setup.py' is needed, pip can be used)
+pybuild = cd $(ddir); rm -rf $(2);                                        \
+	 if ! tar xf $(1); then echo; echo "Tar error"; exit 1; fi;           \
+	 cd $(2);                                                             \
+	 python3 setup.py build &&                                            \
+	 python3 setup.py install &&                                          \
+	 cd .. && rm -rf $(2)
+
+
+
+
+
 # CMake
 # -----
 #
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index e768051..92febb1 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -43,8 +43,10 @@ ildir  = $(BDIR)/dependencies/installed/lib
 ilidir = $(BDIR)/dependencies/installed/lib/built
 
 # Define the top-level programs to build (installed in `.local/bin').
-top-level-programs = astnoisechisel metastore flock zip unzip
-all: $(ddir)/texlive-versions.tex \
+top-level-python   = astropy
+top-level-programs = astnoisechisel metastore flock unzip zip
+all: $(ddir)/texlive-versions.tex                       \
+     $(foreach p, $(top-level-python),   $(ilidir)/$(p)) \
      $(foreach p, $(top-level-programs), $(ibdir)/$(p))
 
 # Other basic environment settings: We are only including the host
@@ -87,24 +89,27 @@ export LDFLAGS           := $(rpath_command) -L$(ildir)
 # convention, but include the name/version in their tarball names with
 # another format, we'll do the modification before the download so the
 # downloaded file has our desired format.
-tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz             \
-                        cmake-$(cmake-version).tar.gz                 \
-                        curl-$(curl-version).tar.gz                   \
-                        flock-$(flock-version).tar.xz                 \
-                        ghostscript-$(ghostscript-version).tar.gz     \
-                        git-$(git-version).tar.xz                     \
-                        gnuastro-$(gnuastro-version).tar.lz           \
-                        gsl-$(gsl-version).tar.gz                     \
-                        install-tl-unx.tar.gz                         \
-                        jpegsrc.$(libjpeg-version).tar.gz             \
-                        libbsd-$(libbsd-version).tar.xz               \
-                        libtool-$(libtool-version).tar.xz             \
-                        libgit2-$(libgit2-version).tar.gz             \
-                        metastore-$(metastore-version).tar.gz         \
-                        unzip-$(unzip-version).tar.gz                 \
-                        tiff-$(libtiff-version).tar.gz                \
-                        wcslib-$(wcslib-version).tar.bz2              \
-                        zip-$(zip-version).tar.gz                     \
+tarballs = $(foreach t, astropy-$(astropy-version).tar.gz                  \
+                        cfitsio-$(cfitsio-version).tar.gz                  \
+                        cmake-$(cmake-version).tar.gz                      \
+                        curl-$(curl-version).tar.gz                        \
+                        flock-$(flock-version).tar.xz                      \
+                        ghostscript-$(ghostscript-version).tar.gz          \
+                        git-$(git-version).tar.xz                          \
+                        gnuastro-$(gnuastro-version).tar.lz                \
+                        gsl-$(gsl-version).tar.gz                          \
+                        install-tl-unx.tar.gz                              \
+                        jpegsrc.$(libjpeg-version).tar.gz                  \
+                        libbsd-$(libbsd-version).tar.xz                    \
+                        libtool-$(libtool-version).tar.xz                  \
+                        libgit2-$(libgit2-version).tar.gz                  \
+                        metastore-$(metastore-version).tar.gz              \
+                        numpy-$(numpy-version).zip                         \
+                        python-$(python-version).tar.gz                    \
+                        unzip-$(unzip-version).tar.gz                      \
+                        tiff-$(libtiff-version).tar.gz                     \
+                        wcslib-$(wcslib-version).tar.bz2                   \
+                        zip-$(zip-version).tar.gz                          \
                       , $(tdir)/$(t) )
 $(tarballs): $(tdir)/%:
 	if [ -f $(DEPENDENCIES-DIR)/$* ]; then
@@ -117,7 +122,8 @@ $(tarballs): $(tdir)/%:
 
           # Set the top download link of the requested tarball.
 	  mergenames=1
-	  if [ $$n = cfitsio     ]; then
+	  if   [ $$n = astropy     ]; then w=https://files.pythonhosted.org/packages/eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352
+	  elif [ $$n = cfitsio     ]; then
 	    mergenames=0
 	    v=$$(echo $(cfitsio-version) | sed -e's/\.//'             \
 	              | awk '{l=length($$1);                          \
@@ -141,6 +147,10 @@ $(tarballs): $(tdir)/%:
 	    mergenames=0
 	    w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
 	  elif [ $$n = metastore   ]; then w=http://akhlaghi.org/src
+	  elif [ $$n = numpy       ]; then w=https://files.pythonhosted.org/packages/2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c
+	  elif [ $$n = python      ]; then
+	    mergenames=0
+	    w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz
 	  elif [ $$n = tiff        ]; then w=https://download.osgeo.org/libtiff
 	  elif [ $$n = unzip       ]; then w=ftp://ftp.info-zip.org/pub/infozip/src
 	    mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//')
@@ -446,6 +456,12 @@ endif
 	               $$staticopts, -j$(numthreads),                \
 	               make check -j$(numthreads))
 
+$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz
+	$(call gbuild, $<, Python-$(python-version))        \
+	&& v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \
+            {printf "%d.%d\n", $$1, $$2}')              \
+    && ln -s $(ildir)/python$$v $(ildir)/python
+
 $(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz
 	v=$$(echo $(unzip-version) | sed -e's/\.//')
 	$(call gbuild, $<, unzip$$v, static,,          \
@@ -466,6 +482,20 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz
 
 
 
+# Python packages
+# ---------------
+$(ilidir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
+                 $(ibdir)/python3
+	pip3 install $< --verbose && echo "numpy is built" > $@
+
+$(ilidir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \
+                   $(ilidir)/numpy
+	pip3 install $< --verbose && echo "astropy is built" > $@
+
+
+
+
+
 # Since we want to avoid complicating the PATH, we are putting a symbolic
 # link of all the TeX Live executables in $(ibdir). But symbolic links are
 # hard to track for Make (as a target). Also, TeX in general is optional
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index ff06dcb..71cdbb8 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -316,7 +316,6 @@ lvcheck = idir=$(BDIR)/dependencies/installed/include;                   \
 
 
 
-
 # Pipeline initialization results
 # -------------------------------
 #
@@ -413,3 +412,8 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
 
         # TeX package versions
 	cat $(BDIR)/dependencies/texlive-versions.tex >> $@
+
+	# Python packages
+	$(call pvcheck, python3, $(python-version), Python, pythonversion)
+	echo "\newcommand{\\numpyversion}{$(numpy-version)}"     >> $@
+	echo "\newcommand{\\astropyversion}{$(astropy-version)}" >> $@