Imported recent work on building Python within the pipeline

Raul Infante-Sainz added the building of Python (along with the Numpy and
Astropy packages) into the pipeline. That work is now being merged into the
main pipeline branch.

There was only this small problem that needed to be fixed: the Python
tarball's name after unpacking is actually `Python-X.X.X' (with a captial
P), not `python-X.X.X'. This has been corrected with this merge.

8 files changed, 225 insertions, 68 deletions

diff --git a/.file-metadata b/.file-metadata
index 45150da..517973d 100644
--- a/.file-metadata
+++ b/.file-metadata
diff --git a/.gitignore b/.gitignore
index 7074d37..5879789 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,12 +5,14 @@
 *.pdf
 *.out
 *.zip
+*.swp
 .nfs*
 mmap_*
 *.tar.gz
 *.auxlock
 
 .tex
+build
 .local
 Makefile
 tex/tikz
@@ -24,4 +26,4 @@ gnuastro-local.conf
 reproduce/config/pipeline/LOCAL.mk
 
 reproduce/BDIR/
-reproduce/SURVEY/
\ No newline at end of file
+reproduce/SURVEY/
diff --git a/paper.tex b/paper.tex
index a36b3eb..16f6eb8 100644
--- a/paper.tex
+++ b/paper.tex
@@ -216,7 +216,9 @@ Scientific Library (GSL) \gslversion, GNU Tar \tarversion, GNU Wget
 Libjpeg \libjpegversion, Libtiff \libtiffversion, Metastore (forked)
 \metastoreversion, OpenSSL \opensslversion, Pkg-config \pkgconfigversion,
 Unzip \unzipversion, WCSLIB \wcslibversion, XZ Utils \xzversion, Zip
-\zipversion, and ZLib \zlibversion. The final paper was produced with
+\zipversion, and ZLib \zlibversion. We use Python {\pythonversion} with the
+following packages: Numpy {\numpyversion} and Astropy {\astropyversion}
+\citep{astropy2013, astropy2018}. The final paper was produced with
 \TeX{} Live \texliveversion, using the following packages: \TeX{}
 \textexversion, EC \texecversion, NewTX \texnewtxversion, Fontaxes
 \texfontaxesversion, Keyval, \texxkeyvalversion, Etoolbox
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index 41a4cf9..f0774bc 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -1,50 +1,59 @@
 # Versions of the various dependnecies
 
 # Programs
-bash-version        = 5.0
-binutils-version    = 2.31.1
-bzip2-version       = 1.0.6
-cmake-version       = 3.12.4
-coreutils-version   = 8.30
-diffutils-version   = 3.7
-findutils-version   = 4.6.0.199-e3fc
-flock-version       = 0.2.3
-gawk-version        = 4.2.1
-gcc-version         = 8.2.0
-ghostscript-version = 9.26
-git-version         = 2.20.1
-gmp-version         = 6.1.2
-gnuastro-version    = 0.8
-grep-version        = 3.3
-gzip-version        = 1.10
-isl-version         = 0.18
-libtool-version     = 2.4.6
-libbsd-version      = 0.9.1
-lzip-version        = 1.20
-make-version        = 4.2.90
-metastore-version   = 1.1.2-23-fa9170b
-mpfr-version        = 4.0.1
-mpc-version         = 1.1.0
-ncurses-version     = 6.1
-openssl-version     = 1.1.1a
-patchelf-version    = 0.9
-pkgconfig-version   = 0.29.2
-readline-version    = 8.0
-sed-version         = 4.7
-tar-version         = 1.31
-unzip-version       = 6.0
-wget-version        = 1.20.1
-which-version       = 2.21
-xz-version          = 5.2.4
-zip-version         = 3.0
+bash-version               = 5.0
+binutils-version           = 2.31.1
+bzip2-version              = 1.0.6
+cmake-version              = 3.12.4
+coreutils-version          = 8.30
+diffutils-version          = 3.7
+findutils-version          = 4.6.0.199-e3fc
+flock-version              = 0.2.3
+gawk-version               = 4.2.1
+gcc-version                = 8.2.0
+ghostscript-version        = 9.26
+git-version                = 2.20.1
+gmp-version                = 6.1.2
+gnuastro-version           = 0.8
+grep-version               = 3.3
+gzip-version               = 1.10
+isl-version                = 0.18
+libtool-version            = 2.4.6
+libbsd-version             = 0.9.1
+lzip-version               = 1.20
+make-version               = 4.2.90
+metastore-version          = 1.1.2-23-fa9170b
+mpfr-version               = 4.0.1
+mpc-version                = 1.1.0
+ncurses-version            = 6.1
+openssl-version            = 1.1.1a
+patchelf-version           = 0.9
+pkgconfig-version          = 0.29.2
+python-version             = 3.6.8
+readline-version           = 8.0
+sed-version                = 4.7
+tar-version                = 1.31
+unzip-version              = 6.0
+wget-version               = 1.20.1
+which-version              = 2.21
+xz-version                 = 5.2.4
+zip-version                = 3.0
 
 # Libraries
-cfitsio-version     = 3.45
-curl-version        = 7.63.0
-gsl-version         = 2.5
-libjpeg-version     = v9b
-libtiff-version     = 4.0.10
-zlib-version        = 1.2.11
+cfitsio-version            = 3.45
+curl-version               = 7.63.0
+gsl-version                = 2.5
+libjpeg-version            = v9b
+libtiff-version            = 4.0.10
+zlib-version               = 1.2.11
+
+# Python packages
+# ---------------
+#
+# IMPORTANT: Fix url in `reproduce/src/make/dependencies.mk'
+# if changing the version
+astropy-version            = 3.1.1
+numpy-version              = 1.16.1
 
 # Special libraries
 # -----------------
@@ -53,5 +62,5 @@ zlib-version        = 1.2.11
 # the software build Makefiles (`reproduce/src/make/dependencies*.mk'). If
 # you change their version, also please change the explicit shared library
 # names also.
-libgit2-version     = 0.26.0
-wcslib-version      = 6.2
+libgit2-version            = 0.26.0
+wcslib-version             = 6.2
diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk
index 0c7262e..2247518 100644
--- a/reproduce/src/make/dependencies-build-rules.mk
+++ b/reproduce/src/make/dependencies-build-rules.mk
@@ -103,6 +103,22 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then        \
 
 
 
+# Python
+# ------
+#
+# To build Python packages with direct access to a `setup.py' (if no direct
+# access to `setup.py' is needed, pip can be used)
+pybuild = cd $(ddir); rm -rf $(2);                                        \
+	 if ! tar xf $(1); then echo; echo "Tar error"; exit 1; fi;           \
+	 cd $(2);                                                             \
+	 python3 setup.py build &&                                            \
+	 python3 setup.py install &&                                          \
+	 cd .. && rm -rf $(2)
+
+
+
+
+
 # CMake
 # -----
 #
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index e768051..92febb1 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -43,8 +43,10 @@ ildir  = $(BDIR)/dependencies/installed/lib
 ilidir = $(BDIR)/dependencies/installed/lib/built
 
 # Define the top-level programs to build (installed in `.local/bin').
-top-level-programs = astnoisechisel metastore flock zip unzip
-all: $(ddir)/texlive-versions.tex \
+top-level-python   = astropy
+top-level-programs = astnoisechisel metastore flock unzip zip
+all: $(ddir)/texlive-versions.tex                       \
+     $(foreach p, $(top-level-python),   $(ilidir)/$(p)) \
      $(foreach p, $(top-level-programs), $(ibdir)/$(p))
 
 # Other basic environment settings: We are only including the host
@@ -87,24 +89,27 @@ export LDFLAGS           := $(rpath_command) -L$(ildir)
 # convention, but include the name/version in their tarball names with
 # another format, we'll do the modification before the download so the
 # downloaded file has our desired format.
-tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz             \
-                        cmake-$(cmake-version).tar.gz                 \
-                        curl-$(curl-version).tar.gz                   \
-                        flock-$(flock-version).tar.xz                 \
-                        ghostscript-$(ghostscript-version).tar.gz     \
-                        git-$(git-version).tar.xz                     \
-                        gnuastro-$(gnuastro-version).tar.lz           \
-                        gsl-$(gsl-version).tar.gz                     \
-                        install-tl-unx.tar.gz                         \
-                        jpegsrc.$(libjpeg-version).tar.gz             \
-                        libbsd-$(libbsd-version).tar.xz               \
-                        libtool-$(libtool-version).tar.xz             \
-                        libgit2-$(libgit2-version).tar.gz             \
-                        metastore-$(metastore-version).tar.gz         \
-                        unzip-$(unzip-version).tar.gz                 \
-                        tiff-$(libtiff-version).tar.gz                \
-                        wcslib-$(wcslib-version).tar.bz2              \
-                        zip-$(zip-version).tar.gz                     \
+tarballs = $(foreach t, astropy-$(astropy-version).tar.gz                  \
+                        cfitsio-$(cfitsio-version).tar.gz                  \
+                        cmake-$(cmake-version).tar.gz                      \
+                        curl-$(curl-version).tar.gz                        \
+                        flock-$(flock-version).tar.xz                      \
+                        ghostscript-$(ghostscript-version).tar.gz          \
+                        git-$(git-version).tar.xz                          \
+                        gnuastro-$(gnuastro-version).tar.lz                \
+                        gsl-$(gsl-version).tar.gz                          \
+                        install-tl-unx.tar.gz                              \
+                        jpegsrc.$(libjpeg-version).tar.gz                  \
+                        libbsd-$(libbsd-version).tar.xz                    \
+                        libtool-$(libtool-version).tar.xz                  \
+                        libgit2-$(libgit2-version).tar.gz                  \
+                        metastore-$(metastore-version).tar.gz              \
+                        numpy-$(numpy-version).zip                         \
+                        python-$(python-version).tar.gz                    \
+                        unzip-$(unzip-version).tar.gz                      \
+                        tiff-$(libtiff-version).tar.gz                     \
+                        wcslib-$(wcslib-version).tar.bz2                   \
+                        zip-$(zip-version).tar.gz                          \
                       , $(tdir)/$(t) )
 $(tarballs): $(tdir)/%:
 	if [ -f $(DEPENDENCIES-DIR)/$* ]; then
@@ -117,7 +122,8 @@ $(tarballs): $(tdir)/%:
 
           # Set the top download link of the requested tarball.
 	  mergenames=1
-	  if [ $$n = cfitsio     ]; then
+	  if   [ $$n = astropy     ]; then w=https://files.pythonhosted.org/packages/eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352
+	  elif [ $$n = cfitsio     ]; then
 	    mergenames=0
 	    v=$$(echo $(cfitsio-version) | sed -e's/\.//'             \
 	              | awk '{l=length($$1);                          \
@@ -141,6 +147,10 @@ $(tarballs): $(tdir)/%:
 	    mergenames=0
 	    w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
 	  elif [ $$n = metastore   ]; then w=http://akhlaghi.org/src
+	  elif [ $$n = numpy       ]; then w=https://files.pythonhosted.org/packages/2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c
+	  elif [ $$n = python      ]; then
+	    mergenames=0
+	    w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz
 	  elif [ $$n = tiff        ]; then w=https://download.osgeo.org/libtiff
 	  elif [ $$n = unzip       ]; then w=ftp://ftp.info-zip.org/pub/infozip/src
 	    mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//')
@@ -446,6 +456,12 @@ endif
 	               $$staticopts, -j$(numthreads),                \
 	               make check -j$(numthreads))
 
+$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz
+	$(call gbuild, $<, Python-$(python-version))        \
+	&& v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \
+            {printf "%d.%d\n", $$1, $$2}')              \
+    && ln -s $(ildir)/python$$v $(ildir)/python
+
 $(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz
 	v=$$(echo $(unzip-version) | sed -e's/\.//')
 	$(call gbuild, $<, unzip$$v, static,,          \
@@ -466,6 +482,20 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz
 
 
 
+# Python packages
+# ---------------
+$(ilidir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
+                 $(ibdir)/python3
+	pip3 install $< --verbose && echo "numpy is built" > $@
+
+$(ilidir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \
+                   $(ilidir)/numpy
+	pip3 install $< --verbose && echo "astropy is built" > $@
+
+
+
+
+
 # Since we want to avoid complicating the PATH, we are putting a symbolic
 # link of all the TeX Live executables in $(ibdir). But symbolic links are
 # hard to track for Make (as a target). Also, TeX in general is optional
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index ff06dcb..71cdbb8 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -316,7 +316,6 @@ lvcheck = idir=$(BDIR)/dependencies/installed/include;                   \
 
 
 
-
 # Pipeline initialization results
 # -------------------------------
 #
@@ -413,3 +412,8 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
 
         # TeX package versions
 	cat $(BDIR)/dependencies/texlive-versions.tex >> $@
+
+	# Python packages
+	$(call pvcheck, python3, $(python-version), Python, pythonversion)
+	echo "\newcommand{\\numpyversion}{$(numpy-version)}"     >> $@
+	echo "\newcommand{\\astropyversion}{$(astropy-version)}" >> $@
diff --git a/tex/src/references.tex b/tex/src/references.tex
index b3998a2..785ab21 100644
--- a/tex/src/references.tex
+++ b/tex/src/references.tex
@@ -1,3 +1,64 @@
+@ARTICLE{astropy2018,
+   author = {{Astropy Collaboration} and {Price-Whelan}, A.~M. and {Sip{\H o}cz}, B.~M. and
+	{G{\"u}nther}, H.~M. and {Lim}, P.~L. and {Crawford}, S.~M. and
+	{Conseil}, S. and {Shupe}, D.~L. and {Craig}, M.~W. and {Dencheva}, N. and
+	{Ginsburg}, A. and {VanderPlas}, J.~T. and {Bradley}, L.~D. and
+	{P{\'e}rez-Su{\'a}rez}, D. and {de Val-Borro}, M. and {Aldcroft}, T.~L. and
+	{Cruz}, K.~L. and {Robitaille}, T.~P. and {Tollerud}, E.~J. and
+	{Ardelean}, C. and {Babej}, T. and {Bach}, Y.~P. and {Bachetti}, M. and
+	{Bakanov}, A.~V. and {Bamford}, S.~P. and {Barentsen}, G. and
+	{Barmby}, P. and {Baumbach}, A. and {Berry}, K.~L. and {Biscani}, F. and
+	{Boquien}, M. and {Bostroem}, K.~A. and {Bouma}, L.~G. and {Brammer}, G.~B. and
+	{Bray}, E.~M. and {Breytenbach}, H. and {Buddelmeijer}, H. and
+	{Burke}, D.~J. and {Calderone}, G. and {Cano Rodr{\'{\i}}guez}, J.~L. and
+	{Cara}, M. and {Cardoso}, J.~V.~M. and {Cheedella}, S. and {Copin}, Y. and
+	{Corrales}, L. and {Crichton}, D. and {D'Avella}, D. and {Deil}, C. and
+	{Depagne}, {\'E}. and {Dietrich}, J.~P. and {Donath}, A. and
+	{Droettboom}, M. and {Earl}, N. and {Erben}, T. and {Fabbro}, S. and
+	{Ferreira}, L.~A. and {Finethy}, T. and {Fox}, R.~T. and {Garrison}, L.~H. and
+	{Gibbons}, S.~L.~J. and {Goldstein}, D.~A. and {Gommers}, R. and
+	{Greco}, J.~P. and {Greenfield}, P. and {Groener}, A.~M. and
+	{Grollier}, F. and {Hagen}, A. and {Hirst}, P. and {Homeier}, D. and
+	{Horton}, A.~J. and {Hosseinzadeh}, G. and {Hu}, L. and {Hunkeler}, J.~S. and
+	{Ivezi{\'c}}, {\v Z}. and {Jain}, A. and {Jenness}, T. and {Kanarek}, G. and
+	{Kendrew}, S. and {Kern}, N.~S. and {Kerzendorf}, W.~E. and
+	{Khvalko}, A. and {King}, J. and {Kirkby}, D. and {Kulkarni}, A.~M. and
+	{Kumar}, A. and {Lee}, A. and {Lenz}, D. and {Littlefair}, S.~P. and
+	{Ma}, Z. and {Macleod}, D.~M. and {Mastropietro}, M. and {McCully}, C. and
+	{Montagnac}, S. and {Morris}, B.~M. and {Mueller}, M. and {Mumford}, S.~J. and
+	{Muna}, D. and {Murphy}, N.~A. and {Nelson}, S. and {Nguyen}, G.~H. and
+	{Ninan}, J.~P. and {N{\"o}the}, M. and {Ogaz}, S. and {Oh}, S. and
+	{Parejko}, J.~K. and {Parley}, N. and {Pascual}, S. and {Patil}, R. and
+	{Patil}, A.~A. and {Plunkett}, A.~L. and {Prochaska}, J.~X. and
+	{Rastogi}, T. and {Reddy Janga}, V. and {Sabater}, J. and {Sakurikar}, P. and
+	{Seifert}, M. and {Sherbert}, L.~E. and {Sherwood-Taylor}, H. and
+	{Shih}, A.~Y. and {Sick}, J. and {Silbiger}, M.~T. and {Singanamalla}, S. and
+	{Singer}, L.~P. and {Sladen}, P.~H. and {Sooley}, K.~A. and
+	{Sornarajah}, S. and {Streicher}, O. and {Teuben}, P. and {Thomas}, S.~W. and
+	{Tremblay}, G.~R. and {Turner}, J.~E.~H. and {Terr{\'o}n}, V. and
+	{van Kerkwijk}, M.~H. and {de la Vega}, A. and {Watkins}, L.~L. and
+	{Weaver}, B.~A. and {Whitmore}, J.~B. and {Woillez}, J. and
+	{Zabalza}, V. and {Astropy Contributors}},
+    title = "{The Astropy Project: Building an Open-science Project and Status of the v2.0 Core Package}",
+  journal = {AJ},
+archivePrefix = "arXiv",
+   eprint = {1801.02634},
+ primaryClass = "astro-ph.IM",
+ keywords = {methods: data analysis, methods: miscellaneous, methods: statistical, reference systems },
+     year = 2018,
+    month = sep,
+   volume = 156,
+      eid = {123},
+    pages = {123},
+      doi = {10.3847/1538-3881/aabc4f},
+   adsurl = {http://adsabs.harvard.edu/abs/2018AJ....156..123A},
+  adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}
+
+
+
+
+
 @ARTICLE{bacon17,
    author = {{Bacon}, R. and {Conseil}, S. and {Mary}, D. and {Brinchmann}, J. and
 	{Shepherd}, M. and {Akhlaghi}, M. and {Weilbacher}, P.~M. and
@@ -43,3 +104,36 @@ archivePrefix = "arXiv",
    adsurl = {http://adsabs.harvard.edu/abs/2015ApJS..220....1A},
   adsnote = {Provided by the SAO/NASA Astrophysics Data System}
 }
+
+
+
+
+@ARTICLE{astropy2013,
+   author = {{Astropy Collaboration} and {Robitaille}, T.~P. and {Tollerud}, E.~J. and
+	{Greenfield}, P. and {Droettboom}, M. and {Bray}, E. and {Aldcroft}, T. and
+	{Davis}, M. and {Ginsburg}, A. and {Price-Whelan}, A.~M. and
+	{Kerzendorf}, W.~E. and {Conley}, A. and {Crighton}, N. and
+	{Barbary}, K. and {Muna}, D. and {Ferguson}, H. and {Grollier}, F. and
+	{Parikh}, M.~M. and {Nair}, P.~H. and {Unther}, H.~M. and {Deil}, C. and
+	{Woillez}, J. and {Conseil}, S. and {Kramer}, R. and {Turner}, J.~E.~H. and
+	{Singer}, L. and {Fox}, R. and {Weaver}, B.~A. and {Zabalza}, V. and
+	{Edwards}, Z.~I. and {Azalee Bostroem}, K. and {Burke}, D.~J. and
+	{Casey}, A.~R. and {Crawford}, S.~M. and {Dencheva}, N. and
+	{Ely}, J. and {Jenness}, T. and {Labrie}, K. and {Lim}, P.~L. and
+	{Pierfederici}, F. and {Pontzen}, A. and {Ptak}, A. and {Refsdal}, B. and
+	{Servillat}, M. and {Streicher}, O.},
+    title = "{Astropy: A community Python package for astronomy}",
+  journal = {A\&A},
+archivePrefix = "arXiv",
+   eprint = {1307.6212},
+ primaryClass = "astro-ph.IM",
+ keywords = {methods: data analysis, methods: miscellaneous, virtual observatory tools},
+     year = 2013,
+    month = oct,
+   volume = 558,
+      eid = {A33},
+    pages = {A33},
+      doi = {10.1051/0004-6361/201322068},
+   adsurl = {http://adsabs.harvard.edu/abs/2013A\%26A...558A..33A},
+  adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}