aboutsummaryrefslogtreecommitdiff
path: root/reproduce
diff options
context:
space:
mode:
authorMohammad Akhlaghi <mohammad@akhlaghi.org>2019-02-13 17:30:02 +0000
committerMohammad Akhlaghi <mohammad@akhlaghi.org>2019-02-13 17:37:17 +0000
commit41b458f8f31bbb5b74f96c5647beae28c49dedaf (patch)
tree64be70cfe6ab20f68a685d430902ab803035308f /reproduce
parentb47b2a3563a0735480e72c64216645fc363030bb (diff)
parent64f519d1b54487fcfc39a0f7c8b4437e660ff6f0 (diff)
Imported recent work on building Python within the pipeline
Raul Infante-Sainz added the building of Python (along with the Numpy and Astropy packages) into the pipeline. That work is now being merged into the main pipeline branch. There was only this small problem that needed to be fixed: the Python tarball's name after unpacking is actually `Python-X.X.X' (with a captial P), not `python-X.X.X'. This has been corrected with this merge.
Diffstat (limited to 'reproduce')
-rw-r--r--reproduce/config/pipeline/dependency-versions.mk97
-rw-r--r--reproduce/src/make/dependencies-build-rules.mk16
-rw-r--r--reproduce/src/make/dependencies.mk72
-rw-r--r--reproduce/src/make/initialize.mk6
4 files changed, 125 insertions, 66 deletions
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index 41a4cf9..f0774bc 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -1,50 +1,59 @@
# Versions of the various dependnecies
# Programs
-bash-version = 5.0
-binutils-version = 2.31.1
-bzip2-version = 1.0.6
-cmake-version = 3.12.4
-coreutils-version = 8.30
-diffutils-version = 3.7
-findutils-version = 4.6.0.199-e3fc
-flock-version = 0.2.3
-gawk-version = 4.2.1
-gcc-version = 8.2.0
-ghostscript-version = 9.26
-git-version = 2.20.1
-gmp-version = 6.1.2
-gnuastro-version = 0.8
-grep-version = 3.3
-gzip-version = 1.10
-isl-version = 0.18
-libtool-version = 2.4.6
-libbsd-version = 0.9.1
-lzip-version = 1.20
-make-version = 4.2.90
-metastore-version = 1.1.2-23-fa9170b
-mpfr-version = 4.0.1
-mpc-version = 1.1.0
-ncurses-version = 6.1
-openssl-version = 1.1.1a
-patchelf-version = 0.9
-pkgconfig-version = 0.29.2
-readline-version = 8.0
-sed-version = 4.7
-tar-version = 1.31
-unzip-version = 6.0
-wget-version = 1.20.1
-which-version = 2.21
-xz-version = 5.2.4
-zip-version = 3.0
+bash-version = 5.0
+binutils-version = 2.31.1
+bzip2-version = 1.0.6
+cmake-version = 3.12.4
+coreutils-version = 8.30
+diffutils-version = 3.7
+findutils-version = 4.6.0.199-e3fc
+flock-version = 0.2.3
+gawk-version = 4.2.1
+gcc-version = 8.2.0
+ghostscript-version = 9.26
+git-version = 2.20.1
+gmp-version = 6.1.2
+gnuastro-version = 0.8
+grep-version = 3.3
+gzip-version = 1.10
+isl-version = 0.18
+libtool-version = 2.4.6
+libbsd-version = 0.9.1
+lzip-version = 1.20
+make-version = 4.2.90
+metastore-version = 1.1.2-23-fa9170b
+mpfr-version = 4.0.1
+mpc-version = 1.1.0
+ncurses-version = 6.1
+openssl-version = 1.1.1a
+patchelf-version = 0.9
+pkgconfig-version = 0.29.2
+python-version = 3.6.8
+readline-version = 8.0
+sed-version = 4.7
+tar-version = 1.31
+unzip-version = 6.0
+wget-version = 1.20.1
+which-version = 2.21
+xz-version = 5.2.4
+zip-version = 3.0
# Libraries
-cfitsio-version = 3.45
-curl-version = 7.63.0
-gsl-version = 2.5
-libjpeg-version = v9b
-libtiff-version = 4.0.10
-zlib-version = 1.2.11
+cfitsio-version = 3.45
+curl-version = 7.63.0
+gsl-version = 2.5
+libjpeg-version = v9b
+libtiff-version = 4.0.10
+zlib-version = 1.2.11
+
+# Python packages
+# ---------------
+#
+# IMPORTANT: Fix url in `reproduce/src/make/dependencies.mk'
+# if changing the version
+astropy-version = 3.1.1
+numpy-version = 1.16.1
# Special libraries
# -----------------
@@ -53,5 +62,5 @@ zlib-version = 1.2.11
# the software build Makefiles (`reproduce/src/make/dependencies*.mk'). If
# you change their version, also please change the explicit shared library
# names also.
-libgit2-version = 0.26.0
-wcslib-version = 6.2
+libgit2-version = 0.26.0
+wcslib-version = 6.2
diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk
index 0c7262e..2247518 100644
--- a/reproduce/src/make/dependencies-build-rules.mk
+++ b/reproduce/src/make/dependencies-build-rules.mk
@@ -103,6 +103,22 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then \
+# Python
+# ------
+#
+# To build Python packages with direct access to a `setup.py' (if no direct
+# access to `setup.py' is needed, pip can be used)
+pybuild = cd $(ddir); rm -rf $(2); \
+ if ! tar xf $(1); then echo; echo "Tar error"; exit 1; fi; \
+ cd $(2); \
+ python3 setup.py build && \
+ python3 setup.py install && \
+ cd .. && rm -rf $(2)
+
+
+
+
+
# CMake
# -----
#
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index e768051..92febb1 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -43,8 +43,10 @@ ildir = $(BDIR)/dependencies/installed/lib
ilidir = $(BDIR)/dependencies/installed/lib/built
# Define the top-level programs to build (installed in `.local/bin').
-top-level-programs = astnoisechisel metastore flock zip unzip
-all: $(ddir)/texlive-versions.tex \
+top-level-python = astropy
+top-level-programs = astnoisechisel metastore flock unzip zip
+all: $(ddir)/texlive-versions.tex \
+ $(foreach p, $(top-level-python), $(ilidir)/$(p)) \
$(foreach p, $(top-level-programs), $(ibdir)/$(p))
# Other basic environment settings: We are only including the host
@@ -87,24 +89,27 @@ export LDFLAGS := $(rpath_command) -L$(ildir)
# convention, but include the name/version in their tarball names with
# another format, we'll do the modification before the download so the
# downloaded file has our desired format.
-tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
- cmake-$(cmake-version).tar.gz \
- curl-$(curl-version).tar.gz \
- flock-$(flock-version).tar.xz \
- ghostscript-$(ghostscript-version).tar.gz \
- git-$(git-version).tar.xz \
- gnuastro-$(gnuastro-version).tar.lz \
- gsl-$(gsl-version).tar.gz \
- install-tl-unx.tar.gz \
- jpegsrc.$(libjpeg-version).tar.gz \
- libbsd-$(libbsd-version).tar.xz \
- libtool-$(libtool-version).tar.xz \
- libgit2-$(libgit2-version).tar.gz \
- metastore-$(metastore-version).tar.gz \
- unzip-$(unzip-version).tar.gz \
- tiff-$(libtiff-version).tar.gz \
- wcslib-$(wcslib-version).tar.bz2 \
- zip-$(zip-version).tar.gz \
+tarballs = $(foreach t, astropy-$(astropy-version).tar.gz \
+ cfitsio-$(cfitsio-version).tar.gz \
+ cmake-$(cmake-version).tar.gz \
+ curl-$(curl-version).tar.gz \
+ flock-$(flock-version).tar.xz \
+ ghostscript-$(ghostscript-version).tar.gz \
+ git-$(git-version).tar.xz \
+ gnuastro-$(gnuastro-version).tar.lz \
+ gsl-$(gsl-version).tar.gz \
+ install-tl-unx.tar.gz \
+ jpegsrc.$(libjpeg-version).tar.gz \
+ libbsd-$(libbsd-version).tar.xz \
+ libtool-$(libtool-version).tar.xz \
+ libgit2-$(libgit2-version).tar.gz \
+ metastore-$(metastore-version).tar.gz \
+ numpy-$(numpy-version).zip \
+ python-$(python-version).tar.gz \
+ unzip-$(unzip-version).tar.gz \
+ tiff-$(libtiff-version).tar.gz \
+ wcslib-$(wcslib-version).tar.bz2 \
+ zip-$(zip-version).tar.gz \
, $(tdir)/$(t) )
$(tarballs): $(tdir)/%:
if [ -f $(DEPENDENCIES-DIR)/$* ]; then
@@ -117,7 +122,8 @@ $(tarballs): $(tdir)/%:
# Set the top download link of the requested tarball.
mergenames=1
- if [ $$n = cfitsio ]; then
+ if [ $$n = astropy ]; then w=https://files.pythonhosted.org/packages/eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352
+ elif [ $$n = cfitsio ]; then
mergenames=0
v=$$(echo $(cfitsio-version) | sed -e's/\.//' \
| awk '{l=length($$1); \
@@ -141,6 +147,10 @@ $(tarballs): $(tdir)/%:
mergenames=0
w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
elif [ $$n = metastore ]; then w=http://akhlaghi.org/src
+ elif [ $$n = numpy ]; then w=https://files.pythonhosted.org/packages/2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c
+ elif [ $$n = python ]; then
+ mergenames=0
+ w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz
elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff
elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src
mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//')
@@ -446,6 +456,12 @@ endif
$$staticopts, -j$(numthreads), \
make check -j$(numthreads))
+$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz
+ $(call gbuild, $<, Python-$(python-version)) \
+ && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \
+ {printf "%d.%d\n", $$1, $$2}') \
+ && ln -s $(ildir)/python$$v $(ildir)/python
+
$(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz
v=$$(echo $(unzip-version) | sed -e's/\.//')
$(call gbuild, $<, unzip$$v, static,, \
@@ -466,6 +482,20 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz
+# Python packages
+# ---------------
+$(ilidir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
+ $(ibdir)/python3
+ pip3 install $< --verbose && echo "numpy is built" > $@
+
+$(ilidir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \
+ $(ilidir)/numpy
+ pip3 install $< --verbose && echo "astropy is built" > $@
+
+
+
+
+
# Since we want to avoid complicating the PATH, we are putting a symbolic
# link of all the TeX Live executables in $(ibdir). But symbolic links are
# hard to track for Make (as a target). Also, TeX in general is optional
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index ff06dcb..71cdbb8 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -316,7 +316,6 @@ lvcheck = idir=$(BDIR)/dependencies/installed/include; \
-
# Pipeline initialization results
# -------------------------------
#
@@ -413,3 +412,8 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
# TeX package versions
cat $(BDIR)/dependencies/texlive-versions.tex >> $@
+
+ # Python packages
+ $(call pvcheck, python3, $(python-version), Python, pythonversion)
+ echo "\newcommand{\\numpyversion}{$(numpy-version)}" >> $@
+ echo "\newcommand{\\astropyversion}{$(astropy-version)}" >> $@