From 71f105dcdc7880919612a3adbe4f99d1706f5ec1 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 7 Feb 2019 16:14:33 +0000 Subject: Python is now in the pipeline Many projects use Python so it is necessary include it in the pipeline. --- reproduce/src/make/dependencies.mk | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 80a787e..101002c 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,7 +43,7 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). -top-level-programs = astnoisechisel metastore flock zip unzip +top-level-programs = astnoisechisel metastore flock python3 unzip zip all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) @@ -101,6 +101,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ + python-$(python-version).tar.gz \ unzip-$(unzip-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ zip-$(zip-version).tar.gz \ @@ -141,6 +142,9 @@ $(tarballs): $(tdir)/%: mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://akhlaghi.org/src + elif [ $$n = python ]; then + mergenames=0 + w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -446,6 +450,9 @@ endif $$staticopts, -j$(numthreads), \ make check -j$(numthreads)) +$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz + $(call gbuild, $<, python-$(python-version)) + $(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz v=$$(echo $(unzip-version) | sed -e's/\.//') $(call gbuild, $<, unzip$$v, static,, \ -- cgit v1.2.1 From ea525171de968707d15afd9dd9da760083722e2a Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 7 Feb 2019 22:14:20 +0000 Subject: Numpy is now in the pipeline Python needs some packages to be really useful. Numpy is the most important package for using Python and a lot of other packages depend on it. In this commit we add numpy to the pipeline. The tarball of numpy right now is fossies. --- reproduce/src/make/dependencies.mk | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 101002c..52b65f4 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -41,10 +41,13 @@ idir = $(BDIR)/dependencies/installed ibdir = $(BDIR)/dependencies/installed/bin ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built +pydir = $(BDIR)/dependencies/installed/lib/python # Define the top-level programs to build (installed in `.local/bin'). -top-level-programs = astnoisechisel metastore flock python3 unzip zip -all: $(ddir)/texlive-versions.tex \ +top-level-python = numpy +top-level-programs = astnoisechisel metastore flock zip +all: $(ddir)/texlive-versions.tex \ + $(foreach p, $(top-level-python), $(pydir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # Other basic environment settings: We are only including the host @@ -101,6 +104,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ + numpy-$(numpy-version).zip \ python-$(python-version).tar.gz \ unzip-$(unzip-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ @@ -142,6 +146,7 @@ $(tarballs): $(tdir)/%: mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://akhlaghi.org/src + elif [ $$n = numpy ]; then w=https://fossies.org/linux/misc elif [ $$n = python ]; then mergenames=0 w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz @@ -451,7 +456,10 @@ endif make check -j$(numthreads)) $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz - $(call gbuild, $<, python-$(python-version)) + $(call gbuild, $<, python-$(python-version)) \ + && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ + {printf "%d.%d\n", $$1, $$2}') \ + && ln -s $(ildir)/python$$v $(ildir)/python $(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -473,6 +481,15 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz +# Python packages +# --------------- +$(pydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ + $(ibdir)/python3 \ + $(ibdir)/unzip + pip3 install $< --verbose + + + # Since we want to avoid complicating the PATH, we are putting a symbolic # link of all the TeX Live executables in $(ibdir). But symbolic links are # hard to track for Make (as a target). Also, TeX in general is optional -- cgit v1.2.1 From 64f519d1b54487fcfc39a0f7c8b4437e660ff6f0 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 13 Feb 2019 15:13:28 +0000 Subject: Astropy installed in the pipeline Astropy was added and one very important thing is that we have to use the pypi tarball (https://pypi.org/) (which is bootstrapped) and not the github tarball. --- reproduce/src/make/dependencies-build-rules.mk | 16 +++++++ reproduce/src/make/dependencies.mk | 66 ++++++++++++++------------ reproduce/src/make/initialize.mk | 7 ++- 3 files changed, 58 insertions(+), 31 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk index 0c7262e..2247518 100644 --- a/reproduce/src/make/dependencies-build-rules.mk +++ b/reproduce/src/make/dependencies-build-rules.mk @@ -103,6 +103,22 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then \ +# Python +# ------ +# +# To build Python packages with direct access to a `setup.py' (if no direct +# access to `setup.py' is needed, pip can be used) +pybuild = cd $(ddir); rm -rf $(2); \ + if ! tar xf $(1); then echo; echo "Tar error"; exit 1; fi; \ + cd $(2); \ + python3 setup.py build && \ + python3 setup.py install && \ + cd .. && rm -rf $(2) + + + + + # CMake # ----- # diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 52b65f4..a28de19 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -41,13 +41,12 @@ idir = $(BDIR)/dependencies/installed ibdir = $(BDIR)/dependencies/installed/bin ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built -pydir = $(BDIR)/dependencies/installed/lib/python # Define the top-level programs to build (installed in `.local/bin'). -top-level-python = numpy -top-level-programs = astnoisechisel metastore flock zip +top-level-python = astropy +top-level-programs = astnoisechisel metastore flock unzip zip all: $(ddir)/texlive-versions.tex \ - $(foreach p, $(top-level-python), $(pydir)/$(p)) \ + $(foreach p, $(top-level-python), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # Other basic environment settings: We are only including the host @@ -90,26 +89,27 @@ export LDFLAGS := $(rpath_command) -L$(ildir) # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. -tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ - cmake-$(cmake-version).tar.gz \ - curl-$(curl-version).tar.gz \ - flock-$(flock-version).tar.xz \ - ghostscript-$(ghostscript-version).tar.gz \ - git-$(git-version).tar.xz \ - gnuastro-$(gnuastro-version).tar.lz \ - gsl-$(gsl-version).tar.gz \ - install-tl-unx.tar.gz \ - jpegsrc.$(libjpeg-version).tar.gz \ - libbsd-$(libbsd-version).tar.xz \ - libtool-$(libtool-version).tar.xz \ - libgit2-$(libgit2-version).tar.gz \ - metastore-$(metastore-version).tar.gz \ - numpy-$(numpy-version).zip \ - python-$(python-version).tar.gz \ - unzip-$(unzip-version).tar.gz \ - tiff-$(libtiff-version).tar.gz \ - zip-$(zip-version).tar.gz \ - wcslib-$(wcslib-version).tar.bz2 \ +tarballs = $(foreach t, astropy-$(astropy-version).tar.gz \ + cfitsio-$(cfitsio-version).tar.gz \ + cmake-$(cmake-version).tar.gz \ + curl-$(curl-version).tar.gz \ + flock-$(flock-version).tar.xz \ + ghostscript-$(ghostscript-version).tar.gz \ + git-$(git-version).tar.xz \ + gnuastro-$(gnuastro-version).tar.lz \ + gsl-$(gsl-version).tar.gz \ + install-tl-unx.tar.gz \ + jpegsrc.$(libjpeg-version).tar.gz \ + libbsd-$(libbsd-version).tar.xz \ + libtool-$(libtool-version).tar.xz \ + libgit2-$(libgit2-version).tar.gz \ + metastore-$(metastore-version).tar.gz \ + numpy-$(numpy-version).zip \ + python-$(python-version).tar.gz \ + unzip-$(unzip-version).tar.gz \ + tiff-$(libtiff-version).tar.gz \ + zip-$(zip-version).tar.gz \ + wcslib-$(wcslib-version).tar.bz2 \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then @@ -122,7 +122,8 @@ $(tarballs): $(tdir)/%: # Set the top download link of the requested tarball. mergenames=1 - if [ $$n = cfitsio ]; then + if [ $$n = astropy ]; then w=https://files.pythonhosted.org/packages/eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 + elif [ $$n = cfitsio ]; then mergenames=0 v=$$(echo $(cfitsio-version) | sed -e's/\.//' \ | awk '{l=length($$1); \ @@ -146,7 +147,7 @@ $(tarballs): $(tdir)/%: mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://akhlaghi.org/src - elif [ $$n = numpy ]; then w=https://fossies.org/linux/misc + elif [ $$n = numpy ]; then w=https://files.pythonhosted.org/packages/2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c elif [ $$n = python ]; then mergenames=0 w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz @@ -483,10 +484,15 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz # Python packages # --------------- -$(pydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ - $(ibdir)/python3 \ - $(ibdir)/unzip - pip3 install $< --verbose +$(ilidir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ + $(ibdir)/python3 + pip3 install $< --verbose && echo "numpy is built" > $@ + +$(ilidir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ + $(ilidir)/numpy + pip3 install $< --verbose && echo "astropy is built" > $@ + + diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index ff06dcb..dfdf161 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -316,7 +316,6 @@ lvcheck = idir=$(BDIR)/dependencies/installed/include; \ - # Pipeline initialization results # ------------------------------- # @@ -413,3 +412,9 @@ $(mtexdir)/initialize.tex: | $(mtexdir) # TeX package versions cat $(BDIR)/dependencies/texlive-versions.tex >> $@ + + # Python packages + $(call pvcheck, python3, $(python-version), Python, pythonversion) + echo "\newcommand{\\numpyversion}{$(numpy-version)}" >> $@ + echo "\newcommand{\\astropyversion}{$(astropy-version)}" >> $@ + -- cgit v1.2.1