From 71f105dcdc7880919612a3adbe4f99d1706f5ec1 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 7 Feb 2019 16:14:33 +0000 Subject: Python is now in the pipeline Many projects use Python so it is necessary include it in the pipeline. --- reproduce/config/pipeline/dependency-versions.mk | 1 + reproduce/src/make/dependencies.mk | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'reproduce') diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 41a4cf9..46dfabd 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -29,6 +29,7 @@ ncurses-version = 6.1 openssl-version = 1.1.1a patchelf-version = 0.9 pkgconfig-version = 0.29.2 +python-version = 3.6.8 readline-version = 8.0 sed-version = 4.7 tar-version = 1.31 diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 80a787e..101002c 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,7 +43,7 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). -top-level-programs = astnoisechisel metastore flock zip unzip +top-level-programs = astnoisechisel metastore flock python3 unzip zip all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) @@ -101,6 +101,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ + python-$(python-version).tar.gz \ unzip-$(unzip-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ zip-$(zip-version).tar.gz \ @@ -141,6 +142,9 @@ $(tarballs): $(tdir)/%: mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://akhlaghi.org/src + elif [ $$n = python ]; then + mergenames=0 + w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -446,6 +450,9 @@ endif $$staticopts, -j$(numthreads), \ make check -j$(numthreads)) +$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz + $(call gbuild, $<, python-$(python-version)) + $(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz v=$$(echo $(unzip-version) | sed -e's/\.//') $(call gbuild, $<, unzip$$v, static,, \ -- cgit v1.2.1 From ea525171de968707d15afd9dd9da760083722e2a Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 7 Feb 2019 22:14:20 +0000 Subject: Numpy is now in the pipeline Python needs some packages to be really useful. Numpy is the most important package for using Python and a lot of other packages depend on it. In this commit we add numpy to the pipeline. The tarball of numpy right now is fossies. --- reproduce/config/pipeline/dependency-versions.mk | 1 + reproduce/src/make/dependencies.mk | 23 ++++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) (limited to 'reproduce') diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 46dfabd..0bea16a 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -26,6 +26,7 @@ metastore-version = 1.1.2-23-fa9170b mpfr-version = 4.0.1 mpc-version = 1.1.0 ncurses-version = 6.1 +numpy-version = 1.16.1 openssl-version = 1.1.1a patchelf-version = 0.9 pkgconfig-version = 0.29.2 diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 101002c..52b65f4 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -41,10 +41,13 @@ idir = $(BDIR)/dependencies/installed ibdir = $(BDIR)/dependencies/installed/bin ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built +pydir = $(BDIR)/dependencies/installed/lib/python # Define the top-level programs to build (installed in `.local/bin'). -top-level-programs = astnoisechisel metastore flock python3 unzip zip -all: $(ddir)/texlive-versions.tex \ +top-level-python = numpy +top-level-programs = astnoisechisel metastore flock zip +all: $(ddir)/texlive-versions.tex \ + $(foreach p, $(top-level-python), $(pydir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # Other basic environment settings: We are only including the host @@ -101,6 +104,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ + numpy-$(numpy-version).zip \ python-$(python-version).tar.gz \ unzip-$(unzip-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ @@ -142,6 +146,7 @@ $(tarballs): $(tdir)/%: mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://akhlaghi.org/src + elif [ $$n = numpy ]; then w=https://fossies.org/linux/misc elif [ $$n = python ]; then mergenames=0 w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz @@ -451,7 +456,10 @@ endif make check -j$(numthreads)) $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz - $(call gbuild, $<, python-$(python-version)) + $(call gbuild, $<, python-$(python-version)) \ + && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ + {printf "%d.%d\n", $$1, $$2}') \ + && ln -s $(ildir)/python$$v $(ildir)/python $(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -473,6 +481,15 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz +# Python packages +# --------------- +$(pydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ + $(ibdir)/python3 \ + $(ibdir)/unzip + pip3 install $< --verbose + + + # Since we want to avoid complicating the PATH, we are putting a symbolic # link of all the TeX Live executables in $(ibdir). But symbolic links are # hard to track for Make (as a target). Also, TeX in general is optional -- cgit v1.2.1 From 64f519d1b54487fcfc39a0f7c8b4437e660ff6f0 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 13 Feb 2019 15:13:28 +0000 Subject: Astropy installed in the pipeline Astropy was added and one very important thing is that we have to use the pypi tarball (https://pypi.org/) (which is bootstrapped) and not the github tarball. --- reproduce/config/pipeline/dependency-versions.mk | 99 +++++++++++++----------- reproduce/src/make/dependencies-build-rules.mk | 16 ++++ reproduce/src/make/dependencies.mk | 66 +++++++++------- reproduce/src/make/initialize.mk | 7 +- 4 files changed, 111 insertions(+), 77 deletions(-) (limited to 'reproduce') diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 0bea16a..f0774bc 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -1,52 +1,59 @@ # Versions of the various dependnecies # Programs -bash-version = 5.0 -binutils-version = 2.31.1 -bzip2-version = 1.0.6 -cmake-version = 3.12.4 -coreutils-version = 8.30 -diffutils-version = 3.7 -findutils-version = 4.6.0.199-e3fc -flock-version = 0.2.3 -gawk-version = 4.2.1 -gcc-version = 8.2.0 -ghostscript-version = 9.26 -git-version = 2.20.1 -gmp-version = 6.1.2 -gnuastro-version = 0.8 -grep-version = 3.3 -gzip-version = 1.10 -isl-version = 0.18 -libtool-version = 2.4.6 -libbsd-version = 0.9.1 -lzip-version = 1.20 -make-version = 4.2.90 -metastore-version = 1.1.2-23-fa9170b -mpfr-version = 4.0.1 -mpc-version = 1.1.0 -ncurses-version = 6.1 -numpy-version = 1.16.1 -openssl-version = 1.1.1a -patchelf-version = 0.9 -pkgconfig-version = 0.29.2 -python-version = 3.6.8 -readline-version = 8.0 -sed-version = 4.7 -tar-version = 1.31 -unzip-version = 6.0 -wget-version = 1.20.1 -which-version = 2.21 -xz-version = 5.2.4 -zip-version = 3.0 +bash-version = 5.0 +binutils-version = 2.31.1 +bzip2-version = 1.0.6 +cmake-version = 3.12.4 +coreutils-version = 8.30 +diffutils-version = 3.7 +findutils-version = 4.6.0.199-e3fc +flock-version = 0.2.3 +gawk-version = 4.2.1 +gcc-version = 8.2.0 +ghostscript-version = 9.26 +git-version = 2.20.1 +gmp-version = 6.1.2 +gnuastro-version = 0.8 +grep-version = 3.3 +gzip-version = 1.10 +isl-version = 0.18 +libtool-version = 2.4.6 +libbsd-version = 0.9.1 +lzip-version = 1.20 +make-version = 4.2.90 +metastore-version = 1.1.2-23-fa9170b +mpfr-version = 4.0.1 +mpc-version = 1.1.0 +ncurses-version = 6.1 +openssl-version = 1.1.1a +patchelf-version = 0.9 +pkgconfig-version = 0.29.2 +python-version = 3.6.8 +readline-version = 8.0 +sed-version = 4.7 +tar-version = 1.31 +unzip-version = 6.0 +wget-version = 1.20.1 +which-version = 2.21 +xz-version = 5.2.4 +zip-version = 3.0 # Libraries -cfitsio-version = 3.45 -curl-version = 7.63.0 -gsl-version = 2.5 -libjpeg-version = v9b -libtiff-version = 4.0.10 -zlib-version = 1.2.11 +cfitsio-version = 3.45 +curl-version = 7.63.0 +gsl-version = 2.5 +libjpeg-version = v9b +libtiff-version = 4.0.10 +zlib-version = 1.2.11 + +# Python packages +# --------------- +# +# IMPORTANT: Fix url in `reproduce/src/make/dependencies.mk' +# if changing the version +astropy-version = 3.1.1 +numpy-version = 1.16.1 # Special libraries # ----------------- @@ -55,5 +62,5 @@ zlib-version = 1.2.11 # the software build Makefiles (`reproduce/src/make/dependencies*.mk'). If # you change their version, also please change the explicit shared library # names also. -libgit2-version = 0.26.0 -wcslib-version = 6.2 +libgit2-version = 0.26.0 +wcslib-version = 6.2 diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk index 0c7262e..2247518 100644 --- a/reproduce/src/make/dependencies-build-rules.mk +++ b/reproduce/src/make/dependencies-build-rules.mk @@ -103,6 +103,22 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then \ +# Python +# ------ +# +# To build Python packages with direct access to a `setup.py' (if no direct +# access to `setup.py' is needed, pip can be used) +pybuild = cd $(ddir); rm -rf $(2); \ + if ! tar xf $(1); then echo; echo "Tar error"; exit 1; fi; \ + cd $(2); \ + python3 setup.py build && \ + python3 setup.py install && \ + cd .. && rm -rf $(2) + + + + + # CMake # ----- # diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 52b65f4..a28de19 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -41,13 +41,12 @@ idir = $(BDIR)/dependencies/installed ibdir = $(BDIR)/dependencies/installed/bin ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built -pydir = $(BDIR)/dependencies/installed/lib/python # Define the top-level programs to build (installed in `.local/bin'). -top-level-python = numpy -top-level-programs = astnoisechisel metastore flock zip +top-level-python = astropy +top-level-programs = astnoisechisel metastore flock unzip zip all: $(ddir)/texlive-versions.tex \ - $(foreach p, $(top-level-python), $(pydir)/$(p)) \ + $(foreach p, $(top-level-python), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # Other basic environment settings: We are only including the host @@ -90,26 +89,27 @@ export LDFLAGS := $(rpath_command) -L$(ildir) # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. -tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ - cmake-$(cmake-version).tar.gz \ - curl-$(curl-version).tar.gz \ - flock-$(flock-version).tar.xz \ - ghostscript-$(ghostscript-version).tar.gz \ - git-$(git-version).tar.xz \ - gnuastro-$(gnuastro-version).tar.lz \ - gsl-$(gsl-version).tar.gz \ - install-tl-unx.tar.gz \ - jpegsrc.$(libjpeg-version).tar.gz \ - libbsd-$(libbsd-version).tar.xz \ - libtool-$(libtool-version).tar.xz \ - libgit2-$(libgit2-version).tar.gz \ - metastore-$(metastore-version).tar.gz \ - numpy-$(numpy-version).zip \ - python-$(python-version).tar.gz \ - unzip-$(unzip-version).tar.gz \ - tiff-$(libtiff-version).tar.gz \ - zip-$(zip-version).tar.gz \ - wcslib-$(wcslib-version).tar.bz2 \ +tarballs = $(foreach t, astropy-$(astropy-version).tar.gz \ + cfitsio-$(cfitsio-version).tar.gz \ + cmake-$(cmake-version).tar.gz \ + curl-$(curl-version).tar.gz \ + flock-$(flock-version).tar.xz \ + ghostscript-$(ghostscript-version).tar.gz \ + git-$(git-version).tar.xz \ + gnuastro-$(gnuastro-version).tar.lz \ + gsl-$(gsl-version).tar.gz \ + install-tl-unx.tar.gz \ + jpegsrc.$(libjpeg-version).tar.gz \ + libbsd-$(libbsd-version).tar.xz \ + libtool-$(libtool-version).tar.xz \ + libgit2-$(libgit2-version).tar.gz \ + metastore-$(metastore-version).tar.gz \ + numpy-$(numpy-version).zip \ + python-$(python-version).tar.gz \ + unzip-$(unzip-version).tar.gz \ + tiff-$(libtiff-version).tar.gz \ + zip-$(zip-version).tar.gz \ + wcslib-$(wcslib-version).tar.bz2 \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then @@ -122,7 +122,8 @@ $(tarballs): $(tdir)/%: # Set the top download link of the requested tarball. mergenames=1 - if [ $$n = cfitsio ]; then + if [ $$n = astropy ]; then w=https://files.pythonhosted.org/packages/eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 + elif [ $$n = cfitsio ]; then mergenames=0 v=$$(echo $(cfitsio-version) | sed -e's/\.//' \ | awk '{l=length($$1); \ @@ -146,7 +147,7 @@ $(tarballs): $(tdir)/%: mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://akhlaghi.org/src - elif [ $$n = numpy ]; then w=https://fossies.org/linux/misc + elif [ $$n = numpy ]; then w=https://files.pythonhosted.org/packages/2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c elif [ $$n = python ]; then mergenames=0 w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz @@ -483,10 +484,15 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz # Python packages # --------------- -$(pydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ - $(ibdir)/python3 \ - $(ibdir)/unzip - pip3 install $< --verbose +$(ilidir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ + $(ibdir)/python3 + pip3 install $< --verbose && echo "numpy is built" > $@ + +$(ilidir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ + $(ilidir)/numpy + pip3 install $< --verbose && echo "astropy is built" > $@ + + diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index ff06dcb..dfdf161 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -316,7 +316,6 @@ lvcheck = idir=$(BDIR)/dependencies/installed/include; \ - # Pipeline initialization results # ------------------------------- # @@ -413,3 +412,9 @@ $(mtexdir)/initialize.tex: | $(mtexdir) # TeX package versions cat $(BDIR)/dependencies/texlive-versions.tex >> $@ + + # Python packages + $(call pvcheck, python3, $(python-version), Python, pythonversion) + echo "\newcommand{\\numpyversion}{$(numpy-version)}" >> $@ + echo "\newcommand{\\astropyversion}{$(astropy-version)}" >> $@ + -- cgit v1.2.1