From 9521434ad2e02904251522b0a4e0194e3d38a1b8 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Mon, 18 Feb 2019 16:07:39 +0000 Subject: Installed astroquery in the pipeline All dependencies for building astroquery package have been done. Until nowthe Python dependencies were built in the same Makefile as the high level libraries and programs. But, because astroquery has many dependencies we split the Python and Python packages installation in a new Makefile. The installation of differents packages are done using Python and not pip, because we found some problems when doing it with pip. Apparently there are some interferences between the packages installed by the pip of the system and the pip installed as part of Python in the pipeline. --- reproduce/src/make/dependencies.mk | 36 +++--------------------------------- 1 file changed, 3 insertions(+), 33 deletions(-) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 92febb1..d227dcc 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,10 +43,8 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astropy -top-level-programs = astnoisechisel metastore flock unzip zip +top-level-programs = astnoisechisel flock metastore unzip zip all: $(ddir)/texlive-versions.tex \ - $(foreach p, $(top-level-python), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # Other basic environment settings: We are only including the host @@ -89,8 +87,7 @@ export LDFLAGS := $(rpath_command) -L$(ildir) # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. -tarballs = $(foreach t, astropy-$(astropy-version).tar.gz \ - cfitsio-$(cfitsio-version).tar.gz \ +tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ flock-$(flock-version).tar.xz \ @@ -104,8 +101,6 @@ tarballs = $(foreach t, astropy-$(astropy-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ - numpy-$(numpy-version).zip \ - python-$(python-version).tar.gz \ unzip-$(unzip-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ @@ -122,8 +117,7 @@ $(tarballs): $(tdir)/%: # Set the top download link of the requested tarball. mergenames=1 - if [ $$n = astropy ]; then w=https://files.pythonhosted.org/packages/eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 - elif [ $$n = cfitsio ]; then + if [ $$n = cfitsio ]; then mergenames=0 v=$$(echo $(cfitsio-version) | sed -e's/\.//' \ | awk '{l=length($$1); \ @@ -147,10 +141,6 @@ $(tarballs): $(tdir)/%: mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://akhlaghi.org/src - elif [ $$n = numpy ]; then w=https://files.pythonhosted.org/packages/2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c - elif [ $$n = python ]; then - mergenames=0 - w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -456,12 +446,6 @@ endif $$staticopts, -j$(numthreads), \ make check -j$(numthreads)) -$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz - $(call gbuild, $<, Python-$(python-version)) \ - && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ - {printf "%d.%d\n", $$1, $$2}') \ - && ln -s $(ildir)/python$$v $(ildir)/python - $(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz v=$$(echo $(unzip-version) | sed -e's/\.//') $(call gbuild, $<, unzip$$v, static,, \ @@ -482,20 +466,6 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz -# Python packages -# --------------- -$(ilidir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ - $(ibdir)/python3 - pip3 install $< --verbose && echo "numpy is built" > $@ - -$(ilidir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ - $(ilidir)/numpy - pip3 install $< --verbose && echo "astropy is built" > $@ - - - - - # Since we want to avoid complicating the PATH, we are putting a symbolic # link of all the TeX Live executables in $(ibdir). But symbolic links are # hard to track for Make (as a target). Also, TeX in general is optional -- cgit v1.2.1 From 69a780ef770034e69db96c2e1e8a4d83626a4d2f Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 28 Feb 2019 12:41:14 +0000 Subject: Python installation and related packages have been split Until now the installation of Python and its packages (numpy, astropy, astroquery, etc.) were done in the same `makefile'. With this commit the installation of Python and its packages have been split and now it is independent of the other programs. The installation of all Python packages needs to be written explicitely because pip is not used anymore. --- reproduce/src/make/dependencies.mk | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index d227dcc..9f4472e 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,9 +43,11 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). -top-level-programs = astnoisechisel flock metastore unzip zip +top-level-programs = astnoisechisel flock metastore unzip zip +top-level-libraries = freetype all: $(ddir)/texlive-versions.tex \ - $(foreach p, $(top-level-programs), $(ibdir)/$(p)) + $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ + $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) # Other basic environment settings: We are only including the host # operating system's PATH environment variable (after our own!) for the @@ -91,6 +93,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ flock-$(flock-version).tar.xz \ + freetype-$(freetype-version).tar.gz \ ghostscript-$(ghostscript-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ @@ -98,6 +101,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ libbsd-$(libbsd-version).tar.xz \ + libpng-$(libpng-version).tar.xz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ @@ -129,6 +133,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) + elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro @@ -136,6 +141,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases + elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool elif [ $$n = libgit ]; then mergenames=0 @@ -222,6 +228,12 @@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, $<, gsl-$(gsl-version), static) \ && echo "GNU Scientific Library is built" > $@ +# Freetype is necessary to install matplotlib +$(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ + $(ilibdir)/libpng + $(call gbuild, $<, freetype-$(freetype-version), static) \ + && echo "freetype is built" > $@ + $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \ && echo "libbsd is built" > $@ @@ -229,6 +241,9 @@ $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz $(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz $(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@ +$(ilidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz + $(call gbuild, $<, libpng, static) && echo "Libpng is built" > $@ + $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/libjpeg $(call gbuild, $<, tiff-$(libtiff-version), static, \ @@ -304,6 +319,13 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ # After searching in `bootstrap', I couldn't find `LIBS', only # `LDFLAGS'. So the extra libraries are being added to `LDFLAGS', # not `LIBS'. + # + # On Mac systems, the build complains about `clang' specific + # features, so we can't use our own GCC build here. + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + export CXX=clang++; \ + fi; \ cd $(ddir) && rm -rf cmake-$(cmake-version) && \ tar xf $< && cd cmake-$(cmake-version) && \ ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ -- cgit v1.2.1 From a57c91cfdb05cf5cd790823f40d443648f3fc21a Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 28 Feb 2019 14:09:31 +0000 Subject: Typo correction in libpng installation In the libpng installation there was `ilibdir' instead of `ilidir'. --- reproduce/src/make/dependencies.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 9f4472e..e1b409e 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -230,7 +230,7 @@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz # Freetype is necessary to install matplotlib $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ - $(ilibdir)/libpng + $(ilidir)/libpng $(call gbuild, $<, freetype-$(freetype-version), static) \ && echo "freetype is built" > $@ -- cgit v1.2.1 From 41865272cd80856eb466daef029a669225dbf139 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 6 Mar 2019 17:12:19 +0000 Subject: Astroquery, astropy, matplotlib and numpy are now in the pipeline Until this commit, we had some of the python packages intalled but they did not work properly because of the `PYTHONPATH' variables. That is, the pipeline's `python' was the `python' of the system instead of the pipeline's `python'. With this commit this issue has been fixed by setting the correct `PYTHONPATH'. In this commit we also modify the installation of `bzip2' because `CMake' was complaining about some libraries built statically. --- reproduce/src/make/dependencies.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index e1b409e..deb9f39 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -242,7 +242,8 @@ $(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz $(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@ $(ilidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz - $(call gbuild, $<, libpng, static) && echo "Libpng is built" > $@ + $(call gbuild, $<, libpng-$(libpng-version), static) \ + && echo "Libpng is built" > $@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/libjpeg -- cgit v1.2.1