From 9521434ad2e02904251522b0a4e0194e3d38a1b8 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Mon, 18 Feb 2019 16:07:39 +0000 Subject: Installed astroquery in the pipeline All dependencies for building astroquery package have been done. Until nowthe Python dependencies were built in the same Makefile as the high level libraries and programs. But, because astroquery has many dependencies we split the Python and Python packages installation in a new Makefile. The installation of differents packages are done using Python and not pip, because we found some problems when doing it with pip. Apparently there are some interferences between the packages installed by the pip of the system and the pip installed as part of Python in the pipeline. --- .file-metadata | Bin 3956 -> 5991097 bytes configure | 18 ++ reproduce/config/pipeline/dependency-versions.mk | 17 ++ reproduce/src/make/dependencies-build-rules.mk | 17 -- reproduce/src/make/dependencies-python.mk | 294 +++++++++++++++++++++++ reproduce/src/make/dependencies.mk | 36 +-- 6 files changed, 332 insertions(+), 50 deletions(-) create mode 100644 reproduce/src/make/dependencies-python.mk diff --git a/.file-metadata b/.file-metadata index 517973d..8d70351 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/configure b/configure index 268ee4c..c8e2b63 100755 --- a/configure +++ b/configure @@ -718,6 +718,24 @@ numthreads=$($instdir/bin/nproc) +# Python dependencies +# ------------------- +# +# Python has its own installation program. To help in managing the +# dependencies we make the installation of packages as a separate +# Makefile. +./.local/bin/make -f reproduce/src/make/dependencies-python.mk \ + rpath_command=$rpath_command \ + static_build=$static_build \ + on_mac_os=$on_mac_os \ + numthreads=$numthreads \ + -j$numthreads + + + + + + # Make sure TeX Live installed successfully # ----------------------------------------- # diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index f0774bc..0aaceaf 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -52,8 +52,25 @@ zlib-version = 1.2.11 # # IMPORTANT: Fix url in `reproduce/src/make/dependencies.mk' # if changing the version +astroquery-version = 0.3.9 astropy-version = 3.1.1 +beautifulsoup4-version = 4.7.1 +certifi-version = 2018.11.29 +chardet-version = 3.0.4 +entrypoints-version = 0.3 +html5lib-version = 1.0.1 +idna-version = 2.8 +keyring-version = 18.0.0 numpy-version = 1.16.1 +pip-version = 19.0.2 +requests-version = 2.21.0 +setuptools-version = 40.8.0 +setuptools_scm-version = 3.2.0 +six-version = 1.12.0 +soupsieve-version = 1.8 +urllib3-version = 1.24.1 +virtualenv-version = 16.4.0 +webencodings-version = 0.5.1 # Special libraries # ----------------- diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk index 2247518..5216488 100644 --- a/reproduce/src/make/dependencies-build-rules.mk +++ b/reproduce/src/make/dependencies-build-rules.mk @@ -102,23 +102,6 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then \ - -# Python -# ------ -# -# To build Python packages with direct access to a `setup.py' (if no direct -# access to `setup.py' is needed, pip can be used) -pybuild = cd $(ddir); rm -rf $(2); \ - if ! tar xf $(1); then echo; echo "Tar error"; exit 1; fi; \ - cd $(2); \ - python3 setup.py build && \ - python3 setup.py install && \ - cd .. && rm -rf $(2) - - - - - # CMake # ----- # diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk new file mode 100644 index 0000000..3e69ac6 --- /dev/null +++ b/reproduce/src/make/dependencies-python.mk @@ -0,0 +1,294 @@ +# Build the reproduction pipeline Python dependencies. +# +# ------------------------------------------------------------------------ +# !!!!! IMPORTANT NOTES !!!!! +# +# This Makefile will be run by the initial `./configure' script. It is not +# included into the reproduction pipe after that. +# +# ------------------------------------------------------------------------ +# +# Original author: +# Raul Infante-Sainz +# Contributing author(s): +# Mohammad Akhlaghi +# Copyright (C) 2019, Your Name. +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# . + + + +# Top level environment +include reproduce/config/pipeline/LOCAL.mk +include reproduce/src/make/dependencies-build-rules.mk +include reproduce/config/pipeline/dependency-versions.mk + +ddir = $(BDIR)/dependencies +tdir = $(BDIR)/dependencies/tarballs +idir = $(BDIR)/dependencies/installed +ibdir = $(BDIR)/dependencies/installed/bin +ildir = $(BDIR)/dependencies/installed/lib +ilidir = $(BDIR)/dependencies/installed/lib/built +ipydir = $(BDIR)/dependencies/installed/lib/built/python + +# Define the top-level programs to build (installed in `.local/bin'). +top-level-python = astroquery +all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) + +# Other basic environment settings: We are only including the host +# operating system's PATH environment variable (after our own!) for the +# compiler and linker. For the library binaries and headers, we are only +# using our internally built libraries. +# +# To investigate: +# +# 1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all +# the parameters defined bellow as `NAME=VALUE' statements before +# calling Bash. This will enable us to completely ignore the user's +# native environment. +# +# 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the +# user's environment. +.ONESHELL: +.SHELLFLAGS := --noprofile --norc -ec +export CCACHE_DISABLE := 1 +export PATH := $(ibdir) +export LD_RUN_PATH := $(ildir) +export LD_LIBRARY_PATH := $(ildir) +export SHELL := $(ibdir)/bash +export CPPFLAGS := -I$(idir)/include +export PKG_CONFIG_PATH := $(ildir)/pkgconfig +export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +export LDFLAGS := $(rpath_command) -L$(ildir) + + + + + +# Tarballs +# -------- +# +# All the necessary tarballs are defined and prepared with this rule. +# +# Note that we want the tarballs to follow the convention of NAME-VERSION +# before the `tar.XX' prefix. For those programs that don't follow this +# convention, but include the name/version in their tarball names with +# another format, we'll do the modification before the download so the +# downloaded file has our desired format. +tarballs = $(foreach t, astroquery-$(astroquery-version).tar.gz \ + astropy-$(astropy-version).tar.gz \ + beautifulsoup4-$(beautifulsoup4-version).tar.gz \ + certifi-$(certifi-version).tar.gz \ + chardet-$(chardet-version).tar.gz \ + entrypoints-$(entrypoints-version).tar.gz \ + html5lib-$(html5lib-version).tar.gz \ + idna-$(idna-version).tar.gz \ + keyring-$(keyring-version).tar.gz \ + numpy-$(numpy-version).zip \ + pip-$(pip-version).tar.gz \ + python-$(python-version).tar.gz \ + requests-$(requests-version).tar.gz \ + setuptools-$(setuptools-version).zip \ + setuptools_scm-$(setuptools_scm-version).tar.gz \ + six-$(six-version).tar.gz \ + soupsieve-$(soupsieve-version).tar.gz \ + urllib3-$(urllib3-version).tar.gz \ + webencodings-$(webencodings-version).tar.gz \ + virtualenv-$(virtualenv-version).tar.gz \ + , $(tdir)/$(t) ) +topurl=https://files.pythonhosted.org/packages +$(tarballs): $(tdir)/%: + if [ -f $(DEPENDENCIES-DIR)/$* ]; then + cp $(DEPENDENCIES-DIR)/$* $@ + else + # Remove all numbers, `-' and `.' from the tarball name so we can + # search more easily only with the program name. + n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ + | awk '{print $$1}' ) + + # Set the top download link of the requested tarball. + mergenames=1 + if [ $$n = python ]; then + mergenames=0 + w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz + elif [ $$n = astroquery ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396 + elif [ $$n = astropy ]; then h=eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 + elif [ $$n = beautifulsoup ]; then h=80/f2/f6aca7f1b209bb9a7ef069d68813b091c8c3620642b568dac4eb0e507748 + elif [ $$n = certifi ]; then h=55/54/3ce77783acba5979ce16674fc98b1920d00b01d337cfaaf5db22543505ed + elif [ $$n = chardet ]; then h=fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d + elif [ $$n = entrypoints ]; then h=b4/ef/063484f1f9ba3081e920ec9972c96664e2edb9fdc3d8669b0e3b8fc0ad7c + elif [ $$n = html ]; then h=85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f + elif [ $$n = idna ]; then h=ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7 + elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2 + elif [ $$n = numpy ]; then h=2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c + elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af + elif [ $$n = requests ]; then h=52/2c/514e4ac25da2b08ca5a464c50463682126385c4272c18193876e91f4bc38 + elif [ $$n = setuptools ]; then h=c2/f7/c7b501b783e5a74cf1768bc174ee4fb0a8a6ee5af6afa92274ff964703e0 + elif [ $$n = setuptools_scm ]; then h=54/85/514ba3ca2a022bddd68819f187ae826986051d130ec5b972076e4f58a9f3 + elif [ $$n = six ]; then h=dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca + elif [ $$n = soupsieve ]; then h=0c/52/e9088bb9b96e2d39fc3b33fcda5b4fde9d71473536ac660a1ca9a0958a2f + elif [ $$n = virtualenv ]; then h=51/aa/c395a6e6eaaedfa5a04723b6446a1df783b16cca6fec66e671cede514688 + elif [ $$n = webencodings ]; then h=0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47 + else + echo; echo; echo; + echo "'$$n' not recognized as a dependency name to download." + echo; echo; echo; + exit 1 + fi + + # Download the requested tarball. Note that some packages may not + # follow our naming convention (where the package name is merged + # with its version number). In such cases, `w' will be the full + # address, not just the top directory address. But since we are + # storing all the tarballs in one directory, we want it to have + # the same naming convention, so we'll download it to a temporary + # name, then rename that. + if [ $$mergenames = 1 ]; then tarballurl=$(topurl)/$$h/"$*" + else tarballurl=$$h + fi + + # If the download fails, Wget will write the error message in the + # target file, so Make will think that its done! To avoid this + # problem, we'll rename the output. + echo "Downloading $$tarballurl" + if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then + rm -f $@ + echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1 + fi + fi + + + + + +# Install without pip +# -------------------- +# +# To build Python packages with direct access to a `setup.py' (if no direct +# access to `setup.py' is needed, pip can be used) +# Arguments of this function are the numbers +# 1) Unpack command +# 2) Package name +# 3) Unpacked directory name after unpacking the tarball +pybuild = cd $(ddir); rm -rf $(3); \ + if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ + cd $(3); \ + python3 setup.py build && \ + python3 setup.py install && \ + cd .. && rm -rf $(3) && \ + echo "done!" > $@ + + + + + +# Python installation +# ------------------- +# +$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz + $(call gbuild, $<, Python-$(python-version)) \ + && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ + {printf "%d.%d\n", $$1, $$2}') \ + && ln -s $(ildir)/python$$v $(ildir)/python \ + && rm -rf $(ipydir) && mkdir $(ipydir) + + + + +# Python packages +# --------------- +# +$(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ + $(ipydir)/astropy \ + $(ipydir)/beautifulsoup4 \ + $(ipydir)/html5lib \ + $(ipydir)/keyring \ + $(ipydir)/numpy \ + $(ipydir)/requests + $(call pybuild, tar xf, $<, astroquery-$(astroquery-version)) + +$(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ + $(ipydir)/numpy + $(call pybuild, tar xf, $<, astropy-$(astropy-version)) + +$(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \ + $(ipydir)/soupsieve + $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version)) + +$(ipydir)/certifi: $(tdir)/certifi-$(certifi-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, certifi-$(certifi-version)) + +$(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, chardet-$(chardet-version)) + +$(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version)) + +$(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \ + $(ipydir)/six \ + $(ipydir)/webencodings + $(call pybuild, tar xf, $<, html5lib-$(html5lib-version)) + +$(ipydir)/idna: $(tdir)/idna-$(idna-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, idna-$(idna-version)) + +$(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \ + $(ipydir)/entrypoints \ + $(ipydir)/setuptools_scm + $(call pybuild, tar xf, $<, keyring-$(keyring-version)) + +$(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ + $(ibdir)/python3 + $(call pybuild, unzip, $<, numpy-$(numpy-version)) + +$(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, pip-$(pip-version)) + +$(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ + $(ipydir)/certifi \ + $(ipydir)/chardet \ + $(ipydir)/idna \ + $(ipydir)/numpy \ + $(ipydir)/urllib3 + $(call pybuild, tar xf, $<, requests-$(requests-version)) + +$(ipydir)/setuptools: $(tdir)/setuptools-$(setuptools-version).zip \ + $(ibdir)/python3 + $(call pybuild, unzip, $<, setuptools-$(setuptools-version)) + +$(ipydir)/setuptools_scm: $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version)) + +$(ipydir)/six: $(tdir)/six-$(six-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, six-$(six-version)) + +$(ipydir)/soupsieve: $(tdir)/soupsieve-$(soupsieve-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version)) + +$(ipydir)/urllib3: $(tdir)/urllib3-$(urllib3-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, urllib3-$(urllib3-version)) + +$(ipydir)/webencodings: $(tdir)/webencodings-$(webencodings-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, webencodings-$(webencodings-version)) + diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 92febb1..d227dcc 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,10 +43,8 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astropy -top-level-programs = astnoisechisel metastore flock unzip zip +top-level-programs = astnoisechisel flock metastore unzip zip all: $(ddir)/texlive-versions.tex \ - $(foreach p, $(top-level-python), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # Other basic environment settings: We are only including the host @@ -89,8 +87,7 @@ export LDFLAGS := $(rpath_command) -L$(ildir) # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. -tarballs = $(foreach t, astropy-$(astropy-version).tar.gz \ - cfitsio-$(cfitsio-version).tar.gz \ +tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ flock-$(flock-version).tar.xz \ @@ -104,8 +101,6 @@ tarballs = $(foreach t, astropy-$(astropy-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ - numpy-$(numpy-version).zip \ - python-$(python-version).tar.gz \ unzip-$(unzip-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ @@ -122,8 +117,7 @@ $(tarballs): $(tdir)/%: # Set the top download link of the requested tarball. mergenames=1 - if [ $$n = astropy ]; then w=https://files.pythonhosted.org/packages/eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 - elif [ $$n = cfitsio ]; then + if [ $$n = cfitsio ]; then mergenames=0 v=$$(echo $(cfitsio-version) | sed -e's/\.//' \ | awk '{l=length($$1); \ @@ -147,10 +141,6 @@ $(tarballs): $(tdir)/%: mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://akhlaghi.org/src - elif [ $$n = numpy ]; then w=https://files.pythonhosted.org/packages/2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c - elif [ $$n = python ]; then - mergenames=0 - w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -456,12 +446,6 @@ endif $$staticopts, -j$(numthreads), \ make check -j$(numthreads)) -$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz - $(call gbuild, $<, Python-$(python-version)) \ - && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ - {printf "%d.%d\n", $$1, $$2}') \ - && ln -s $(ildir)/python$$v $(ildir)/python - $(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz v=$$(echo $(unzip-version) | sed -e's/\.//') $(call gbuild, $<, unzip$$v, static,, \ @@ -482,20 +466,6 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz -# Python packages -# --------------- -$(ilidir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ - $(ibdir)/python3 - pip3 install $< --verbose && echo "numpy is built" > $@ - -$(ilidir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ - $(ilidir)/numpy - pip3 install $< --verbose && echo "astropy is built" > $@ - - - - - # Since we want to avoid complicating the PATH, we are putting a symbolic # link of all the TeX Live executables in $(ibdir). But symbolic links are # hard to track for Make (as a target). Also, TeX in general is optional -- cgit v1.2.1 From a92b25adb31bde17fe4db4f37be11bbe389fdf4b Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 21 Feb 2019 15:34:33 +0000 Subject: Matplotlib is now in the pipeline As matplotlib is a general package for plotting and it is widely used in science, we have added it to the pipeline. When installing a dependency of matplotlib `python-dateutil', we found a conflict in the download of the tarball. This is because the name has a dash (-) in the middle. In addition, the name starts with 'python', so it is the same as the python itself. Now it is possible to install any package with any name, just adding an elif in before the URL direction. --- .file-metadata | Bin 5991097 -> 16853304 bytes reproduce/config/pipeline/dependency-versions.mk | 6 ++ reproduce/src/make/dependencies-python.mk | 68 +++++++++++++++++++++-- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/.file-metadata b/.file-metadata index 8d70351..525a090 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 0aaceaf..f242ac9 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -57,13 +57,19 @@ astropy-version = 3.1.1 beautifulsoup4-version = 4.7.1 certifi-version = 2018.11.29 chardet-version = 3.0.4 +cycler-version = 0.10.0 entrypoints-version = 0.3 html5lib-version = 1.0.1 idna-version = 2.8 +kiwisolver-version = 1.0.1 keyring-version = 18.0.0 +matplotlib-version = 3.0.2 numpy-version = 1.16.1 pip-version = 19.0.2 +pyparsing-version = 2.3.1 +python-dateutil-version = 2.8.0 requests-version = 2.21.0 +scipy-version = 1.2.1 setuptools-version = 40.8.0 setuptools_scm-version = 3.2.0 six-version = 1.12.0 diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 3e69ac6..0f45fe2 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -43,7 +43,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astroquery +top-level-python = astroquery matplotlib #scipy all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host @@ -91,14 +91,20 @@ tarballs = $(foreach t, astroquery-$(astroquery-version).tar.gz \ beautifulsoup4-$(beautifulsoup4-version).tar.gz \ certifi-$(certifi-version).tar.gz \ chardet-$(chardet-version).tar.gz \ + cycler-$(cycler-version).tar.gz \ entrypoints-$(entrypoints-version).tar.gz \ html5lib-$(html5lib-version).tar.gz \ idna-$(idna-version).tar.gz \ + kiwisolver-$(kiwisolver-version).tar.gz \ keyring-$(keyring-version).tar.gz \ + matplotlib-$(matplotlib-version).tar.gz \ numpy-$(numpy-version).zip \ pip-$(pip-version).tar.gz \ python-$(python-version).tar.gz \ + python-dateutil-$(python-dateutil-version).tar.gz \ + pyparsing-$(pyparsing-version).tar.gz \ requests-$(requests-version).tar.gz \ + scipy-$(scipy-version).tar.gz \ setuptools-$(setuptools-version).zip \ setuptools_scm-$(setuptools_scm-version).tar.gz \ six-$(six-version).tar.gz \ @@ -112,10 +118,27 @@ $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* $@ else - # Remove all numbers, `-' and `.' from the tarball name so we can - # search more easily only with the program name. - n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ - | awk '{print $$1}' ) + + # Convenience variable + # -------------------- + # + # `n' is just for convenience and to avoid having to repeat the + # package tarball name in the conditional to find its URL. + # + # For some packages (for example `python-dateutil', or those with + # a number or dash in their name), we need special consideration + # because the tokenization above will produce `python' as the + # first string. + if [ $* = python-dateutil-$(python-dateutil-version).tar.gz ]; then + n=dateutil + # elif [ $* = strange-tarball5name-version.tar.gz ]; then + # n=strange5-name + else + # Remove all numbers, `-' and `.' from the tarball name so we can + # search more easily only with the program name. + n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ + | awk '{print $$1}') + fi # Set the top download link of the requested tarball. mergenames=1 @@ -127,19 +150,26 @@ $(tarballs): $(tdir)/%: elif [ $$n = beautifulsoup ]; then h=80/f2/f6aca7f1b209bb9a7ef069d68813b091c8c3620642b568dac4eb0e507748 elif [ $$n = certifi ]; then h=55/54/3ce77783acba5979ce16674fc98b1920d00b01d337cfaaf5db22543505ed elif [ $$n = chardet ]; then h=fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d + elif [ $$n = cycler ]; then h=c2/4b/137dea450d6e1e3d474e1d873cd1d4f7d3beed7e0dc973b06e8e10d32488 elif [ $$n = entrypoints ]; then h=b4/ef/063484f1f9ba3081e920ec9972c96664e2edb9fdc3d8669b0e3b8fc0ad7c elif [ $$n = html ]; then h=85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f elif [ $$n = idna ]; then h=ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7 elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2 + elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864 + elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f elif [ $$n = numpy ]; then h=2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af + elif [ $$n = pyparsing ]; then h=b9/b8/6b32b3e84014148dcd60dd05795e35c2e7f4b72f918616c61fdce83d27fc + elif [ $$n = dateutil ]; then h=ad/99/5b2e99737edeb28c71bcbec5b5dda19d0d9ef3ca3e92e3e925e7c0bb364c elif [ $$n = requests ]; then h=52/2c/514e4ac25da2b08ca5a464c50463682126385c4272c18193876e91f4bc38 + elif [ $$n = scipy ]; then h=a9/b4/5598a706697d1e2929eaf7fe68898ef4bea76e4950b9efbe1ef396b8813a elif [ $$n = setuptools ]; then h=c2/f7/c7b501b783e5a74cf1768bc174ee4fb0a8a6ee5af6afa92274ff964703e0 elif [ $$n = setuptools_scm ]; then h=54/85/514ba3ca2a022bddd68819f187ae826986051d130ec5b972076e4f58a9f3 elif [ $$n = six ]; then h=dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca elif [ $$n = soupsieve ]; then h=0c/52/e9088bb9b96e2d39fc3b33fcda5b4fde9d71473536ac660a1ca9a0958a2f elif [ $$n = virtualenv ]; then h=51/aa/c395a6e6eaaedfa5a04723b6446a1df783b16cca6fec66e671cede514688 elif [ $$n = webencodings ]; then h=0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47 +# elif [ $$n = strange5-name ]; then h=XXXXX else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." @@ -234,6 +264,10 @@ $(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \ $(ibdir)/python3 $(call pybuild, tar xf, $<, chardet-$(chardet-version)) +$(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \ + $(ipydir)/six + $(call pybuild, tar xf, $<, cycler-$(cycler-version)) + $(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ $(ibdir)/python3 $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version)) @@ -252,6 +286,18 @@ $(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \ $(ipydir)/setuptools_scm $(call pybuild, tar xf, $<, keyring-$(keyring-version)) +$(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version)) + + +$(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ + $(ipydir)/cycler \ + $(ipydir)/pyparsing \ + $(ipydir)/python-dateutil \ + $(ipydir)/kiwisolver + $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) + $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ibdir)/python3 $(call pybuild, unzip, $<, numpy-$(numpy-version)) @@ -260,6 +306,14 @@ $(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ $(ibdir)/python3 $(call pybuild, tar xf, $<, pip-$(pip-version)) +$(ipydir)/pyparsing: $(tdir)/pyparsing-$(pyparsing-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version)) + +$(ipydir)/python-dateutil: $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz \ + $(ipydir)/six + $(call pybuild, tar xf, $<, python-dateutil-$(python-dateutil-version)) + $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ $(ipydir)/certifi \ $(ipydir)/chardet \ @@ -268,6 +322,10 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ $(ipydir)/urllib3 $(call pybuild, tar xf, $<, requests-$(requests-version)) +$(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ + $(ipydir)/numpy + $(call pybuild, tar xf, $<, scipy-$(scipy-version)) + $(ipydir)/setuptools: $(tdir)/setuptools-$(setuptools-version).zip \ $(ibdir)/python3 $(call pybuild, unzip, $<, setuptools-$(setuptools-version)) -- cgit v1.2.1 From 62cb377a954921ef7940059e6dfb8521f9698c32 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 28 Feb 2019 12:32:19 +0000 Subject: Git hooks for metastore check for the existance of metastore Until now, once the Git hooks have been installed (after the installation of Metastore), if metastore doesn't exist (for example by manually deleting the build directory for a re-build with same configurations as before) we can't run `git commit' and `git checkout' will print an ugly warning. With this commit, the two Git hooks check for the existance of Metastore and if it doesn't exist, they won't do anything. --- reproduce/src/bash/git-post-checkout | 9 +++++++-- reproduce/src/bash/git-pre-commit | 11 ++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/reproduce/src/bash/git-post-checkout b/reproduce/src/bash/git-post-checkout index e2a0fd0..153b277 100644 --- a/reproduce/src/bash/git-post-checkout +++ b/reproduce/src/bash/git-post-checkout @@ -9,8 +9,13 @@ # special characters for the installation location of meta-store so our own # installation is found by Git. -# File containig the metadata. +# File containig the metadata and metastore executable. MSFILE=".file-metadata" +MSBIN=@BINDIR@/metastore + +# If metastore is not installed, then ignore this script (exit with a +# status of 0). +if [ ! -f $MSBIN ]; then exit 0; fi # Delete all temporary files find @TOP_PROJECT_DIR@/ -name "*~" -type f -delete @@ -32,7 +37,7 @@ fi # Run metastore. exit_on_fail \ - @BINDIR@/metastore -a -m -e -E -q -O @USER@ -G @GROUP@ -f "$MSFILE" + $MSBIN -a -m -e -E -q -O @USER@ -G @GROUP@ -f "$MSFILE" # Return with a success code (0). exit 0 diff --git a/reproduce/src/bash/git-pre-commit b/reproduce/src/bash/git-pre-commit index ca28757..0fa6c52 100644 --- a/reproduce/src/bash/git-pre-commit +++ b/reproduce/src/bash/git-pre-commit @@ -20,8 +20,13 @@ # git reset HEAD -- .metadata # git checkout HEAD -- .metadata -# File containig the metadata. +# File containig the metadata and metastore executable. MSFILE=".file-metadata" +MSBIN=@BINDIR@/metastore + +# If metastore is not installed, then ignore this script (exit with a +# status of 0). +if [ ! -f $MSBIN ]; then exit 0; fi # Function to help in reporting a crash. exit_on_fail() { @@ -34,12 +39,12 @@ exit_on_fail() { # Run metastore. exit_on_fail \ - @BINDIR@/metastore -O @USER@ -G @GROUP@ -s -f "$MSFILE" + $MSBIN -O @USER@ -G @GROUP@ -s -f "$MSFILE" # If it's first metastore commit, store again to include $MSFILE in $MSFILE. if ! git-ls-tree --name-only HEAD 2>/dev/null | grep -Fqx "$MSFILE"; then exit_on_fail \ - @BINDIR@/metastore -O @USER@ -G @GROUP@ -s -f "$MSFILE" + $MSBIN -O @USER@ -G @GROUP@ -s -f "$MSFILE" fi # Check if the metadata file exists. -- cgit v1.2.1 From 69a780ef770034e69db96c2e1e8a4d83626a4d2f Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 28 Feb 2019 12:41:14 +0000 Subject: Python installation and related packages have been split Until now the installation of Python and its packages (numpy, astropy, astroquery, etc.) were done in the same `makefile'. With this commit the installation of Python and its packages have been split and now it is independent of the other programs. The installation of all Python packages needs to be written explicitely because pip is not used anymore. --- reproduce/config/pipeline/dependency-versions.mk | 8 +++++--- reproduce/src/make/dependencies-python.mk | 14 ++++++++++++- reproduce/src/make/dependencies.mk | 26 ++++++++++++++++++++++-- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index f242ac9..655d44e 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -9,8 +9,9 @@ coreutils-version = 8.30 diffutils-version = 3.7 findutils-version = 4.6.0.199-e3fc flock-version = 0.2.3 +freetype-version = 2.9 gawk-version = 4.2.1 -gcc-version = 8.2.0 +gcc-version = 8.3.0 ghostscript-version = 9.26 git-version = 2.20.1 gmp-version = 6.1.2 @@ -18,12 +19,13 @@ gnuastro-version = 0.8 grep-version = 3.3 gzip-version = 1.10 isl-version = 0.18 -libtool-version = 2.4.6 libbsd-version = 0.9.1 +libpng-version = 1.6.36 +libtool-version = 2.4.6 lzip-version = 1.20 make-version = 4.2.90 metastore-version = 1.1.2-23-fa9170b -mpfr-version = 4.0.1 +mpfr-version = 4.0.2 mpc-version = 1.1.0 ncurses-version = 6.1 openssl-version = 1.1.1a diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 0f45fe2..c45cc13 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -144,7 +144,7 @@ $(tarballs): $(tdir)/%: mergenames=1 if [ $$n = python ]; then mergenames=0 - w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz + h=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz elif [ $$n = astroquery ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396 elif [ $$n = astropy ]; then h=eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 elif [ $$n = beautifulsoup ]; then h=80/f2/f6aca7f1b209bb9a7ef069d68813b091c8c3620642b568dac4eb0e507748 @@ -167,6 +167,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = setuptools_scm ]; then h=54/85/514ba3ca2a022bddd68819f187ae826986051d130ec5b972076e4f58a9f3 elif [ $$n = six ]; then h=dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca elif [ $$n = soupsieve ]; then h=0c/52/e9088bb9b96e2d39fc3b33fcda5b4fde9d71473536ac660a1ca9a0958a2f + elif [ $$n = urllib ]; then h=b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1 elif [ $$n = virtualenv ]; then h=51/aa/c395a6e6eaaedfa5a04723b6446a1df783b16cca6fec66e671cede514688 elif [ $$n = webencodings ]; then h=0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47 # elif [ $$n = strange5-name ]; then h=XXXXX @@ -227,6 +228,13 @@ pybuild = cd $(ddir); rm -rf $(3); \ # ------------------- # $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz + # On Mac systems, the build complains about `clang' specific + # features, so we can't use our own GCC build here. +# if [ x$(on_mac_os) = xyes ]; then \ +# export CC=clang; \ +# export CXX=clang++; \ +# fi; \ + $(call gbuild, $<, Python-$(python-version)) \ && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') \ @@ -293,6 +301,7 @@ $(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/cycler \ + $(ilidir)/freetype \ $(ipydir)/pyparsing \ $(ipydir)/python-dateutil \ $(ipydir)/kiwisolver @@ -300,6 +309,9 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ibdir)/python3 + export BLAS=None; \ + export ATLAS=None; \ + export LAPACK=None; \ $(call pybuild, unzip, $<, numpy-$(numpy-version)) $(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index d227dcc..9f4472e 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,9 +43,11 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). -top-level-programs = astnoisechisel flock metastore unzip zip +top-level-programs = astnoisechisel flock metastore unzip zip +top-level-libraries = freetype all: $(ddir)/texlive-versions.tex \ - $(foreach p, $(top-level-programs), $(ibdir)/$(p)) + $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ + $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) # Other basic environment settings: We are only including the host # operating system's PATH environment variable (after our own!) for the @@ -91,6 +93,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ flock-$(flock-version).tar.xz \ + freetype-$(freetype-version).tar.gz \ ghostscript-$(ghostscript-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ @@ -98,6 +101,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ libbsd-$(libbsd-version).tar.xz \ + libpng-$(libpng-version).tar.xz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ @@ -129,6 +133,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) + elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro @@ -136,6 +141,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases + elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool elif [ $$n = libgit ]; then mergenames=0 @@ -222,6 +228,12 @@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, $<, gsl-$(gsl-version), static) \ && echo "GNU Scientific Library is built" > $@ +# Freetype is necessary to install matplotlib +$(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ + $(ilibdir)/libpng + $(call gbuild, $<, freetype-$(freetype-version), static) \ + && echo "freetype is built" > $@ + $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \ && echo "libbsd is built" > $@ @@ -229,6 +241,9 @@ $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz $(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz $(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@ +$(ilidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz + $(call gbuild, $<, libpng, static) && echo "Libpng is built" > $@ + $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/libjpeg $(call gbuild, $<, tiff-$(libtiff-version), static, \ @@ -304,6 +319,13 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ # After searching in `bootstrap', I couldn't find `LIBS', only # `LDFLAGS'. So the extra libraries are being added to `LDFLAGS', # not `LIBS'. + # + # On Mac systems, the build complains about `clang' specific + # features, so we can't use our own GCC build here. + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + export CXX=clang++; \ + fi; \ cd $(ddir) && rm -rf cmake-$(cmake-version) && \ tar xf $< && cd cmake-$(cmake-version) && \ ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ -- cgit v1.2.1 From 4f50f0acf3e06f6a1ade0532df0b95a39c23629a Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 28 Feb 2019 12:54:29 +0000 Subject: GNU GCC is now in the pipeline Until now, the pipeline was not installing its own `gcc' but using the system one by making a symbolic link. With this commit, GNU GCC has been added into the pipeline. Right now the installation does not work on Mac OS system beause of some conflicts with `clang', but in principle it should work on GNU Linux distributions. --- reproduce/src/make/dependencies-basic.mk | 120 +++++++++++++++++-------------- 1 file changed, 65 insertions(+), 55 deletions(-) diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 184d089..f02f68f 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -65,8 +65,7 @@ export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) # Define the programs that don't depend on any other. -top-level-programs = low-level-links ls sed gawk grep diff find \ - wget which +top-level-programs = low-level-links wget gcc all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) @@ -141,7 +140,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = diffutils ]; then w=http://ftpmirror.gnu.org/gnu/diffutils;\ elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \ elif [ $$n = gawk ]; then w=http://ftpmirror.gnu.org/gnu/gawk; \ - elif [ $$n = gcc ]; then w=http://ftpmirror.gnu.org/gcc/gcc-$(gcc-version); \ + elif [ $$n = gcc ]; then w=http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version); \ elif [ $$n = gmp ]; then w=https://gmplib.org/download/gmp; \ elif [ $$n = grep ]; then w=http://ftpmirror.gnu.org/gnu/grep; \ elif [ $$n = gzip ]; then w=http://ftpmirror.gnu.org/gnu/gzip; \ @@ -210,14 +209,14 @@ makelink = export PATH=$$(echo $(syspath)| tr : '\n' |grep -v ccache \ if [ x$$a != x ]; then ln -s $$a $(ibdir)/$(1); fi $(ibdir) $(ildir):; mkdir $@ $(ibdir)/low-level-links: | $(ibdir) $(ildir) + # The Assembler $(call makelink,as) - # The compiler + # Compiler (Cmake needs the clang compiler which we aren't building + # yet in the pipeline). $(call makelink,clang) - $(call makelink,gcc) - $(call makelink,g++) - $(call makelink,cc) + $(call makelink,clang++) # The linker $(call makelink,ar) @@ -227,7 +226,9 @@ $(ibdir)/low-level-links: | $(ibdir) $(ildir) $(call makelink,ranlib) # Mac OS specific + $(call makelink,sysctl) $(call makelink,sw_vers) + $(call makelink,dsymutil) $(call makelink,install_name_tool) # On Mac OS, libtool is different compared to GNU Libtool. The @@ -342,6 +343,13 @@ $(ilidir)/ncurses: $(tdir)/ncurses-$(ncurses-version).tar.gz \ # the build process completed afterwards and reset the links). rm -f $(ildir)/libncursesw* + # Delete the (possibly existing) low-level programs that depend on + # `readline', and thus `ncurses'. Since these programs are actually + # used during the building of `ncurses', we need to delete them so + # the build process doesn't use the pipeline's Bash and AWK, but + # the host systems. + rm -f $(ibdir)/bash* $(ibdir)/awk* $(ibdir)/gawk* + # Standard build process. $(call gbuild, $<, ncurses-$(ncurses-version), static, \ --with-shared --enable-rpath --without-normal \ @@ -662,20 +670,10 @@ $(ilidir)/isl: $(tdir)/isl-$(isl-version).tar.bz2 \ $(call gbuild, $<, isl-$(isl-version), static) \ && echo "GCC's ISL library is built" > $@ -# On non-GNU systems, the default linker is different and we don't want our -# new linker to be mixed with that during the building of libraries and -# programs before GCC. -$(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz \ - $(ibdir)/ls \ - $(ibdir)/sed \ - $(ilidir)/isl \ - $(ilidir)/mpc \ - $(ibdir)/gawk \ - $(ibdir)/grep \ - $(ibdir)/diff \ - $(ibdir)/find \ - $(ibdir)/bash \ - $(ibdir)/which +# Binutils' linker `ld' is apparently only good for GNU/Linux systems and +# other OSs have their own. So for now we aren't actually building +# Binutils (`ld' isn't a prerequisite of GCC). +$(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz $(call gbuild, $<, binutils-$(binutils-version), static) @@ -701,41 +699,53 @@ $(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz \ # We want to build GCC after building all the basic tools that are often # used in a configure script to enable GCC's configure script to work as # smoothly/robustly as possible. +# Including `objc, obj-c++' is necessary for installing matplotlib. $(ibdir)/gcc: $(tdir)/gcc-$(gcc-version).tar.xz \ - $(ibdir)/ld + $(ibdir)/ls \ + $(ibdir)/sed \ + $(ilidir)/isl \ + $(ilidir)/mpc \ + $(ibdir)/gawk \ + $(ibdir)/grep \ + $(ibdir)/diff \ + $(ibdir)/find \ + $(ibdir)/bash \ + $(ibdir)/which + + # Clean up (possibly existing) gcc installation + rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov* + rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc* \ + $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64* # Un-pack all the necessary tools in the top building directory - cd $(ddir); \ - rm -rf gcc-build gcc-$(gcc-version); \ - tar xf $< && \ - mkdir $(ddir)/gcc-build && \ - cd $(ddir)/gcc-build && \ - ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ - --prefix=$(idir) \ - --with-mpc=$(idir) \ - --with-mpfr=$(idir) \ - --with-gmp=$(idir) \ - --with-isl=$(idir) \ - --with-build-time-tools=$(idir) \ - --enable-shared \ - --disable-multilib \ - --disable-multiarch \ - --enable-threads=posix \ - --enable-libmpx \ - --with-local-prefix=$(idir) \ - --enable-linker-build-id \ - --with-gnu-as \ - --with-gnu-ld \ - --enable-lto \ - --with-linker-hash-style=gnu \ - --enable-languages=c,c++ \ - --disable-libada \ - --disable-nls \ - --enable-default-pie \ - --enable-default-ssp \ - --enable-cet=auto \ - --enable-decimal-float && \ - make SHELL=$(ibdir)/bash -j$$(nproc) && \ - make SHELL=$(ibdir)/bash install && \ - cd .. && \ + cd $(ddir); \ + rm -rf gcc-build gcc-$(gcc-version); \ + tar xf $< && \ + mkdir $(ddir)/gcc-build && \ + cd $(ddir)/gcc-build && \ + ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ + --prefix=$(idir) \ + --with-mpc=$(idir) \ + --with-mpfr=$(idir) \ + --with-gmp=$(idir) \ + --with-isl=$(idir) \ + --with-build-time-tools=$(idir) \ + --enable-shared \ + --disable-multilib \ + --disable-multiarch \ + --enable-threads=posix \ + --with-local-prefix=$(idir) \ + --enable-linker-build-id \ + --enable-lto \ + --enable-languages=c,c++,fortran,objc,obj-c++ \ + --disable-libada \ + --disable-nls \ + --enable-default-pie \ + --enable-default-ssp \ + --enable-cet=auto \ + --enable-decimal-float && \ + make SHELL=$(ibdir)/bash -j$$(nproc) && \ + make SHELL=$(ibdir)/bash install && \ + cd .. && \ rm -rf gcc-build gcc-$(gcc-version) + -- cgit v1.2.1 From a57c91cfdb05cf5cd790823f40d443648f3fc21a Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 28 Feb 2019 14:09:31 +0000 Subject: Typo correction in libpng installation In the libpng installation there was `ilibdir' instead of `ilidir'. --- reproduce/src/make/dependencies.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 9f4472e..e1b409e 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -230,7 +230,7 @@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz # Freetype is necessary to install matplotlib $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ - $(ilibdir)/libpng + $(ilidir)/libpng $(call gbuild, $<, freetype-$(freetype-version), static) \ && echo "freetype is built" > $@ -- cgit v1.2.1 From 41865272cd80856eb466daef029a669225dbf139 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 6 Mar 2019 17:12:19 +0000 Subject: Astroquery, astropy, matplotlib and numpy are now in the pipeline Until this commit, we had some of the python packages intalled but they did not work properly because of the `PYTHONPATH' variables. That is, the pipeline's `python' was the `python' of the system instead of the pipeline's `python'. With this commit this issue has been fixed by setting the correct `PYTHONPATH'. In this commit we also modify the installation of `bzip2' because `CMake' was complaining about some libraries built statically. --- .file-metadata | Bin 16853304 -> 3721 bytes reproduce/config/pipeline/dependency-versions.mk | 9 +++--- reproduce/src/make/dependencies-basic.mk | 37 +++++++++++++++++------ reproduce/src/make/dependencies-python.mk | 1 + reproduce/src/make/dependencies.mk | 3 +- reproduce/src/make/initialize.mk | 13 ++++++++ 6 files changed, 48 insertions(+), 15 deletions(-) diff --git a/.file-metadata b/.file-metadata index 525a090..1176014 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 655d44e..9104852 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -3,7 +3,6 @@ # Programs bash-version = 5.0 binutils-version = 2.31.1 -bzip2-version = 1.0.6 cmake-version = 3.12.4 coreutils-version = 8.30 diffutils-version = 3.7 @@ -83,9 +82,9 @@ webencodings-version = 0.5.1 # Special libraries # ----------------- # -# The shared library name of the following libraries is explicity mentioned -# the software build Makefiles (`reproduce/src/make/dependencies*.mk'). If -# you change their version, also please change the explicit shared library -# names also. +# When updating the version of these libraries, please look into the build +# rule first: In one way or another, the version string becomes necessary +# during their build and must be accounted for. +bzip2-version = 1.0.6 libgit2-version = 0.26.0 wcslib-version = 6.2 diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index f02f68f..fefda6f 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -288,15 +288,34 @@ $(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz $(call gbuild, $<, xz-$(xz-version), static) $(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz - tdir=bzip2-$(bzip2-version); \ - if [ $(static_build) = yes ]; then \ - makecommand="make LDFLAGS=-static"; \ - else \ - makecommand="make"; \ - fi; \ - cd $(ddir) && rm -rf $$tdir && tar xf $< && cd $$tdir && \ - $$makecommand && make install PREFIX=$(idir) && \ - cd .. && rm -rf $$tdir + # Bzip2 doesn't have a `./configure' script, and its Makefile + # doesn't build a shared library. So we can't use the `gbuild' + # function here and we need to take some extra steps (inspired + # from the "Linux from Scratch" guide for Bzip2): + # 1) The `sed' call is for relative installed symbolic links. + # 2) The special Makefile-libbz2_so builds the shared library. + # + # NOTE: the major version number appears in the final symbolic + # link. + tdir=bzip2-$(bzip2-version); \ + if [ $(static_build) = yes ]; then \ + makecommand="make LDFLAGS=-static"; \ + makeshared="echo no-shared"; \ + else \ + makecommand="make"; \ + makeshared="make -f Makefile-libbz2_so"; \ + fi; \ + cd $(ddir) && rm -rf $$tdir && tar xf $< && cd $$tdir \ + && sed -i 's@\(ln -s -f \)$$(PREFIX)/bin/@\1@' Makefile \ + && $$makeshared \ + && cp -a libbz2* $(ildir)/ \ + && make clean \ + && $$makecommand \ + && make install PREFIX=$(idir) \ + && cd .. \ + && rm -rf $$tdir \ + && cd $(ildir) \ + && ln -fs libbz2.so.1.0 libbz2.so # GNU Tar: When built statically, tar gives a segmentation fault on # unpacking Bash. So we'll build it dynamically. diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index c45cc13..1e4f611 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -302,6 +302,7 @@ $(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/cycler \ $(ilidir)/freetype \ + $(ipydir)/numpy \ $(ipydir)/pyparsing \ $(ipydir)/python-dateutil \ $(ipydir)/kiwisolver diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index e1b409e..deb9f39 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -242,7 +242,8 @@ $(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz $(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@ $(ilidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz - $(call gbuild, $<, libpng, static) && echo "Libpng is built" > $@ + $(call gbuild, $<, libpng-$(libpng-version), static) \ + && echo "Libpng is built" > $@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/libjpeg diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 71cdbb8..638338b 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -116,6 +116,19 @@ export LDFLAGS := -L$(installdir)/lib export SHELL := $(installdir)/bin/bash export CPPFLAGS := -I$(installdir)/include +# Python enviroment +# So far we have found several other Python-related environment +# variables which might interfere. So we are just removing all +# of their values within the pipeline. +export PYTHONPATH := $(installdir)/lib/python/site-packages +export PYTHONPATH3 := $(PYTHONPATH) +export _LMFILES_ := +export PYTHONPATH2 := +export LOADEDMODULES := +export MPI_PYTHON_SITEARCH := +export MPI_PYTHON2_SITEARCH := +export MPI_PYTHON3_SITEARCH := + -- cgit v1.2.1