From 3e583296cb8ed2784f6cb71b53cbe3be4931fa46 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 21 Mar 2019 15:39:06 +0000 Subject: ATLAS and Scipy working on GNU/Linux Numpy needs ATLAS as shared libraries. So we also need to build Python with shared libraries. We also need to define site.cfg for numpy and scipy so we define a master template: `reproduce/config/pipeline/dependency-numpy-scipy.cfg' Also `Openssl' did not have rpath so we added with this commit. --- .../config/pipeline/dependency-numpy-scipy.cfg | 47 ++++++++++++++++ reproduce/config/pipeline/dependency-versions.mk | 2 +- reproduce/src/make/dependencies-basic.mk | 11 +++- reproduce/src/make/dependencies-python.mk | 51 ++++++++++++------ reproduce/src/make/dependencies.mk | 63 +++++++++++----------- 5 files changed, 124 insertions(+), 50 deletions(-) create mode 100644 reproduce/config/pipeline/dependency-numpy-scipy.cfg (limited to 'reproduce') diff --git a/reproduce/config/pipeline/dependency-numpy-scipy.cfg b/reproduce/config/pipeline/dependency-numpy-scipy.cfg new file mode 100644 index 0000000..55e93bc --- /dev/null +++ b/reproduce/config/pipeline/dependency-numpy-scipy.cfg @@ -0,0 +1,47 @@ +# THIS IS A COPY OF NUMPY'S site.cfg.example, CUSTOMIZED FOR THIS PIPELINE +# ------------------------------------------------------------------------ + +# This file provides configuration information about non-Python dependencies for +# numpy.distutils-using packages. Create a file like this called "site.cfg" next +# to your package's setup.py file and fill in the appropriate sections. Not all +# packages will use all sections so you should leave out sections that your +# package does not use. + +# IMPORTANT NOTE +# -------------- +# +# The `ALL' grouping does't apply to ATLAS!!!!! + + [ALL] + library_dirs = @LIBDIR@ + include_dirs = @INCDIR@ + +# Atlas +# ----- +# Atlas is an open source optimized implementation of the BLAS and Lapack +# routines. NumPy will try to build against Atlas by default when available in +# the system library dirs. To build numpy against a custom installation of +# Atlas you can add an explicit section such as the following. Here we assume +# that Atlas was configured with ``prefix=/opt/atlas``. +# + [atlas] + library_dirs = @LIBDIR@ + include_dirs = @INCDIR@ + +# FFT libraries +# ------------- +# There are two FFT libraries that we can configure here: FFTW (2 and 3) and djbfft. +# Note that these libraries are not used by numpy or scipy. +# +# http://fftw.org/ +# https://cr.yp.to/djbfft.html +# +# Given only this section, numpy.distutils will try to figure out which version +# of FFTW you are using. +#[fftw] +#libraries = fftw3 +# +# For djbfft, numpy.distutils will look for either djbfft.a or libdjbfft.a . +#[djbfft] +#include_dirs = /usr/local/djbfft/include +#library_dirs = /usr/local/djbfft/lib diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 609d1cb..79ecdce 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -82,7 +82,7 @@ jeepney-version = 0.4 kiwisolver-version = 1.0.1 keyring-version = 18.0.0 matplotlib-version = 3.0.2 -numpy-version = 1.16.1 +numpy-version = 1.16.2 pip-version = 19.0.2 pycparser-version = 2.19 pyparsing-version = 2.3.1 diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index a5cda52..853f6d7 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -566,8 +566,15 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ --with-zlib-lib=$(ildir) \ --with-zlib-include=$(idir)/include, , , \ ./config ) && \ - cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem && \ - echo "OpenSSL is built and ready" > $@ + cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem; \ + if [ $$? = 0 ]; then \ + if [ x$(on_mac_os) = xyes ]; then \ + echo "No need to fix rpath in libssl"; \ + else \ + patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \ + fi; \ + echo "OpenSSL is built and ready" > $@; \ + fi # GNU Wget # diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 3b92f7e..e4c2324 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -43,7 +43,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astroquery matplotlib #scipy +top-level-python = astroquery matplotlib scipy all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host @@ -249,18 +249,24 @@ $(tarballs): $(tdir)/%: # -------------------- # # To build Python packages with direct access to a `setup.py' (if no direct -# access to `setup.py' is needed, pip can be used) +# access to `setup.py' is needed, pip can be used). # Arguments of this function are the numbers # 1) Unpack command # 2) Package name # 3) Unpacked directory name after unpacking the tarball -pybuild = cd $(ddir); rm -rf $(3); \ - if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ - cd $(3); \ - python3 setup.py build && \ - python3 setup.py install && \ - cd .. && rm -rf $(3) && \ - echo "done!" > $@ +# 4) site.cfg file (optional) +pybuild = cd $(ddir); rm -rf $(3); \ + if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ + cd $(3); \ + if [ "x$(4)" != x ]; then \ + sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ + -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ + $(4) > site.cfg; \ + fi; \ + python3 setup.py build \ + && python3 setup.py install \ + && cd .. && rm -rf $(3) \ + && echo "done!" > $@ @@ -271,7 +277,12 @@ pybuild = cd $(ddir); rm -rf $(3); \ # # While this Makefile is for Python programs, in some cases, we need # certain programs (like Python itself), or libraries for the modules. -$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz +$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz + $(call gbuild, $<, libffi-$(libffi-version)) \ + echo "libffi is built" > $@ + +$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \ + $(ilidir)/libffi # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. # if [ x$(on_mac_os) = xyes ]; then \ @@ -279,16 +290,18 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz # export CXX=clang++; \ # fi; \ - $(call gbuild, $<, Python-$(python-version)) \ + $(call gbuild, $<, Python-$(python-version),, \ + --enable-optimizations \ + --without-ensurepip \ + --with-system-ffi \ + --enable-shared \ + --with-threads \ + --with-lto ) \ && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') \ && ln -s $(ildir)/python$$v $(ildir)/python \ && rm -rf $(ipydir) && mkdir $(ipydir) -$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz - $(call gbuild, $<, libffi-$(libffi-version)) \ - echo "libffi is built" > $@ - @@ -378,7 +391,9 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ibdir)/python3 - $(call pybuild, unzip, $<, numpy-$(numpy-version)) + export LDFLAGS="$$LDFLAGS -shared"; \ + conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf) $(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ $(ibdir)/python3 @@ -407,7 +422,9 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ $(ipydir)/numpy - $(call pybuild, tar xf, $<, scipy-$(scipy-version)) + export LDFLAGS="$$LDFLAGS -shared"; \ + conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf) $(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \ $(ipydir)/cryptography \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 7b34b31..5dbbd65 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -284,36 +284,39 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ # `rpath_command'. export LDFLAGS=-L$(ildir) - cd $(ddir) - tar xf $< - cd ATLAS - rm -rf build - mkdir build - cd build - ../configure -b 64 -D c -DPentiumCPS=$$core \ - --with-netlib-lapack-tarfile=$(word 2, $^) \ - --cripple-atlas-performance \ - -Fa alg -fPIC --shared \ - --prefix=$(idir) - - # Do the basic build and the extra shared libraries. - make - cd lib && make -f $$sharedmk && cd .. - - # Add RPATH to all the shared libraries. - for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done - - # Install the libraires. - make install - cp -d lib/*.so* $(ildir) - [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir) - ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3 - ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so - ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3 - exit 1 - cd $(ddir) - rm -rf ATLAS - echo "Atlas is built" > $@ + cd $(ddir) \ + && tar xf $< \ + && cd ATLAS \ + && rm -rf build \ + && mkdir build \ + && cd build \ + && ../configure -b 64 -D c -DPentiumCPS=$$core \ + --with-netlib-lapack-tarfile=$(word 2, $^) \ + --cripple-atlas-performance \ + -Fa alg -fPIC --shared \ + --prefix=$(idir) \ + && make \ + && cd lib && make -f $$sharedmk && cd .. \ + && for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done \ + && make install \ + && cp -d lib/*.so* $(ildir) \ + && ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3 \ + && ln -fs $(ildir)/libf77blas.so $(ildir)/libf77blas.so.3 \ + && ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so \ + && ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3; + + # We need to check the existance of `libptlapack.a', but we can't + # do this in the `&&' steps above (it will conflict). So we'll do + # the check after seeing if `libtatlas.so' is installed, then we'll + # finalize the build (delete the untarred directory). + if [ -e $(ildir)/libtatlas.so ]; then \ + [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \ + cd $(ddir); \ + rm -rf ATLAS; \ + echo "Atlas is built" > $@; \ + else \ + echo; echo "ATLAS wasn't installed!!!!"; exit 1; \ + fi -- cgit v1.2.1