From 3e583296cb8ed2784f6cb71b53cbe3be4931fa46 Mon Sep 17 00:00:00 2001
From: Raul Infante-Sainz <infantesainz@gmail.com>
Date: Thu, 21 Mar 2019 15:39:06 +0000
Subject: ATLAS and Scipy working on GNU/Linux
Numpy needs ATLAS as shared libraries. So we also need to build Python with
shared libraries. We also need to define site.cfg for numpy and scipy so we
define a master template:
`reproduce/config/pipeline/dependency-numpy-scipy.cfg'
Also `Openssl' did not have rpath so we added with this commit.
---
.file-metadata | Bin 4012 -> 3993 bytes
.../config/pipeline/dependency-numpy-scipy.cfg | 47 +++++++++++++++
reproduce/config/pipeline/dependency-versions.mk | 2 +-
reproduce/src/make/dependencies-basic.mk | 11 +++-
reproduce/src/make/dependencies-python.mk | 51 +++++++++++------
reproduce/src/make/dependencies.mk | 63 +++++++++++----------
6 files changed, 124 insertions(+), 50 deletions(-)
create mode 100644 reproduce/config/pipeline/dependency-numpy-scipy.cfg
diff --git a/.file-metadata b/.file-metadata
index 5a39552..b56ff6c 100644
Binary files a/.file-metadata and b/.file-metadata differ
diff --git a/reproduce/config/pipeline/dependency-numpy-scipy.cfg b/reproduce/config/pipeline/dependency-numpy-scipy.cfg
new file mode 100644
index 0000000..55e93bc
--- /dev/null
+++ b/reproduce/config/pipeline/dependency-numpy-scipy.cfg
@@ -0,0 +1,47 @@
+# THIS IS A COPY OF NUMPY'S site.cfg.example, CUSTOMIZED FOR THIS PIPELINE
+# ------------------------------------------------------------------------
+
+# This file provides configuration information about non-Python dependencies for
+# numpy.distutils-using packages. Create a file like this called "site.cfg" next
+# to your package's setup.py file and fill in the appropriate sections. Not all
+# packages will use all sections so you should leave out sections that your
+# package does not use.
+
+# IMPORTANT NOTE
+# --------------
+#
+# The `ALL' grouping does't apply to ATLAS!!!!!
+
+ [ALL]
+ library_dirs = @LIBDIR@
+ include_dirs = @INCDIR@
+
+# Atlas
+# -----
+# Atlas is an open source optimized implementation of the BLAS and Lapack
+# routines. NumPy will try to build against Atlas by default when available in
+# the system library dirs. To build numpy against a custom installation of
+# Atlas you can add an explicit section such as the following. Here we assume
+# that Atlas was configured with ``prefix=/opt/atlas``.
+#
+ [atlas]
+ library_dirs = @LIBDIR@
+ include_dirs = @INCDIR@
+
+# FFT libraries
+# -------------
+# There are two FFT libraries that we can configure here: FFTW (2 and 3) and djbfft.
+# Note that these libraries are not used by numpy or scipy.
+#
+# http://fftw.org/
+# https://cr.yp.to/djbfft.html
+#
+# Given only this section, numpy.distutils will try to figure out which version
+# of FFTW you are using.
+#[fftw]
+#libraries = fftw3
+#
+# For djbfft, numpy.distutils will look for either djbfft.a or libdjbfft.a .
+#[djbfft]
+#include_dirs = /usr/local/djbfft/include
+#library_dirs = /usr/local/djbfft/lib
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index 609d1cb..79ecdce 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -82,7 +82,7 @@ jeepney-version = 0.4
kiwisolver-version = 1.0.1
keyring-version = 18.0.0
matplotlib-version = 3.0.2
-numpy-version = 1.16.1
+numpy-version = 1.16.2
pip-version = 19.0.2
pycparser-version = 2.19
pyparsing-version = 2.3.1
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index a5cda52..853f6d7 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -566,8 +566,15 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \
--with-zlib-lib=$(ildir) \
--with-zlib-include=$(idir)/include, , , \
./config ) && \
- cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem && \
- echo "OpenSSL is built and ready" > $@
+ cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem; \
+ if [ $$? = 0 ]; then \
+ if [ x$(on_mac_os) = xyes ]; then \
+ echo "No need to fix rpath in libssl"; \
+ else \
+ patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \
+ fi; \
+ echo "OpenSSL is built and ready" > $@; \
+ fi
# GNU Wget
#
diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk
index 3b92f7e..e4c2324 100644
--- a/reproduce/src/make/dependencies-python.mk
+++ b/reproduce/src/make/dependencies-python.mk
@@ -43,7 +43,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built
ipydir = $(BDIR)/dependencies/installed/lib/built/python
# Define the top-level programs to build (installed in `.local/bin').
-top-level-python = astroquery matplotlib #scipy
+top-level-python = astroquery matplotlib scipy
all: $(foreach p, $(top-level-python), $(ipydir)/$(p))
# Other basic environment settings: We are only including the host
@@ -249,18 +249,24 @@ $(tarballs): $(tdir)/%:
# --------------------
#
# To build Python packages with direct access to a `setup.py' (if no direct
-# access to `setup.py' is needed, pip can be used)
+# access to `setup.py' is needed, pip can be used).
# Arguments of this function are the numbers
# 1) Unpack command
# 2) Package name
# 3) Unpacked directory name after unpacking the tarball
-pybuild = cd $(ddir); rm -rf $(3); \
- if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \
- cd $(3); \
- python3 setup.py build && \
- python3 setup.py install && \
- cd .. && rm -rf $(3) && \
- echo "done!" > $@
+# 4) site.cfg file (optional)
+pybuild = cd $(ddir); rm -rf $(3); \
+ if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \
+ cd $(3); \
+ if [ "x$(4)" != x ]; then \
+ sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \
+ -e 's|@INCDIR[@]|'"$(idir)/include"'|' \
+ $(4) > site.cfg; \
+ fi; \
+ python3 setup.py build \
+ && python3 setup.py install \
+ && cd .. && rm -rf $(3) \
+ && echo "done!" > $@
@@ -271,7 +277,12 @@ pybuild = cd $(ddir); rm -rf $(3); \
#
# While this Makefile is for Python programs, in some cases, we need
# certain programs (like Python itself), or libraries for the modules.
-$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz
+$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz
+ $(call gbuild, $<, libffi-$(libffi-version)) \
+ echo "libffi is built" > $@
+
+$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \
+ $(ilidir)/libffi
# On Mac systems, the build complains about `clang' specific
# features, so we can't use our own GCC build here.
# if [ x$(on_mac_os) = xyes ]; then \
@@ -279,16 +290,18 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz
# export CXX=clang++; \
# fi; \
- $(call gbuild, $<, Python-$(python-version)) \
+ $(call gbuild, $<, Python-$(python-version),, \
+ --enable-optimizations \
+ --without-ensurepip \
+ --with-system-ffi \
+ --enable-shared \
+ --with-threads \
+ --with-lto ) \
&& v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \
{printf "%d.%d\n", $$1, $$2}') \
&& ln -s $(ildir)/python$$v $(ildir)/python \
&& rm -rf $(ipydir) && mkdir $(ipydir)
-$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz
- $(call gbuild, $<, libffi-$(libffi-version)) \
- echo "libffi is built" > $@
-
@@ -378,7 +391,9 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
$(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
$(ibdir)/python3
- $(call pybuild, unzip, $<, numpy-$(numpy-version))
+ export LDFLAGS="$$LDFLAGS -shared"; \
+ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
+ $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf)
$(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \
$(ibdir)/python3
@@ -407,7 +422,9 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \
$(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \
$(ipydir)/numpy
- $(call pybuild, tar xf, $<, scipy-$(scipy-version))
+ export LDFLAGS="$$LDFLAGS -shared"; \
+ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
+ $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf)
$(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \
$(ipydir)/cryptography \
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index 7b34b31..5dbbd65 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -284,36 +284,39 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
# `rpath_command'.
export LDFLAGS=-L$(ildir)
- cd $(ddir)
- tar xf $<
- cd ATLAS
- rm -rf build
- mkdir build
- cd build
- ../configure -b 64 -D c -DPentiumCPS=$$core \
- --with-netlib-lapack-tarfile=$(word 2, $^) \
- --cripple-atlas-performance \
- -Fa alg -fPIC --shared \
- --prefix=$(idir)
-
- # Do the basic build and the extra shared libraries.
- make
- cd lib && make -f $$sharedmk && cd ..
-
- # Add RPATH to all the shared libraries.
- for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done
-
- # Install the libraires.
- make install
- cp -d lib/*.so* $(ildir)
- [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir)
- ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3
- ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so
- ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3
- exit 1
- cd $(ddir)
- rm -rf ATLAS
- echo "Atlas is built" > $@
+ cd $(ddir) \
+ && tar xf $< \
+ && cd ATLAS \
+ && rm -rf build \
+ && mkdir build \
+ && cd build \
+ && ../configure -b 64 -D c -DPentiumCPS=$$core \
+ --with-netlib-lapack-tarfile=$(word 2, $^) \
+ --cripple-atlas-performance \
+ -Fa alg -fPIC --shared \
+ --prefix=$(idir) \
+ && make \
+ && cd lib && make -f $$sharedmk && cd .. \
+ && for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done \
+ && make install \
+ && cp -d lib/*.so* $(ildir) \
+ && ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3 \
+ && ln -fs $(ildir)/libf77blas.so $(ildir)/libf77blas.so.3 \
+ && ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so \
+ && ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3;
+
+ # We need to check the existance of `libptlapack.a', but we can't
+ # do this in the `&&' steps above (it will conflict). So we'll do
+ # the check after seeing if `libtatlas.so' is installed, then we'll
+ # finalize the build (delete the untarred directory).
+ if [ -e $(ildir)/libtatlas.so ]; then \
+ [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \
+ cd $(ddir); \
+ rm -rf ATLAS; \
+ echo "Atlas is built" > $@; \
+ else \
+ echo; echo "ATLAS wasn't installed!!!!"; exit 1; \
+ fi
--
cgit v1.2.1