ATLAS and Scipy working on GNU/Linux

Numpy needs ATLAS as shared libraries. So we also need to build Python with
shared libraries.  We also need to define site.cfg for numpy and scipy so we
define a master template:
`reproduce/config/pipeline/dependency-numpy-scipy.cfg'

Also `Openssl' did not have rpath so we added with this commit.

5 files changed, 124 insertions, 50 deletions

diff --git a/reproduce/config/pipeline/dependency-numpy-scipy.cfg b/reproduce/config/pipeline/dependency-numpy-scipy.cfg
new file mode 100644
index 0000000..55e93bc
--- /dev/null
+++ b/reproduce/config/pipeline/dependency-numpy-scipy.cfg
@@ -0,0 +1,47 @@
+# THIS IS A COPY OF NUMPY'S site.cfg.example, CUSTOMIZED FOR THIS PIPELINE
+# ------------------------------------------------------------------------
+
+# This file provides configuration information about non-Python dependencies for
+# numpy.distutils-using packages. Create a file like this called "site.cfg" next
+# to your package's setup.py file and fill in the appropriate sections. Not all
+# packages will use all sections so you should leave out sections that your
+# package does not use.
+
+# IMPORTANT NOTE
+# --------------
+#
+# The `ALL' grouping does't apply to ATLAS!!!!!
+
+ [ALL]
+ library_dirs = @LIBDIR@
+ include_dirs = @INCDIR@
+
+# Atlas
+# -----
+# Atlas is an open source optimized implementation of the BLAS and Lapack
+# routines. NumPy will try to build against Atlas by default when available in
+# the system library dirs. To build numpy against a custom installation of
+# Atlas you can add an explicit section such as the following. Here we assume
+# that Atlas was configured with ``prefix=/opt/atlas``.
+#
+ [atlas]
+ library_dirs = @LIBDIR@
+ include_dirs = @INCDIR@
+
+# FFT libraries
+# -------------
+# There are two FFT libraries that we can configure here: FFTW (2 and 3) and djbfft.
+# Note that these libraries are not used by numpy or scipy.
+#
+#   http://fftw.org/
+#   https://cr.yp.to/djbfft.html
+#
+# Given only this section, numpy.distutils will try to figure out which version
+# of FFTW you are using.
+#[fftw]
+#libraries = fftw3
+#
+# For djbfft, numpy.distutils will look for either djbfft.a or libdjbfft.a . 
+#[djbfft]
+#include_dirs = /usr/local/djbfft/include
+#library_dirs = /usr/local/djbfft/lib
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index 609d1cb..79ecdce 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -82,7 +82,7 @@ jeepney-version            = 0.4
 kiwisolver-version         = 1.0.1
 keyring-version            = 18.0.0
 matplotlib-version         = 3.0.2
-numpy-version              = 1.16.1
+numpy-version              = 1.16.2
 pip-version                = 19.0.2
 pycparser-version          = 2.19
 pyparsing-version          = 2.3.1
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index a5cda52..853f6d7 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -566,8 +566,15 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz         \
 	               --with-zlib-lib=$(ildir)                      \
                        --with-zlib-include=$(idir)/include, , ,      \
 	               ./config ) &&                                 \
-	cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem &&              \
-	echo "OpenSSL is built and ready" > $@
+	cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem;                \
+	if [ $$? = 0 ]; then                                         \
+	  if [ x$(on_mac_os) = xyes ]; then                          \
+	    echo "No need to fix rpath in libssl";                   \
+	  else                                                       \
+	    patchelf --set-rpath $(ildir) $(ildir)/libssl.so;        \
+	  fi;                                                        \
+	  echo "OpenSSL is built and ready" > $@;                    \
+	fi
 
 # GNU Wget
 #
diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk
index 3b92f7e..e4c2324 100644
--- a/reproduce/src/make/dependencies-python.mk
+++ b/reproduce/src/make/dependencies-python.mk
@@ -43,7 +43,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built
 ipydir = $(BDIR)/dependencies/installed/lib/built/python
 
 # Define the top-level programs to build (installed in `.local/bin').
-top-level-python   = astroquery matplotlib #scipy
+top-level-python   = astroquery matplotlib scipy
 all: $(foreach p, $(top-level-python), $(ipydir)/$(p))
 
 # Other basic environment settings: We are only including the host
@@ -249,18 +249,24 @@ $(tarballs): $(tdir)/%:
 # --------------------
 #
 # To build Python packages with direct access to a `setup.py' (if no direct
-# access to `setup.py' is needed, pip can be used)
+# access to `setup.py' is needed, pip can be used).
 # Arguments of this function are the numbers
 #   1) Unpack command
 #   2) Package name
 #   3) Unpacked directory name after unpacking the tarball
-pybuild = cd $(ddir); rm -rf $(3);                                        \
-	 if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi;             \
-	 cd $(3);                                                             \
-	 python3 setup.py build &&                                            \
-	 python3 setup.py install &&                                          \
-	 cd .. && rm -rf $(3) &&                                              \
-	 echo "done!" > $@
+#   4) site.cfg file (optional)
+pybuild = cd $(ddir); rm -rf $(3);                                \
+	 if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \
+	 cd $(3);                                                 \
+	 if [ "x$(4)" != x ]; then                                \
+	   sed -e 's|@LIBDIR[@]|'"$(ildir)"'|'                    \
+	       -e 's|@INCDIR[@]|'"$(idir)/include"'|'             \
+	       $(4) > site.cfg;                                   \
+	 fi;                                                      \
+	 python3 setup.py build                                   \
+	 && python3 setup.py install                              \
+	 && cd .. && rm -rf $(3)                                  \
+	 && echo "done!" > $@
 
 
 
@@ -271,7 +277,12 @@ pybuild = cd $(ddir); rm -rf $(3);                                        \
 #
 # While this Makefile is for Python programs, in some cases, we need
 # certain programs (like Python itself), or libraries for the modules.
-$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz
+$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz
+	$(call gbuild, $<, libffi-$(libffi-version))        \
+	echo "libffi is built" > $@
+
+$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \
+                  $(ilidir)/libffi
         # On Mac systems, the build complains about `clang' specific
         # features, so we can't use our own GCC build here.
 #	if [ x$(on_mac_os) = xyes ]; then                   \
@@ -279,16 +290,18 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz
 #	  export CXX=clang++;                               \
 #	fi;                                                 \
 
-	$(call gbuild, $<, Python-$(python-version))        \
+	$(call gbuild, $<, Python-$(python-version),,       \
+	       --enable-optimizations                       \
+	       --without-ensurepip                          \
+	       --with-system-ffi                            \
+	       --enable-shared                              \
+	       --with-threads                               \
+	       --with-lto )                                 \
 	&& v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \
 	    {printf "%d.%d\n", $$1, $$2}')                  \
 	&& ln -s $(ildir)/python$$v $(ildir)/python         \
 	&& rm -rf $(ipydir) && mkdir $(ipydir)
 
-$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz
-	$(call gbuild, $<, libffi-$(libffi-version))        \
-	echo "libffi is built" > $@
-
 
 
 
@@ -378,7 +391,9 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz   \
 
 $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
                  $(ibdir)/python3
-	$(call pybuild, unzip, $<, numpy-$(numpy-version))
+	export LDFLAGS="$$LDFLAGS -shared"; \
+	conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
+	$(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf)
 
 $(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \
                $(ibdir)/python3
@@ -407,7 +422,9 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz   \
 
 $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \
                  $(ipydir)/numpy
-	$(call pybuild, tar xf, $<, scipy-$(scipy-version))
+	export LDFLAGS="$$LDFLAGS -shared"; \
+	conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
+	$(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf)
 
 $(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \
                          $(ipydir)/cryptography                                \
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index 7b34b31..5dbbd65 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -284,36 +284,39 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
         # `rpath_command'.
 	export LDFLAGS=-L$(ildir)
 
-	cd $(ddir)
-	tar xf $<
-	cd ATLAS
-	rm -rf build
-	mkdir build
-	cd build
-	../configure -b 64 -D c -DPentiumCPS=$$core  \
-	             --with-netlib-lapack-tarfile=$(word 2, $^) \
-	             --cripple-atlas-performance     \
-	             -Fa alg -fPIC --shared          \
-	             --prefix=$(idir)
-
-        # Do the basic build and the extra shared libraries.
-	make
-	cd lib && make -f $$sharedmk && cd ..
-
-        # Add RPATH to all the shared libraries.
-	for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done
-
-        # Install the libraires.
-	make install
-	cp -d lib/*.so* $(ildir)
-	[ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir)
-	ln -fs $(ildir)/libblas.so         $(ildir)/libblas.so.3
-	ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so
-	ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3
-	exit 1
-	cd $(ddir)
-	rm -rf ATLAS
-	echo "Atlas is built" > $@
+	cd $(ddir)                                                \
+	&& tar xf $<                                              \
+	&& cd ATLAS                                               \
+	&& rm -rf build                                           \
+	&& mkdir build                                            \
+	&& cd build                                               \
+	&& ../configure -b 64 -D c -DPentiumCPS=$$core            \
+	             --with-netlib-lapack-tarfile=$(word 2, $^)   \
+	             --cripple-atlas-performance                  \
+	             -Fa alg -fPIC --shared                       \
+	             --prefix=$(idir)                             \
+	&& make                                                   \
+	&& cd lib && make -f $$sharedmk && cd ..                  \
+	&& for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done \
+	&& make install                                           \
+	&& cp -d lib/*.so* $(ildir)                               \
+	&& ln -fs $(ildir)/libblas.so         $(ildir)/libblas.so.3 \
+	&& ln -fs $(ildir)/libf77blas.so      $(ildir)/libf77blas.so.3 \
+	&& ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so \
+	&& ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3;
+
+        # We need to check the existance of `libptlapack.a', but we can't
+        # do this in the `&&' steps above (it will conflict). So we'll do
+        # the check after seeing if `libtatlas.so' is installed, then we'll
+        # finalize the build (delete the untarred directory).
+	if [ -e $(ildir)/libtatlas.so ]; then                      \
+	  [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \
+	  cd $(ddir);                                              \
+	  rm -rf ATLAS;                                            \
+	  echo "Atlas is built" > $@;                              \
+	else                                                       \
+	  echo; echo "ATLAS wasn't installed!!!!"; exit 1;         \
+	fi