Numpy and Scipy using OpenBLAS instead of ATLAS

We could not get ATLAS shared libraries on Mac (while the static ATLAS
libraries are built and can be used successfully on Mac). So, the
pipeline now builds OpenBLAS, which both Numpy and Scipy can use on Mac
and GNU/Linux.

We also added FFTW as a dependency of Numpy. Altough Numpy is not linking to
FFTW for some reason. However, since FFTW is a low level library used by
many programs, we have kept it as a dependency of Numpy anyway for now.

4 files changed, 54 insertions, 24 deletions

diff --git a/reproduce/config/pipeline/dependency-numpy-scipy.cfg b/reproduce/config/pipeline/dependency-numpy-scipy.cfg
index 55e93bc..0eecf3a 100644
--- a/reproduce/config/pipeline/dependency-numpy-scipy.cfg
+++ b/reproduce/config/pipeline/dependency-numpy-scipy.cfg
@@ -16,17 +16,6 @@
  library_dirs = @LIBDIR@
  include_dirs = @INCDIR@
 
-# Atlas
-# -----
-# Atlas is an open source optimized implementation of the BLAS and Lapack
-# routines. NumPy will try to build against Atlas by default when available in
-# the system library dirs. To build numpy against a custom installation of
-# Atlas you can add an explicit section such as the following. Here we assume
-# that Atlas was configured with ``prefix=/opt/atlas``.
-#
- [atlas]
- library_dirs = @LIBDIR@
- include_dirs = @INCDIR@
 
 # FFT libraries
 # -------------
@@ -38,10 +27,13 @@
 #
 # Given only this section, numpy.distutils will try to figure out which version
 # of FFTW you are using.
-#[fftw]
-#libraries = fftw3
+
+# Even with this section, Numpy apparently does not use fftw. But we will
+# keep it here for the future (if a solution is found).
+[fftw]
+libraries = fftw3
 #
-# For djbfft, numpy.distutils will look for either djbfft.a or libdjbfft.a . 
+# For djbfft, numpy.distutils will look for either djbfft.a or libdjbfft.a .
 #[djbfft]
 #include_dirs = /usr/local/djbfft/include
 #library_dirs = /usr/local/djbfft/lib
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index 211ebbd..cb5920e 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -6,6 +6,7 @@ binutils-version           = 2.31.1
 cmake-version              = 3.12.4
 coreutils-version          = 8.30
 diffutils-version          = 3.7
+fftw-version               = 3.3.8
 findutils-version          = 4.6.0.199-e3fc
 flock-version              = 0.2.3
 freetype-version           = 2.9
@@ -48,6 +49,7 @@ curl-version               = 7.63.0
 gsl-version                = 2.5
 libjpeg-version            = v9b
 libtiff-version            = 4.0.10
+openblas-version           = 0.3.5
 zlib-version               = 1.2.11
 
 # Special libraries
diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk
index c7f861b..642316c 100644
--- a/reproduce/src/make/dependencies-python.mk
+++ b/reproduce/src/make/dependencies-python.mk
@@ -43,10 +43,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built
 ipydir = $(BDIR)/dependencies/installed/lib/built/python
 
 # Define the top-level programs to build (installed in `.local/bin').
-ifneq ($(on_mac_os),yes)
-withscipy=scipy
-endif
-top-level-python   = astroquery matplotlib $(withscipy)
+top-level-python   = astroquery matplotlib
 all: $(foreach p, $(top-level-python), $(ipydir)/$(p))
 
 # Other basic environment settings: We are only including the host
@@ -328,7 +325,8 @@ $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz  \
 
 $(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \
                    $(ipydir)/h5py                            \
-                   $(ipydir)/numpy
+                   $(ipydir)/numpy                           \
+                   $(ipydir)/scipy
 	$(call pybuild, tar xf, $<, astropy-$(astropy-version))
 
 $(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \
@@ -398,8 +396,14 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz   \
 	$(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version))
 
 $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
-                 $(ipydir)/setuptools
-	export LDFLAGS="$$LDFLAGS -shared"; \
+                 $(ipydir)/setuptools               \
+                 $(ilidir)/openblas                 \
+                 $(ilidir)/fftw
+	if [ x$(on_mac_os) = xyes ]; then                                    \
+	  export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle";     \
+	else                                                                 \
+	  export LDFLAGS="$(LDFLAGS) -shared";                               \
+	fi;                                                                  \
 	conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
 	$(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf)
 
@@ -430,7 +434,11 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz   \
 
 $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \
                  $(ipydir)/numpy
-	export LDFLAGS="$$LDFLAGS -shared"; \
+	if [ x$(on_mac_os) = xyes ]; then                                    \
+	  export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle";     \
+	else                                                                 \
+	  export LDFLAGS="$(LDFLAGS) -shared";                               \
+	fi;                                                                  \
 	conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
 	$(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf)
 
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index 9953311..ee95be9 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -43,8 +43,13 @@ ildir  = $(BDIR)/dependencies/installed/lib
 ilidir = $(BDIR)/dependencies/installed/lib/built
 
 # Define the top-level programs to build (installed in `.local/bin').
+#
+# About ATLAS: currently the core pipeline does not depend on ATLAS but many
+# high level software depend on it. The current rule for ATLAS is tested
+# successfully on Mac (only static) and GNU/Linux (shared and static). But,
+# since it takes a few hours to build, it is not currently a target.
 top-level-programs  = astnoisechisel flock metastore unzip zip
-top-level-libraries = freetype atlas
+top-level-libraries = freetype openblas fftw  # atlas
 all: $(ddir)/texlive-versions.tex                       \
      $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \
      $(foreach p, $(top-level-libraries), $(ilidir)/$(p))
@@ -95,6 +100,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz                  \
                         curl-$(curl-version).tar.gz                        \
                         flock-$(flock-version).tar.xz                      \
                         freetype-$(freetype-version).tar.gz                \
+                        fftw-$(fftw-version).tar.gz                        \
                         ghostscript-$(ghostscript-version).tar.gz          \
                         git-$(git-version).tar.xz                          \
                         gnuastro-$(gnuastro-version).tar.lz                \
@@ -108,6 +114,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz                  \
                         libgit2-$(libgit2-version).tar.gz                  \
                         metastore-$(metastore-version).tar.gz              \
                         unzip-$(unzip-version).tar.gz                      \
+                        openblas-$(openblas-version).tar.gz                \
                         tiff-$(libtiff-version).tar.gz                     \
                         wcslib-$(wcslib-version).tar.bz2                   \
                         zip-$(zip-version).tar.gz                          \
@@ -138,6 +145,7 @@ $(tarballs): $(tdir)/%:
 	  elif [ $$n = cmake       ]; then w=https://cmake.org/files/v3.12
 	  elif [ $$n = curl        ]; then w=https://curl.haxx.se/download
 	  elif [ $$n = flock       ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version)
+	  elif [ $$n = fftw        ]; then w=ftp://ftp.fftw.org/pub/fftw
 	  elif [ $$n = freetype    ]; then w=https://download.savannah.gnu.org/releases/freetype
 	  elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926
 	  elif [ $$n = git         ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git
@@ -153,6 +161,9 @@ $(tarballs): $(tdir)/%:
 	    mergenames=0
 	    w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
 	  elif [ $$n = metastore   ]; then w=http://akhlaghi.org/src
+	  elif [ $$n = openblas    ]; then
+	    mergenames=0
+	    w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz
 	  elif [ $$n = tiff        ]; then w=https://download.osgeo.org/libtiff
 	  elif [ $$n = unzip       ]; then w=ftp://ftp.info-zip.org/pub/infozip/src
 	    mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//')
@@ -234,6 +245,11 @@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz
 	$(call gbuild, $<, gsl-$(gsl-version), static) \
 	&& echo "GNU Scientific Library is built" > $@
 
+$(ilidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz
+	$(call gbuild, $<, fftw-$(fftw-version), static,  \
+	               --enable-shared)                   \
+	&& echo "FFTW is built" > $@
+
 # Freetype is necessary to install matplotlib
 $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \
 	                $(ilidir)/libpng
@@ -303,7 +319,7 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
 	&& rm -rf build                                           \
 	&& mkdir build                                            \
 	&& cd build                                               \
-	&& echo ../configure -b 64 -D c -DPentiumCPS=$$core       \
+	&& ../configure -b 64 -D c -DPentiumCPS=$$core            \
 	             --with-netlib-lapack-tarfile=$(word 2, $^)   \
 	             --cripple-atlas-performance                  \
 	             -Fa alg -fPIC --shared $$clangflag           \
@@ -335,6 +351,18 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
         # currently building shared library on Mac.
 	if [ -f $(ildir)/libatlas.a ]; then echo "Atlas is built" > $@; fi
 
+$(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz
+	if [ x$(on_mac_os) = xyes ]; then                           \
+	  export CC=clang                                           \
+	fi;                                                         \
+	cd $(ddir)                                                  \
+	&& tar xf $<                                                \
+	&& cd OpenBLAS-$(openblas-version)                          \
+	&& make                                                     \
+	&& make PREFIX=$(idir) install                              \
+	&& cd ..                                                    \
+	&& rm -rf OpenBLAS-$(openblas-version)                      \
+	&& echo "Libtiff is built" > $@