aboutsummaryrefslogtreecommitdiff
path: root/reproduce/src
diff options
context:
space:
mode:
authorRaul Infante-Sainz <infantesainz@gmail.com>2019-04-02 16:26:43 +0100
committerRaul Infante-Sainz <infantesainz@gmail.com>2019-04-02 16:26:43 +0100
commitf0030ea8974b41ef4b26c3ce41c3294b224fb358 (patch)
tree56b6051d54c14e2b8fb34bf38d6c14dfa73c9768 /reproduce/src
parentd26372bb074b1e4ac5ff758d7716129975fe4963 (diff)
Numpy and Scipy using OpenBLAS instead of ATLAS
We could not get ATLAS shared libraries on Mac (while the static ATLAS libraries are built and can be used successfully on Mac). So, the pipeline now builds OpenBLAS, which both Numpy and Scipy can use on Mac and GNU/Linux. We also added FFTW as a dependency of Numpy. Altough Numpy is not linking to FFTW for some reason. However, since FFTW is a low level library used by many programs, we have kept it as a dependency of Numpy anyway for now.
Diffstat (limited to 'reproduce/src')
-rw-r--r--reproduce/src/make/dependencies-python.mk24
-rw-r--r--reproduce/src/make/dependencies.mk32
2 files changed, 46 insertions, 10 deletions
diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk
index c7f861b..642316c 100644
--- a/reproduce/src/make/dependencies-python.mk
+++ b/reproduce/src/make/dependencies-python.mk
@@ -43,10 +43,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built
ipydir = $(BDIR)/dependencies/installed/lib/built/python
# Define the top-level programs to build (installed in `.local/bin').
-ifneq ($(on_mac_os),yes)
-withscipy=scipy
-endif
-top-level-python = astroquery matplotlib $(withscipy)
+top-level-python = astroquery matplotlib
all: $(foreach p, $(top-level-python), $(ipydir)/$(p))
# Other basic environment settings: We are only including the host
@@ -328,7 +325,8 @@ $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \
$(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \
$(ipydir)/h5py \
- $(ipydir)/numpy
+ $(ipydir)/numpy \
+ $(ipydir)/scipy
$(call pybuild, tar xf, $<, astropy-$(astropy-version))
$(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \
@@ -398,8 +396,14 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
$(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version))
$(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
- $(ipydir)/setuptools
- export LDFLAGS="$$LDFLAGS -shared"; \
+ $(ipydir)/setuptools \
+ $(ilidir)/openblas \
+ $(ilidir)/fftw
+ if [ x$(on_mac_os) = xyes ]; then \
+ export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \
+ else \
+ export LDFLAGS="$(LDFLAGS) -shared"; \
+ fi; \
conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
$(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf)
@@ -430,7 +434,11 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \
$(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \
$(ipydir)/numpy
- export LDFLAGS="$$LDFLAGS -shared"; \
+ if [ x$(on_mac_os) = xyes ]; then \
+ export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \
+ else \
+ export LDFLAGS="$(LDFLAGS) -shared"; \
+ fi; \
conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
$(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf)
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index 9953311..ee95be9 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -43,8 +43,13 @@ ildir = $(BDIR)/dependencies/installed/lib
ilidir = $(BDIR)/dependencies/installed/lib/built
# Define the top-level programs to build (installed in `.local/bin').
+#
+# About ATLAS: currently the core pipeline does not depend on ATLAS but many
+# high level software depend on it. The current rule for ATLAS is tested
+# successfully on Mac (only static) and GNU/Linux (shared and static). But,
+# since it takes a few hours to build, it is not currently a target.
top-level-programs = astnoisechisel flock metastore unzip zip
-top-level-libraries = freetype atlas
+top-level-libraries = freetype openblas fftw # atlas
all: $(ddir)/texlive-versions.tex \
$(foreach p, $(top-level-programs), $(ibdir)/$(p)) \
$(foreach p, $(top-level-libraries), $(ilidir)/$(p))
@@ -95,6 +100,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
curl-$(curl-version).tar.gz \
flock-$(flock-version).tar.xz \
freetype-$(freetype-version).tar.gz \
+ fftw-$(fftw-version).tar.gz \
ghostscript-$(ghostscript-version).tar.gz \
git-$(git-version).tar.xz \
gnuastro-$(gnuastro-version).tar.lz \
@@ -108,6 +114,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
libgit2-$(libgit2-version).tar.gz \
metastore-$(metastore-version).tar.gz \
unzip-$(unzip-version).tar.gz \
+ openblas-$(openblas-version).tar.gz \
tiff-$(libtiff-version).tar.gz \
wcslib-$(wcslib-version).tar.bz2 \
zip-$(zip-version).tar.gz \
@@ -138,6 +145,7 @@ $(tarballs): $(tdir)/%:
elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12
elif [ $$n = curl ]; then w=https://curl.haxx.se/download
elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version)
+ elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw
elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype
elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926
elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git
@@ -153,6 +161,9 @@ $(tarballs): $(tdir)/%:
mergenames=0
w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
elif [ $$n = metastore ]; then w=http://akhlaghi.org/src
+ elif [ $$n = openblas ]; then
+ mergenames=0
+ w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz
elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff
elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src
mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//')
@@ -234,6 +245,11 @@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz
$(call gbuild, $<, gsl-$(gsl-version), static) \
&& echo "GNU Scientific Library is built" > $@
+$(ilidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz
+ $(call gbuild, $<, fftw-$(fftw-version), static, \
+ --enable-shared) \
+ && echo "FFTW is built" > $@
+
# Freetype is necessary to install matplotlib
$(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \
$(ilidir)/libpng
@@ -303,7 +319,7 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
&& rm -rf build \
&& mkdir build \
&& cd build \
- && echo ../configure -b 64 -D c -DPentiumCPS=$$core \
+ && ../configure -b 64 -D c -DPentiumCPS=$$core \
--with-netlib-lapack-tarfile=$(word 2, $^) \
--cripple-atlas-performance \
-Fa alg -fPIC --shared $$clangflag \
@@ -335,6 +351,18 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
# currently building shared library on Mac.
if [ -f $(ildir)/libatlas.a ]; then echo "Atlas is built" > $@; fi
+$(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz
+ if [ x$(on_mac_os) = xyes ]; then \
+ export CC=clang \
+ fi; \
+ cd $(ddir) \
+ && tar xf $< \
+ && cd OpenBLAS-$(openblas-version) \
+ && make \
+ && make PREFIX=$(idir) install \
+ && cd .. \
+ && rm -rf OpenBLAS-$(openblas-version) \
+ && echo "Libtiff is built" > $@