From 75571a551f794223bea2995dc7775a49abd63654 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 20 Mar 2019 17:41:50 +0000 Subject: Including ATLAS in the pipeline, not yet complete An initial installation of atlas is now included in the pipeline, but we are still trying to make it compile and build smoothly. In the process, we found that GCC also needs some modifications (for example rpath issues). --- reproduce/src/make/dependencies.mk | 70 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 3 deletions(-) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index deb9f39..7b34b31 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -44,7 +44,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). top-level-programs = astnoisechisel flock metastore unzip zip -top-level-libraries = freetype +top-level-libraries = atlas freetype all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) @@ -67,12 +67,12 @@ all: $(ddir)/texlive-versions.tex \ .SHELLFLAGS := --noprofile --norc -ec export CCACHE_DISABLE := 1 export PATH := $(ibdir) -export LD_RUN_PATH := $(ildir) -export LD_LIBRARY_PATH := $(ildir) export SHELL := $(ibdir)/bash export CPPFLAGS := -I$(idir)/include export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +export LD_RUN_PATH := $(ildir):$(il64dir) +export LD_LIBRARY_PATH := $(ildir):$(il64dir) export LDFLAGS := $(rpath_command) -L$(ildir) @@ -90,6 +90,7 @@ export LDFLAGS := $(rpath_command) -L$(ildir) # another format, we'll do the modification before the download so the # downloaded file has our desired format. tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ + atlas-$(atlas-version).tar.bz2 \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ flock-$(flock-version).tar.xz \ @@ -100,6 +101,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ gsl-$(gsl-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ + lapack-$(lapack-version).tar.gz \ libbsd-$(libbsd-version).tar.xz \ libpng-$(libpng-version).tar.xz \ libtool-$(libtool-version).tar.xz \ @@ -130,6 +132,9 @@ $(tarballs): $(tdir)/%: : (l==2 ? "%d00\n" \ : "%d000\n") ), $$1)}') w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz + elif [ $$n = atlas ]; then + mergenames=0 + w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) @@ -140,6 +145,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = gsl ]; then w=http://ftpmirror.gnu.org/gnu/gsl elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files + elif [ $$n = lapack ]; then w=http://www.netlib.org/lapack elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool @@ -251,6 +257,64 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ --disable-webp --disable-zstd) \ && echo "Libtiff is built" > $@ +$(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ + $(tdir)/lapack-$(lapack-version).tar.gz + + + # Get the CPU frequency. + if [ x$(on_mac_os) = xyes ]; then + core=2400 + else + core=$$(cat /proc/cpuinfo | grep "cpu MHz" \ + | head -n 1 \ + | sed "s/.*: \([0-9.]*\).*/\1/") + fi + + # See if the shared libraries should be build for a single CPU + # thread or multiple threads. + N=$$(nproc) + srcdir=$$(pwd)/reproduce/src/make + if [ $$N = 1 ]; then + sharedmk=$$srcdir/dependencies-atlas-single.mk + else + sharedmk=$$srcdir/dependencies-atlas-multiple.mk + fi + + # The linking step here doesn't recognize the `-Wl' in the + # `rpath_command'. + export LDFLAGS=-L$(ildir) + + cd $(ddir) + tar xf $< + cd ATLAS + rm -rf build + mkdir build + cd build + ../configure -b 64 -D c -DPentiumCPS=$$core \ + --with-netlib-lapack-tarfile=$(word 2, $^) \ + --cripple-atlas-performance \ + -Fa alg -fPIC --shared \ + --prefix=$(idir) + + # Do the basic build and the extra shared libraries. + make + cd lib && make -f $$sharedmk && cd .. + + # Add RPATH to all the shared libraries. + for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done + + # Install the libraires. + make install + cp -d lib/*.so* $(ildir) + [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir) + ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3 + ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so + ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3 + exit 1 + cd $(ddir) + rm -rf ATLAS + echo "Atlas is built" > $@ + -- cgit v1.2.1 From 3e583296cb8ed2784f6cb71b53cbe3be4931fa46 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 21 Mar 2019 15:39:06 +0000 Subject: ATLAS and Scipy working on GNU/Linux Numpy needs ATLAS as shared libraries. So we also need to build Python with shared libraries. We also need to define site.cfg for numpy and scipy so we define a master template: `reproduce/config/pipeline/dependency-numpy-scipy.cfg' Also `Openssl' did not have rpath so we added with this commit. --- reproduce/src/make/dependencies.mk | 63 ++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 30 deletions(-) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 7b34b31..5dbbd65 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -284,36 +284,39 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ # `rpath_command'. export LDFLAGS=-L$(ildir) - cd $(ddir) - tar xf $< - cd ATLAS - rm -rf build - mkdir build - cd build - ../configure -b 64 -D c -DPentiumCPS=$$core \ - --with-netlib-lapack-tarfile=$(word 2, $^) \ - --cripple-atlas-performance \ - -Fa alg -fPIC --shared \ - --prefix=$(idir) - - # Do the basic build and the extra shared libraries. - make - cd lib && make -f $$sharedmk && cd .. - - # Add RPATH to all the shared libraries. - for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done - - # Install the libraires. - make install - cp -d lib/*.so* $(ildir) - [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir) - ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3 - ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so - ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3 - exit 1 - cd $(ddir) - rm -rf ATLAS - echo "Atlas is built" > $@ + cd $(ddir) \ + && tar xf $< \ + && cd ATLAS \ + && rm -rf build \ + && mkdir build \ + && cd build \ + && ../configure -b 64 -D c -DPentiumCPS=$$core \ + --with-netlib-lapack-tarfile=$(word 2, $^) \ + --cripple-atlas-performance \ + -Fa alg -fPIC --shared \ + --prefix=$(idir) \ + && make \ + && cd lib && make -f $$sharedmk && cd .. \ + && for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done \ + && make install \ + && cp -d lib/*.so* $(ildir) \ + && ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3 \ + && ln -fs $(ildir)/libf77blas.so $(ildir)/libf77blas.so.3 \ + && ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so \ + && ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3; + + # We need to check the existance of `libptlapack.a', but we can't + # do this in the `&&' steps above (it will conflict). So we'll do + # the check after seeing if `libtatlas.so' is installed, then we'll + # finalize the build (delete the untarred directory). + if [ -e $(ildir)/libtatlas.so ]; then \ + [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \ + cd $(ddir); \ + rm -rf ATLAS; \ + echo "Atlas is built" > $@; \ + else \ + echo; echo "ATLAS wasn't installed!!!!"; exit 1; \ + fi -- cgit v1.2.1 From 4ecd180c77850b85441d46ed56512f63f11d61eb Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 22 Mar 2019 16:26:06 +0000 Subject: Modifying ATLAS scripts for building on Mac We generalized the libraries suffixes to work on Mac and GNU/Linux. --- reproduce/src/make/dependencies.mk | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 5dbbd65..f89c176 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -261,10 +261,21 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(tdir)/lapack-$(lapack-version).tar.gz - # Get the CPU frequency. + # Get the operating system specific features (how to get + # CPU frequency and the library suffixes). To make the steps + # more readable, the different library version suffixes are + # named with a single character: `s' for no version in the + # name, `m' for the major version suffix, and `f' for the + # full version suffix. if [ x$(on_mac_os) = xyes ]; then - core=2400 + s=dylib + m=3.dylib + f=3.6.1.dylib + core=$$(sysctl hw.cpufrequency | awk '{print $$2/1000000}') else + s=so + m=so.3 + f=so.3.6.1 core=$$(cat /proc/cpuinfo | grep "cpu MHz" \ | head -n 1 \ | sed "s/.*: \([0-9.]*\).*/\1/") @@ -297,19 +308,22 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ --prefix=$(idir) \ && make \ && cd lib && make -f $$sharedmk && cd .. \ - && for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done \ + && if [ "x$(on_mac_os)" != xyes ]; then \ + for l in lib/*.$$s*; do \ + patchelf --set-rpath $(ildir) $$l; done \ + fi \ && make install \ - && cp -d lib/*.so* $(ildir) \ - && ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3 \ - && ln -fs $(ildir)/libf77blas.so $(ildir)/libf77blas.so.3 \ - && ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so \ - && ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3; + && cp -d lib/*.$$s* $(ildir) \ + && ln -fs $(ildir)/libblas.$$s $(ildir)/libblas.$$m \ + && ln -fs $(ildir)/libf77blas.$$s $(ildir)/libf77blas.$$m \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$s \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$m; # We need to check the existance of `libptlapack.a', but we can't # do this in the `&&' steps above (it will conflict). So we'll do # the check after seeing if `libtatlas.so' is installed, then we'll # finalize the build (delete the untarred directory). - if [ -e $(ildir)/libtatlas.so ]; then \ + if [ -e $(ildir)/libtatlas.$$s ]; then \ [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \ cd $(ddir); \ rm -rf ATLAS; \ -- cgit v1.2.1 From 8748d3e93fbe60511912fbe6ac424a6d6352de4f Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 22 Mar 2019 17:58:45 +0000 Subject: On Mac systems, using a copy of GCC, not a link Until now we were using a symbolic link to replace GCC, but Make doesn't treat symbolic links like files. So it would rebuild the links every time. With this commit, only for GCC on Mac systems, we are actually copying the host's GCC executable to avoid this problem. Also, a wrong comment for cURL was removed. --- reproduce/src/make/dependencies.mk | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index f89c176..d797cdb 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -416,11 +416,6 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ make install && \ cd ..&& rm -rf cmake-$(cmake-version) -# Some programs that depend on cURL (in particular CMake) don't necessarily -# have easiy ways to explicity tell them to also link with libcurl's -# dependencies (libssl, libcrypto, and libz). So we won't force curl to -# only be static. -# # cURL (and its library, which is needed by several programs here) can # optionally link with many different network-related libraries on the host # system that we are not yet building in the pipeline. Many of these are @@ -446,7 +441,7 @@ $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz --without-axtls \ --disable-ldaps \ --disable-ldap \ - --without-nss ) + --without-nss, V=1) # On Mac OS, libtool does different things, so to avoid confusion, we'll # prefix GNU's libtool executables with `glibtool'. -- cgit v1.2.1 From 12648438cf9f75738cef045fd02f4093eab7990d Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 27 Mar 2019 17:16:21 +0000 Subject: GCC is now built on a Mac, not yet ATLAS Until now, we were simply using the host's GCC for Mac systems. But we found that except for a single step (to fixing `rpath'), it works on Mac!!! So, GCC is now part of the Mac build as well. However, we are still having some problems in building ATLAS on Mac. It works on GNU/Linux, but not in Mac. So for the time being (just temporarily), we are avoiding ATLAS (and thus Scipy) on Mac systems. We just filed an issue on the ATLAS discussion list to hopefully fix the problem soon. --- reproduce/src/make/dependencies.mk | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index d797cdb..02fe18a 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -44,7 +44,10 @@ ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). top-level-programs = astnoisechisel flock metastore unzip zip -top-level-libraries = atlas freetype +ifneq ($(on_mac_os),yes) +withatlas = atlas +endif +top-level-libraries = freetype $(withatlas) all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) @@ -260,6 +263,11 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(tdir)/lapack-$(lapack-version).tar.gz + if [ x$(on_mac_os) = xyes ]; then + echo; echo; + echo "ATLAS build instructions not yet working on Mac" + exit 1 + fi # Get the operating system specific features (how to get # CPU frequency and the library suffixes). To make the steps @@ -294,7 +302,6 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ # The linking step here doesn't recognize the `-Wl' in the # `rpath_command'. export LDFLAGS=-L$(ildir) - cd $(ddir) \ && tar xf $< \ && cd ATLAS \ @@ -305,6 +312,14 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ --with-netlib-lapack-tarfile=$(word 2, $^) \ --cripple-atlas-performance \ -Fa alg -fPIC --shared \ + -C xc $(ibdir)/gcc \ + -C gc $(ibdir)/gcc \ + -C if $(ibdir)/gfortran \ + -C ic $(ibdir)/gcc \ + -C dm $(ibdir)/gcc \ + -C sm $(ibdir)/gcc \ + -C dk $(ibdir)/gcc \ + -C sk $(ibdir)/gcc \ --prefix=$(idir) \ && make \ && cd lib && make -f $$sharedmk && cd .. \ -- cgit v1.2.1 From d26372bb074b1e4ac5ff758d7716129975fe4963 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 29 Mar 2019 10:49:31 +0000 Subject: Disabling ATLAS shared libraries on Mac OS The Makefile that build the shared libraries comes from Arch Linux so it does not work easily on Mac. But the full ATLAS build goes successfully for static libraries. For now we are disabling shared libraries on Mac. Python was built explicity with `clang' on Mac. --- reproduce/src/make/dependencies.mk | 54 +++++++++++++++----------------------- 1 file changed, 21 insertions(+), 33 deletions(-) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 02fe18a..9953311 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -44,10 +44,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). top-level-programs = astnoisechisel flock metastore unzip zip -ifneq ($(on_mac_os),yes) -withatlas = atlas -endif -top-level-libraries = freetype $(withatlas) +top-level-libraries = freetype atlas all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) @@ -263,27 +260,25 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(tdir)/lapack-$(lapack-version).tar.gz - if [ x$(on_mac_os) = xyes ]; then - echo; echo; - echo "ATLAS build instructions not yet working on Mac" - exit 1 - fi - # Get the operating system specific features (how to get # CPU frequency and the library suffixes). To make the steps # more readable, the different library version suffixes are # named with a single character: `s' for no version in the # name, `m' for the major version suffix, and `f' for the # full version suffix. + # GCC in Mac OS doesn't work. To work around this issue, on Mac + # systems we force ATLAS to use `clang' instead of `gcc'. if [ x$(on_mac_os) = xyes ]; then s=dylib m=3.dylib f=3.6.1.dylib core=$$(sysctl hw.cpufrequency | awk '{print $$2/1000000}') + clangflag="--force-clang=$(ibdir)/clang" else s=so m=so.3 f=so.3.6.1 + clangflag= core=$$(cat /proc/cpuinfo | grep "cpu MHz" \ | head -n 1 \ | sed "s/.*: \([0-9.]*\).*/\1/") @@ -308,45 +303,38 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ && rm -rf build \ && mkdir build \ && cd build \ - && ../configure -b 64 -D c -DPentiumCPS=$$core \ + && echo ../configure -b 64 -D c -DPentiumCPS=$$core \ --with-netlib-lapack-tarfile=$(word 2, $^) \ --cripple-atlas-performance \ - -Fa alg -fPIC --shared \ - -C xc $(ibdir)/gcc \ - -C gc $(ibdir)/gcc \ - -C if $(ibdir)/gfortran \ - -C ic $(ibdir)/gcc \ - -C dm $(ibdir)/gcc \ - -C sm $(ibdir)/gcc \ - -C dk $(ibdir)/gcc \ - -C sk $(ibdir)/gcc \ + -Fa alg -fPIC --shared $$clangflag \ --prefix=$(idir) \ && make \ - && cd lib && make -f $$sharedmk && cd .. \ && if [ "x$(on_mac_os)" != xyes ]; then \ - for l in lib/*.$$s*; do \ - patchelf --set-rpath $(ildir) $$l; done \ + cd lib && make -f $$sharedmk && cd .. \ + && for l in lib/*.$$s*; do \ + patchelf --set-rpath $(ildir) $$l; done \ + && cp -d lib/*.$$s* $(ildir) \ + && ln -fs $(ildir)/libblas.$$s $(ildir)/libblas.$$m \ + && ln -fs $(ildir)/libf77blas.$$s $(ildir)/libf77blas.$$m \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$s \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$m; \ fi \ - && make install \ - && cp -d lib/*.$$s* $(ildir) \ - && ln -fs $(ildir)/libblas.$$s $(ildir)/libblas.$$m \ - && ln -fs $(ildir)/libf77blas.$$s $(ildir)/libf77blas.$$m \ - && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$s \ - && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$m; + && make install # We need to check the existance of `libptlapack.a', but we can't # do this in the `&&' steps above (it will conflict). So we'll do # the check after seeing if `libtatlas.so' is installed, then we'll # finalize the build (delete the untarred directory). - if [ -e $(ildir)/libtatlas.$$s ]; then \ + if [ "x$(on_mac_os)" != xyes ]; then \ [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \ cd $(ddir); \ rm -rf ATLAS; \ - echo "Atlas is built" > $@; \ - else \ - echo; echo "ATLAS wasn't installed!!!!"; exit 1; \ fi + # We'll check the full installation with the static library (not + # currently building shared library on Mac. + if [ -f $(ildir)/libatlas.a ]; then echo "Atlas is built" > $@; fi + -- cgit v1.2.1 From f0030ea8974b41ef4b26c3ce41c3294b224fb358 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 2 Apr 2019 16:26:43 +0100 Subject: Numpy and Scipy using OpenBLAS instead of ATLAS We could not get ATLAS shared libraries on Mac (while the static ATLAS libraries are built and can be used successfully on Mac). So, the pipeline now builds OpenBLAS, which both Numpy and Scipy can use on Mac and GNU/Linux. We also added FFTW as a dependency of Numpy. Altough Numpy is not linking to FFTW for some reason. However, since FFTW is a low level library used by many programs, we have kept it as a dependency of Numpy anyway for now. --- reproduce/src/make/dependencies.mk | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 9953311..ee95be9 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,8 +43,13 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). +# +# About ATLAS: currently the core pipeline does not depend on ATLAS but many +# high level software depend on it. The current rule for ATLAS is tested +# successfully on Mac (only static) and GNU/Linux (shared and static). But, +# since it takes a few hours to build, it is not currently a target. top-level-programs = astnoisechisel flock metastore unzip zip -top-level-libraries = freetype atlas +top-level-libraries = freetype openblas fftw # atlas all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) @@ -95,6 +100,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ curl-$(curl-version).tar.gz \ flock-$(flock-version).tar.xz \ freetype-$(freetype-version).tar.gz \ + fftw-$(fftw-version).tar.gz \ ghostscript-$(ghostscript-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ @@ -108,6 +114,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ unzip-$(unzip-version).tar.gz \ + openblas-$(openblas-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ zip-$(zip-version).tar.gz \ @@ -138,6 +145,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) + elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git @@ -153,6 +161,9 @@ $(tarballs): $(tdir)/%: mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://akhlaghi.org/src + elif [ $$n = openblas ]; then + mergenames=0 + w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -234,6 +245,11 @@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, $<, gsl-$(gsl-version), static) \ && echo "GNU Scientific Library is built" > $@ +$(ilidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz + $(call gbuild, $<, fftw-$(fftw-version), static, \ + --enable-shared) \ + && echo "FFTW is built" > $@ + # Freetype is necessary to install matplotlib $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ $(ilidir)/libpng @@ -303,7 +319,7 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ && rm -rf build \ && mkdir build \ && cd build \ - && echo ../configure -b 64 -D c -DPentiumCPS=$$core \ + && ../configure -b 64 -D c -DPentiumCPS=$$core \ --with-netlib-lapack-tarfile=$(word 2, $^) \ --cripple-atlas-performance \ -Fa alg -fPIC --shared $$clangflag \ @@ -335,6 +351,18 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ # currently building shared library on Mac. if [ -f $(ildir)/libatlas.a ]; then echo "Atlas is built" > $@; fi +$(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang \ + fi; \ + cd $(ddir) \ + && tar xf $< \ + && cd OpenBLAS-$(openblas-version) \ + && make \ + && make PREFIX=$(idir) install \ + && cd .. \ + && rm -rf OpenBLAS-$(openblas-version) \ + && echo "Libtiff is built" > $@ -- cgit v1.2.1 From aa9de365ae7f85488e8e53acd642cf657de251a8 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 2 Apr 2019 22:43:29 +0100 Subject: Python packages are installed as high level program dependencies Until this commit, the installation of all Python packages were done in a separate Makefile. With this commit, the pipeline install Python packages as part of the hight level software. All Python packages rules them remain in a separate Makefile, but this Makefile is included in the high level dependency `reproduce/src/make/dependencies.mk'. --- reproduce/src/make/dependencies.mk | 43 +++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 19 deletions(-) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index ee95be9..61d0701 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -35,12 +35,14 @@ include reproduce/src/make/dependencies-build-rules.mk include reproduce/config/pipeline/dependency-texlive.mk include reproduce/config/pipeline/dependency-versions.mk + ddir = $(BDIR)/dependencies tdir = $(BDIR)/dependencies/tarballs idir = $(BDIR)/dependencies/installed ibdir = $(BDIR)/dependencies/installed/bin ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built +ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). # @@ -48,11 +50,13 @@ ilidir = $(BDIR)/dependencies/installed/lib/built # high level software depend on it. The current rule for ATLAS is tested # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. +top-level-libraries = # atlas top-level-programs = astnoisechisel flock metastore unzip zip -top-level-libraries = freetype openblas fftw # atlas -all: $(ddir)/texlive-versions.tex \ - $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ - $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) +top-level-python = astroquery matplotlib +all: $(ddir)/texlive-versions.tex \ + $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ + $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ + $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host # operating system's PATH environment variable (after our own!) for the @@ -81,7 +85,8 @@ export LD_LIBRARY_PATH := $(ildir):$(il64dir) export LDFLAGS := $(rpath_command) -L$(ildir) - +# Python packages +include reproduce/src/make/dependencies-python.mk # Tarballs @@ -123,12 +128,12 @@ $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* $@ else - # Remove all numbers, `-' and `.' from the tarball name so we can - # search more easily only with the program name. + # Remove all numbers, `-' and `.' from the tarball name so we can + # search more easily only with the program name. n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ | awk '{print $$1}' ) - # Set the top download link of the requested tarball. + # Set the top download link of the requested tarball. mergenames=1 if [ $$n = cfitsio ]; then mergenames=0 @@ -179,20 +184,20 @@ $(tarballs): $(tdir)/%: exit 1 fi - # Download the requested tarball. Note that some packages may not - # follow our naming convention (where the package name is merged - # with its version number). In such cases, `w' will be the full - # address, not just the top directory address. But since we are - # storing all the tarballs in one directory, we want it to have - # the same naming convention, so we'll download it to a temporary - # name, then rename that. + # Download the requested tarball. Note that some packages may not + # follow our naming convention (where the package name is merged + # with its version number). In such cases, `w' will be the full + # address, not just the top directory address. But since we are + # storing all the tarballs in one directory, we want it to have + # the same naming convention, so we'll download it to a temporary + # name, then rename that. if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*" else tarballurl=$$w fi - # If the download fails, Wget will write the error message in the - # target file, so Make will think that its done! To avoid this - # problem, we'll rename the output. + # If the download fails, Wget will write the error message in the + # target file, so Make will think that its done! To avoid this + # problem, we'll rename the output. echo "Downloading $$tarballurl" if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then rm -f $@ @@ -353,7 +358,7 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz if [ x$(on_mac_os) = xyes ]; then \ - export CC=clang \ + export CC=clang; \ fi; \ cd $(ddir) \ && tar xf $< \ -- cgit v1.2.1 From 2d500f26e9777fc395131301404cf06c0d1a2951 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 3 Apr 2019 15:23:44 +0100 Subject: OpenMPI, HDF5 libraries added for h5py After trying the build a system with no Python library, I noticed that Python's HDF5 module (`h5py') needs the HDF5 library and OpenMPI (to work in parallel). So they were added. Finally `h5py' uses the `mpi4py' module to communicate with OpenMPI, so it was also added. However, for some reason, mpi4py doesn't work with this version of OpenMPI (as described in the comments above). So for now, h5py doesn't use it and can only work on a single thread, while the HDF5 C library links with OpenMPI with no problem. --- reproduce/src/make/dependencies.mk | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 61d0701..a1f1b23 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -110,6 +110,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ gsl-$(gsl-version).tar.gz \ + hdf5-$(hdf5-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ lapack-$(lapack-version).tar.gz \ @@ -118,6 +119,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ + openmpi-$(openmpi-version).tar.gz \ unzip-$(unzip-version).tar.gz \ openblas-$(openblas-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ @@ -152,6 +154,10 @@ $(tarballs): $(tdir)/%: elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype + elif [ $$n = hdf ]; then + mergenames=0 + majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') + w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$* elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro @@ -169,6 +175,10 @@ $(tarballs): $(tdir)/%: elif [ $$n = openblas ]; then mergenames=0 w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz + elif [ $$n = openmpi ]; then + mergenames=0 + majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') + w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -261,6 +271,13 @@ $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ $(call gbuild, $<, freetype-$(freetype-version), static) \ && echo "freetype is built" > $@ +$(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ + $(ilidir)/openmpi + $(call gbuild, $<, hdf5-$(hdf5-version), static, \ + --enable-parallel \ + --enable-fortran, V=1) \ + && echo "HDF5 library is built" > $@ + $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \ && echo "libbsd is built" > $@ @@ -278,6 +295,10 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ --disable-webp --disable-zstd) \ && echo "Libtiff is built" > $@ +$(ilidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz + $(call gbuild, $<, openmpi-$(openmpi-version), static, , V=1) \ + && echo "OpenMPI is built" > $@ + $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(tdir)/lapack-$(lapack-version).tar.gz -- cgit v1.2.1 From ef19dbeb3aa7131754e03ce6d3ccd66db9fd81fd Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 4 Apr 2019 12:30:33 +0100 Subject: Using host GCC on Mac, defining compilers in HDF5 library We wer not able to build `gcc' on Mac, so we are using links to the host compilers. In this commit we also found that on Mac the HDF5 library needs an explicit definition of the compilers. --- reproduce/src/make/dependencies.mk | 2 ++ 1 file changed, 2 insertions(+) (limited to 'reproduce/src/make/dependencies.mk') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index a1f1b23..fe29d1b 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -273,6 +273,8 @@ $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ $(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ $(ilidir)/openmpi + export CC=mpicc; \ + export FC=mpif90; \ $(call gbuild, $<, hdf5-$(hdf5-version), static, \ --enable-parallel \ --enable-fortran, V=1) \ -- cgit v1.2.1