From 75571a551f794223bea2995dc7775a49abd63654 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 20 Mar 2019 17:41:50 +0000 Subject: Including ATLAS in the pipeline, not yet complete An initial installation of atlas is now included in the pipeline, but we are still trying to make it compile and build smoothly. In the process, we found that GCC also needs some modifications (for example rpath issues). --- reproduce/src/make/dependencies-atlas-multiple.mk | 47 +++++++++++++++ reproduce/src/make/dependencies-atlas-single.mk | 29 ++++++++++ reproduce/src/make/dependencies-basic.mk | 33 ++++++++--- reproduce/src/make/dependencies-python.mk | 3 - reproduce/src/make/dependencies.mk | 70 ++++++++++++++++++++++- 5 files changed, 167 insertions(+), 15 deletions(-) create mode 100755 reproduce/src/make/dependencies-atlas-multiple.mk create mode 100755 reproduce/src/make/dependencies-atlas-single.mk (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-atlas-multiple.mk b/reproduce/src/make/dependencies-atlas-multiple.mk new file mode 100755 index 0000000..d66890f --- /dev/null +++ b/reproduce/src/make/dependencies-atlas-multiple.mk @@ -0,0 +1,47 @@ +ORIGLDFLAGS := $(LDFLAGS) + +include Make.inc + +all: libatlas.so libf77blas.so libptf77blas.so libstcblas.so libptcblas.so \ + libblas.so libcblas.so liblapack.so.3.6.1 libptlapack.so.3.6.1 liblapack.so.3 + +libatlas.so: libatlas.a + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \ + --whole-archive libatlas.a --no-whole-archive -lc $(LIBS) + +libf77blas.so : libf77blas.a libatlas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libf77blas.so.3 \ + -o $@ --whole-archive libf77blas.a --no-whole-archive \ + $(F77SYSLIB) -L. -latlas + +libptf77blas.so : libptf77blas.a libatlas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libblas.so.3 \ + -o $@ --whole-archive libptf77blas.a --no-whole-archive \ + $(F77SYSLIB) -L. -latlas + +libstcblas.so : libcblas.a libatlas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libstcblas.so \ + -o $@ --whole-archive libcblas.a -L. -latlas -lblas + +libptcblas.so : libptcblas.a libatlas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libcblas.so \ + -o $@ --whole-archive libptcblas.a -L. -latlas -lblas + +libblas.so: libptf77blas.so + ln -s $< $@ + +libcblas.so: libptcblas.so + ln -s $< $@ + +liblapack.so.3.6.1 : liblapack.a libstcblas.so libf77blas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libstlapack.so.3 \ + -o $@ --whole-archive liblapack.a --no-whole-archive \ + $(F77SYSLIB) -L. -lstcblas -lf77blas + +libptlapack.so.3.6.1 : libptlapack.a libcblas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname liblapack.so.3 \ + -o $@ --whole-archive libptlapack.a --no-whole-archive \ + $(F77SYSLIB) -L. -lcblas -lblas + +liblapack.so.3: libptlapack.so.3.6.1 + ln -s $< $@ diff --git a/reproduce/src/make/dependencies-atlas-single.mk b/reproduce/src/make/dependencies-atlas-single.mk new file mode 100755 index 0000000..12dd19b --- /dev/null +++ b/reproduce/src/make/dependencies-atlas-single.mk @@ -0,0 +1,29 @@ +ORIGLDFLAGS := $(LDFLAGS) + +include Make.inc + +all: libatlas.so libf77blas.so libcblas.so libblas.so liblapack.so.3.6.1 + +libatlas.so: libatlas.a + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \ + --whole-archive libatlas.a --no-whole-archive -lc $(LIBS) + +libf77blas.so : libf77blas.a libatlas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libblas.so.3 \ + -o $@ --whole-archive libf77blas.a --no-whole-archive \ + $(F77SYSLIB) -L. -latlas + +libcblas.so : libcblas.a libatlas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \ + --whole-archive libcblas.a -L. -latlas -lblas + +libblas.so: libf77blas.so + ln -s $< $@ + +liblapack.so.3.6.1 : liblapack.a libcblas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname liblapack.so.3 \ + -o $@ --whole-archive liblapack.a --no-whole-archive \ + $(F77SYSLIB) -L. -lcblas -lblas + +liblapack.so.3: liblapack.so.3.6.1 + ln -s $< $@ diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 5965f00..a5cda52 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -744,6 +744,12 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ # On a macOS, we (currently!) won't build GCC because of some # errors we are still trying to find. So, we'll just make a # symbolic link to the host's executables. + # + # GCC builds is own libraries in '$(idir)/lib64'. But all other + # libraries are in '$(idir)/lib'. Since this pipeline is only for a + # single architecture, we can trick GCC into building its libraries + # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic + # link to '$(idir)/lib'. if [ "x$(on_mac_os)" = xyes ]; then \ $(call makelink,gfortran); \ $(call makelink,gcc); \ @@ -754,12 +760,14 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \ rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\ \ + ln -fs $(ildir) $(idir)/lib64; \ + \ cd $(ddir); \ rm -rf gcc-build gcc-$(gcc-version); \ - tar xf $< && \ - mkdir $(ddir)/gcc-build && \ - cd $(ddir)/gcc-build && \ - ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ + tar xf $< \ + && mkdir $(ddir)/gcc-build \ + && cd $(ddir)/gcc-build \ + && ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ --prefix=$(idir) \ --with-mpc=$(idir) \ --with-mpfr=$(idir) \ @@ -777,9 +785,16 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ --enable-default-pie \ --enable-default-ssp \ --enable-cet=auto \ - --enable-decimal-float && \ - make SHELL=$(ibdir)/bash -j$$(nproc) && \ - make SHELL=$(ibdir)/bash install && \ - cd .. && \ - rm -rf gcc-build gcc-$(gcc-version); \ + --enable-decimal-float \ + && make SHELL=$(ibdir)/bash -j$$(nproc) \ + && make SHELL=$(ibdir)/bash install \ + && cd .. \ + && rm -rf gcc-build gcc-$(gcc-version) \ + \ + && for f in $$(find $(idir)/libexec/gcc); do \ + if ldd $$f &> /dev/null; then \ + patchelf --set-rpath $(ildir) $$f; \ + fi; \ + done; \ fi + diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 946c047..3b92f7e 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -378,9 +378,6 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ibdir)/python3 - export BLAS=None; \ - export ATLAS=None; \ - export LAPACK=None; \ $(call pybuild, unzip, $<, numpy-$(numpy-version)) $(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index deb9f39..7b34b31 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -44,7 +44,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). top-level-programs = astnoisechisel flock metastore unzip zip -top-level-libraries = freetype +top-level-libraries = atlas freetype all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) @@ -67,12 +67,12 @@ all: $(ddir)/texlive-versions.tex \ .SHELLFLAGS := --noprofile --norc -ec export CCACHE_DISABLE := 1 export PATH := $(ibdir) -export LD_RUN_PATH := $(ildir) -export LD_LIBRARY_PATH := $(ildir) export SHELL := $(ibdir)/bash export CPPFLAGS := -I$(idir)/include export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +export LD_RUN_PATH := $(ildir):$(il64dir) +export LD_LIBRARY_PATH := $(ildir):$(il64dir) export LDFLAGS := $(rpath_command) -L$(ildir) @@ -90,6 +90,7 @@ export LDFLAGS := $(rpath_command) -L$(ildir) # another format, we'll do the modification before the download so the # downloaded file has our desired format. tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ + atlas-$(atlas-version).tar.bz2 \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ flock-$(flock-version).tar.xz \ @@ -100,6 +101,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ gsl-$(gsl-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ + lapack-$(lapack-version).tar.gz \ libbsd-$(libbsd-version).tar.xz \ libpng-$(libpng-version).tar.xz \ libtool-$(libtool-version).tar.xz \ @@ -130,6 +132,9 @@ $(tarballs): $(tdir)/%: : (l==2 ? "%d00\n" \ : "%d000\n") ), $$1)}') w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz + elif [ $$n = atlas ]; then + mergenames=0 + w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) @@ -140,6 +145,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = gsl ]; then w=http://ftpmirror.gnu.org/gnu/gsl elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files + elif [ $$n = lapack ]; then w=http://www.netlib.org/lapack elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool @@ -251,6 +257,64 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ --disable-webp --disable-zstd) \ && echo "Libtiff is built" > $@ +$(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ + $(tdir)/lapack-$(lapack-version).tar.gz + + + # Get the CPU frequency. + if [ x$(on_mac_os) = xyes ]; then + core=2400 + else + core=$$(cat /proc/cpuinfo | grep "cpu MHz" \ + | head -n 1 \ + | sed "s/.*: \([0-9.]*\).*/\1/") + fi + + # See if the shared libraries should be build for a single CPU + # thread or multiple threads. + N=$$(nproc) + srcdir=$$(pwd)/reproduce/src/make + if [ $$N = 1 ]; then + sharedmk=$$srcdir/dependencies-atlas-single.mk + else + sharedmk=$$srcdir/dependencies-atlas-multiple.mk + fi + + # The linking step here doesn't recognize the `-Wl' in the + # `rpath_command'. + export LDFLAGS=-L$(ildir) + + cd $(ddir) + tar xf $< + cd ATLAS + rm -rf build + mkdir build + cd build + ../configure -b 64 -D c -DPentiumCPS=$$core \ + --with-netlib-lapack-tarfile=$(word 2, $^) \ + --cripple-atlas-performance \ + -Fa alg -fPIC --shared \ + --prefix=$(idir) + + # Do the basic build and the extra shared libraries. + make + cd lib && make -f $$sharedmk && cd .. + + # Add RPATH to all the shared libraries. + for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done + + # Install the libraires. + make install + cp -d lib/*.so* $(ildir) + [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir) + ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3 + ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so + ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3 + exit 1 + cd $(ddir) + rm -rf ATLAS + echo "Atlas is built" > $@ + -- cgit v1.2.1 From 3e583296cb8ed2784f6cb71b53cbe3be4931fa46 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 21 Mar 2019 15:39:06 +0000 Subject: ATLAS and Scipy working on GNU/Linux Numpy needs ATLAS as shared libraries. So we also need to build Python with shared libraries. We also need to define site.cfg for numpy and scipy so we define a master template: `reproduce/config/pipeline/dependency-numpy-scipy.cfg' Also `Openssl' did not have rpath so we added with this commit. --- reproduce/src/make/dependencies-basic.mk | 11 +++++- reproduce/src/make/dependencies-python.mk | 51 ++++++++++++++++--------- reproduce/src/make/dependencies.mk | 63 ++++++++++++++++--------------- 3 files changed, 76 insertions(+), 49 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index a5cda52..853f6d7 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -566,8 +566,15 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ --with-zlib-lib=$(ildir) \ --with-zlib-include=$(idir)/include, , , \ ./config ) && \ - cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem && \ - echo "OpenSSL is built and ready" > $@ + cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem; \ + if [ $$? = 0 ]; then \ + if [ x$(on_mac_os) = xyes ]; then \ + echo "No need to fix rpath in libssl"; \ + else \ + patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \ + fi; \ + echo "OpenSSL is built and ready" > $@; \ + fi # GNU Wget # diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 3b92f7e..e4c2324 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -43,7 +43,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astroquery matplotlib #scipy +top-level-python = astroquery matplotlib scipy all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host @@ -249,18 +249,24 @@ $(tarballs): $(tdir)/%: # -------------------- # # To build Python packages with direct access to a `setup.py' (if no direct -# access to `setup.py' is needed, pip can be used) +# access to `setup.py' is needed, pip can be used). # Arguments of this function are the numbers # 1) Unpack command # 2) Package name # 3) Unpacked directory name after unpacking the tarball -pybuild = cd $(ddir); rm -rf $(3); \ - if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ - cd $(3); \ - python3 setup.py build && \ - python3 setup.py install && \ - cd .. && rm -rf $(3) && \ - echo "done!" > $@ +# 4) site.cfg file (optional) +pybuild = cd $(ddir); rm -rf $(3); \ + if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ + cd $(3); \ + if [ "x$(4)" != x ]; then \ + sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ + -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ + $(4) > site.cfg; \ + fi; \ + python3 setup.py build \ + && python3 setup.py install \ + && cd .. && rm -rf $(3) \ + && echo "done!" > $@ @@ -271,7 +277,12 @@ pybuild = cd $(ddir); rm -rf $(3); \ # # While this Makefile is for Python programs, in some cases, we need # certain programs (like Python itself), or libraries for the modules. -$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz +$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz + $(call gbuild, $<, libffi-$(libffi-version)) \ + echo "libffi is built" > $@ + +$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \ + $(ilidir)/libffi # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. # if [ x$(on_mac_os) = xyes ]; then \ @@ -279,16 +290,18 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz # export CXX=clang++; \ # fi; \ - $(call gbuild, $<, Python-$(python-version)) \ + $(call gbuild, $<, Python-$(python-version),, \ + --enable-optimizations \ + --without-ensurepip \ + --with-system-ffi \ + --enable-shared \ + --with-threads \ + --with-lto ) \ && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') \ && ln -s $(ildir)/python$$v $(ildir)/python \ && rm -rf $(ipydir) && mkdir $(ipydir) -$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz - $(call gbuild, $<, libffi-$(libffi-version)) \ - echo "libffi is built" > $@ - @@ -378,7 +391,9 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ibdir)/python3 - $(call pybuild, unzip, $<, numpy-$(numpy-version)) + export LDFLAGS="$$LDFLAGS -shared"; \ + conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf) $(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ $(ibdir)/python3 @@ -407,7 +422,9 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ $(ipydir)/numpy - $(call pybuild, tar xf, $<, scipy-$(scipy-version)) + export LDFLAGS="$$LDFLAGS -shared"; \ + conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf) $(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \ $(ipydir)/cryptography \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 7b34b31..5dbbd65 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -284,36 +284,39 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ # `rpath_command'. export LDFLAGS=-L$(ildir) - cd $(ddir) - tar xf $< - cd ATLAS - rm -rf build - mkdir build - cd build - ../configure -b 64 -D c -DPentiumCPS=$$core \ - --with-netlib-lapack-tarfile=$(word 2, $^) \ - --cripple-atlas-performance \ - -Fa alg -fPIC --shared \ - --prefix=$(idir) - - # Do the basic build and the extra shared libraries. - make - cd lib && make -f $$sharedmk && cd .. - - # Add RPATH to all the shared libraries. - for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done - - # Install the libraires. - make install - cp -d lib/*.so* $(ildir) - [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir) - ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3 - ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so - ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3 - exit 1 - cd $(ddir) - rm -rf ATLAS - echo "Atlas is built" > $@ + cd $(ddir) \ + && tar xf $< \ + && cd ATLAS \ + && rm -rf build \ + && mkdir build \ + && cd build \ + && ../configure -b 64 -D c -DPentiumCPS=$$core \ + --with-netlib-lapack-tarfile=$(word 2, $^) \ + --cripple-atlas-performance \ + -Fa alg -fPIC --shared \ + --prefix=$(idir) \ + && make \ + && cd lib && make -f $$sharedmk && cd .. \ + && for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done \ + && make install \ + && cp -d lib/*.so* $(ildir) \ + && ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3 \ + && ln -fs $(ildir)/libf77blas.so $(ildir)/libf77blas.so.3 \ + && ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so \ + && ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3; + + # We need to check the existance of `libptlapack.a', but we can't + # do this in the `&&' steps above (it will conflict). So we'll do + # the check after seeing if `libtatlas.so' is installed, then we'll + # finalize the build (delete the untarred directory). + if [ -e $(ildir)/libtatlas.so ]; then \ + [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \ + cd $(ddir); \ + rm -rf ATLAS; \ + echo "Atlas is built" > $@; \ + else \ + echo; echo "ATLAS wasn't installed!!!!"; exit 1; \ + fi -- cgit v1.2.1 From 5b008fbd6760014283d18986ae26585daf473e0f Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 22 Mar 2019 11:59:04 +0000 Subject: H5py added and setuptools as main Python module dependency In this commit we add `h5py' Python package. We also include `setuptools' as a main dependency of Python because with the previous commit it (as well as `pip') is no longer installed with Python. Numpy version also has been incremented. --- reproduce/src/make/dependencies-python.mk | 63 ++++++++++++++++++------------- reproduce/src/make/initialize.mk | 15 +++++--- 2 files changed, 45 insertions(+), 33 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index e4c2324..54d3a56 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -116,6 +116,7 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ cryptography-$(cryptography-version).tar.gz \ cycler-$(cycler-version).tar.gz \ entrypoints-$(entrypoints-version).tar.gz \ + h5py-$(h5py-version).tar.gz \ html5lib-$(html5lib-version).tar.gz \ idna-$(idna-version).tar.gz \ jeepney-$(jeepney-version).tar.gz \ @@ -157,7 +158,10 @@ $(tarballs): $(tdir)/%: # because the tokenization above will produce `python' as the # first string. if [ $* = python-dateutil-$(python-dateutil-version).tar.gz ]; then - n=dateutil + n=dateutil + elif [ $* = h5py-$(h5py-version).tar.gz ]; then + n=h5py + # elif [ $* = strange-tarball5name-version.tar.gz ]; then # n=strange5-name else @@ -191,13 +195,14 @@ $(tarballs): $(tdir)/%: elif [ $$n = cryptography ]; then h=07/ca/bc827c5e55918ad223d59d299fff92f3563476c3b00d0a9157d9c0217449 elif [ $$n = cycler ]; then h=c2/4b/137dea450d6e1e3d474e1d873cd1d4f7d3beed7e0dc973b06e8e10d32488 elif [ $$n = entrypoints ]; then h=b4/ef/063484f1f9ba3081e920ec9972c96664e2edb9fdc3d8669b0e3b8fc0ad7c + elif [ $$n = h5py ]; then h=43/27/a6e7dcb8ae20a4dbf3725321058923fec262b6f7835179d78ccc8d98deec elif [ $$n = html ]; then h=85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f elif [ $$n = idna ]; then h=ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7 elif [ $$n = jeepney ]; then h=16/1d/74adf3b164a8d19a60d0fcf706a751ffa2a1eaa8e5bbb1b6705c92a05263 elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2 elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864 elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f - elif [ $$n = numpy ]; then h=2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c + elif [ $$n = numpy ]; then h=cf/8d/6345b4f32b37945fedc1e027e83970005fc9c699068d2f566b82826515f2 elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af elif [ $$n = pycparser ]; then h=68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a elif [ $$n = pyparsing ]; then h=b9/b8/6b32b3e84014148dcd60dd05795e35c2e7f4b72f918616c61fdce83d27fc @@ -212,7 +217,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = urllib ]; then h=b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1 elif [ $$n = virtualenv ]; then h=51/aa/c395a6e6eaaedfa5a04723b6446a1df783b16cca6fec66e671cede514688 elif [ $$n = webencodings ]; then h=0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47 -# elif [ $$n = strange5-name ]; then h=XXXXX +# elif [ $$n = strange5-name ]; then h=XXXXX else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." @@ -310,7 +315,7 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \ # # All the necessary Python modules go here. $(ipydir)/asn1crypto: $(tdir)/asn1crypto-$(asn1crypto-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, asn1crypto-$(asn1crypto-version)) $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ @@ -323,6 +328,7 @@ $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ $(call pybuild, tar xf, $<, astroquery-$(astroquery-version)) $(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ + $(ipydir)/h5py \ $(ipydir)/numpy $(call pybuild, tar xf, $<, astropy-$(astropy-version)) @@ -331,80 +337,83 @@ $(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.g $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version)) $(ipydir)/certifi: $(tdir)/certifi-$(certifi-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, certifi-$(certifi-version)) $(ipydir)/cffi: $(tdir)/cffi-$(cffi-version).tar.gz \ - $(ipydir)/pycparser \ - $(ilidir)/libffi + $(ilidir)/libffi \ + $(ipydir)/pycparser $(call pybuild, tar xf, $<, cffi-$(cffi-version)) $(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, chardet-$(chardet-version)) $(ipydir)/cryptography: $(tdir)/cryptography-$(cryptography-version).tar.gz \ - $(ipydir)/cffi \ - $(ipydir)/asn1crypto + $(ipydir)/asn1crypto \ + $(ipydir)/cffi $(call pybuild, tar xf, $<, cryptography-$(cryptography-version)) $(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \ - $(ipydir)/six + $(ipydir)/six $(call pybuild, tar xf, $<, cycler-$(cycler-version)) $(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version)) +$(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, h5py-$(h5py-version)) + $(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \ $(ipydir)/six \ $(ipydir)/webencodings $(call pybuild, tar xf, $<, html5lib-$(html5lib-version)) $(ipydir)/idna: $(tdir)/idna-$(idna-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, idna-$(idna-version)) $(ipydir)/jeepney: $(tdir)/jeepney-$(jeepney-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, jeepney-$(jeepney-version)) $(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \ - $(ipydir)/setuptools_scm \ + $(ipydir)/entrypoints \ $(ipydir)/secretstorage \ - $(ipydir)/entrypoints + $(ipydir)/setuptools_scm $(call pybuild, tar xf, $<, keyring-$(keyring-version)) $(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ $(ipydir)/setuptools $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version)) - $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/cycler \ $(ilidir)/freetype \ + $(ipydir)/kiwisolver \ $(ipydir)/numpy \ $(ipydir)/pyparsing \ - $(ipydir)/python-dateutil \ - $(ipydir)/kiwisolver + $(ipydir)/python-dateutil $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ - $(ibdir)/python3 + $(ipydir)/setuptools export LDFLAGS="$$LDFLAGS -shared"; \ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf) $(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, pip-$(pip-version)) $(ipydir)/pycparser: $(tdir)/pycparser-$(pycparser-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, pycparser-$(pycparser-version)) $(ipydir)/pyparsing: $(tdir)/pyparsing-$(pyparsing-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version)) $(ipydir)/python-dateutil: $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz \ @@ -440,17 +449,17 @@ $(ipydir)/setuptools_scm: $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.g $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version)) $(ipydir)/six: $(tdir)/six-$(six-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, six-$(six-version)) $(ipydir)/soupsieve: $(tdir)/soupsieve-$(soupsieve-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version)) $(ipydir)/urllib3: $(tdir)/urllib3-$(urllib3-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, urllib3-$(urllib3-version)) $(ipydir)/webencodings: $(tdir)/webencodings-$(webencodings-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, webencodings-$(webencodings-version)) diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 545fcc1..66e7c77 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -369,10 +369,6 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\\grepversion}{$(grep-version)}" >> $@ echo "\newcommand{\\gzipversion}{$(gzip-version)}" >> $@ echo "\newcommand{\\islversion}{$(isl-version)}" >> $@ - echo "\newcommand{\\libbsdversion}{$(libbsd-version)}" >> $@ - echo "\newcommand{\\libffiversion}{$(libffi-version)}" >> $@ - echo "\newcommand{\\libpngversion}{$(libpng-version)}" >> $@ - echo "\newcommand{\\libtoolversion}{$(libtool-version)}" >> $@ echo "\newcommand{\\lzipversion}{$(lzip-version)}" >> $@ echo "\newcommand{\\makeversion}{$(make-version)}" >> $@ echo "\newcommand{\\metastoreversion}{$(metastore-version)}" >> $@ @@ -394,12 +390,18 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\\zipversion}{$(zip-version)}" >> $@ # Libraries. + echo "\newcommand{\\atlasversion}{$(cfitsio-version)}" >> $@ echo "\newcommand{\\cfitsioversion}{$(cfitsio-version)}" >> $@ echo "\newcommand{\\curlversion}{$(curl-version)}" >> $@ echo "\newcommand{\\gslversion}{$(gsl-version)}" >> $@ + echo "\newcommand{\\lapack}{$(lapack-version)}" >> $@ + echo "\newcommand{\\libbsdversion}{$(libbsd-version)}" >> $@ + echo "\newcommand{\\libffiversion}{$(libffi-version)}" >> $@ echo "\newcommand{\\libgittwoversion}{$(libgit2-version)}" >> $@ echo "\newcommand{\\libjpegversion}{$(libjpeg-version)}" >> $@ + echo "\newcommand{\\libpngversion}{$(libpng-version)}" >> $@ echo "\newcommand{\\libtiffversion}{$(libtiff-version)}" >> $@ + echo "\newcommand{\\libtoolversion}{$(libtool-version)}" >> $@ echo "\newcommand{\\wcslibversion}{$(wcslib-version)}" >> $@ echo "\newcommand{\\zlibversion}{$(zlib-version)}" >> $@ @@ -414,8 +416,9 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\\cryptographyversion}{$(cryptography-version)}" >> $@ echo "\newcommand{\\cyclerversion}{$(cycler-version)}" >> $@ echo "\newcommand{\\entrypointsversion}{$(entrypoints-version)}" >> $@ + echo "\newcommand{\\hpyversion}{$(h5py-version)}" >> $@ echo "\newcommand{\\htmlfivelibversion}{$(html5lib-version)}" >> $@ - echo "\newcommand{\\indaversion}{$(idna-version)}" >> $@ + echo "\newcommand{\\idaversion}{$(idna-version)}" >> $@ echo "\newcommand{\\jeepneyversion}{$(jeepney-version)}" >> $@ echo "\newcommand{\\kiwisolverversion}{$(kiwisolver-version)}" >> $@ echo "\newcommand{\\keyringversion}{$(keyring-version)}" >> $@ @@ -433,7 +436,7 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\\sixversion}{$(six-version)}" >> $@ echo "\newcommand{\\soupsieveversion}{$(soupsieve-version)}" >> $@ echo "\newcommand{\\urllibthreeversion}{$(urllib3-version)}" >> $@ - echo "\newcommand{\\virtualenvversion}{$(virtualenv-version)}" >> $@ +# echo "\newcommand{\\virtualenvversion}{$(virtualenv-version)}" >> $@ echo "\newcommand{\\webencodingsversion}{$(webencodings-version)}" >> $@ # TeX package versions -- cgit v1.2.1 From 4ecd180c77850b85441d46ed56512f63f11d61eb Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 22 Mar 2019 16:26:06 +0000 Subject: Modifying ATLAS scripts for building on Mac We generalized the libraries suffixes to work on Mac and GNU/Linux. --- reproduce/src/make/dependencies.mk | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 5dbbd65..f89c176 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -261,10 +261,21 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(tdir)/lapack-$(lapack-version).tar.gz - # Get the CPU frequency. + # Get the operating system specific features (how to get + # CPU frequency and the library suffixes). To make the steps + # more readable, the different library version suffixes are + # named with a single character: `s' for no version in the + # name, `m' for the major version suffix, and `f' for the + # full version suffix. if [ x$(on_mac_os) = xyes ]; then - core=2400 + s=dylib + m=3.dylib + f=3.6.1.dylib + core=$$(sysctl hw.cpufrequency | awk '{print $$2/1000000}') else + s=so + m=so.3 + f=so.3.6.1 core=$$(cat /proc/cpuinfo | grep "cpu MHz" \ | head -n 1 \ | sed "s/.*: \([0-9.]*\).*/\1/") @@ -297,19 +308,22 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ --prefix=$(idir) \ && make \ && cd lib && make -f $$sharedmk && cd .. \ - && for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done \ + && if [ "x$(on_mac_os)" != xyes ]; then \ + for l in lib/*.$$s*; do \ + patchelf --set-rpath $(ildir) $$l; done \ + fi \ && make install \ - && cp -d lib/*.so* $(ildir) \ - && ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3 \ - && ln -fs $(ildir)/libf77blas.so $(ildir)/libf77blas.so.3 \ - && ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so \ - && ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3; + && cp -d lib/*.$$s* $(ildir) \ + && ln -fs $(ildir)/libblas.$$s $(ildir)/libblas.$$m \ + && ln -fs $(ildir)/libf77blas.$$s $(ildir)/libf77blas.$$m \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$s \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$m; # We need to check the existance of `libptlapack.a', but we can't # do this in the `&&' steps above (it will conflict). So we'll do # the check after seeing if `libtatlas.so' is installed, then we'll # finalize the build (delete the untarred directory). - if [ -e $(ildir)/libtatlas.so ]; then \ + if [ -e $(ildir)/libtatlas.$$s ]; then \ [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \ cd $(ddir); \ rm -rf ATLAS; \ -- cgit v1.2.1 From 8748d3e93fbe60511912fbe6ac424a6d6352de4f Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 22 Mar 2019 17:58:45 +0000 Subject: On Mac systems, using a copy of GCC, not a link Until now we were using a symbolic link to replace GCC, but Make doesn't treat symbolic links like files. So it would rebuild the links every time. With this commit, only for GCC on Mac systems, we are actually copying the host's GCC executable to avoid this problem. Also, a wrong comment for cURL was removed. --- reproduce/src/make/dependencies-basic.mk | 7 ++++--- reproduce/src/make/dependencies.mk | 7 +------ 2 files changed, 5 insertions(+), 9 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 4d39141..76bfa16 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -206,8 +206,9 @@ makelink = origpath="$$PATH"; \ export PATH=$$(echo $(syspath) | tr : '\n' | grep -v ccache \ | paste -s -d:); \ a=$$(which $(1) 2> /dev/null); \ - if [ -f $(ibdir)/$(1) ]; then rm $(ibdir)/$(1); fi; \ - if [ x$$a != x ]; then ln -s $$a $(ibdir)/$(1); fi; \ + if [ -e $(ibdir)/$(1) ]; then rm $(ibdir)/$(1); fi; \ + if [ x"$(2)" = xcopy ]; then c=cp; else c="ln -s"; fi; \ + if [ x$$a != x ]; then $$c $$a $(ibdir)/$(1); fi; \ export PATH="$$origpath" $(ibdir) $(ildir):; mkdir $@ $(ibdir)/low-level-links: | $(ibdir) $(ildir) @@ -768,7 +769,7 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ if [ "x$(on_mac_os)" = xyes ]; then \ $(call makelink,gfortran); \ $(call makelink,g++); \ - $(call makelink,gcc); \ + $(call makelink,gcc,copy); \ else \ \ rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index f89c176..d797cdb 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -416,11 +416,6 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ make install && \ cd ..&& rm -rf cmake-$(cmake-version) -# Some programs that depend on cURL (in particular CMake) don't necessarily -# have easiy ways to explicity tell them to also link with libcurl's -# dependencies (libssl, libcrypto, and libz). So we won't force curl to -# only be static. -# # cURL (and its library, which is needed by several programs here) can # optionally link with many different network-related libraries on the host # system that we are not yet building in the pipeline. Many of these are @@ -446,7 +441,7 @@ $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz --without-axtls \ --disable-ldaps \ --disable-ldap \ - --without-nss ) + --without-nss, V=1) # On Mac OS, libtool does different things, so to avoid confusion, we'll # prefix GNU's libtool executables with `glibtool'. -- cgit v1.2.1 From 7d1d6f88fb28630cc581ea41c487b2b0c0da269f Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Mon, 25 Mar 2019 14:45:47 +0000 Subject: Corrected makelink command to avoid paste command We just noticed that recently the `paste' command on macOS doesn't work with a pipe. So we are now simply using the `tr' command in reverse to re-create the PATH (to find where to link to). --- reproduce/src/make/dependencies-basic.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 76bfa16..b51f35d 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -202,9 +202,9 @@ $(tarballs): $(tdir)/%: # is very annoying and can cause many complications. We thus remove any # part of PATH of that has `ccache' in it before making symbolic links to # the programs we are not building ourselves. -makelink = origpath="$$PATH"; \ +makelink = origpath="$$PATH"; \ export PATH=$$(echo $(syspath) | tr : '\n' | grep -v ccache \ - | paste -s -d:); \ + | tr '\n' :); \ a=$$(which $(1) 2> /dev/null); \ if [ -e $(ibdir)/$(1) ]; then rm $(ibdir)/$(1); fi; \ if [ x"$(2)" = xcopy ]; then c=cp; else c="ln -s"; fi; \ -- cgit v1.2.1 From 12648438cf9f75738cef045fd02f4093eab7990d Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 27 Mar 2019 17:16:21 +0000 Subject: GCC is now built on a Mac, not yet ATLAS Until now, we were simply using the host's GCC for Mac systems. But we found that except for a single step (to fixing `rpath'), it works on Mac!!! So, GCC is now part of the Mac build as well. However, we are still having some problems in building ATLAS on Mac. It works on GNU/Linux, but not in Mac. So for the time being (just temporarily), we are avoiding ATLAS (and thus Scipy) on Mac systems. We just filed an issue on the ATLAS discussion list to hopefully fix the problem soon. --- reproduce/src/make/dependencies-basic.mk | 37 ++++++++++++++++++------------- reproduce/src/make/dependencies-python.mk | 5 ++++- reproduce/src/make/dependencies.mk | 19 ++++++++++++++-- 3 files changed, 42 insertions(+), 19 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index b51f35d..69d2d84 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -740,13 +740,13 @@ $(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz # # We are currently having problems installing GCC on macOS, so for the time # being, if the pipeline is being run on a macOS, we'll just set a link. -ifeq ($(on_mac_os),yes) -gcc-prerequisites = -else +#ifeq ($(on_mac_os),yes) +#gcc-prerequisites = +#else gcc-prerequisites = $(tdir)/gcc-$(gcc-version).tar.xz \ $(ilidir)/isl \ $(ilidir)/mpc -endif +#endif $(ibdir)/gcc: $(gcc-prerequisites) \ $(ibdir)/ls \ $(ibdir)/sed \ @@ -766,12 +766,15 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ # single architecture, we can trick GCC into building its libraries # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic # link to '$(idir)/lib'. - if [ "x$(on_mac_os)" = xyes ]; then \ - $(call makelink,gfortran); \ - $(call makelink,g++); \ - $(call makelink,gcc,copy); \ - else \ - \ + +# SO FAR IT SEEMS TO BE WORKING ON MAC, BUT MORE TESTS ARE NEEDED TO TOTALLY +# REMOVE THE STEP WHERE WE JUST USE THE HOST'S GCC. +# if [ "x$(on_mac_os)" = xyesno ]; then \ +# $(call makelink,gfortran); \ +# $(call makelink,g++); \ +# $(call makelink,gcc,copy); \ +# else \ + rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\ rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \ rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\ @@ -807,10 +810,12 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ && cd .. \ && rm -rf gcc-build gcc-$(gcc-version) \ \ - && for f in $$(find $(idir)/libexec/gcc); do \ - if ldd $$f &> /dev/null; then \ - patchelf --set-rpath $(ildir) $$f; \ - fi; \ - done; \ - fi + && if [ "x$(on_mac_os)" != xyes ]; then \ + for f in $$(find $(idir)/libexec/gcc); do \ + if ldd $$f &> /dev/null; then \ + patchelf --set-rpath $(ildir) $$f; \ + fi; \ + done; \ + fi; \ +# fi diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 54d3a56..fe82175 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -43,7 +43,10 @@ ilidir = $(BDIR)/dependencies/installed/lib/built ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astroquery matplotlib scipy +ifneq ($(on_mac_os),yes) +withscipy=scipy +endif +top-level-python = astroquery matplotlib $(withscipy) all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index d797cdb..02fe18a 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -44,7 +44,10 @@ ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). top-level-programs = astnoisechisel flock metastore unzip zip -top-level-libraries = atlas freetype +ifneq ($(on_mac_os),yes) +withatlas = atlas +endif +top-level-libraries = freetype $(withatlas) all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) @@ -260,6 +263,11 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(tdir)/lapack-$(lapack-version).tar.gz + if [ x$(on_mac_os) = xyes ]; then + echo; echo; + echo "ATLAS build instructions not yet working on Mac" + exit 1 + fi # Get the operating system specific features (how to get # CPU frequency and the library suffixes). To make the steps @@ -294,7 +302,6 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ # The linking step here doesn't recognize the `-Wl' in the # `rpath_command'. export LDFLAGS=-L$(ildir) - cd $(ddir) \ && tar xf $< \ && cd ATLAS \ @@ -305,6 +312,14 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ --with-netlib-lapack-tarfile=$(word 2, $^) \ --cripple-atlas-performance \ -Fa alg -fPIC --shared \ + -C xc $(ibdir)/gcc \ + -C gc $(ibdir)/gcc \ + -C if $(ibdir)/gfortran \ + -C ic $(ibdir)/gcc \ + -C dm $(ibdir)/gcc \ + -C sm $(ibdir)/gcc \ + -C dk $(ibdir)/gcc \ + -C sk $(ibdir)/gcc \ --prefix=$(idir) \ && make \ && cd lib && make -f $$sharedmk && cd .. \ -- cgit v1.2.1 From d26372bb074b1e4ac5ff758d7716129975fe4963 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 29 Mar 2019 10:49:31 +0000 Subject: Disabling ATLAS shared libraries on Mac OS The Makefile that build the shared libraries comes from Arch Linux so it does not work easily on Mac. But the full ATLAS build goes successfully for static libraries. For now we are disabling shared libraries on Mac. Python was built explicity with `clang' on Mac. --- reproduce/src/make/dependencies-python.mk | 14 +++----- reproduce/src/make/dependencies.mk | 54 ++++++++++++------------------- 2 files changed, 26 insertions(+), 42 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index fe82175..c7f861b 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -293,18 +293,14 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \ $(ilidir)/libffi # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. -# if [ x$(on_mac_os) = xyes ]; then \ -# export CC=clang; \ -# export CXX=clang++; \ -# fi; \ - + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + export CXX=clang++; \ + fi; \ $(call gbuild, $<, Python-$(python-version),, \ - --enable-optimizations \ --without-ensurepip \ --with-system-ffi \ - --enable-shared \ - --with-threads \ - --with-lto ) \ + --enable-shared) \ && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') \ && ln -s $(ildir)/python$$v $(ildir)/python \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 02fe18a..9953311 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -44,10 +44,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). top-level-programs = astnoisechisel flock metastore unzip zip -ifneq ($(on_mac_os),yes) -withatlas = atlas -endif -top-level-libraries = freetype $(withatlas) +top-level-libraries = freetype atlas all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) @@ -263,27 +260,25 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(tdir)/lapack-$(lapack-version).tar.gz - if [ x$(on_mac_os) = xyes ]; then - echo; echo; - echo "ATLAS build instructions not yet working on Mac" - exit 1 - fi - # Get the operating system specific features (how to get # CPU frequency and the library suffixes). To make the steps # more readable, the different library version suffixes are # named with a single character: `s' for no version in the # name, `m' for the major version suffix, and `f' for the # full version suffix. + # GCC in Mac OS doesn't work. To work around this issue, on Mac + # systems we force ATLAS to use `clang' instead of `gcc'. if [ x$(on_mac_os) = xyes ]; then s=dylib m=3.dylib f=3.6.1.dylib core=$$(sysctl hw.cpufrequency | awk '{print $$2/1000000}') + clangflag="--force-clang=$(ibdir)/clang" else s=so m=so.3 f=so.3.6.1 + clangflag= core=$$(cat /proc/cpuinfo | grep "cpu MHz" \ | head -n 1 \ | sed "s/.*: \([0-9.]*\).*/\1/") @@ -308,45 +303,38 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ && rm -rf build \ && mkdir build \ && cd build \ - && ../configure -b 64 -D c -DPentiumCPS=$$core \ + && echo ../configure -b 64 -D c -DPentiumCPS=$$core \ --with-netlib-lapack-tarfile=$(word 2, $^) \ --cripple-atlas-performance \ - -Fa alg -fPIC --shared \ - -C xc $(ibdir)/gcc \ - -C gc $(ibdir)/gcc \ - -C if $(ibdir)/gfortran \ - -C ic $(ibdir)/gcc \ - -C dm $(ibdir)/gcc \ - -C sm $(ibdir)/gcc \ - -C dk $(ibdir)/gcc \ - -C sk $(ibdir)/gcc \ + -Fa alg -fPIC --shared $$clangflag \ --prefix=$(idir) \ && make \ - && cd lib && make -f $$sharedmk && cd .. \ && if [ "x$(on_mac_os)" != xyes ]; then \ - for l in lib/*.$$s*; do \ - patchelf --set-rpath $(ildir) $$l; done \ + cd lib && make -f $$sharedmk && cd .. \ + && for l in lib/*.$$s*; do \ + patchelf --set-rpath $(ildir) $$l; done \ + && cp -d lib/*.$$s* $(ildir) \ + && ln -fs $(ildir)/libblas.$$s $(ildir)/libblas.$$m \ + && ln -fs $(ildir)/libf77blas.$$s $(ildir)/libf77blas.$$m \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$s \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$m; \ fi \ - && make install \ - && cp -d lib/*.$$s* $(ildir) \ - && ln -fs $(ildir)/libblas.$$s $(ildir)/libblas.$$m \ - && ln -fs $(ildir)/libf77blas.$$s $(ildir)/libf77blas.$$m \ - && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$s \ - && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$m; + && make install # We need to check the existance of `libptlapack.a', but we can't # do this in the `&&' steps above (it will conflict). So we'll do # the check after seeing if `libtatlas.so' is installed, then we'll # finalize the build (delete the untarred directory). - if [ -e $(ildir)/libtatlas.$$s ]; then \ + if [ "x$(on_mac_os)" != xyes ]; then \ [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \ cd $(ddir); \ rm -rf ATLAS; \ - echo "Atlas is built" > $@; \ - else \ - echo; echo "ATLAS wasn't installed!!!!"; exit 1; \ fi + # We'll check the full installation with the static library (not + # currently building shared library on Mac. + if [ -f $(ildir)/libatlas.a ]; then echo "Atlas is built" > $@; fi + -- cgit v1.2.1 From f0030ea8974b41ef4b26c3ce41c3294b224fb358 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 2 Apr 2019 16:26:43 +0100 Subject: Numpy and Scipy using OpenBLAS instead of ATLAS We could not get ATLAS shared libraries on Mac (while the static ATLAS libraries are built and can be used successfully on Mac). So, the pipeline now builds OpenBLAS, which both Numpy and Scipy can use on Mac and GNU/Linux. We also added FFTW as a dependency of Numpy. Altough Numpy is not linking to FFTW for some reason. However, since FFTW is a low level library used by many programs, we have kept it as a dependency of Numpy anyway for now. --- reproduce/src/make/dependencies-python.mk | 24 +++++++++++++++-------- reproduce/src/make/dependencies.mk | 32 +++++++++++++++++++++++++++++-- 2 files changed, 46 insertions(+), 10 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index c7f861b..642316c 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -43,10 +43,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). -ifneq ($(on_mac_os),yes) -withscipy=scipy -endif -top-level-python = astroquery matplotlib $(withscipy) +top-level-python = astroquery matplotlib all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host @@ -328,7 +325,8 @@ $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ $(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ $(ipydir)/h5py \ - $(ipydir)/numpy + $(ipydir)/numpy \ + $(ipydir)/scipy $(call pybuild, tar xf, $<, astropy-$(astropy-version)) $(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \ @@ -398,8 +396,14 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ - $(ipydir)/setuptools - export LDFLAGS="$$LDFLAGS -shared"; \ + $(ipydir)/setuptools \ + $(ilidir)/openblas \ + $(ilidir)/fftw + if [ x$(on_mac_os) = xyes ]; then \ + export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ + else \ + export LDFLAGS="$(LDFLAGS) -shared"; \ + fi; \ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf) @@ -430,7 +434,11 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ $(ipydir)/numpy - export LDFLAGS="$$LDFLAGS -shared"; \ + if [ x$(on_mac_os) = xyes ]; then \ + export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ + else \ + export LDFLAGS="$(LDFLAGS) -shared"; \ + fi; \ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf) diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 9953311..ee95be9 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,8 +43,13 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). +# +# About ATLAS: currently the core pipeline does not depend on ATLAS but many +# high level software depend on it. The current rule for ATLAS is tested +# successfully on Mac (only static) and GNU/Linux (shared and static). But, +# since it takes a few hours to build, it is not currently a target. top-level-programs = astnoisechisel flock metastore unzip zip -top-level-libraries = freetype atlas +top-level-libraries = freetype openblas fftw # atlas all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) @@ -95,6 +100,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ curl-$(curl-version).tar.gz \ flock-$(flock-version).tar.xz \ freetype-$(freetype-version).tar.gz \ + fftw-$(fftw-version).tar.gz \ ghostscript-$(ghostscript-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ @@ -108,6 +114,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ unzip-$(unzip-version).tar.gz \ + openblas-$(openblas-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ zip-$(zip-version).tar.gz \ @@ -138,6 +145,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) + elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git @@ -153,6 +161,9 @@ $(tarballs): $(tdir)/%: mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://akhlaghi.org/src + elif [ $$n = openblas ]; then + mergenames=0 + w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -234,6 +245,11 @@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, $<, gsl-$(gsl-version), static) \ && echo "GNU Scientific Library is built" > $@ +$(ilidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz + $(call gbuild, $<, fftw-$(fftw-version), static, \ + --enable-shared) \ + && echo "FFTW is built" > $@ + # Freetype is necessary to install matplotlib $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ $(ilidir)/libpng @@ -303,7 +319,7 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ && rm -rf build \ && mkdir build \ && cd build \ - && echo ../configure -b 64 -D c -DPentiumCPS=$$core \ + && ../configure -b 64 -D c -DPentiumCPS=$$core \ --with-netlib-lapack-tarfile=$(word 2, $^) \ --cripple-atlas-performance \ -Fa alg -fPIC --shared $$clangflag \ @@ -335,6 +351,18 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ # currently building shared library on Mac. if [ -f $(ildir)/libatlas.a ]; then echo "Atlas is built" > $@; fi +$(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang \ + fi; \ + cd $(ddir) \ + && tar xf $< \ + && cd OpenBLAS-$(openblas-version) \ + && make \ + && make PREFIX=$(idir) install \ + && cd .. \ + && rm -rf OpenBLAS-$(openblas-version) \ + && echo "Libtiff is built" > $@ -- cgit v1.2.1 From aa9de365ae7f85488e8e53acd642cf657de251a8 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 2 Apr 2019 22:43:29 +0100 Subject: Python packages are installed as high level program dependencies Until this commit, the installation of all Python packages were done in a separate Makefile. With this commit, the pipeline install Python packages as part of the hight level software. All Python packages rules them remain in a separate Makefile, but this Makefile is included in the high level dependency `reproduce/src/make/dependencies.mk'. --- reproduce/src/make/dependencies-python.mk | 57 ++++--------------------------- reproduce/src/make/dependencies.mk | 43 ++++++++++++----------- 2 files changed, 30 insertions(+), 70 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 642316c..30e10c1 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -29,51 +29,6 @@ -# Top level environment -include reproduce/config/pipeline/LOCAL.mk -include reproduce/src/make/dependencies-build-rules.mk -include reproduce/config/pipeline/dependency-versions.mk - -ddir = $(BDIR)/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib -ilidir = $(BDIR)/dependencies/installed/lib/built -ipydir = $(BDIR)/dependencies/installed/lib/built/python - -# Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astroquery matplotlib -all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) - -# Other basic environment settings: We are only including the host -# operating system's PATH environment variable (after our own!) for the -# compiler and linker. For the library binaries and headers, we are only -# using our internally built libraries. -# -# To investigate: -# -# 1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all -# the parameters defined bellow as `NAME=VALUE' statements before -# calling Bash. This will enable us to completely ignore the user's -# native environment. -# -# 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the -# user's environment. -.ONESHELL: -.SHELLFLAGS := --noprofile --norc -ec -export CCACHE_DISABLE := 1 -export PATH := $(ibdir) -export LD_RUN_PATH := $(ildir) -export LD_LIBRARY_PATH := $(ildir) -export SHELL := $(ibdir)/bash -export CPPFLAGS := -I$(idir)/include -export PKG_CONFIG_PATH := $(ildir)/pkgconfig -export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export LDFLAGS := $(rpath_command) -L$(ildir) - - - # Python enviroment @@ -106,7 +61,7 @@ export MPI_PYTHON3_SITEARCH := # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. -tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ +pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ astroquery-$(astroquery-version).tar.gz \ astropy-$(astropy-version).tar.gz \ beautifulsoup4-$(beautifulsoup4-version).tar.gz \ @@ -141,8 +96,8 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ webencodings-$(webencodings-version).tar.gz \ virtualenv-$(virtualenv-version).tar.gz \ , $(tdir)/$(t) ) -topurl=https://files.pythonhosted.org/packages -$(tarballs): $(tdir)/%: +pytopurl=https://files.pythonhosted.org/packages +$(pytarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* $@ else @@ -184,7 +139,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = secretstorage ]; then mergenames=0 hash=a6/89/df343dbc2957a317127e7ff2983230dc5336273be34f2e1911519d85aeb5 - h=$(topurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz + h=$(pytopurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz elif [ $$n = asn ]; then h=fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4 elif [ $$n = astroquery ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396 elif [ $$n = astropy ]; then h=eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 @@ -232,7 +187,7 @@ $(tarballs): $(tdir)/%: # storing all the tarballs in one directory, we want it to have # the same naming convention, so we'll download it to a temporary # name, then rename that. - if [ $$mergenames = 1 ]; then tarballurl=$(topurl)/$$h/"$*" + if [ $$mergenames = 1 ]; then tarballurl=$(pytopurl)/$$h/"$*" else tarballurl=$$h fi @@ -383,7 +338,7 @@ $(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \ $(call pybuild, tar xf, $<, keyring-$(keyring-version)) $(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ - $(ipydir)/setuptools + $(ipydir)/setuptools $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version)) $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index ee95be9..61d0701 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -35,12 +35,14 @@ include reproduce/src/make/dependencies-build-rules.mk include reproduce/config/pipeline/dependency-texlive.mk include reproduce/config/pipeline/dependency-versions.mk + ddir = $(BDIR)/dependencies tdir = $(BDIR)/dependencies/tarballs idir = $(BDIR)/dependencies/installed ibdir = $(BDIR)/dependencies/installed/bin ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built +ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). # @@ -48,11 +50,13 @@ ilidir = $(BDIR)/dependencies/installed/lib/built # high level software depend on it. The current rule for ATLAS is tested # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. +top-level-libraries = # atlas top-level-programs = astnoisechisel flock metastore unzip zip -top-level-libraries = freetype openblas fftw # atlas -all: $(ddir)/texlive-versions.tex \ - $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ - $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) +top-level-python = astroquery matplotlib +all: $(ddir)/texlive-versions.tex \ + $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ + $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ + $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host # operating system's PATH environment variable (after our own!) for the @@ -81,7 +85,8 @@ export LD_LIBRARY_PATH := $(ildir):$(il64dir) export LDFLAGS := $(rpath_command) -L$(ildir) - +# Python packages +include reproduce/src/make/dependencies-python.mk # Tarballs @@ -123,12 +128,12 @@ $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* $@ else - # Remove all numbers, `-' and `.' from the tarball name so we can - # search more easily only with the program name. + # Remove all numbers, `-' and `.' from the tarball name so we can + # search more easily only with the program name. n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ | awk '{print $$1}' ) - # Set the top download link of the requested tarball. + # Set the top download link of the requested tarball. mergenames=1 if [ $$n = cfitsio ]; then mergenames=0 @@ -179,20 +184,20 @@ $(tarballs): $(tdir)/%: exit 1 fi - # Download the requested tarball. Note that some packages may not - # follow our naming convention (where the package name is merged - # with its version number). In such cases, `w' will be the full - # address, not just the top directory address. But since we are - # storing all the tarballs in one directory, we want it to have - # the same naming convention, so we'll download it to a temporary - # name, then rename that. + # Download the requested tarball. Note that some packages may not + # follow our naming convention (where the package name is merged + # with its version number). In such cases, `w' will be the full + # address, not just the top directory address. But since we are + # storing all the tarballs in one directory, we want it to have + # the same naming convention, so we'll download it to a temporary + # name, then rename that. if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*" else tarballurl=$$w fi - # If the download fails, Wget will write the error message in the - # target file, so Make will think that its done! To avoid this - # problem, we'll rename the output. + # If the download fails, Wget will write the error message in the + # target file, so Make will think that its done! To avoid this + # problem, we'll rename the output. echo "Downloading $$tarballurl" if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then rm -f $@ @@ -353,7 +358,7 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz if [ x$(on_mac_os) = xyes ]; then \ - export CC=clang \ + export CC=clang; \ fi; \ cd $(ddir) \ && tar xf $< \ -- cgit v1.2.1 From 2d500f26e9777fc395131301404cf06c0d1a2951 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 3 Apr 2019 15:23:44 +0100 Subject: OpenMPI, HDF5 libraries added for h5py After trying the build a system with no Python library, I noticed that Python's HDF5 module (`h5py') needs the HDF5 library and OpenMPI (to work in parallel). So they were added. Finally `h5py' uses the `mpi4py' module to communicate with OpenMPI, so it was also added. However, for some reason, mpi4py doesn't work with this version of OpenMPI (as described in the comments above). So for now, h5py doesn't use it and can only work on a single thread, while the HDF5 C library links with OpenMPI with no problem. --- reproduce/src/make/dependencies-python.mk | 24 ++++++++++++++++++++++-- reproduce/src/make/dependencies.mk | 21 +++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 30e10c1..6711d17 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -39,9 +39,9 @@ # systems which might interfere. To be safe, we are removing all their # values. export PYTHONPATH := $(installdir)/lib/python/site-packages +export PYTHONPATH2 := $(PYTHONPATH) export PYTHONPATH3 := $(PYTHONPATH) export _LMFILES_ := -export PYTHONPATH2 := export LOADEDMODULES := export MPI_PYTHON_SITEARCH := export MPI_PYTHON2_SITEARCH := @@ -79,6 +79,7 @@ pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ keyring-$(keyring-version).tar.gz \ libffi-$(libffi-version).tar.gz \ matplotlib-$(matplotlib-version).tar.gz \ + mpi4py-$(mpi4py-version).tar.gz \ numpy-$(numpy-version).zip \ pip-$(pip-version).tar.gz \ pycparser-$(pycparser-version).tar.gz \ @@ -157,6 +158,7 @@ $(pytarballs): $(tdir)/%: elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2 elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864 elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f + elif [ $$n = mpi ]; then h=55/a2/c827b196070e161357b49287fa46d69f25641930fd5f854722319d431843 elif [ $$n = numpy ]; then h=cf/8d/6345b4f32b37945fedc1e027e83970005fc9c699068d2f566b82826515f2 elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af elif [ $$n = pycparser ]; then h=68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a @@ -315,7 +317,11 @@ $(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version)) $(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \ - $(ipydir)/setuptools + $(ipydir)/setuptools \ + $(ilidir)/hdf5 + # $(ipydir)/mpi4py # AFTER its problem is fixed. + #export HDF5_MPI=ON; # AFTER its problem is fixed. + export HDF5_DIR=$(ildir); \ $(call pybuild, tar xf, $<, h5py-$(h5py-version)) $(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \ @@ -350,6 +356,20 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/python-dateutil $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) +# Currently mpi4py doesn't build because of some conflict with OpenMPI: +# +# In file included from src/mpi4py.MPI.c:591, +# from src/MPI.c:4: +# src/mpi4py.MPI.c: In function '__pyx_f_6mpi4py_3MPI_del_Datatype': +# src/mpi4py.MPI.c:15094:36: error: expected expression before '_Static_assert' +# __pyx_t_1 = (((__pyx_v_ob[0]) == MPI_UB) != 0); +# +# But atleast on my system it fails. +$(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \ + $(ipydir)/setuptools \ + $(ilidir)/openmpi + $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version)) + $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ipydir)/setuptools \ $(ilidir)/openblas \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 61d0701..a1f1b23 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -110,6 +110,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ gsl-$(gsl-version).tar.gz \ + hdf5-$(hdf5-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ lapack-$(lapack-version).tar.gz \ @@ -118,6 +119,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ + openmpi-$(openmpi-version).tar.gz \ unzip-$(unzip-version).tar.gz \ openblas-$(openblas-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ @@ -152,6 +154,10 @@ $(tarballs): $(tdir)/%: elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype + elif [ $$n = hdf ]; then + mergenames=0 + majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') + w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$* elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro @@ -169,6 +175,10 @@ $(tarballs): $(tdir)/%: elif [ $$n = openblas ]; then mergenames=0 w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz + elif [ $$n = openmpi ]; then + mergenames=0 + majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') + w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -261,6 +271,13 @@ $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ $(call gbuild, $<, freetype-$(freetype-version), static) \ && echo "freetype is built" > $@ +$(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ + $(ilidir)/openmpi + $(call gbuild, $<, hdf5-$(hdf5-version), static, \ + --enable-parallel \ + --enable-fortran, V=1) \ + && echo "HDF5 library is built" > $@ + $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \ && echo "libbsd is built" > $@ @@ -278,6 +295,10 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ --disable-webp --disable-zstd) \ && echo "Libtiff is built" > $@ +$(ilidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz + $(call gbuild, $<, openmpi-$(openmpi-version), static, , V=1) \ + && echo "OpenMPI is built" > $@ + $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(tdir)/lapack-$(lapack-version).tar.gz -- cgit v1.2.1 From ef19dbeb3aa7131754e03ce6d3ccd66db9fd81fd Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 4 Apr 2019 12:30:33 +0100 Subject: Using host GCC on Mac, defining compilers in HDF5 library We wer not able to build `gcc' on Mac, so we are using links to the host compilers. In this commit we also found that on Mac the HDF5 library needs an explicit definition of the compilers. --- reproduce/src/make/dependencies-basic.mk | 19 ++++++++++--------- reproduce/src/make/dependencies.mk | 2 ++ 2 files changed, 12 insertions(+), 9 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 69d2d84..74a04ed 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -767,14 +767,15 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic # link to '$(idir)/lib'. -# SO FAR IT SEEMS TO BE WORKING ON MAC, BUT MORE TESTS ARE NEEDED TO TOTALLY -# REMOVE THE STEP WHERE WE JUST USE THE HOST'S GCC. -# if [ "x$(on_mac_os)" = xyesno ]; then \ -# $(call makelink,gfortran); \ -# $(call makelink,g++); \ -# $(call makelink,gcc,copy); \ -# else \ - +# SO FAR WE HAVEN'T BEEN ABLE TO GET A CONSISTENT BUILD OF GCC ON MAC +# (SOMETIMES IT CRASHES IN libiberty with g++) AND SOMETIMES IT FINISHES, +# SO, MORE TESTS ARE NEEDED ON MAC AND WE'LL USE THE HOST'S COMPILER UNTIL +# THEN. + if [ "x$(on_mac_os)" = xyesno ]; then \ + $(call makelink,gfortran); \ + $(call makelink,g++); \ + $(call makelink,gcc,copy); \ + else \ rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\ rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \ rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\ @@ -817,5 +818,5 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ fi; \ done; \ fi; \ -# fi + fi diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index a1f1b23..fe29d1b 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -273,6 +273,8 @@ $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ $(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ $(ilidir)/openmpi + export CC=mpicc; \ + export FC=mpif90; \ $(call gbuild, $<, hdf5-$(hdf5-version), static, \ --enable-parallel \ --enable-fortran, V=1) \ -- cgit v1.2.1