From 75571a551f794223bea2995dc7775a49abd63654 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 20 Mar 2019 17:41:50 +0000 Subject: Including ATLAS in the pipeline, not yet complete An initial installation of atlas is now included in the pipeline, but we are still trying to make it compile and build smoothly. In the process, we found that GCC also needs some modifications (for example rpath issues). --- reproduce/config/pipeline/dependency-versions.mk | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) (limited to 'reproduce/config/pipeline') diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 7cdddac..609d1cb 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -42,6 +42,7 @@ xz-version = 5.2.4 zip-version = 3.0 # Libraries +atlas-version = 3.10.3 cfitsio-version = 3.45 curl-version = 7.63.0 gsl-version = 2.5 @@ -49,6 +50,17 @@ libjpeg-version = v9b libtiff-version = 4.0.10 zlib-version = 1.2.11 +# Special libraries +# ----------------- +# +# When updating the version of these libraries, please look into the build +# rule first: In one way or another, the version string becomes necessary +# during their build and must be accounted for. +bzip2-version = 1.0.6 +lapack-version = 3.8.0 +libgit2-version = 0.26.0 +wcslib-version = 6.2 + # Python packages # --------------- # @@ -86,12 +98,3 @@ urllib3-version = 1.24.1 virtualenv-version = 16.4.0 webencodings-version = 0.5.1 -# Special libraries -# ----------------- -# -# When updating the version of these libraries, please look into the build -# rule first: In one way or another, the version string becomes necessary -# during their build and must be accounted for. -bzip2-version = 1.0.6 -libgit2-version = 0.26.0 -wcslib-version = 6.2 -- cgit v1.2.1 From 3e583296cb8ed2784f6cb71b53cbe3be4931fa46 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 21 Mar 2019 15:39:06 +0000 Subject: ATLAS and Scipy working on GNU/Linux Numpy needs ATLAS as shared libraries. So we also need to build Python with shared libraries. We also need to define site.cfg for numpy and scipy so we define a master template: `reproduce/config/pipeline/dependency-numpy-scipy.cfg' Also `Openssl' did not have rpath so we added with this commit. --- .../config/pipeline/dependency-numpy-scipy.cfg | 47 ++++++++++++++++++++++ reproduce/config/pipeline/dependency-versions.mk | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 reproduce/config/pipeline/dependency-numpy-scipy.cfg (limited to 'reproduce/config/pipeline') diff --git a/reproduce/config/pipeline/dependency-numpy-scipy.cfg b/reproduce/config/pipeline/dependency-numpy-scipy.cfg new file mode 100644 index 0000000..55e93bc --- /dev/null +++ b/reproduce/config/pipeline/dependency-numpy-scipy.cfg @@ -0,0 +1,47 @@ +# THIS IS A COPY OF NUMPY'S site.cfg.example, CUSTOMIZED FOR THIS PIPELINE +# ------------------------------------------------------------------------ + +# This file provides configuration information about non-Python dependencies for +# numpy.distutils-using packages. Create a file like this called "site.cfg" next +# to your package's setup.py file and fill in the appropriate sections. Not all +# packages will use all sections so you should leave out sections that your +# package does not use. + +# IMPORTANT NOTE +# -------------- +# +# The `ALL' grouping does't apply to ATLAS!!!!! + + [ALL] + library_dirs = @LIBDIR@ + include_dirs = @INCDIR@ + +# Atlas +# ----- +# Atlas is an open source optimized implementation of the BLAS and Lapack +# routines. NumPy will try to build against Atlas by default when available in +# the system library dirs. To build numpy against a custom installation of +# Atlas you can add an explicit section such as the following. Here we assume +# that Atlas was configured with ``prefix=/opt/atlas``. +# + [atlas] + library_dirs = @LIBDIR@ + include_dirs = @INCDIR@ + +# FFT libraries +# ------------- +# There are two FFT libraries that we can configure here: FFTW (2 and 3) and djbfft. +# Note that these libraries are not used by numpy or scipy. +# +# http://fftw.org/ +# https://cr.yp.to/djbfft.html +# +# Given only this section, numpy.distutils will try to figure out which version +# of FFTW you are using. +#[fftw] +#libraries = fftw3 +# +# For djbfft, numpy.distutils will look for either djbfft.a or libdjbfft.a . +#[djbfft] +#include_dirs = /usr/local/djbfft/include +#library_dirs = /usr/local/djbfft/lib diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 609d1cb..79ecdce 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -82,7 +82,7 @@ jeepney-version = 0.4 kiwisolver-version = 1.0.1 keyring-version = 18.0.0 matplotlib-version = 3.0.2 -numpy-version = 1.16.1 +numpy-version = 1.16.2 pip-version = 19.0.2 pycparser-version = 2.19 pyparsing-version = 2.3.1 -- cgit v1.2.1 From 5b008fbd6760014283d18986ae26585daf473e0f Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 22 Mar 2019 11:59:04 +0000 Subject: H5py added and setuptools as main Python module dependency In this commit we add `h5py' Python package. We also include `setuptools' as a main dependency of Python because with the previous commit it (as well as `pip') is no longer installed with Python. Numpy version also has been incremented. --- reproduce/config/pipeline/dependency-versions.mk | 1 + 1 file changed, 1 insertion(+) (limited to 'reproduce/config/pipeline') diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 79ecdce..211ebbd 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -76,6 +76,7 @@ chardet-version = 3.0.4 cryptography-version = 2.6.1 cycler-version = 0.10.0 entrypoints-version = 0.3 +h5py-version = 2.9.0 html5lib-version = 1.0.1 idna-version = 2.8 jeepney-version = 0.4 -- cgit v1.2.1 From f0030ea8974b41ef4b26c3ce41c3294b224fb358 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 2 Apr 2019 16:26:43 +0100 Subject: Numpy and Scipy using OpenBLAS instead of ATLAS We could not get ATLAS shared libraries on Mac (while the static ATLAS libraries are built and can be used successfully on Mac). So, the pipeline now builds OpenBLAS, which both Numpy and Scipy can use on Mac and GNU/Linux. We also added FFTW as a dependency of Numpy. Altough Numpy is not linking to FFTW for some reason. However, since FFTW is a low level library used by many programs, we have kept it as a dependency of Numpy anyway for now. --- reproduce/config/pipeline/dependency-numpy-scipy.cfg | 20 ++++++-------------- reproduce/config/pipeline/dependency-versions.mk | 2 ++ 2 files changed, 8 insertions(+), 14 deletions(-) (limited to 'reproduce/config/pipeline') diff --git a/reproduce/config/pipeline/dependency-numpy-scipy.cfg b/reproduce/config/pipeline/dependency-numpy-scipy.cfg index 55e93bc..0eecf3a 100644 --- a/reproduce/config/pipeline/dependency-numpy-scipy.cfg +++ b/reproduce/config/pipeline/dependency-numpy-scipy.cfg @@ -16,17 +16,6 @@ library_dirs = @LIBDIR@ include_dirs = @INCDIR@ -# Atlas -# ----- -# Atlas is an open source optimized implementation of the BLAS and Lapack -# routines. NumPy will try to build against Atlas by default when available in -# the system library dirs. To build numpy against a custom installation of -# Atlas you can add an explicit section such as the following. Here we assume -# that Atlas was configured with ``prefix=/opt/atlas``. -# - [atlas] - library_dirs = @LIBDIR@ - include_dirs = @INCDIR@ # FFT libraries # ------------- @@ -38,10 +27,13 @@ # # Given only this section, numpy.distutils will try to figure out which version # of FFTW you are using. -#[fftw] -#libraries = fftw3 + +# Even with this section, Numpy apparently does not use fftw. But we will +# keep it here for the future (if a solution is found). +[fftw] +libraries = fftw3 # -# For djbfft, numpy.distutils will look for either djbfft.a or libdjbfft.a . +# For djbfft, numpy.distutils will look for either djbfft.a or libdjbfft.a . #[djbfft] #include_dirs = /usr/local/djbfft/include #library_dirs = /usr/local/djbfft/lib diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 211ebbd..cb5920e 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -6,6 +6,7 @@ binutils-version = 2.31.1 cmake-version = 3.12.4 coreutils-version = 8.30 diffutils-version = 3.7 +fftw-version = 3.3.8 findutils-version = 4.6.0.199-e3fc flock-version = 0.2.3 freetype-version = 2.9 @@ -48,6 +49,7 @@ curl-version = 7.63.0 gsl-version = 2.5 libjpeg-version = v9b libtiff-version = 4.0.10 +openblas-version = 0.3.5 zlib-version = 1.2.11 # Special libraries -- cgit v1.2.1 From 2d500f26e9777fc395131301404cf06c0d1a2951 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 3 Apr 2019 15:23:44 +0100 Subject: OpenMPI, HDF5 libraries added for h5py After trying the build a system with no Python library, I noticed that Python's HDF5 module (`h5py') needs the HDF5 library and OpenMPI (to work in parallel). So they were added. Finally `h5py' uses the `mpi4py' module to communicate with OpenMPI, so it was also added. However, for some reason, mpi4py doesn't work with this version of OpenMPI (as described in the comments above). So for now, h5py doesn't use it and can only work on a single thread, while the HDF5 C library links with OpenMPI with no problem. --- reproduce/config/pipeline/dependency-versions.mk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'reproduce/config/pipeline') diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index cb5920e..4cb8a71 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -18,6 +18,7 @@ gmp-version = 6.1.2 gnuastro-version = 0.8 grep-version = 3.3 gzip-version = 1.10 +hdf5-version = 1.10.5 isl-version = 0.18 libbsd-version = 0.9.1 libffi-version = 3.2.1 @@ -29,6 +30,7 @@ metastore-version = 1.1.2-23-fa9170b mpfr-version = 4.0.2 mpc-version = 1.1.0 ncurses-version = 6.1 +openmpi-version = 4.0.1 openssl-version = 1.1.1a patchelf-version = 0.9 pkgconfig-version = 0.29.2 @@ -85,6 +87,7 @@ jeepney-version = 0.4 kiwisolver-version = 1.0.1 keyring-version = 18.0.0 matplotlib-version = 3.0.2 +mpi4py-version = 3.0.1 numpy-version = 1.16.2 pip-version = 19.0.2 pycparser-version = 2.19 @@ -100,4 +103,3 @@ soupsieve-version = 1.8 urllib3-version = 1.24.1 virtualenv-version = 16.4.0 webencodings-version = 0.5.1 - -- cgit v1.2.1