From 2d500f26e9777fc395131301404cf06c0d1a2951 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 3 Apr 2019 15:23:44 +0100 Subject: OpenMPI, HDF5 libraries added for h5py After trying the build a system with no Python library, I noticed that Python's HDF5 module (`h5py') needs the HDF5 library and OpenMPI (to work in parallel). So they were added. Finally `h5py' uses the `mpi4py' module to communicate with OpenMPI, so it was also added. However, for some reason, mpi4py doesn't work with this version of OpenMPI (as described in the comments above). So for now, h5py doesn't use it and can only work on a single thread, while the HDF5 C library links with OpenMPI with no problem. --- reproduce/config/pipeline/dependency-versions.mk | 4 +++- reproduce/src/make/dependencies-python.mk | 24 ++++++++++++++++++++++-- reproduce/src/make/dependencies.mk | 21 +++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) (limited to 'reproduce') diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index cb5920e..4cb8a71 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -18,6 +18,7 @@ gmp-version = 6.1.2 gnuastro-version = 0.8 grep-version = 3.3 gzip-version = 1.10 +hdf5-version = 1.10.5 isl-version = 0.18 libbsd-version = 0.9.1 libffi-version = 3.2.1 @@ -29,6 +30,7 @@ metastore-version = 1.1.2-23-fa9170b mpfr-version = 4.0.2 mpc-version = 1.1.0 ncurses-version = 6.1 +openmpi-version = 4.0.1 openssl-version = 1.1.1a patchelf-version = 0.9 pkgconfig-version = 0.29.2 @@ -85,6 +87,7 @@ jeepney-version = 0.4 kiwisolver-version = 1.0.1 keyring-version = 18.0.0 matplotlib-version = 3.0.2 +mpi4py-version = 3.0.1 numpy-version = 1.16.2 pip-version = 19.0.2 pycparser-version = 2.19 @@ -100,4 +103,3 @@ soupsieve-version = 1.8 urllib3-version = 1.24.1 virtualenv-version = 16.4.0 webencodings-version = 0.5.1 - diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 30e10c1..6711d17 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -39,9 +39,9 @@ # systems which might interfere. To be safe, we are removing all their # values. export PYTHONPATH := $(installdir)/lib/python/site-packages +export PYTHONPATH2 := $(PYTHONPATH) export PYTHONPATH3 := $(PYTHONPATH) export _LMFILES_ := -export PYTHONPATH2 := export LOADEDMODULES := export MPI_PYTHON_SITEARCH := export MPI_PYTHON2_SITEARCH := @@ -79,6 +79,7 @@ pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ keyring-$(keyring-version).tar.gz \ libffi-$(libffi-version).tar.gz \ matplotlib-$(matplotlib-version).tar.gz \ + mpi4py-$(mpi4py-version).tar.gz \ numpy-$(numpy-version).zip \ pip-$(pip-version).tar.gz \ pycparser-$(pycparser-version).tar.gz \ @@ -157,6 +158,7 @@ $(pytarballs): $(tdir)/%: elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2 elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864 elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f + elif [ $$n = mpi ]; then h=55/a2/c827b196070e161357b49287fa46d69f25641930fd5f854722319d431843 elif [ $$n = numpy ]; then h=cf/8d/6345b4f32b37945fedc1e027e83970005fc9c699068d2f566b82826515f2 elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af elif [ $$n = pycparser ]; then h=68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a @@ -315,7 +317,11 @@ $(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version)) $(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \ - $(ipydir)/setuptools + $(ipydir)/setuptools \ + $(ilidir)/hdf5 + # $(ipydir)/mpi4py # AFTER its problem is fixed. + #export HDF5_MPI=ON; # AFTER its problem is fixed. + export HDF5_DIR=$(ildir); \ $(call pybuild, tar xf, $<, h5py-$(h5py-version)) $(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \ @@ -350,6 +356,20 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/python-dateutil $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) +# Currently mpi4py doesn't build because of some conflict with OpenMPI: +# +# In file included from src/mpi4py.MPI.c:591, +# from src/MPI.c:4: +# src/mpi4py.MPI.c: In function '__pyx_f_6mpi4py_3MPI_del_Datatype': +# src/mpi4py.MPI.c:15094:36: error: expected expression before '_Static_assert' +# __pyx_t_1 = (((__pyx_v_ob[0]) == MPI_UB) != 0); +# +# But atleast on my system it fails. +$(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \ + $(ipydir)/setuptools \ + $(ilidir)/openmpi + $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version)) + $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ipydir)/setuptools \ $(ilidir)/openblas \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 61d0701..a1f1b23 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -110,6 +110,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ gsl-$(gsl-version).tar.gz \ + hdf5-$(hdf5-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ lapack-$(lapack-version).tar.gz \ @@ -118,6 +119,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ + openmpi-$(openmpi-version).tar.gz \ unzip-$(unzip-version).tar.gz \ openblas-$(openblas-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ @@ -152,6 +154,10 @@ $(tarballs): $(tdir)/%: elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype + elif [ $$n = hdf ]; then + mergenames=0 + majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') + w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$* elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro @@ -169,6 +175,10 @@ $(tarballs): $(tdir)/%: elif [ $$n = openblas ]; then mergenames=0 w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz + elif [ $$n = openmpi ]; then + mergenames=0 + majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') + w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -261,6 +271,13 @@ $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ $(call gbuild, $<, freetype-$(freetype-version), static) \ && echo "freetype is built" > $@ +$(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ + $(ilidir)/openmpi + $(call gbuild, $<, hdf5-$(hdf5-version), static, \ + --enable-parallel \ + --enable-fortran, V=1) \ + && echo "HDF5 library is built" > $@ + $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \ && echo "libbsd is built" > $@ @@ -278,6 +295,10 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ --disable-webp --disable-zstd) \ && echo "Libtiff is built" > $@ +$(ilidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz + $(call gbuild, $<, openmpi-$(openmpi-version), static, , V=1) \ + && echo "OpenMPI is built" > $@ + $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(tdir)/lapack-$(lapack-version).tar.gz -- cgit v1.2.1