diff options
author | Mohammad Akhlaghi <mohammad@akhlaghi.org> | 2019-04-03 15:23:44 +0100 |
---|---|---|
committer | Mohammad Akhlaghi <mohammad@akhlaghi.org> | 2019-04-03 15:28:12 +0100 |
commit | 2d500f26e9777fc395131301404cf06c0d1a2951 (patch) | |
tree | 1f4f0a68b09534acb84b16c7286d71c900924597 | |
parent | aa9de365ae7f85488e8e53acd642cf657de251a8 (diff) |
OpenMPI, HDF5 libraries added for h5py
After trying the build a system with no Python library, I noticed that
Python's HDF5 module (`h5py') needs the HDF5 library and OpenMPI (to work
in parallel). So they were added. Finally `h5py' uses the `mpi4py' module
to communicate with OpenMPI, so it was also added. However, for some
reason, mpi4py doesn't work with this version of OpenMPI (as described in
the comments above).
So for now, h5py doesn't use it and can only work on a single thread, while
the HDF5 C library links with OpenMPI with no problem.
-rw-r--r-- | .file-metadata | bin | 4800 -> 4340 bytes | |||
-rw-r--r-- | reproduce/config/pipeline/dependency-versions.mk | 4 | ||||
-rw-r--r-- | reproduce/src/make/dependencies-python.mk | 24 | ||||
-rw-r--r-- | reproduce/src/make/dependencies.mk | 21 |
4 files changed, 46 insertions, 3 deletions
diff --git a/.file-metadata b/.file-metadata Binary files differindex 7e8d12c..27bfed5 100644 --- a/.file-metadata +++ b/.file-metadata diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index cb5920e..4cb8a71 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -18,6 +18,7 @@ gmp-version = 6.1.2 gnuastro-version = 0.8 grep-version = 3.3 gzip-version = 1.10 +hdf5-version = 1.10.5 isl-version = 0.18 libbsd-version = 0.9.1 libffi-version = 3.2.1 @@ -29,6 +30,7 @@ metastore-version = 1.1.2-23-fa9170b mpfr-version = 4.0.2 mpc-version = 1.1.0 ncurses-version = 6.1 +openmpi-version = 4.0.1 openssl-version = 1.1.1a patchelf-version = 0.9 pkgconfig-version = 0.29.2 @@ -85,6 +87,7 @@ jeepney-version = 0.4 kiwisolver-version = 1.0.1 keyring-version = 18.0.0 matplotlib-version = 3.0.2 +mpi4py-version = 3.0.1 numpy-version = 1.16.2 pip-version = 19.0.2 pycparser-version = 2.19 @@ -100,4 +103,3 @@ soupsieve-version = 1.8 urllib3-version = 1.24.1 virtualenv-version = 16.4.0 webencodings-version = 0.5.1 - diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 30e10c1..6711d17 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -39,9 +39,9 @@ # systems which might interfere. To be safe, we are removing all their # values. export PYTHONPATH := $(installdir)/lib/python/site-packages +export PYTHONPATH2 := $(PYTHONPATH) export PYTHONPATH3 := $(PYTHONPATH) export _LMFILES_ := -export PYTHONPATH2 := export LOADEDMODULES := export MPI_PYTHON_SITEARCH := export MPI_PYTHON2_SITEARCH := @@ -79,6 +79,7 @@ pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ keyring-$(keyring-version).tar.gz \ libffi-$(libffi-version).tar.gz \ matplotlib-$(matplotlib-version).tar.gz \ + mpi4py-$(mpi4py-version).tar.gz \ numpy-$(numpy-version).zip \ pip-$(pip-version).tar.gz \ pycparser-$(pycparser-version).tar.gz \ @@ -157,6 +158,7 @@ $(pytarballs): $(tdir)/%: elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2 elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864 elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f + elif [ $$n = mpi ]; then h=55/a2/c827b196070e161357b49287fa46d69f25641930fd5f854722319d431843 elif [ $$n = numpy ]; then h=cf/8d/6345b4f32b37945fedc1e027e83970005fc9c699068d2f566b82826515f2 elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af elif [ $$n = pycparser ]; then h=68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a @@ -315,7 +317,11 @@ $(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version)) $(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \ - $(ipydir)/setuptools + $(ipydir)/setuptools \ + $(ilidir)/hdf5 + # $(ipydir)/mpi4py # AFTER its problem is fixed. + #export HDF5_MPI=ON; # AFTER its problem is fixed. + export HDF5_DIR=$(ildir); \ $(call pybuild, tar xf, $<, h5py-$(h5py-version)) $(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \ @@ -350,6 +356,20 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/python-dateutil $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) +# Currently mpi4py doesn't build because of some conflict with OpenMPI: +# +# In file included from src/mpi4py.MPI.c:591, +# from src/MPI.c:4: +# src/mpi4py.MPI.c: In function '__pyx_f_6mpi4py_3MPI_del_Datatype': +# src/mpi4py.MPI.c:15094:36: error: expected expression before '_Static_assert' +# __pyx_t_1 = (((__pyx_v_ob[0]) == MPI_UB) != 0); +# +# But atleast on my system it fails. +$(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \ + $(ipydir)/setuptools \ + $(ilidir)/openmpi + $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version)) + $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ipydir)/setuptools \ $(ilidir)/openblas \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 61d0701..a1f1b23 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -110,6 +110,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ gsl-$(gsl-version).tar.gz \ + hdf5-$(hdf5-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ lapack-$(lapack-version).tar.gz \ @@ -118,6 +119,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ + openmpi-$(openmpi-version).tar.gz \ unzip-$(unzip-version).tar.gz \ openblas-$(openblas-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ @@ -152,6 +154,10 @@ $(tarballs): $(tdir)/%: elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype + elif [ $$n = hdf ]; then + mergenames=0 + majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') + w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$* elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro @@ -169,6 +175,10 @@ $(tarballs): $(tdir)/%: elif [ $$n = openblas ]; then mergenames=0 w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz + elif [ $$n = openmpi ]; then + mergenames=0 + majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') + w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -261,6 +271,13 @@ $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ $(call gbuild, $<, freetype-$(freetype-version), static) \ && echo "freetype is built" > $@ +$(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ + $(ilidir)/openmpi + $(call gbuild, $<, hdf5-$(hdf5-version), static, \ + --enable-parallel \ + --enable-fortran, V=1) \ + && echo "HDF5 library is built" > $@ + $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \ && echo "libbsd is built" > $@ @@ -278,6 +295,10 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ --disable-webp --disable-zstd) \ && echo "Libtiff is built" > $@ +$(ilidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz + $(call gbuild, $<, openmpi-$(openmpi-version), static, , V=1) \ + && echo "OpenMPI is built" > $@ + $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(tdir)/lapack-$(lapack-version).tar.gz |