aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMohammad Akhlaghi <mohammad@akhlaghi.org>2019-04-03 15:23:44 +0100
committerMohammad Akhlaghi <mohammad@akhlaghi.org>2019-04-03 15:28:12 +0100
commit2d500f26e9777fc395131301404cf06c0d1a2951 (patch)
tree1f4f0a68b09534acb84b16c7286d71c900924597
parentaa9de365ae7f85488e8e53acd642cf657de251a8 (diff)
OpenMPI, HDF5 libraries added for h5py
After trying the build a system with no Python library, I noticed that Python's HDF5 module (`h5py') needs the HDF5 library and OpenMPI (to work in parallel). So they were added. Finally `h5py' uses the `mpi4py' module to communicate with OpenMPI, so it was also added. However, for some reason, mpi4py doesn't work with this version of OpenMPI (as described in the comments above). So for now, h5py doesn't use it and can only work on a single thread, while the HDF5 C library links with OpenMPI with no problem.
-rw-r--r--.file-metadatabin4800 -> 4340 bytes
-rw-r--r--reproduce/config/pipeline/dependency-versions.mk4
-rw-r--r--reproduce/src/make/dependencies-python.mk24
-rw-r--r--reproduce/src/make/dependencies.mk21
4 files changed, 46 insertions, 3 deletions
diff --git a/.file-metadata b/.file-metadata
index 7e8d12c..27bfed5 100644
--- a/.file-metadata
+++ b/.file-metadata
Binary files differ
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index cb5920e..4cb8a71 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -18,6 +18,7 @@ gmp-version = 6.1.2
gnuastro-version = 0.8
grep-version = 3.3
gzip-version = 1.10
+hdf5-version = 1.10.5
isl-version = 0.18
libbsd-version = 0.9.1
libffi-version = 3.2.1
@@ -29,6 +30,7 @@ metastore-version = 1.1.2-23-fa9170b
mpfr-version = 4.0.2
mpc-version = 1.1.0
ncurses-version = 6.1
+openmpi-version = 4.0.1
openssl-version = 1.1.1a
patchelf-version = 0.9
pkgconfig-version = 0.29.2
@@ -85,6 +87,7 @@ jeepney-version = 0.4
kiwisolver-version = 1.0.1
keyring-version = 18.0.0
matplotlib-version = 3.0.2
+mpi4py-version = 3.0.1
numpy-version = 1.16.2
pip-version = 19.0.2
pycparser-version = 2.19
@@ -100,4 +103,3 @@ soupsieve-version = 1.8
urllib3-version = 1.24.1
virtualenv-version = 16.4.0
webencodings-version = 0.5.1
-
diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk
index 30e10c1..6711d17 100644
--- a/reproduce/src/make/dependencies-python.mk
+++ b/reproduce/src/make/dependencies-python.mk
@@ -39,9 +39,9 @@
# systems which might interfere. To be safe, we are removing all their
# values.
export PYTHONPATH := $(installdir)/lib/python/site-packages
+export PYTHONPATH2 := $(PYTHONPATH)
export PYTHONPATH3 := $(PYTHONPATH)
export _LMFILES_ :=
-export PYTHONPATH2 :=
export LOADEDMODULES :=
export MPI_PYTHON_SITEARCH :=
export MPI_PYTHON2_SITEARCH :=
@@ -79,6 +79,7 @@ pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \
keyring-$(keyring-version).tar.gz \
libffi-$(libffi-version).tar.gz \
matplotlib-$(matplotlib-version).tar.gz \
+ mpi4py-$(mpi4py-version).tar.gz \
numpy-$(numpy-version).zip \
pip-$(pip-version).tar.gz \
pycparser-$(pycparser-version).tar.gz \
@@ -157,6 +158,7 @@ $(pytarballs): $(tdir)/%:
elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2
elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864
elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f
+ elif [ $$n = mpi ]; then h=55/a2/c827b196070e161357b49287fa46d69f25641930fd5f854722319d431843
elif [ $$n = numpy ]; then h=cf/8d/6345b4f32b37945fedc1e027e83970005fc9c699068d2f566b82826515f2
elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af
elif [ $$n = pycparser ]; then h=68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a
@@ -315,7 +317,11 @@ $(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \
$(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version))
$(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \
- $(ipydir)/setuptools
+ $(ipydir)/setuptools \
+ $(ilidir)/hdf5
+ # $(ipydir)/mpi4py # AFTER its problem is fixed.
+ #export HDF5_MPI=ON; # AFTER its problem is fixed.
+ export HDF5_DIR=$(ildir); \
$(call pybuild, tar xf, $<, h5py-$(h5py-version))
$(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \
@@ -350,6 +356,20 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
$(ipydir)/python-dateutil
$(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version))
+# Currently mpi4py doesn't build because of some conflict with OpenMPI:
+#
+# In file included from src/mpi4py.MPI.c:591,
+# from src/MPI.c:4:
+# src/mpi4py.MPI.c: In function '__pyx_f_6mpi4py_3MPI_del_Datatype':
+# src/mpi4py.MPI.c:15094:36: error: expected expression before '_Static_assert'
+# __pyx_t_1 = (((__pyx_v_ob[0]) == MPI_UB) != 0);
+#
+# But atleast on my system it fails.
+$(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \
+ $(ipydir)/setuptools \
+ $(ilidir)/openmpi
+ $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version))
+
$(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
$(ipydir)/setuptools \
$(ilidir)/openblas \
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index 61d0701..a1f1b23 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -110,6 +110,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
git-$(git-version).tar.xz \
gnuastro-$(gnuastro-version).tar.lz \
gsl-$(gsl-version).tar.gz \
+ hdf5-$(hdf5-version).tar.gz \
install-tl-unx.tar.gz \
jpegsrc.$(libjpeg-version).tar.gz \
lapack-$(lapack-version).tar.gz \
@@ -118,6 +119,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
libtool-$(libtool-version).tar.xz \
libgit2-$(libgit2-version).tar.gz \
metastore-$(metastore-version).tar.gz \
+ openmpi-$(openmpi-version).tar.gz \
unzip-$(unzip-version).tar.gz \
openblas-$(openblas-version).tar.gz \
tiff-$(libtiff-version).tar.gz \
@@ -152,6 +154,10 @@ $(tarballs): $(tdir)/%:
elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version)
elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw
elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype
+ elif [ $$n = hdf ]; then
+ mergenames=0
+ majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}')
+ w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$*
elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926
elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git
elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro
@@ -169,6 +175,10 @@ $(tarballs): $(tdir)/%:
elif [ $$n = openblas ]; then
mergenames=0
w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz
+ elif [ $$n = openmpi ]; then
+ mergenames=0
+ majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}')
+ w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$*
elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff
elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src
mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//')
@@ -261,6 +271,13 @@ $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \
$(call gbuild, $<, freetype-$(freetype-version), static) \
&& echo "freetype is built" > $@
+$(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \
+ $(ilidir)/openmpi
+ $(call gbuild, $<, hdf5-$(hdf5-version), static, \
+ --enable-parallel \
+ --enable-fortran, V=1) \
+ && echo "HDF5 library is built" > $@
+
$(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz
$(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \
&& echo "libbsd is built" > $@
@@ -278,6 +295,10 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \
--disable-webp --disable-zstd) \
&& echo "Libtiff is built" > $@
+$(ilidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz
+ $(call gbuild, $<, openmpi-$(openmpi-version), static, , V=1) \
+ && echo "OpenMPI is built" > $@
+
$(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
$(tdir)/lapack-$(lapack-version).tar.gz