From 75571a551f794223bea2995dc7775a49abd63654 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 20 Mar 2019 17:41:50 +0000 Subject: Including ATLAS in the pipeline, not yet complete An initial installation of atlas is now included in the pipeline, but we are still trying to make it compile and build smoothly. In the process, we found that GCC also needs some modifications (for example rpath issues). --- reproduce/src/make/dependencies-python.mk | 3 --- 1 file changed, 3 deletions(-) (limited to 'reproduce/src/make/dependencies-python.mk') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 946c047..3b92f7e 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -378,9 +378,6 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ibdir)/python3 - export BLAS=None; \ - export ATLAS=None; \ - export LAPACK=None; \ $(call pybuild, unzip, $<, numpy-$(numpy-version)) $(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ -- cgit v1.2.1 From 3e583296cb8ed2784f6cb71b53cbe3be4931fa46 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 21 Mar 2019 15:39:06 +0000 Subject: ATLAS and Scipy working on GNU/Linux Numpy needs ATLAS as shared libraries. So we also need to build Python with shared libraries. We also need to define site.cfg for numpy and scipy so we define a master template: `reproduce/config/pipeline/dependency-numpy-scipy.cfg' Also `Openssl' did not have rpath so we added with this commit. --- reproduce/src/make/dependencies-python.mk | 51 ++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 17 deletions(-) (limited to 'reproduce/src/make/dependencies-python.mk') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 3b92f7e..e4c2324 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -43,7 +43,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astroquery matplotlib #scipy +top-level-python = astroquery matplotlib scipy all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host @@ -249,18 +249,24 @@ $(tarballs): $(tdir)/%: # -------------------- # # To build Python packages with direct access to a `setup.py' (if no direct -# access to `setup.py' is needed, pip can be used) +# access to `setup.py' is needed, pip can be used). # Arguments of this function are the numbers # 1) Unpack command # 2) Package name # 3) Unpacked directory name after unpacking the tarball -pybuild = cd $(ddir); rm -rf $(3); \ - if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ - cd $(3); \ - python3 setup.py build && \ - python3 setup.py install && \ - cd .. && rm -rf $(3) && \ - echo "done!" > $@ +# 4) site.cfg file (optional) +pybuild = cd $(ddir); rm -rf $(3); \ + if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ + cd $(3); \ + if [ "x$(4)" != x ]; then \ + sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ + -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ + $(4) > site.cfg; \ + fi; \ + python3 setup.py build \ + && python3 setup.py install \ + && cd .. && rm -rf $(3) \ + && echo "done!" > $@ @@ -271,7 +277,12 @@ pybuild = cd $(ddir); rm -rf $(3); \ # # While this Makefile is for Python programs, in some cases, we need # certain programs (like Python itself), or libraries for the modules. -$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz +$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz + $(call gbuild, $<, libffi-$(libffi-version)) \ + echo "libffi is built" > $@ + +$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \ + $(ilidir)/libffi # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. # if [ x$(on_mac_os) = xyes ]; then \ @@ -279,16 +290,18 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz # export CXX=clang++; \ # fi; \ - $(call gbuild, $<, Python-$(python-version)) \ + $(call gbuild, $<, Python-$(python-version),, \ + --enable-optimizations \ + --without-ensurepip \ + --with-system-ffi \ + --enable-shared \ + --with-threads \ + --with-lto ) \ && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') \ && ln -s $(ildir)/python$$v $(ildir)/python \ && rm -rf $(ipydir) && mkdir $(ipydir) -$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz - $(call gbuild, $<, libffi-$(libffi-version)) \ - echo "libffi is built" > $@ - @@ -378,7 +391,9 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ibdir)/python3 - $(call pybuild, unzip, $<, numpy-$(numpy-version)) + export LDFLAGS="$$LDFLAGS -shared"; \ + conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf) $(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ $(ibdir)/python3 @@ -407,7 +422,9 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ $(ipydir)/numpy - $(call pybuild, tar xf, $<, scipy-$(scipy-version)) + export LDFLAGS="$$LDFLAGS -shared"; \ + conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf) $(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \ $(ipydir)/cryptography \ -- cgit v1.2.1 From 5b008fbd6760014283d18986ae26585daf473e0f Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 22 Mar 2019 11:59:04 +0000 Subject: H5py added and setuptools as main Python module dependency In this commit we add `h5py' Python package. We also include `setuptools' as a main dependency of Python because with the previous commit it (as well as `pip') is no longer installed with Python. Numpy version also has been incremented. --- reproduce/src/make/dependencies-python.mk | 63 ++++++++++++++++++------------- 1 file changed, 36 insertions(+), 27 deletions(-) (limited to 'reproduce/src/make/dependencies-python.mk') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index e4c2324..54d3a56 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -116,6 +116,7 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ cryptography-$(cryptography-version).tar.gz \ cycler-$(cycler-version).tar.gz \ entrypoints-$(entrypoints-version).tar.gz \ + h5py-$(h5py-version).tar.gz \ html5lib-$(html5lib-version).tar.gz \ idna-$(idna-version).tar.gz \ jeepney-$(jeepney-version).tar.gz \ @@ -157,7 +158,10 @@ $(tarballs): $(tdir)/%: # because the tokenization above will produce `python' as the # first string. if [ $* = python-dateutil-$(python-dateutil-version).tar.gz ]; then - n=dateutil + n=dateutil + elif [ $* = h5py-$(h5py-version).tar.gz ]; then + n=h5py + # elif [ $* = strange-tarball5name-version.tar.gz ]; then # n=strange5-name else @@ -191,13 +195,14 @@ $(tarballs): $(tdir)/%: elif [ $$n = cryptography ]; then h=07/ca/bc827c5e55918ad223d59d299fff92f3563476c3b00d0a9157d9c0217449 elif [ $$n = cycler ]; then h=c2/4b/137dea450d6e1e3d474e1d873cd1d4f7d3beed7e0dc973b06e8e10d32488 elif [ $$n = entrypoints ]; then h=b4/ef/063484f1f9ba3081e920ec9972c96664e2edb9fdc3d8669b0e3b8fc0ad7c + elif [ $$n = h5py ]; then h=43/27/a6e7dcb8ae20a4dbf3725321058923fec262b6f7835179d78ccc8d98deec elif [ $$n = html ]; then h=85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f elif [ $$n = idna ]; then h=ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7 elif [ $$n = jeepney ]; then h=16/1d/74adf3b164a8d19a60d0fcf706a751ffa2a1eaa8e5bbb1b6705c92a05263 elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2 elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864 elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f - elif [ $$n = numpy ]; then h=2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c + elif [ $$n = numpy ]; then h=cf/8d/6345b4f32b37945fedc1e027e83970005fc9c699068d2f566b82826515f2 elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af elif [ $$n = pycparser ]; then h=68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a elif [ $$n = pyparsing ]; then h=b9/b8/6b32b3e84014148dcd60dd05795e35c2e7f4b72f918616c61fdce83d27fc @@ -212,7 +217,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = urllib ]; then h=b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1 elif [ $$n = virtualenv ]; then h=51/aa/c395a6e6eaaedfa5a04723b6446a1df783b16cca6fec66e671cede514688 elif [ $$n = webencodings ]; then h=0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47 -# elif [ $$n = strange5-name ]; then h=XXXXX +# elif [ $$n = strange5-name ]; then h=XXXXX else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." @@ -310,7 +315,7 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \ # # All the necessary Python modules go here. $(ipydir)/asn1crypto: $(tdir)/asn1crypto-$(asn1crypto-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, asn1crypto-$(asn1crypto-version)) $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ @@ -323,6 +328,7 @@ $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ $(call pybuild, tar xf, $<, astroquery-$(astroquery-version)) $(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ + $(ipydir)/h5py \ $(ipydir)/numpy $(call pybuild, tar xf, $<, astropy-$(astropy-version)) @@ -331,80 +337,83 @@ $(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.g $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version)) $(ipydir)/certifi: $(tdir)/certifi-$(certifi-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, certifi-$(certifi-version)) $(ipydir)/cffi: $(tdir)/cffi-$(cffi-version).tar.gz \ - $(ipydir)/pycparser \ - $(ilidir)/libffi + $(ilidir)/libffi \ + $(ipydir)/pycparser $(call pybuild, tar xf, $<, cffi-$(cffi-version)) $(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, chardet-$(chardet-version)) $(ipydir)/cryptography: $(tdir)/cryptography-$(cryptography-version).tar.gz \ - $(ipydir)/cffi \ - $(ipydir)/asn1crypto + $(ipydir)/asn1crypto \ + $(ipydir)/cffi $(call pybuild, tar xf, $<, cryptography-$(cryptography-version)) $(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \ - $(ipydir)/six + $(ipydir)/six $(call pybuild, tar xf, $<, cycler-$(cycler-version)) $(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version)) +$(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, h5py-$(h5py-version)) + $(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \ $(ipydir)/six \ $(ipydir)/webencodings $(call pybuild, tar xf, $<, html5lib-$(html5lib-version)) $(ipydir)/idna: $(tdir)/idna-$(idna-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, idna-$(idna-version)) $(ipydir)/jeepney: $(tdir)/jeepney-$(jeepney-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, jeepney-$(jeepney-version)) $(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \ - $(ipydir)/setuptools_scm \ + $(ipydir)/entrypoints \ $(ipydir)/secretstorage \ - $(ipydir)/entrypoints + $(ipydir)/setuptools_scm $(call pybuild, tar xf, $<, keyring-$(keyring-version)) $(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ $(ipydir)/setuptools $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version)) - $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/cycler \ $(ilidir)/freetype \ + $(ipydir)/kiwisolver \ $(ipydir)/numpy \ $(ipydir)/pyparsing \ - $(ipydir)/python-dateutil \ - $(ipydir)/kiwisolver + $(ipydir)/python-dateutil $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ - $(ibdir)/python3 + $(ipydir)/setuptools export LDFLAGS="$$LDFLAGS -shared"; \ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf) $(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, pip-$(pip-version)) $(ipydir)/pycparser: $(tdir)/pycparser-$(pycparser-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, pycparser-$(pycparser-version)) $(ipydir)/pyparsing: $(tdir)/pyparsing-$(pyparsing-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version)) $(ipydir)/python-dateutil: $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz \ @@ -440,17 +449,17 @@ $(ipydir)/setuptools_scm: $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.g $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version)) $(ipydir)/six: $(tdir)/six-$(six-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, six-$(six-version)) $(ipydir)/soupsieve: $(tdir)/soupsieve-$(soupsieve-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version)) $(ipydir)/urllib3: $(tdir)/urllib3-$(urllib3-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, urllib3-$(urllib3-version)) $(ipydir)/webencodings: $(tdir)/webencodings-$(webencodings-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, webencodings-$(webencodings-version)) -- cgit v1.2.1 From 12648438cf9f75738cef045fd02f4093eab7990d Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 27 Mar 2019 17:16:21 +0000 Subject: GCC is now built on a Mac, not yet ATLAS Until now, we were simply using the host's GCC for Mac systems. But we found that except for a single step (to fixing `rpath'), it works on Mac!!! So, GCC is now part of the Mac build as well. However, we are still having some problems in building ATLAS on Mac. It works on GNU/Linux, but not in Mac. So for the time being (just temporarily), we are avoiding ATLAS (and thus Scipy) on Mac systems. We just filed an issue on the ATLAS discussion list to hopefully fix the problem soon. --- reproduce/src/make/dependencies-python.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'reproduce/src/make/dependencies-python.mk') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 54d3a56..fe82175 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -43,7 +43,10 @@ ilidir = $(BDIR)/dependencies/installed/lib/built ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astroquery matplotlib scipy +ifneq ($(on_mac_os),yes) +withscipy=scipy +endif +top-level-python = astroquery matplotlib $(withscipy) all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host -- cgit v1.2.1 From d26372bb074b1e4ac5ff758d7716129975fe4963 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 29 Mar 2019 10:49:31 +0000 Subject: Disabling ATLAS shared libraries on Mac OS The Makefile that build the shared libraries comes from Arch Linux so it does not work easily on Mac. But the full ATLAS build goes successfully for static libraries. For now we are disabling shared libraries on Mac. Python was built explicity with `clang' on Mac. --- reproduce/src/make/dependencies-python.mk | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'reproduce/src/make/dependencies-python.mk') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index fe82175..c7f861b 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -293,18 +293,14 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \ $(ilidir)/libffi # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. -# if [ x$(on_mac_os) = xyes ]; then \ -# export CC=clang; \ -# export CXX=clang++; \ -# fi; \ - + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + export CXX=clang++; \ + fi; \ $(call gbuild, $<, Python-$(python-version),, \ - --enable-optimizations \ --without-ensurepip \ --with-system-ffi \ - --enable-shared \ - --with-threads \ - --with-lto ) \ + --enable-shared) \ && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') \ && ln -s $(ildir)/python$$v $(ildir)/python \ -- cgit v1.2.1 From f0030ea8974b41ef4b26c3ce41c3294b224fb358 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 2 Apr 2019 16:26:43 +0100 Subject: Numpy and Scipy using OpenBLAS instead of ATLAS We could not get ATLAS shared libraries on Mac (while the static ATLAS libraries are built and can be used successfully on Mac). So, the pipeline now builds OpenBLAS, which both Numpy and Scipy can use on Mac and GNU/Linux. We also added FFTW as a dependency of Numpy. Altough Numpy is not linking to FFTW for some reason. However, since FFTW is a low level library used by many programs, we have kept it as a dependency of Numpy anyway for now. --- reproduce/src/make/dependencies-python.mk | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) (limited to 'reproduce/src/make/dependencies-python.mk') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index c7f861b..642316c 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -43,10 +43,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). -ifneq ($(on_mac_os),yes) -withscipy=scipy -endif -top-level-python = astroquery matplotlib $(withscipy) +top-level-python = astroquery matplotlib all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host @@ -328,7 +325,8 @@ $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ $(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ $(ipydir)/h5py \ - $(ipydir)/numpy + $(ipydir)/numpy \ + $(ipydir)/scipy $(call pybuild, tar xf, $<, astropy-$(astropy-version)) $(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \ @@ -398,8 +396,14 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ - $(ipydir)/setuptools - export LDFLAGS="$$LDFLAGS -shared"; \ + $(ipydir)/setuptools \ + $(ilidir)/openblas \ + $(ilidir)/fftw + if [ x$(on_mac_os) = xyes ]; then \ + export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ + else \ + export LDFLAGS="$(LDFLAGS) -shared"; \ + fi; \ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf) @@ -430,7 +434,11 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ $(ipydir)/numpy - export LDFLAGS="$$LDFLAGS -shared"; \ + if [ x$(on_mac_os) = xyes ]; then \ + export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ + else \ + export LDFLAGS="$(LDFLAGS) -shared"; \ + fi; \ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf) -- cgit v1.2.1 From aa9de365ae7f85488e8e53acd642cf657de251a8 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 2 Apr 2019 22:43:29 +0100 Subject: Python packages are installed as high level program dependencies Until this commit, the installation of all Python packages were done in a separate Makefile. With this commit, the pipeline install Python packages as part of the hight level software. All Python packages rules them remain in a separate Makefile, but this Makefile is included in the high level dependency `reproduce/src/make/dependencies.mk'. --- reproduce/src/make/dependencies-python.mk | 57 ++++--------------------------- 1 file changed, 6 insertions(+), 51 deletions(-) (limited to 'reproduce/src/make/dependencies-python.mk') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 642316c..30e10c1 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -29,51 +29,6 @@ -# Top level environment -include reproduce/config/pipeline/LOCAL.mk -include reproduce/src/make/dependencies-build-rules.mk -include reproduce/config/pipeline/dependency-versions.mk - -ddir = $(BDIR)/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib -ilidir = $(BDIR)/dependencies/installed/lib/built -ipydir = $(BDIR)/dependencies/installed/lib/built/python - -# Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astroquery matplotlib -all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) - -# Other basic environment settings: We are only including the host -# operating system's PATH environment variable (after our own!) for the -# compiler and linker. For the library binaries and headers, we are only -# using our internally built libraries. -# -# To investigate: -# -# 1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all -# the parameters defined bellow as `NAME=VALUE' statements before -# calling Bash. This will enable us to completely ignore the user's -# native environment. -# -# 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the -# user's environment. -.ONESHELL: -.SHELLFLAGS := --noprofile --norc -ec -export CCACHE_DISABLE := 1 -export PATH := $(ibdir) -export LD_RUN_PATH := $(ildir) -export LD_LIBRARY_PATH := $(ildir) -export SHELL := $(ibdir)/bash -export CPPFLAGS := -I$(idir)/include -export PKG_CONFIG_PATH := $(ildir)/pkgconfig -export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export LDFLAGS := $(rpath_command) -L$(ildir) - - - # Python enviroment @@ -106,7 +61,7 @@ export MPI_PYTHON3_SITEARCH := # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. -tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ +pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ astroquery-$(astroquery-version).tar.gz \ astropy-$(astropy-version).tar.gz \ beautifulsoup4-$(beautifulsoup4-version).tar.gz \ @@ -141,8 +96,8 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ webencodings-$(webencodings-version).tar.gz \ virtualenv-$(virtualenv-version).tar.gz \ , $(tdir)/$(t) ) -topurl=https://files.pythonhosted.org/packages -$(tarballs): $(tdir)/%: +pytopurl=https://files.pythonhosted.org/packages +$(pytarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* $@ else @@ -184,7 +139,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = secretstorage ]; then mergenames=0 hash=a6/89/df343dbc2957a317127e7ff2983230dc5336273be34f2e1911519d85aeb5 - h=$(topurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz + h=$(pytopurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz elif [ $$n = asn ]; then h=fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4 elif [ $$n = astroquery ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396 elif [ $$n = astropy ]; then h=eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 @@ -232,7 +187,7 @@ $(tarballs): $(tdir)/%: # storing all the tarballs in one directory, we want it to have # the same naming convention, so we'll download it to a temporary # name, then rename that. - if [ $$mergenames = 1 ]; then tarballurl=$(topurl)/$$h/"$*" + if [ $$mergenames = 1 ]; then tarballurl=$(pytopurl)/$$h/"$*" else tarballurl=$$h fi @@ -383,7 +338,7 @@ $(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \ $(call pybuild, tar xf, $<, keyring-$(keyring-version)) $(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ - $(ipydir)/setuptools + $(ipydir)/setuptools $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version)) $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ -- cgit v1.2.1 From 2d500f26e9777fc395131301404cf06c0d1a2951 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 3 Apr 2019 15:23:44 +0100 Subject: OpenMPI, HDF5 libraries added for h5py After trying the build a system with no Python library, I noticed that Python's HDF5 module (`h5py') needs the HDF5 library and OpenMPI (to work in parallel). So they were added. Finally `h5py' uses the `mpi4py' module to communicate with OpenMPI, so it was also added. However, for some reason, mpi4py doesn't work with this version of OpenMPI (as described in the comments above). So for now, h5py doesn't use it and can only work on a single thread, while the HDF5 C library links with OpenMPI with no problem. --- reproduce/src/make/dependencies-python.mk | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'reproduce/src/make/dependencies-python.mk') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 30e10c1..6711d17 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -39,9 +39,9 @@ # systems which might interfere. To be safe, we are removing all their # values. export PYTHONPATH := $(installdir)/lib/python/site-packages +export PYTHONPATH2 := $(PYTHONPATH) export PYTHONPATH3 := $(PYTHONPATH) export _LMFILES_ := -export PYTHONPATH2 := export LOADEDMODULES := export MPI_PYTHON_SITEARCH := export MPI_PYTHON2_SITEARCH := @@ -79,6 +79,7 @@ pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ keyring-$(keyring-version).tar.gz \ libffi-$(libffi-version).tar.gz \ matplotlib-$(matplotlib-version).tar.gz \ + mpi4py-$(mpi4py-version).tar.gz \ numpy-$(numpy-version).zip \ pip-$(pip-version).tar.gz \ pycparser-$(pycparser-version).tar.gz \ @@ -157,6 +158,7 @@ $(pytarballs): $(tdir)/%: elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2 elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864 elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f + elif [ $$n = mpi ]; then h=55/a2/c827b196070e161357b49287fa46d69f25641930fd5f854722319d431843 elif [ $$n = numpy ]; then h=cf/8d/6345b4f32b37945fedc1e027e83970005fc9c699068d2f566b82826515f2 elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af elif [ $$n = pycparser ]; then h=68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a @@ -315,7 +317,11 @@ $(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version)) $(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \ - $(ipydir)/setuptools + $(ipydir)/setuptools \ + $(ilidir)/hdf5 + # $(ipydir)/mpi4py # AFTER its problem is fixed. + #export HDF5_MPI=ON; # AFTER its problem is fixed. + export HDF5_DIR=$(ildir); \ $(call pybuild, tar xf, $<, h5py-$(h5py-version)) $(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \ @@ -350,6 +356,20 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/python-dateutil $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) +# Currently mpi4py doesn't build because of some conflict with OpenMPI: +# +# In file included from src/mpi4py.MPI.c:591, +# from src/MPI.c:4: +# src/mpi4py.MPI.c: In function '__pyx_f_6mpi4py_3MPI_del_Datatype': +# src/mpi4py.MPI.c:15094:36: error: expected expression before '_Static_assert' +# __pyx_t_1 = (((__pyx_v_ob[0]) == MPI_UB) != 0); +# +# But atleast on my system it fails. +$(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \ + $(ipydir)/setuptools \ + $(ilidir)/openmpi + $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version)) + $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ipydir)/setuptools \ $(ilidir)/openblas \ -- cgit v1.2.1