From d0a51f78d4705c2496c42a02c49b0bc70e931cf7 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Mon, 6 Apr 2020 09:12:47 +0100 Subject: Astropy now depending on the Expat library to fix internal conflict Until now, Astropy was instructed to build its own internal copy of the Expat library. However, with the recent commits before, Maneage now includes an installation of Expat and Astropy can't keep the two (its internal version and the project's version) separate, so they conflict and don't let Astropy get built. With this commit, the problem is fixed by setting the Expat library as an explicit dependency of Astropy and asking Astropy to ignore its internal copy. While doing this, I recognized that it is much easier and elegant to add steps in various stages of the `pybuild' function through hooks instead of variables. So the fifth argument of the `pybuild' function was removed and now it actually checks if hooks are defined as functions and if so, they will be called. The `pyhook_after' function was also implemented in the installation of `pybind11' (which needed it, given that the 5th argument of `pybuild' was removed) and after doing a test-build, I noticed that two lines were not ending with a `\' in `boost' (a dependency of `pybind11'). Commit written originally by Mohammad Akghlaghi --- reproduce/software/make/python.mk | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) (limited to 'reproduce/software/make/python.mk') diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index fef1605..9c79abc 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -335,7 +335,10 @@ $(ibidir)/python: $(ibidir)/libffi \ # 2) Unpacked directory name after unpacking the tarball # 3) site.cfg file (optional). # 4) Official software name (for paper). -# 5) Manual step after installation. +# +# Hooks: +# pyhook_before: optional steps before running `python setup.py build' +# pyhook_after: optional steps after running `python setup.py install' pybuild = cd $(ddir); rm -rf $(2); \ if ! $(1) $(word 1,$(filter $(tdir)/%,$|)); then \ echo; echo "Tar error"; exit 1; \ @@ -346,11 +349,10 @@ pybuild = cd $(ddir); rm -rf $(2); \ -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ $(3) > site.cfg; \ fi; \ - if [ x"$(5)" = x ]; then after="echo no after"; \ - else after="$(5)"; fi \ + if type pyhook_before &>/dev/null; then pyhook_before; fi \ && python setup.py build \ && python setup.py install \ - && $$after \ + && if type pyhook_after &>/dev/null; then pyhook_after; fi \ && cd .. \ && rm -rf $(2) \ && echo "$(4)" > $@ @@ -381,12 +383,24 @@ $(ipydir)/astroquery: $(ipydir)/astropy \ Astroquery $(astroquery-version)) $(ipydir)/astropy: $(ipydir)/h5py \ + $(ibidir)/expat \ $(ipydir)/scipy \ $(ipydir)/numpy \ $(ipydir)/pyyaml \ $(ipydir)/html5lib \ $(ipydir)/beautifulsoup4 \ | $(tdir)/astropy-$(astropy-version).tar.gz + # Currently, when the Expat library is already built in a project + # (for example as a dependency of another program), Astropy's + # internal building of Expat will conflict with the project's. So + # we have added Expat as a dependency of Astropy (so it is always + # built before it, and we tell Astropy to use the project's + # libexpat. + pyhook_before () { + echo "" >> setup.cfg + echo "[build]" >> setup.cfg + echo "use_system_expat=1" >> setup.cfg + } $(call pybuild, tar xf, astropy-$(astropy-version)) \ && cp $(dtexdir)/astropy.tex $(ictdir)/ \ && echo "Astropy $(astropy-version) \citep{astropy2013,astropy2018}" > $@ @@ -474,7 +488,7 @@ $(ipydir)/galsim: $(ipydir)/future \ $(ipydir)/pybind11 \ $(ipydir)/lsstdesccoord \ | $(tdir)/galsim-$(galsim-version).tar.gz - $(call pybuild, tar xf, GalSim-$(galsim-version), ,) \ + $(call pybuild, tar xf, GalSim-$(galsim-version)) \ && cp $(dtexdir)/galsim.tex $(ictdir)/ \ && echo "Galsim $(galsim-version) \citep{galsim}" > $@ @@ -610,9 +624,11 @@ $(ipydir)/pybind11: $(ibidir)/eigen \ $(ibidir)/boost \ $(ipydir)/setuptools \ | $(tdir)/pybind11-$(pybind11-version).tar.gz + pyhook_after() { + cp -r include/pybind11 $(iidir)/python$(python-major-version)m/ + } $(call pybuild, tar xf, pybind11-$(pybind11-version), ,\ - pybind11 $(pybind11-version), - cp -r include/pybind11 $(iidir)/python$(python-major-version)m/) + pybind11 $(pybind11-version)) $(ipydir)/pycparser: $(ipydir)/setuptools \ | $(tdir)/pycparser-$(pycparser-version).tar.gz -- cgit v1.2.1 From fb041fe844074d2556cfc350b52f5742bfb866b3 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 12 Apr 2020 15:26:37 +0100 Subject: Configure (numpy): added --std=c99 to CFLAGS to fix error Elham Saremi recently reported the following errors when building Numpy in numpy/core/src/npysort/radixsort.c.src: "error: 'for' loop initial declarations are only allowed in C99 or C11 mode". After some searching, I found Issue 14147[1] on Numpy's main repository about the same problem. As described there, apparently Numpy needs C99 compiler, but doesn't check for it or set it manually (for some strange reason, leaving it to the packagers to check if they want!!!). Any way, after a check with Elham, we were able to fix it by adding the `--std=c99' to CFLAGS of Numpy's build and with this commit, it is now being implemented in the core Maneage to not cause a problem in any other project. [1] https://github.com/numpy/numpy/issues/14147 --- reproduce/software/make/python.mk | 1 + 1 file changed, 1 insertion(+) (limited to 'reproduce/software/make/python.mk') diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 9c79abc..a15841c 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -599,6 +599,7 @@ $(ipydir)/numpy: $(ibidir)/unzip \ else \ export LDFLAGS="$(LDFLAGS) -shared"; \ fi; \ + export CFLAGS="--std=c99 $$CFLAGS"; \ conf="$$(pwd)/reproduce/software/config/installation/numpy-scipy.cfg"; \ $(call pybuild, unzip, numpy-$(numpy-version),$$conf, \ Numpy $(numpy-version)) \ -- cgit v1.2.1 From 8eb0892e179b4970a9835d29b5f57dd912ca4464 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 17 Apr 2020 01:06:20 +0100 Subject: IMPORTANT: software config directly under reproduce/software/config Until now the software configuration parameters were defined under the `reproduce/software/config/installation/' directory. This was because the configuration parameters of analysis software (for example Gnuastro's configurations) were placed under there too. But this was terribly confusing, because the run-time options of programs falls under the "analysis" phase of the project. With this commit, the Gnuastro configuration files have been moved under the new `reproduce/analysis/config/gnuastro' directory and the software configuration files are directly under `reproduce/software/config'. A clean build was done with this change and it didn't crash, but it may cause crashes in derived projects, so after merging with Maneage, please re-configure your project to see if anything has been missed. Please let us know if there is a problem. --- reproduce/software/make/python.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'reproduce/software/make/python.mk') diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index a15841c..1ef1bf2 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -600,7 +600,7 @@ $(ipydir)/numpy: $(ibidir)/unzip \ export LDFLAGS="$(LDFLAGS) -shared"; \ fi; \ export CFLAGS="--std=c99 $$CFLAGS"; \ - conf="$$(pwd)/reproduce/software/config/installation/numpy-scipy.cfg"; \ + conf="$$(pwd)/reproduce/software/config/numpy-scipy.cfg"; \ $(call pybuild, unzip, numpy-$(numpy-version),$$conf, \ Numpy $(numpy-version)) \ && cp $(dtexdir)/numpy.tex $(ictdir)/ \ @@ -679,7 +679,7 @@ $(ipydir)/scipy: $(ipydir)/numpy \ else \ export LDFLAGS="$(LDFLAGS) -shared"; \ fi; \ - conf="$$(pwd)/reproduce/software/config/installation/numpy-scipy.cfg"; \ + conf="$$(pwd)/reproduce/software/config/numpy-scipy.cfg"; \ $(call pybuild, tar xf, scipy-$(scipy-version),$$conf) \ && cp $(dtexdir)/scipy.tex $(ictdir)/ \ && echo "Scipy $(scipy-version) \citep{scipy2007,scipy2011}" > $@ -- cgit v1.2.1