Astropy now depending on the Expat library to fix internal conflict

Until now, Astropy was instructed to build its own internal copy of the
Expat library. However, with the recent commits before, Maneage now
includes an installation of Expat and Astropy can't keep the two (its
internal version and the project's version) separate, so they conflict and
don't let Astropy get built.

When trying to build Manage (the actual project, not this paper) after
applying the commits before there, Raul discovered this problem.

With this commit, the problem is fixed by setting the Expat library as an
explicit dependency of Astropy and asking Astropy to ignore its internal
copy.

While doing this, I recognized that it is much easier and elegant to add
steps in various stages of the `pybuild' function through hooks instead of
variables. So the fifth argument of the `pybuild' function was removed and
now it actually checks if hooks are defined as functions and if so, they
will be called.

The `pyhook_after' function was also implemented in the installation of
`pybind11' (which needed it, given that the 5th argument of `pybuild' was
removed) and after doing a test-build, I noticed that two lines were not
ending with a `\' in `boost' (a dependency of `pybind11').

2 files changed, 25 insertions, 9 deletions

diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk
index 605ca74..cdaaf7d 100644
--- a/reproduce/software/make/high-level.mk
+++ b/reproduce/software/make/high-level.mk
@@ -520,8 +520,8 @@ $(ibidir)/boost: $(ibidir)/openmpi \
 	vstr=$$(echo $(boost-version) | sed -e's/\./_/g')
 	rm -rf $(ddir)/boost_$$vstr
 	topdir=$(pwd); cd $(ddir);
-	tar xf $(word 1,$(filter $(tdir)/%,$|))
-	&& cd boost_$$vstr
+	tar xf $(word 1,$(filter $(tdir)/%,$|)) \
+	&& cd boost_$$vstr \
 	&& ./bootstrap.sh --prefix=$(idir) --with-libraries=all \
 	                  --with-python=python3 \
 	&& echo "using mpi ;" > project-config.jam \
diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk
index fef1605..77743fc 100644
--- a/reproduce/software/make/python.mk
+++ b/reproduce/software/make/python.mk
@@ -335,7 +335,10 @@ $(ibidir)/python: $(ibidir)/libffi \
 #   2) Unpacked directory name after unpacking the tarball
 #   3) site.cfg file (optional).
 #   4) Official software name (for paper).
-#   5) Manual step after installation.
+#
+# Hooks:
+#   pyhook_before: optional steps before running `python setup.py build'
+#   pyhook_after: optional steps after running `python setup.py install'
 pybuild = cd $(ddir); rm -rf $(2); \
 	 if ! $(1) $(word 1,$(filter $(tdir)/%,$|)); then \
 	   echo; echo "Tar error"; exit 1; \
@@ -346,11 +349,10 @@ pybuild = cd $(ddir); rm -rf $(2); \
 	       -e 's|@INCDIR[@]|'"$(idir)/include"'|' \
 	       $(3) > site.cfg; \
 	 fi; \
-	 if [ x"$(5)" = x ]; then after="echo no after"; \
-	 else after="$(5)"; fi \
+	 && if type pyhook_before &>/dev/null; then pyhook_before; fi \
 	 && python setup.py build \
 	 && python setup.py install \
-	 && $$after \
+	 && if type pyhook_after &>/dev/null; then pyhook_after; fi \
 	 && cd .. \
 	 && rm -rf $(2) \
 	 && echo "$(4)" > $@
@@ -381,12 +383,24 @@ $(ipydir)/astroquery: $(ipydir)/astropy \
 	                Astroquery $(astroquery-version))
 
 $(ipydir)/astropy: $(ipydir)/h5py \
+                   $(ibidir)/expat \
                    $(ipydir)/scipy \
                    $(ipydir)/numpy \
                    $(ipydir)/pyyaml \
                    $(ipydir)/html5lib \
                    $(ipydir)/beautifulsoup4 \
                    | $(tdir)/astropy-$(astropy-version).tar.gz
+        # Currently, when the Expat library is already built in a project
+        # (for example as a dependency of another program), Astropy's
+        # internal building of Expat will conflict with the project's. So
+        # we have added Expat as a dependency of astropy (so it is always
+        # built before it, and we tell Astropy to use the project's
+        # libexpat.
+	pyhook_before () {
+	  echo ""                   >> setup.cfg
+	  echo "[build]"            >> setup.cfg
+	  echo "use_system_expat=1" >> setup.cfg
+	}
 	$(call pybuild, tar xf, astropy-$(astropy-version)) \
 	&& cp $(dtexdir)/astropy.tex $(ictdir)/ \
 	&& echo "Astropy $(astropy-version) \citep{astropy2013,astropy2018}" > $@
@@ -474,7 +488,7 @@ $(ipydir)/galsim: $(ipydir)/future \
                   $(ipydir)/pybind11 \
                   $(ipydir)/lsstdesccoord \
                   | $(tdir)/galsim-$(galsim-version).tar.gz
-	$(call pybuild, tar xf, GalSim-$(galsim-version), ,) \
+	$(call pybuild, tar xf, GalSim-$(galsim-version)) \
 	&& cp $(dtexdir)/galsim.tex $(ictdir)/ \
 	&& echo "Galsim $(galsim-version) \citep{galsim}" > $@
 
@@ -610,9 +624,11 @@ $(ipydir)/pybind11: $(ibidir)/eigen \
                     $(ibidir)/boost \
                     $(ipydir)/setuptools \
                     | $(tdir)/pybind11-$(pybind11-version).tar.gz
+	pyhook_after() {
+	  cp -r include/pybind11 $(iidir)/python$(python-major-version)m/
+	}
 	$(call pybuild, tar xf, pybind11-$(pybind11-version), ,\
-	                pybind11 $(pybind11-version),
-	                cp -r include/pybind11 $(iidir)/python$(python-major-version)m/)
+	                pybind11 $(pybind11-version))
 
 $(ipydir)/pycparser: $(ipydir)/setuptools \
                      | $(tdir)/pycparser-$(pycparser-version).tar.gz