Python packages are installed as high level program dependencies

Until this commit, the installation of all Python packages were
done in a separate Makefile.

With this commit, the pipeline install Python packages as part of the
hight level software. All Python packages rules  them remain in a
separate Makefile, but this Makefile is included in the high level
dependency `reproduce/src/make/dependencies.mk'.

2 files changed, 30 insertions, 70 deletions

diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk
index 642316c..30e10c1 100644
--- a/reproduce/src/make/dependencies-python.mk
+++ b/reproduce/src/make/dependencies-python.mk
@@ -29,51 +29,6 @@
 
 
 
-# Top level environment
-include reproduce/config/pipeline/LOCAL.mk
-include reproduce/src/make/dependencies-build-rules.mk
-include reproduce/config/pipeline/dependency-versions.mk
-
-ddir   = $(BDIR)/dependencies
-tdir   = $(BDIR)/dependencies/tarballs
-idir   = $(BDIR)/dependencies/installed
-ibdir  = $(BDIR)/dependencies/installed/bin
-ildir  = $(BDIR)/dependencies/installed/lib
-ilidir = $(BDIR)/dependencies/installed/lib/built
-ipydir = $(BDIR)/dependencies/installed/lib/built/python
-
-# Define the top-level programs to build (installed in `.local/bin').
-top-level-python   = astroquery matplotlib
-all: $(foreach p, $(top-level-python), $(ipydir)/$(p))
-
-# Other basic environment settings: We are only including the host
-# operating system's PATH environment variable (after our own!) for the
-# compiler and linker. For the library binaries and headers, we are only
-# using our internally built libraries.
-#
-# To investigate:
-#
-#    1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all
-#       the parameters defined bellow as `NAME=VALUE' statements before
-#       calling Bash. This will enable us to completely ignore the user's
-#       native environment.
-#
-#    2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the
-#       user's environment.
-.ONESHELL:
-.SHELLFLAGS              := --noprofile --norc -ec
-export CCACHE_DISABLE    := 1
-export PATH              := $(ibdir)
-export LD_RUN_PATH       := $(ildir)
-export LD_LIBRARY_PATH   := $(ildir)
-export SHELL             := $(ibdir)/bash
-export CPPFLAGS          := -I$(idir)/include
-export PKG_CONFIG_PATH   := $(ildir)/pkgconfig
-export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig
-export LDFLAGS           := $(rpath_command) -L$(ildir)
-
-
-
 
 
 # Python enviroment
@@ -106,7 +61,7 @@ export MPI_PYTHON3_SITEARCH   :=
 # convention, but include the name/version in their tarball names with
 # another format, we'll do the modification before the download so the
 # downloaded file has our desired format.
-tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz           \
+pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz         \
                         astroquery-$(astroquery-version).tar.gz           \
                         astropy-$(astropy-version).tar.gz                 \
                         beautifulsoup4-$(beautifulsoup4-version).tar.gz   \
@@ -141,8 +96,8 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz           \
                         webencodings-$(webencodings-version).tar.gz       \
                         virtualenv-$(virtualenv-version).tar.gz           \
                       , $(tdir)/$(t) )
-topurl=https://files.pythonhosted.org/packages
-$(tarballs): $(tdir)/%:
+pytopurl=https://files.pythonhosted.org/packages
+$(pytarballs): $(tdir)/%:
 	if [ -f $(DEPENDENCIES-DIR)/$* ]; then
 	  cp $(DEPENDENCIES-DIR)/$* $@
 	else
@@ -184,7 +139,7 @@ $(tarballs): $(tdir)/%:
 	  elif [ $$n = secretstorage  ]; then
 	    mergenames=0
 	    hash=a6/89/df343dbc2957a317127e7ff2983230dc5336273be34f2e1911519d85aeb5
-	    h=$(topurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz
+	    h=$(pytopurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz
 	  elif [ $$n = asn            ]; then h=fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4
 	  elif [ $$n = astroquery     ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396
 	  elif [ $$n = astropy        ]; then h=eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352
@@ -232,7 +187,7 @@ $(tarballs): $(tdir)/%:
           # storing all the tarballs in one directory, we want it to have
           # the same naming convention, so we'll download it to a temporary
           # name, then rename that.
-	  if [ $$mergenames = 1 ]; then  tarballurl=$(topurl)/$$h/"$*"
+	  if [ $$mergenames = 1 ]; then  tarballurl=$(pytopurl)/$$h/"$*"
 	  else                           tarballurl=$$h
 	  fi
 
@@ -383,7 +338,7 @@ $(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz    \
 	$(call pybuild, tar xf, $<, keyring-$(keyring-version))
 
 $(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz    \
-                   $(ipydir)/setuptools
+                      $(ipydir)/setuptools
 	$(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version))
 
 $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz   \
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index ee95be9..61d0701 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -35,12 +35,14 @@ include reproduce/src/make/dependencies-build-rules.mk
 include reproduce/config/pipeline/dependency-texlive.mk
 include reproduce/config/pipeline/dependency-versions.mk
 
+
 ddir   = $(BDIR)/dependencies
 tdir   = $(BDIR)/dependencies/tarballs
 idir   = $(BDIR)/dependencies/installed
 ibdir  = $(BDIR)/dependencies/installed/bin
 ildir  = $(BDIR)/dependencies/installed/lib
 ilidir = $(BDIR)/dependencies/installed/lib/built
+ipydir = $(BDIR)/dependencies/installed/lib/built/python
 
 # Define the top-level programs to build (installed in `.local/bin').
 #
@@ -48,11 +50,13 @@ ilidir = $(BDIR)/dependencies/installed/lib/built
 # high level software depend on it. The current rule for ATLAS is tested
 # successfully on Mac (only static) and GNU/Linux (shared and static). But,
 # since it takes a few hours to build, it is not currently a target.
+top-level-libraries = # atlas
 top-level-programs  = astnoisechisel flock metastore unzip zip
-top-level-libraries = freetype openblas fftw  # atlas
-all: $(ddir)/texlive-versions.tex                       \
-     $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \
-     $(foreach p, $(top-level-libraries), $(ilidir)/$(p))
+top-level-python    = astroquery matplotlib
+all: $(ddir)/texlive-versions.tex                         \
+     $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \
+     $(foreach p, $(top-level-programs),  $(ibdir)/$(p))  \
+     $(foreach p, $(top-level-python),    $(ipydir)/$(p))
 
 # Other basic environment settings: We are only including the host
 # operating system's PATH environment variable (after our own!) for the
@@ -81,7 +85,8 @@ export LD_LIBRARY_PATH   := $(ildir):$(il64dir)
 export LDFLAGS           := $(rpath_command) -L$(ildir)
 
 
-
+# Python packages
+include reproduce/src/make/dependencies-python.mk
 
 
 # Tarballs
@@ -123,12 +128,12 @@ $(tarballs): $(tdir)/%:
 	if [ -f $(DEPENDENCIES-DIR)/$* ]; then
 	  cp $(DEPENDENCIES-DIR)/$* $@
 	else
-          # Remove all numbers, `-' and `.' from the tarball name so we can
-          # search more easily only with the program name.
+	  # Remove all numbers, `-' and `.' from the tarball name so we can
+	  # search more easily only with the program name.
 	  n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g'           \
 	               | awk '{print $$1}' )
 
-          # Set the top download link of the requested tarball.
+	  # Set the top download link of the requested tarball.
 	  mergenames=1
 	  if [ $$n = cfitsio     ]; then
 	    mergenames=0
@@ -179,20 +184,20 @@ $(tarballs): $(tdir)/%:
 	    exit 1
 	  fi
 
-          # Download the requested tarball. Note that some packages may not
-          # follow our naming convention (where the package name is merged
-          # with its version number). In such cases, `w' will be the full
-          # address, not just the top directory address. But since we are
-          # storing all the tarballs in one directory, we want it to have
-          # the same naming convention, so we'll download it to a temporary
-          # name, then rename that.
+	  # Download the requested tarball. Note that some packages may not
+	  # follow our naming convention (where the package name is merged
+	  # with its version number). In such cases, `w' will be the full
+	  # address, not just the top directory address. But since we are
+	  # storing all the tarballs in one directory, we want it to have
+	  # the same naming convention, so we'll download it to a temporary
+	  # name, then rename that.
 	  if [ $$mergenames = 1 ]; then  tarballurl=$$w/"$*"
 	  else                           tarballurl=$$w
 	  fi
 
-          # If the download fails, Wget will write the error message in the
-          # target file, so Make will think that its done! To avoid this
-          # problem, we'll rename the output.
+	  # If the download fails, Wget will write the error message in the
+	  # target file, so Make will think that its done! To avoid this
+	  # problem, we'll rename the output.
 	  echo "Downloading $$tarballurl"
 	  if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then
 	     rm -f $@
@@ -353,7 +358,7 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
 
 $(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz
 	if [ x$(on_mac_os) = xyes ]; then                           \
-	  export CC=clang                                           \
+	  export CC=clang;                                          \
 	fi;                                                         \
 	cd $(ddir)                                                  \
 	&& tar xf $<                                                \