aboutsummaryrefslogtreecommitdiff
path: root/reproduce/src
diff options
context:
space:
mode:
authorRaul Infante-Sainz <infantesainz@gmail.com>2019-04-02 22:43:29 +0100
committerRaul Infante-Sainz <infantesainz@gmail.com>2019-04-02 22:43:29 +0100
commitaa9de365ae7f85488e8e53acd642cf657de251a8 (patch)
treece6f68cf89a289e2e6d9cab3e0e1693a48c01d5c /reproduce/src
parentf0030ea8974b41ef4b26c3ce41c3294b224fb358 (diff)
Python packages are installed as high level program dependencies
Until this commit, the installation of all Python packages were done in a separate Makefile. With this commit, the pipeline install Python packages as part of the hight level software. All Python packages rules them remain in a separate Makefile, but this Makefile is included in the high level dependency `reproduce/src/make/dependencies.mk'.
Diffstat (limited to 'reproduce/src')
-rw-r--r--reproduce/src/make/dependencies-python.mk57
-rw-r--r--reproduce/src/make/dependencies.mk43
2 files changed, 30 insertions, 70 deletions
diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk
index 642316c..30e10c1 100644
--- a/reproduce/src/make/dependencies-python.mk
+++ b/reproduce/src/make/dependencies-python.mk
@@ -29,51 +29,6 @@
-# Top level environment
-include reproduce/config/pipeline/LOCAL.mk
-include reproduce/src/make/dependencies-build-rules.mk
-include reproduce/config/pipeline/dependency-versions.mk
-
-ddir = $(BDIR)/dependencies
-tdir = $(BDIR)/dependencies/tarballs
-idir = $(BDIR)/dependencies/installed
-ibdir = $(BDIR)/dependencies/installed/bin
-ildir = $(BDIR)/dependencies/installed/lib
-ilidir = $(BDIR)/dependencies/installed/lib/built
-ipydir = $(BDIR)/dependencies/installed/lib/built/python
-
-# Define the top-level programs to build (installed in `.local/bin').
-top-level-python = astroquery matplotlib
-all: $(foreach p, $(top-level-python), $(ipydir)/$(p))
-
-# Other basic environment settings: We are only including the host
-# operating system's PATH environment variable (after our own!) for the
-# compiler and linker. For the library binaries and headers, we are only
-# using our internally built libraries.
-#
-# To investigate:
-#
-# 1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all
-# the parameters defined bellow as `NAME=VALUE' statements before
-# calling Bash. This will enable us to completely ignore the user's
-# native environment.
-#
-# 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the
-# user's environment.
-.ONESHELL:
-.SHELLFLAGS := --noprofile --norc -ec
-export CCACHE_DISABLE := 1
-export PATH := $(ibdir)
-export LD_RUN_PATH := $(ildir)
-export LD_LIBRARY_PATH := $(ildir)
-export SHELL := $(ibdir)/bash
-export CPPFLAGS := -I$(idir)/include
-export PKG_CONFIG_PATH := $(ildir)/pkgconfig
-export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig
-export LDFLAGS := $(rpath_command) -L$(ildir)
-
-
-
# Python enviroment
@@ -106,7 +61,7 @@ export MPI_PYTHON3_SITEARCH :=
# convention, but include the name/version in their tarball names with
# another format, we'll do the modification before the download so the
# downloaded file has our desired format.
-tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \
+pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \
astroquery-$(astroquery-version).tar.gz \
astropy-$(astropy-version).tar.gz \
beautifulsoup4-$(beautifulsoup4-version).tar.gz \
@@ -141,8 +96,8 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \
webencodings-$(webencodings-version).tar.gz \
virtualenv-$(virtualenv-version).tar.gz \
, $(tdir)/$(t) )
-topurl=https://files.pythonhosted.org/packages
-$(tarballs): $(tdir)/%:
+pytopurl=https://files.pythonhosted.org/packages
+$(pytarballs): $(tdir)/%:
if [ -f $(DEPENDENCIES-DIR)/$* ]; then
cp $(DEPENDENCIES-DIR)/$* $@
else
@@ -184,7 +139,7 @@ $(tarballs): $(tdir)/%:
elif [ $$n = secretstorage ]; then
mergenames=0
hash=a6/89/df343dbc2957a317127e7ff2983230dc5336273be34f2e1911519d85aeb5
- h=$(topurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz
+ h=$(pytopurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz
elif [ $$n = asn ]; then h=fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4
elif [ $$n = astroquery ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396
elif [ $$n = astropy ]; then h=eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352
@@ -232,7 +187,7 @@ $(tarballs): $(tdir)/%:
# storing all the tarballs in one directory, we want it to have
# the same naming convention, so we'll download it to a temporary
# name, then rename that.
- if [ $$mergenames = 1 ]; then tarballurl=$(topurl)/$$h/"$*"
+ if [ $$mergenames = 1 ]; then tarballurl=$(pytopurl)/$$h/"$*"
else tarballurl=$$h
fi
@@ -383,7 +338,7 @@ $(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \
$(call pybuild, tar xf, $<, keyring-$(keyring-version))
$(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \
- $(ipydir)/setuptools
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version))
$(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index ee95be9..61d0701 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -35,12 +35,14 @@ include reproduce/src/make/dependencies-build-rules.mk
include reproduce/config/pipeline/dependency-texlive.mk
include reproduce/config/pipeline/dependency-versions.mk
+
ddir = $(BDIR)/dependencies
tdir = $(BDIR)/dependencies/tarballs
idir = $(BDIR)/dependencies/installed
ibdir = $(BDIR)/dependencies/installed/bin
ildir = $(BDIR)/dependencies/installed/lib
ilidir = $(BDIR)/dependencies/installed/lib/built
+ipydir = $(BDIR)/dependencies/installed/lib/built/python
# Define the top-level programs to build (installed in `.local/bin').
#
@@ -48,11 +50,13 @@ ilidir = $(BDIR)/dependencies/installed/lib/built
# high level software depend on it. The current rule for ATLAS is tested
# successfully on Mac (only static) and GNU/Linux (shared and static). But,
# since it takes a few hours to build, it is not currently a target.
+top-level-libraries = # atlas
top-level-programs = astnoisechisel flock metastore unzip zip
-top-level-libraries = freetype openblas fftw # atlas
-all: $(ddir)/texlive-versions.tex \
- $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \
- $(foreach p, $(top-level-libraries), $(ilidir)/$(p))
+top-level-python = astroquery matplotlib
+all: $(ddir)/texlive-versions.tex \
+ $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \
+ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \
+ $(foreach p, $(top-level-python), $(ipydir)/$(p))
# Other basic environment settings: We are only including the host
# operating system's PATH environment variable (after our own!) for the
@@ -81,7 +85,8 @@ export LD_LIBRARY_PATH := $(ildir):$(il64dir)
export LDFLAGS := $(rpath_command) -L$(ildir)
-
+# Python packages
+include reproduce/src/make/dependencies-python.mk
# Tarballs
@@ -123,12 +128,12 @@ $(tarballs): $(tdir)/%:
if [ -f $(DEPENDENCIES-DIR)/$* ]; then
cp $(DEPENDENCIES-DIR)/$* $@
else
- # Remove all numbers, `-' and `.' from the tarball name so we can
- # search more easily only with the program name.
+ # Remove all numbers, `-' and `.' from the tarball name so we can
+ # search more easily only with the program name.
n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \
| awk '{print $$1}' )
- # Set the top download link of the requested tarball.
+ # Set the top download link of the requested tarball.
mergenames=1
if [ $$n = cfitsio ]; then
mergenames=0
@@ -179,20 +184,20 @@ $(tarballs): $(tdir)/%:
exit 1
fi
- # Download the requested tarball. Note that some packages may not
- # follow our naming convention (where the package name is merged
- # with its version number). In such cases, `w' will be the full
- # address, not just the top directory address. But since we are
- # storing all the tarballs in one directory, we want it to have
- # the same naming convention, so we'll download it to a temporary
- # name, then rename that.
+ # Download the requested tarball. Note that some packages may not
+ # follow our naming convention (where the package name is merged
+ # with its version number). In such cases, `w' will be the full
+ # address, not just the top directory address. But since we are
+ # storing all the tarballs in one directory, we want it to have
+ # the same naming convention, so we'll download it to a temporary
+ # name, then rename that.
if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*"
else tarballurl=$$w
fi
- # If the download fails, Wget will write the error message in the
- # target file, so Make will think that its done! To avoid this
- # problem, we'll rename the output.
+ # If the download fails, Wget will write the error message in the
+ # target file, so Make will think that its done! To avoid this
+ # problem, we'll rename the output.
echo "Downloading $$tarballurl"
if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then
rm -f $@
@@ -353,7 +358,7 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
$(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz
if [ x$(on_mac_os) = xyes ]; then \
- export CC=clang \
+ export CC=clang; \
fi; \
cd $(ddir) \
&& tar xf $< \