From aa9de365ae7f85488e8e53acd642cf657de251a8 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 2 Apr 2019 22:43:29 +0100 Subject: Python packages are installed as high level program dependencies Until this commit, the installation of all Python packages were done in a separate Makefile. With this commit, the pipeline install Python packages as part of the hight level software. All Python packages rules them remain in a separate Makefile, but this Makefile is included in the high level dependency `reproduce/src/make/dependencies.mk'. --- reproduce/src/make/dependencies-python.mk | 57 ++++--------------------------- reproduce/src/make/dependencies.mk | 43 ++++++++++++----------- 2 files changed, 30 insertions(+), 70 deletions(-) (limited to 'reproduce') diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 642316c..30e10c1 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -29,51 +29,6 @@ -# Top level environment -include reproduce/config/pipeline/LOCAL.mk -include reproduce/src/make/dependencies-build-rules.mk -include reproduce/config/pipeline/dependency-versions.mk - -ddir = $(BDIR)/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib -ilidir = $(BDIR)/dependencies/installed/lib/built -ipydir = $(BDIR)/dependencies/installed/lib/built/python - -# Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astroquery matplotlib -all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) - -# Other basic environment settings: We are only including the host -# operating system's PATH environment variable (after our own!) for the -# compiler and linker. For the library binaries and headers, we are only -# using our internally built libraries. -# -# To investigate: -# -# 1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all -# the parameters defined bellow as `NAME=VALUE' statements before -# calling Bash. This will enable us to completely ignore the user's -# native environment. -# -# 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the -# user's environment. -.ONESHELL: -.SHELLFLAGS := --noprofile --norc -ec -export CCACHE_DISABLE := 1 -export PATH := $(ibdir) -export LD_RUN_PATH := $(ildir) -export LD_LIBRARY_PATH := $(ildir) -export SHELL := $(ibdir)/bash -export CPPFLAGS := -I$(idir)/include -export PKG_CONFIG_PATH := $(ildir)/pkgconfig -export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export LDFLAGS := $(rpath_command) -L$(ildir) - - - # Python enviroment @@ -106,7 +61,7 @@ export MPI_PYTHON3_SITEARCH := # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. -tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ +pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ astroquery-$(astroquery-version).tar.gz \ astropy-$(astropy-version).tar.gz \ beautifulsoup4-$(beautifulsoup4-version).tar.gz \ @@ -141,8 +96,8 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ webencodings-$(webencodings-version).tar.gz \ virtualenv-$(virtualenv-version).tar.gz \ , $(tdir)/$(t) ) -topurl=https://files.pythonhosted.org/packages -$(tarballs): $(tdir)/%: +pytopurl=https://files.pythonhosted.org/packages +$(pytarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* $@ else @@ -184,7 +139,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = secretstorage ]; then mergenames=0 hash=a6/89/df343dbc2957a317127e7ff2983230dc5336273be34f2e1911519d85aeb5 - h=$(topurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz + h=$(pytopurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz elif [ $$n = asn ]; then h=fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4 elif [ $$n = astroquery ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396 elif [ $$n = astropy ]; then h=eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 @@ -232,7 +187,7 @@ $(tarballs): $(tdir)/%: # storing all the tarballs in one directory, we want it to have # the same naming convention, so we'll download it to a temporary # name, then rename that. - if [ $$mergenames = 1 ]; then tarballurl=$(topurl)/$$h/"$*" + if [ $$mergenames = 1 ]; then tarballurl=$(pytopurl)/$$h/"$*" else tarballurl=$$h fi @@ -383,7 +338,7 @@ $(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \ $(call pybuild, tar xf, $<, keyring-$(keyring-version)) $(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ - $(ipydir)/setuptools + $(ipydir)/setuptools $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version)) $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index ee95be9..61d0701 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -35,12 +35,14 @@ include reproduce/src/make/dependencies-build-rules.mk include reproduce/config/pipeline/dependency-texlive.mk include reproduce/config/pipeline/dependency-versions.mk + ddir = $(BDIR)/dependencies tdir = $(BDIR)/dependencies/tarballs idir = $(BDIR)/dependencies/installed ibdir = $(BDIR)/dependencies/installed/bin ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built +ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). # @@ -48,11 +50,13 @@ ilidir = $(BDIR)/dependencies/installed/lib/built # high level software depend on it. The current rule for ATLAS is tested # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. +top-level-libraries = # atlas top-level-programs = astnoisechisel flock metastore unzip zip -top-level-libraries = freetype openblas fftw # atlas -all: $(ddir)/texlive-versions.tex \ - $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ - $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) +top-level-python = astroquery matplotlib +all: $(ddir)/texlive-versions.tex \ + $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ + $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ + $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host # operating system's PATH environment variable (after our own!) for the @@ -81,7 +85,8 @@ export LD_LIBRARY_PATH := $(ildir):$(il64dir) export LDFLAGS := $(rpath_command) -L$(ildir) - +# Python packages +include reproduce/src/make/dependencies-python.mk # Tarballs @@ -123,12 +128,12 @@ $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* $@ else - # Remove all numbers, `-' and `.' from the tarball name so we can - # search more easily only with the program name. + # Remove all numbers, `-' and `.' from the tarball name so we can + # search more easily only with the program name. n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ | awk '{print $$1}' ) - # Set the top download link of the requested tarball. + # Set the top download link of the requested tarball. mergenames=1 if [ $$n = cfitsio ]; then mergenames=0 @@ -179,20 +184,20 @@ $(tarballs): $(tdir)/%: exit 1 fi - # Download the requested tarball. Note that some packages may not - # follow our naming convention (where the package name is merged - # with its version number). In such cases, `w' will be the full - # address, not just the top directory address. But since we are - # storing all the tarballs in one directory, we want it to have - # the same naming convention, so we'll download it to a temporary - # name, then rename that. + # Download the requested tarball. Note that some packages may not + # follow our naming convention (where the package name is merged + # with its version number). In such cases, `w' will be the full + # address, not just the top directory address. But since we are + # storing all the tarballs in one directory, we want it to have + # the same naming convention, so we'll download it to a temporary + # name, then rename that. if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*" else tarballurl=$$w fi - # If the download fails, Wget will write the error message in the - # target file, so Make will think that its done! To avoid this - # problem, we'll rename the output. + # If the download fails, Wget will write the error message in the + # target file, so Make will think that its done! To avoid this + # problem, we'll rename the output. echo "Downloading $$tarballurl" if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then rm -f $@ @@ -353,7 +358,7 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz if [ x$(on_mac_os) = xyes ]; then \ - export CC=clang \ + export CC=clang; \ fi; \ cd $(ddir) \ && tar xf $< \ -- cgit v1.2.1