diff options
| -rw-r--r-- | .file-metadata | bin | 3923 -> 4007 bytes | |||
| -rwxr-xr-x | configure | 18 | ||||
| -rw-r--r-- | reproduce/config/pipeline/dependency-versions.mk | 38 | ||||
| -rw-r--r-- | reproduce/src/bash/git-post-checkout | 9 | ||||
| -rw-r--r-- | reproduce/src/bash/git-pre-commit | 11 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies-basic.mk | 122 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies-build-rules.mk | 17 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies-python.mk | 365 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies.mk | 61 | ||||
| -rw-r--r-- | reproduce/src/make/initialize.mk | 13 |
10 files changed, 542 insertions, 112 deletions
diff --git a/.file-metadata b/.file-metadata Binary files differindex a8b09db..d58756a 100644 --- a/.file-metadata +++ b/.file-metadata @@ -718,6 +718,24 @@ numthreads=$($instdir/bin/nproc) +# Python dependencies +# ------------------- +# +# Python has its own installation program. To help in managing the +# dependencies we make the installation of packages as a separate +# Makefile. +./.local/bin/make -f reproduce/src/make/dependencies-python.mk \ + rpath_command=$rpath_command \ + static_build=$static_build \ + on_mac_os=$on_mac_os \ + numthreads=$numthreads \ + -j$numthreads + + + + + + # Make sure TeX Live installed successfully # ----------------------------------------- # diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index c704e27..9104852 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -3,14 +3,14 @@ # Programs bash-version = 5.0 binutils-version = 2.31.1 -bzip2-version = 1.0.6 cmake-version = 3.12.4 coreutils-version = 8.30 diffutils-version = 3.7 findutils-version = 4.6.0.199-e3fc flock-version = 0.2.3 +freetype-version = 2.9 gawk-version = 4.2.1 -gcc-version = 8.2.0 +gcc-version = 8.3.0 ghostscript-version = 9.26 git-version = 2.20.1 gmp-version = 6.1.2 @@ -18,8 +18,9 @@ gnuastro-version = 0.8 grep-version = 3.3 gzip-version = 1.10 isl-version = 0.18 -libtool-version = 2.4.6 libbsd-version = 0.9.1 +libpng-version = 1.6.36 +libtool-version = 2.4.6 lzip-version = 1.20 make-version = 4.2.90 metastore-version = 1.1.2-23-fa9170b @@ -52,15 +53,38 @@ zlib-version = 1.2.11 # # IMPORTANT: Fix url in `reproduce/src/make/dependencies.mk' # if changing the version +astroquery-version = 0.3.9 astropy-version = 3.1.1 +beautifulsoup4-version = 4.7.1 +certifi-version = 2018.11.29 +chardet-version = 3.0.4 +cycler-version = 0.10.0 +entrypoints-version = 0.3 +html5lib-version = 1.0.1 +idna-version = 2.8 +kiwisolver-version = 1.0.1 +keyring-version = 18.0.0 +matplotlib-version = 3.0.2 numpy-version = 1.16.1 +pip-version = 19.0.2 +pyparsing-version = 2.3.1 +python-dateutil-version = 2.8.0 +requests-version = 2.21.0 +scipy-version = 1.2.1 +setuptools-version = 40.8.0 +setuptools_scm-version = 3.2.0 +six-version = 1.12.0 +soupsieve-version = 1.8 +urllib3-version = 1.24.1 +virtualenv-version = 16.4.0 +webencodings-version = 0.5.1 # Special libraries # ----------------- # -# The shared library name of the following libraries is explicity mentioned -# the software build Makefiles (`reproduce/src/make/dependencies*.mk'). If -# you change their version, also please change the explicit shared library -# names also. +# When updating the version of these libraries, please look into the build +# rule first: In one way or another, the version string becomes necessary +# during their build and must be accounted for. +bzip2-version = 1.0.6 libgit2-version = 0.26.0 wcslib-version = 6.2 diff --git a/reproduce/src/bash/git-post-checkout b/reproduce/src/bash/git-post-checkout index e2a0fd0..153b277 100644 --- a/reproduce/src/bash/git-post-checkout +++ b/reproduce/src/bash/git-post-checkout @@ -9,8 +9,13 @@ # special characters for the installation location of meta-store so our own # installation is found by Git. -# File containig the metadata. +# File containig the metadata and metastore executable. MSFILE=".file-metadata" +MSBIN=@BINDIR@/metastore + +# If metastore is not installed, then ignore this script (exit with a +# status of 0). +if [ ! -f $MSBIN ]; then exit 0; fi # Delete all temporary files find @TOP_PROJECT_DIR@/ -name "*~" -type f -delete @@ -32,7 +37,7 @@ fi # Run metastore. exit_on_fail \ - @BINDIR@/metastore -a -m -e -E -q -O @USER@ -G @GROUP@ -f "$MSFILE" + $MSBIN -a -m -e -E -q -O @USER@ -G @GROUP@ -f "$MSFILE" # Return with a success code (0). exit 0 diff --git a/reproduce/src/bash/git-pre-commit b/reproduce/src/bash/git-pre-commit index ca28757..0fa6c52 100644 --- a/reproduce/src/bash/git-pre-commit +++ b/reproduce/src/bash/git-pre-commit @@ -20,8 +20,13 @@ # git reset HEAD -- .metadata # git checkout HEAD -- .metadata -# File containig the metadata. +# File containig the metadata and metastore executable. MSFILE=".file-metadata" +MSBIN=@BINDIR@/metastore + +# If metastore is not installed, then ignore this script (exit with a +# status of 0). +if [ ! -f $MSBIN ]; then exit 0; fi # Function to help in reporting a crash. exit_on_fail() { @@ -34,12 +39,12 @@ exit_on_fail() { # Run metastore. exit_on_fail \ - @BINDIR@/metastore -O @USER@ -G @GROUP@ -s -f "$MSFILE" + $MSBIN -O @USER@ -G @GROUP@ -s -f "$MSFILE" # If it's first metastore commit, store again to include $MSFILE in $MSFILE. if ! git-ls-tree --name-only HEAD 2>/dev/null | grep -Fqx "$MSFILE"; then exit_on_fail \ - @BINDIR@/metastore -O @USER@ -G @GROUP@ -s -f "$MSFILE" + $MSBIN -O @USER@ -G @GROUP@ -s -f "$MSFILE" fi # Check if the metadata file exists. diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 4f5362f..cb3a7f2 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -65,8 +65,7 @@ export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) # Define the programs that don't depend on any other. -top-level-programs = low-level-links ls sed gawk grep diff find \ - wget which +top-level-programs = low-level-links wget gcc all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) @@ -141,7 +140,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = diffutils ]; then w=http://ftpmirror.gnu.org/gnu/diffutils;\ elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \ elif [ $$n = gawk ]; then w=http://ftpmirror.gnu.org/gnu/gawk; \ - elif [ $$n = gcc ]; then w=http://ftpmirror.gnu.org/gcc/gcc-$(gcc-version); \ + elif [ $$n = gcc ]; then w=http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version); \ elif [ $$n = gmp ]; then w=https://gmplib.org/download/gmp; \ elif [ $$n = grep ]; then w=http://ftpmirror.gnu.org/gnu/grep; \ elif [ $$n = gzip ]; then w=http://ftpmirror.gnu.org/gnu/gzip; \ @@ -210,14 +209,14 @@ makelink = export PATH=$$(echo $(syspath)| tr : '\n' |grep -v ccache \ if [ x$$a != x ]; then ln -s $$a $(ibdir)/$(1); fi $(ibdir) $(ildir):; mkdir $@ $(ibdir)/low-level-links: | $(ibdir) $(ildir) + # The Assembler $(call makelink,as) - # The compiler + # Compiler (Cmake needs the clang compiler which we aren't building + # yet in the pipeline). $(call makelink,clang) - $(call makelink,gcc) - $(call makelink,g++) - $(call makelink,cc) + $(call makelink,clang++) # The linker $(call makelink,ar) @@ -227,7 +226,9 @@ $(ibdir)/low-level-links: | $(ibdir) $(ildir) $(call makelink,ranlib) # Mac OS specific + $(call makelink,sysctl) $(call makelink,sw_vers) + $(call makelink,dsymutil) $(call makelink,install_name_tool) # On Mac OS, libtool is different compared to GNU Libtool. The @@ -287,15 +288,34 @@ $(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz $(call gbuild, $<, xz-$(xz-version), static) $(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz - tdir=bzip2-$(bzip2-version); \ - if [ $(static_build) = yes ]; then \ - makecommand="make LDFLAGS=-static"; \ - else \ - makecommand="make"; \ - fi; \ - cd $(ddir) && rm -rf $$tdir && tar xf $< && cd $$tdir && \ - $$makecommand && make install PREFIX=$(idir) && \ - cd .. && rm -rf $$tdir + # Bzip2 doesn't have a `./configure' script, and its Makefile + # doesn't build a shared library. So we can't use the `gbuild' + # function here and we need to take some extra steps (inspired + # from the "Linux from Scratch" guide for Bzip2): + # 1) The `sed' call is for relative installed symbolic links. + # 2) The special Makefile-libbz2_so builds the shared library. + # + # NOTE: the major version number appears in the final symbolic + # link. + tdir=bzip2-$(bzip2-version); \ + if [ $(static_build) = yes ]; then \ + makecommand="make LDFLAGS=-static"; \ + makeshared="echo no-shared"; \ + else \ + makecommand="make"; \ + makeshared="make -f Makefile-libbz2_so"; \ + fi; \ + cd $(ddir) && rm -rf $$tdir && tar xf $< && cd $$tdir \ + && sed -i 's@\(ln -s -f \)$$(PREFIX)/bin/@\1@' Makefile \ + && $$makeshared \ + && cp -a libbz2* $(ildir)/ \ + && make clean \ + && $$makecommand \ + && make install PREFIX=$(idir) \ + && cd .. \ + && rm -rf $$tdir \ + && cd $(ildir) \ + && ln -fs libbz2.so.1.0 libbz2.so # GNU Tar: When built statically, tar gives a segmentation fault on # unpacking Bash. So we'll build it dynamically. @@ -698,6 +718,9 @@ $(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz # We want to build GCC after building all the basic tools that are often # used in a configure script to enable GCC's configure script to work as # smoothly/robustly as possible. +# +# Including Objective C and Objective C++ is necessary for installing +# `matplotlib'. $(ibdir)/gcc: $(tdir)/gcc-$(gcc-version).tar.xz \ $(ibdir)/ls \ $(ibdir)/sed \ @@ -710,38 +733,39 @@ $(ibdir)/gcc: $(tdir)/gcc-$(gcc-version).tar.xz \ $(ibdir)/bash \ $(ibdir)/which + # Clean up (possibly existing) gcc installation + rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov* + rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc* \ + $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64* + # Un-pack all the necessary tools in the top building directory - cd $(ddir); \ - rm -rf gcc-build gcc-$(gcc-version); \ - tar xf $< && \ - mkdir $(ddir)/gcc-build && \ - cd $(ddir)/gcc-build && \ - ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ - --prefix=$(idir) \ - --with-mpc=$(idir) \ - --with-mpfr=$(idir) \ - --with-gmp=$(idir) \ - --with-isl=$(idir) \ - --with-build-time-tools=$(idir) \ - --enable-shared \ - --disable-multilib \ - --disable-multiarch \ - --enable-threads=posix \ - --enable-libmpx \ - --with-local-prefix=$(idir) \ - --enable-linker-build-id \ - --with-gnu-as \ - --with-gnu-ld \ - --enable-lto \ - --with-linker-hash-style=gnu \ - --enable-languages=c,c++,fortran\ - --disable-libada \ - --disable-nls \ - --enable-default-pie \ - --enable-default-ssp \ - --enable-cet=auto \ - --enable-decimal-float && \ - make SHELL=$(ibdir)/bash -j$$(nproc) && \ - make SHELL=$(ibdir)/bash install && \ - cd .. && \ + cd $(ddir); \ + rm -rf gcc-build gcc-$(gcc-version); \ + tar xf $< && \ + mkdir $(ddir)/gcc-build && \ + cd $(ddir)/gcc-build && \ + ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ + --prefix=$(idir) \ + --with-mpc=$(idir) \ + --with-mpfr=$(idir) \ + --with-gmp=$(idir) \ + --with-isl=$(idir) \ + --with-build-time-tools=$(idir) \ + --enable-shared \ + --disable-multilib \ + --disable-multiarch \ + --enable-threads=posix \ + --with-local-prefix=$(idir) \ + --enable-linker-build-id \ + --enable-lto \ + --enable-languages=c,c++,fortran,objc,obj-c++ \ + --disable-libada \ + --disable-nls \ + --enable-default-pie \ + --enable-default-ssp \ + --enable-cet=auto \ + --enable-decimal-float && \ + make SHELL=$(ibdir)/bash -j$$(nproc) && \ + make SHELL=$(ibdir)/bash install && \ + cd .. && \ rm -rf gcc-build gcc-$(gcc-version) diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk index 2247518..5216488 100644 --- a/reproduce/src/make/dependencies-build-rules.mk +++ b/reproduce/src/make/dependencies-build-rules.mk @@ -102,23 +102,6 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then \ - -# Python -# ------ -# -# To build Python packages with direct access to a `setup.py' (if no direct -# access to `setup.py' is needed, pip can be used) -pybuild = cd $(ddir); rm -rf $(2); \ - if ! tar xf $(1); then echo; echo "Tar error"; exit 1; fi; \ - cd $(2); \ - python3 setup.py build && \ - python3 setup.py install && \ - cd .. && rm -rf $(2) - - - - - # CMake # ----- # diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk new file mode 100644 index 0000000..1e4f611 --- /dev/null +++ b/reproduce/src/make/dependencies-python.mk @@ -0,0 +1,365 @@ +# Build the reproduction pipeline Python dependencies. +# +# ------------------------------------------------------------------------ +# !!!!! IMPORTANT NOTES !!!!! +# +# This Makefile will be run by the initial `./configure' script. It is not +# included into the reproduction pipe after that. +# +# ------------------------------------------------------------------------ +# +# Original author: +# Raul Infante-Sainz <infantesainz@gmail.com> +# Contributing author(s): +# Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019, Your Name. +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + + + +# Top level environment +include reproduce/config/pipeline/LOCAL.mk +include reproduce/src/make/dependencies-build-rules.mk +include reproduce/config/pipeline/dependency-versions.mk + +ddir = $(BDIR)/dependencies +tdir = $(BDIR)/dependencies/tarballs +idir = $(BDIR)/dependencies/installed +ibdir = $(BDIR)/dependencies/installed/bin +ildir = $(BDIR)/dependencies/installed/lib +ilidir = $(BDIR)/dependencies/installed/lib/built +ipydir = $(BDIR)/dependencies/installed/lib/built/python + +# Define the top-level programs to build (installed in `.local/bin'). +top-level-python = astroquery matplotlib #scipy +all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) + +# Other basic environment settings: We are only including the host +# operating system's PATH environment variable (after our own!) for the +# compiler and linker. For the library binaries and headers, we are only +# using our internally built libraries. +# +# To investigate: +# +# 1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all +# the parameters defined bellow as `NAME=VALUE' statements before +# calling Bash. This will enable us to completely ignore the user's +# native environment. +# +# 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the +# user's environment. +.ONESHELL: +.SHELLFLAGS := --noprofile --norc -ec +export CCACHE_DISABLE := 1 +export PATH := $(ibdir) +export LD_RUN_PATH := $(ildir) +export LD_LIBRARY_PATH := $(ildir) +export SHELL := $(ibdir)/bash +export CPPFLAGS := -I$(idir)/include +export PKG_CONFIG_PATH := $(ildir)/pkgconfig +export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +export LDFLAGS := $(rpath_command) -L$(ildir) + + + + + +# Tarballs +# -------- +# +# All the necessary tarballs are defined and prepared with this rule. +# +# Note that we want the tarballs to follow the convention of NAME-VERSION +# before the `tar.XX' prefix. For those programs that don't follow this +# convention, but include the name/version in their tarball names with +# another format, we'll do the modification before the download so the +# downloaded file has our desired format. +tarballs = $(foreach t, astroquery-$(astroquery-version).tar.gz \ + astropy-$(astropy-version).tar.gz \ + beautifulsoup4-$(beautifulsoup4-version).tar.gz \ + certifi-$(certifi-version).tar.gz \ + chardet-$(chardet-version).tar.gz \ + cycler-$(cycler-version).tar.gz \ + entrypoints-$(entrypoints-version).tar.gz \ + html5lib-$(html5lib-version).tar.gz \ + idna-$(idna-version).tar.gz \ + kiwisolver-$(kiwisolver-version).tar.gz \ + keyring-$(keyring-version).tar.gz \ + matplotlib-$(matplotlib-version).tar.gz \ + numpy-$(numpy-version).zip \ + pip-$(pip-version).tar.gz \ + python-$(python-version).tar.gz \ + python-dateutil-$(python-dateutil-version).tar.gz \ + pyparsing-$(pyparsing-version).tar.gz \ + requests-$(requests-version).tar.gz \ + scipy-$(scipy-version).tar.gz \ + setuptools-$(setuptools-version).zip \ + setuptools_scm-$(setuptools_scm-version).tar.gz \ + six-$(six-version).tar.gz \ + soupsieve-$(soupsieve-version).tar.gz \ + urllib3-$(urllib3-version).tar.gz \ + webencodings-$(webencodings-version).tar.gz \ + virtualenv-$(virtualenv-version).tar.gz \ + , $(tdir)/$(t) ) +topurl=https://files.pythonhosted.org/packages +$(tarballs): $(tdir)/%: + if [ -f $(DEPENDENCIES-DIR)/$* ]; then + cp $(DEPENDENCIES-DIR)/$* $@ + else + + # Convenience variable + # -------------------- + # + # `n' is just for convenience and to avoid having to repeat the + # package tarball name in the conditional to find its URL. + # + # For some packages (for example `python-dateutil', or those with + # a number or dash in their name), we need special consideration + # because the tokenization above will produce `python' as the + # first string. + if [ $* = python-dateutil-$(python-dateutil-version).tar.gz ]; then + n=dateutil + # elif [ $* = strange-tarball5name-version.tar.gz ]; then + # n=strange5-name + else + # Remove all numbers, `-' and `.' from the tarball name so we can + # search more easily only with the program name. + n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ + | awk '{print $$1}') + fi + + # Set the top download link of the requested tarball. + mergenames=1 + if [ $$n = python ]; then + mergenames=0 + h=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz + elif [ $$n = astroquery ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396 + elif [ $$n = astropy ]; then h=eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 + elif [ $$n = beautifulsoup ]; then h=80/f2/f6aca7f1b209bb9a7ef069d68813b091c8c3620642b568dac4eb0e507748 + elif [ $$n = certifi ]; then h=55/54/3ce77783acba5979ce16674fc98b1920d00b01d337cfaaf5db22543505ed + elif [ $$n = chardet ]; then h=fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d + elif [ $$n = cycler ]; then h=c2/4b/137dea450d6e1e3d474e1d873cd1d4f7d3beed7e0dc973b06e8e10d32488 + elif [ $$n = entrypoints ]; then h=b4/ef/063484f1f9ba3081e920ec9972c96664e2edb9fdc3d8669b0e3b8fc0ad7c + elif [ $$n = html ]; then h=85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f + elif [ $$n = idna ]; then h=ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7 + elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2 + elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864 + elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f + elif [ $$n = numpy ]; then h=2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c + elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af + elif [ $$n = pyparsing ]; then h=b9/b8/6b32b3e84014148dcd60dd05795e35c2e7f4b72f918616c61fdce83d27fc + elif [ $$n = dateutil ]; then h=ad/99/5b2e99737edeb28c71bcbec5b5dda19d0d9ef3ca3e92e3e925e7c0bb364c + elif [ $$n = requests ]; then h=52/2c/514e4ac25da2b08ca5a464c50463682126385c4272c18193876e91f4bc38 + elif [ $$n = scipy ]; then h=a9/b4/5598a706697d1e2929eaf7fe68898ef4bea76e4950b9efbe1ef396b8813a + elif [ $$n = setuptools ]; then h=c2/f7/c7b501b783e5a74cf1768bc174ee4fb0a8a6ee5af6afa92274ff964703e0 + elif [ $$n = setuptools_scm ]; then h=54/85/514ba3ca2a022bddd68819f187ae826986051d130ec5b972076e4f58a9f3 + elif [ $$n = six ]; then h=dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca + elif [ $$n = soupsieve ]; then h=0c/52/e9088bb9b96e2d39fc3b33fcda5b4fde9d71473536ac660a1ca9a0958a2f + elif [ $$n = urllib ]; then h=b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1 + elif [ $$n = virtualenv ]; then h=51/aa/c395a6e6eaaedfa5a04723b6446a1df783b16cca6fec66e671cede514688 + elif [ $$n = webencodings ]; then h=0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47 +# elif [ $$n = strange5-name ]; then h=XXXXX + else + echo; echo; echo; + echo "'$$n' not recognized as a dependency name to download." + echo; echo; echo; + exit 1 + fi + + # Download the requested tarball. Note that some packages may not + # follow our naming convention (where the package name is merged + # with its version number). In such cases, `w' will be the full + # address, not just the top directory address. But since we are + # storing all the tarballs in one directory, we want it to have + # the same naming convention, so we'll download it to a temporary + # name, then rename that. + if [ $$mergenames = 1 ]; then tarballurl=$(topurl)/$$h/"$*" + else tarballurl=$$h + fi + + # If the download fails, Wget will write the error message in the + # target file, so Make will think that its done! To avoid this + # problem, we'll rename the output. + echo "Downloading $$tarballurl" + if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then + rm -f $@ + echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1 + fi + fi + + + + + +# Install without pip +# -------------------- +# +# To build Python packages with direct access to a `setup.py' (if no direct +# access to `setup.py' is needed, pip can be used) +# Arguments of this function are the numbers +# 1) Unpack command +# 2) Package name +# 3) Unpacked directory name after unpacking the tarball +pybuild = cd $(ddir); rm -rf $(3); \ + if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ + cd $(3); \ + python3 setup.py build && \ + python3 setup.py install && \ + cd .. && rm -rf $(3) && \ + echo "done!" > $@ + + + + + +# Python installation +# ------------------- +# +$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz + # On Mac systems, the build complains about `clang' specific + # features, so we can't use our own GCC build here. +# if [ x$(on_mac_os) = xyes ]; then \ +# export CC=clang; \ +# export CXX=clang++; \ +# fi; \ + + $(call gbuild, $<, Python-$(python-version)) \ + && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ + {printf "%d.%d\n", $$1, $$2}') \ + && ln -s $(ildir)/python$$v $(ildir)/python \ + && rm -rf $(ipydir) && mkdir $(ipydir) + + + + +# Python packages +# --------------- +# +$(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ + $(ipydir)/astropy \ + $(ipydir)/beautifulsoup4 \ + $(ipydir)/html5lib \ + $(ipydir)/keyring \ + $(ipydir)/numpy \ + $(ipydir)/requests + $(call pybuild, tar xf, $<, astroquery-$(astroquery-version)) + +$(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ + $(ipydir)/numpy + $(call pybuild, tar xf, $<, astropy-$(astropy-version)) + +$(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \ + $(ipydir)/soupsieve + $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version)) + +$(ipydir)/certifi: $(tdir)/certifi-$(certifi-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, certifi-$(certifi-version)) + +$(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, chardet-$(chardet-version)) + +$(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \ + $(ipydir)/six + $(call pybuild, tar xf, $<, cycler-$(cycler-version)) + +$(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version)) + +$(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \ + $(ipydir)/six \ + $(ipydir)/webencodings + $(call pybuild, tar xf, $<, html5lib-$(html5lib-version)) + +$(ipydir)/idna: $(tdir)/idna-$(idna-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, idna-$(idna-version)) + +$(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \ + $(ipydir)/entrypoints \ + $(ipydir)/setuptools_scm + $(call pybuild, tar xf, $<, keyring-$(keyring-version)) + +$(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version)) + + +$(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ + $(ipydir)/cycler \ + $(ilidir)/freetype \ + $(ipydir)/numpy \ + $(ipydir)/pyparsing \ + $(ipydir)/python-dateutil \ + $(ipydir)/kiwisolver + $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) + +$(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ + $(ibdir)/python3 + export BLAS=None; \ + export ATLAS=None; \ + export LAPACK=None; \ + $(call pybuild, unzip, $<, numpy-$(numpy-version)) + +$(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, pip-$(pip-version)) + +$(ipydir)/pyparsing: $(tdir)/pyparsing-$(pyparsing-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version)) + +$(ipydir)/python-dateutil: $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz \ + $(ipydir)/six + $(call pybuild, tar xf, $<, python-dateutil-$(python-dateutil-version)) + +$(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ + $(ipydir)/certifi \ + $(ipydir)/chardet \ + $(ipydir)/idna \ + $(ipydir)/numpy \ + $(ipydir)/urllib3 + $(call pybuild, tar xf, $<, requests-$(requests-version)) + +$(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ + $(ipydir)/numpy + $(call pybuild, tar xf, $<, scipy-$(scipy-version)) + +$(ipydir)/setuptools: $(tdir)/setuptools-$(setuptools-version).zip \ + $(ibdir)/python3 + $(call pybuild, unzip, $<, setuptools-$(setuptools-version)) + +$(ipydir)/setuptools_scm: $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version)) + +$(ipydir)/six: $(tdir)/six-$(six-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, six-$(six-version)) + +$(ipydir)/soupsieve: $(tdir)/soupsieve-$(soupsieve-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version)) + +$(ipydir)/urllib3: $(tdir)/urllib3-$(urllib3-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, urllib3-$(urllib3-version)) + +$(ipydir)/webencodings: $(tdir)/webencodings-$(webencodings-version).tar.gz \ + $(ibdir)/python3 + $(call pybuild, tar xf, $<, webencodings-$(webencodings-version)) + diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 92febb1..deb9f39 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,11 +43,11 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astropy -top-level-programs = astnoisechisel metastore flock unzip zip +top-level-programs = astnoisechisel flock metastore unzip zip +top-level-libraries = freetype all: $(ddir)/texlive-versions.tex \ - $(foreach p, $(top-level-python), $(ilidir)/$(p)) \ - $(foreach p, $(top-level-programs), $(ibdir)/$(p)) + $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ + $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) # Other basic environment settings: We are only including the host # operating system's PATH environment variable (after our own!) for the @@ -89,11 +89,11 @@ export LDFLAGS := $(rpath_command) -L$(ildir) # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. -tarballs = $(foreach t, astropy-$(astropy-version).tar.gz \ - cfitsio-$(cfitsio-version).tar.gz \ +tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ flock-$(flock-version).tar.xz \ + freetype-$(freetype-version).tar.gz \ ghostscript-$(ghostscript-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ @@ -101,11 +101,10 @@ tarballs = $(foreach t, astropy-$(astropy-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ libbsd-$(libbsd-version).tar.xz \ + libpng-$(libpng-version).tar.xz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ - numpy-$(numpy-version).zip \ - python-$(python-version).tar.gz \ unzip-$(unzip-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ @@ -122,8 +121,7 @@ $(tarballs): $(tdir)/%: # Set the top download link of the requested tarball. mergenames=1 - if [ $$n = astropy ]; then w=https://files.pythonhosted.org/packages/eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 - elif [ $$n = cfitsio ]; then + if [ $$n = cfitsio ]; then mergenames=0 v=$$(echo $(cfitsio-version) | sed -e's/\.//' \ | awk '{l=length($$1); \ @@ -135,6 +133,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) + elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro @@ -142,15 +141,12 @@ $(tarballs): $(tdir)/%: elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases + elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool elif [ $$n = libgit ]; then mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://akhlaghi.org/src - elif [ $$n = numpy ]; then w=https://files.pythonhosted.org/packages/2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c - elif [ $$n = python ]; then - mergenames=0 - w=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -232,6 +228,12 @@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, $<, gsl-$(gsl-version), static) \ && echo "GNU Scientific Library is built" > $@ +# Freetype is necessary to install matplotlib +$(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ + $(ilidir)/libpng + $(call gbuild, $<, freetype-$(freetype-version), static) \ + && echo "freetype is built" > $@ + $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \ && echo "libbsd is built" > $@ @@ -239,6 +241,10 @@ $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz $(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz $(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@ +$(ilidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz + $(call gbuild, $<, libpng-$(libpng-version), static) \ + && echo "Libpng is built" > $@ + $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/libjpeg $(call gbuild, $<, tiff-$(libtiff-version), static, \ @@ -314,6 +320,13 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ # After searching in `bootstrap', I couldn't find `LIBS', only # `LDFLAGS'. So the extra libraries are being added to `LDFLAGS', # not `LIBS'. + # + # On Mac systems, the build complains about `clang' specific + # features, so we can't use our own GCC build here. + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + export CXX=clang++; \ + fi; \ cd $(ddir) && rm -rf cmake-$(cmake-version) && \ tar xf $< && cd cmake-$(cmake-version) && \ ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ @@ -456,12 +469,6 @@ endif $$staticopts, -j$(numthreads), \ make check -j$(numthreads)) -$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz - $(call gbuild, $<, Python-$(python-version)) \ - && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ - {printf "%d.%d\n", $$1, $$2}') \ - && ln -s $(ildir)/python$$v $(ildir)/python - $(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz v=$$(echo $(unzip-version) | sed -e's/\.//') $(call gbuild, $<, unzip$$v, static,, \ @@ -482,20 +489,6 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz -# Python packages -# --------------- -$(ilidir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ - $(ibdir)/python3 - pip3 install $< --verbose && echo "numpy is built" > $@ - -$(ilidir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ - $(ilidir)/numpy - pip3 install $< --verbose && echo "astropy is built" > $@ - - - - - # Since we want to avoid complicating the PATH, we are putting a symbolic # link of all the TeX Live executables in $(ibdir). But symbolic links are # hard to track for Make (as a target). Also, TeX in general is optional diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 71cdbb8..638338b 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -116,6 +116,19 @@ export LDFLAGS := -L$(installdir)/lib export SHELL := $(installdir)/bin/bash export CPPFLAGS := -I$(installdir)/include +# Python enviroment +# So far we have found several other Python-related environment +# variables which might interfere. So we are just removing all +# of their values within the pipeline. +export PYTHONPATH := $(installdir)/lib/python/site-packages +export PYTHONPATH3 := $(PYTHONPATH) +export _LMFILES_ := +export PYTHONPATH2 := +export LOADEDMODULES := +export MPI_PYTHON_SITEARCH := +export MPI_PYTHON2_SITEARCH := +export MPI_PYTHON3_SITEARCH := + |