diff options
Diffstat (limited to 'reproduce')
| -rw-r--r-- | reproduce/config/pipeline/dependency-numpy-scipy.cfg | 39 | ||||
| -rw-r--r-- | reproduce/config/pipeline/dependency-versions.mk | 30 | ||||
| -rwxr-xr-x | reproduce/src/make/dependencies-atlas-multiple.mk | 47 | ||||
| -rwxr-xr-x | reproduce/src/make/dependencies-atlas-single.mk | 29 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies-basic.mk | 77 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies-python.mk | 213 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies.mk | 189 | ||||
| -rw-r--r-- | reproduce/src/make/initialize.mk | 15 |
8 files changed, 467 insertions, 172 deletions
diff --git a/reproduce/config/pipeline/dependency-numpy-scipy.cfg b/reproduce/config/pipeline/dependency-numpy-scipy.cfg new file mode 100644 index 0000000..0eecf3a --- /dev/null +++ b/reproduce/config/pipeline/dependency-numpy-scipy.cfg @@ -0,0 +1,39 @@ +# THIS IS A COPY OF NUMPY'S site.cfg.example, CUSTOMIZED FOR THIS PIPELINE +# ------------------------------------------------------------------------ + +# This file provides configuration information about non-Python dependencies for +# numpy.distutils-using packages. Create a file like this called "site.cfg" next +# to your package's setup.py file and fill in the appropriate sections. Not all +# packages will use all sections so you should leave out sections that your +# package does not use. + +# IMPORTANT NOTE +# -------------- +# +# The `ALL' grouping does't apply to ATLAS!!!!! + + [ALL] + library_dirs = @LIBDIR@ + include_dirs = @INCDIR@ + + +# FFT libraries +# ------------- +# There are two FFT libraries that we can configure here: FFTW (2 and 3) and djbfft. +# Note that these libraries are not used by numpy or scipy. +# +# http://fftw.org/ +# https://cr.yp.to/djbfft.html +# +# Given only this section, numpy.distutils will try to figure out which version +# of FFTW you are using. + +# Even with this section, Numpy apparently does not use fftw. But we will +# keep it here for the future (if a solution is found). +[fftw] +libraries = fftw3 +# +# For djbfft, numpy.distutils will look for either djbfft.a or libdjbfft.a . +#[djbfft] +#include_dirs = /usr/local/djbfft/include +#library_dirs = /usr/local/djbfft/lib diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 87b837c..2e74a45 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -25,6 +25,7 @@ binutils-version = 2.31.1 cmake-version = 3.12.4 coreutils-version = 8.30 diffutils-version = 3.7 +fftw-version = 3.3.8 findutils-version = 4.6.0.199-e3fc flock-version = 0.2.3 freetype-version = 2.9 @@ -36,6 +37,7 @@ gmp-version = 6.1.2 gnuastro-version = 0.8 grep-version = 3.3 gzip-version = 1.10 +hdf5-version = 1.10.5 isl-version = 0.18 libbsd-version = 0.9.1 libffi-version = 3.2.1 @@ -47,6 +49,7 @@ metastore-version = 1.1.2-23-fa9170b mpfr-version = 4.0.2 mpc-version = 1.1.0 ncurses-version = 6.1 +openmpi-version = 4.0.1 openssl-version = 1.1.1a patchelf-version = 0.9 pkgconfig-version = 0.29.2 @@ -61,13 +64,26 @@ xz-version = 5.2.4 zip-version = 3.0 # Libraries +atlas-version = 3.10.3 cfitsio-version = 3.45 curl-version = 7.63.0 gsl-version = 2.5 libjpeg-version = v9b libtiff-version = 4.0.10 +openblas-version = 0.3.5 zlib-version = 1.2.11 +# Special libraries +# ----------------- +# +# When updating the version of these libraries, please look into the build +# rule first: In one way or another, the version string becomes necessary +# during their build and must be accounted for. +bzip2-version = 1.0.6 +lapack-version = 3.8.0 +libgit2-version = 0.26.0 +wcslib-version = 6.2 + # Python packages # --------------- # @@ -83,13 +99,15 @@ chardet-version = 3.0.4 cryptography-version = 2.6.1 cycler-version = 0.10.0 entrypoints-version = 0.3 +h5py-version = 2.9.0 html5lib-version = 1.0.1 idna-version = 2.8 jeepney-version = 0.4 kiwisolver-version = 1.0.1 keyring-version = 18.0.0 matplotlib-version = 3.0.2 -numpy-version = 1.16.1 +mpi4py-version = 3.0.1 +numpy-version = 1.16.2 pip-version = 19.0.2 pycparser-version = 2.19 pyparsing-version = 2.3.1 @@ -104,13 +122,3 @@ soupsieve-version = 1.8 urllib3-version = 1.24.1 virtualenv-version = 16.4.0 webencodings-version = 0.5.1 - -# Special libraries -# ----------------- -# -# When updating the version of these libraries, please look into the build -# rule first: In one way or another, the version string becomes necessary -# during their build and must be accounted for. -bzip2-version = 1.0.6 -libgit2-version = 0.26.0 -wcslib-version = 6.2 diff --git a/reproduce/src/make/dependencies-atlas-multiple.mk b/reproduce/src/make/dependencies-atlas-multiple.mk new file mode 100755 index 0000000..d66890f --- /dev/null +++ b/reproduce/src/make/dependencies-atlas-multiple.mk @@ -0,0 +1,47 @@ +ORIGLDFLAGS := $(LDFLAGS) + +include Make.inc + +all: libatlas.so libf77blas.so libptf77blas.so libstcblas.so libptcblas.so \ + libblas.so libcblas.so liblapack.so.3.6.1 libptlapack.so.3.6.1 liblapack.so.3 + +libatlas.so: libatlas.a + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \ + --whole-archive libatlas.a --no-whole-archive -lc $(LIBS) + +libf77blas.so : libf77blas.a libatlas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libf77blas.so.3 \ + -o $@ --whole-archive libf77blas.a --no-whole-archive \ + $(F77SYSLIB) -L. -latlas + +libptf77blas.so : libptf77blas.a libatlas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libblas.so.3 \ + -o $@ --whole-archive libptf77blas.a --no-whole-archive \ + $(F77SYSLIB) -L. -latlas + +libstcblas.so : libcblas.a libatlas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libstcblas.so \ + -o $@ --whole-archive libcblas.a -L. -latlas -lblas + +libptcblas.so : libptcblas.a libatlas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libcblas.so \ + -o $@ --whole-archive libptcblas.a -L. -latlas -lblas + +libblas.so: libptf77blas.so + ln -s $< $@ + +libcblas.so: libptcblas.so + ln -s $< $@ + +liblapack.so.3.6.1 : liblapack.a libstcblas.so libf77blas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libstlapack.so.3 \ + -o $@ --whole-archive liblapack.a --no-whole-archive \ + $(F77SYSLIB) -L. -lstcblas -lf77blas + +libptlapack.so.3.6.1 : libptlapack.a libcblas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname liblapack.so.3 \ + -o $@ --whole-archive libptlapack.a --no-whole-archive \ + $(F77SYSLIB) -L. -lcblas -lblas + +liblapack.so.3: libptlapack.so.3.6.1 + ln -s $< $@ diff --git a/reproduce/src/make/dependencies-atlas-single.mk b/reproduce/src/make/dependencies-atlas-single.mk new file mode 100755 index 0000000..12dd19b --- /dev/null +++ b/reproduce/src/make/dependencies-atlas-single.mk @@ -0,0 +1,29 @@ +ORIGLDFLAGS := $(LDFLAGS) + +include Make.inc + +all: libatlas.so libf77blas.so libcblas.so libblas.so liblapack.so.3.6.1 + +libatlas.so: libatlas.a + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \ + --whole-archive libatlas.a --no-whole-archive -lc $(LIBS) + +libf77blas.so : libf77blas.a libatlas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libblas.so.3 \ + -o $@ --whole-archive libf77blas.a --no-whole-archive \ + $(F77SYSLIB) -L. -latlas + +libcblas.so : libcblas.a libatlas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \ + --whole-archive libcblas.a -L. -latlas -lblas + +libblas.so: libf77blas.so + ln -s $< $@ + +liblapack.so.3.6.1 : liblapack.a libcblas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname liblapack.so.3 \ + -o $@ --whole-archive liblapack.a --no-whole-archive \ + $(F77SYSLIB) -L. -lcblas -lblas + +liblapack.so.3: liblapack.so.3.6.1 + ln -s $< $@ diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index cdb4a8c..52a41d9 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -204,12 +204,13 @@ $(tarballs): $(tdir)/%: $(lockdir) # is very annoying and can cause many complications. We thus remove any # part of PATH of that has `ccache' in it before making symbolic links to # the programs we are not building ourselves. -makelink = origpath="$$PATH"; \ +makelink = origpath="$$PATH"; \ export PATH=$$(echo $(syspath) | tr : '\n' | grep -v ccache \ - | paste -s -d:); \ + | tr '\n' :); \ a=$$(which $(1) 2> /dev/null); \ - if [ -f $(ibdir)/$(1) ]; then rm $(ibdir)/$(1); fi; \ - if [ x$$a != x ]; then ln -s $$a $(ibdir)/$(1); fi; \ + if [ -e $(ibdir)/$(1) ]; then rm $(ibdir)/$(1); fi; \ + if [ x"$(2)" = xcopy ]; then c=cp; else c="ln -s"; fi; \ + if [ x$$a != x ]; then $$c $$a $(ibdir)/$(1); fi; \ export PATH="$$origpath" $(ibdir) $(ildir):; mkdir $@ $(ibdir)/low-level-links: | $(ibdir) $(ildir) @@ -576,8 +577,15 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ --with-zlib-lib=$(ildir) \ --with-zlib-include=$(idir)/include, , , \ ./config ) && \ - cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem && \ - echo "OpenSSL is built and ready" > $@ + cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem; \ + if [ $$? = 0 ]; then \ + if [ x$(on_mac_os) = xyes ]; then \ + echo "No need to fix rpath in libssl"; \ + else \ + patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \ + fi; \ + echo "OpenSSL is built and ready" > $@; \ + fi # GNU Wget # @@ -734,13 +742,13 @@ $(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz # # We are currently having problems installing GCC on macOS, so for the time # being, if the pipeline is being run on a macOS, we'll just set a link. -ifeq ($(on_mac_os),yes) -gcc-prerequisites = -else +#ifeq ($(on_mac_os),yes) +#gcc-prerequisites = +#else gcc-prerequisites = $(tdir)/gcc-$(gcc-version).tar.xz \ $(ilidir)/isl \ $(ilidir)/mpc -endif +#endif $(ibdir)/gcc: $(gcc-prerequisites) \ $(ibdir)/ls \ $(ibdir)/sed \ @@ -752,24 +760,36 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ $(ibdir)/which # On a macOS, we (currently!) won't build GCC because of some - # errors we are still trying to fix. So, we'll just make a symbolic - # link to the host's executables. - if [ "x$(on_mac_os)" = xyes ]; then \ + # errors we are still trying to find. So, we'll just make a + # symbolic link to the host's executables. + # + # GCC builds is own libraries in '$(idir)/lib64'. But all other + # libraries are in '$(idir)/lib'. Since this pipeline is only for a + # single architecture, we can trick GCC into building its libraries + # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic + # link to '$(idir)/lib'. + +# SO FAR WE HAVEN'T BEEN ABLE TO GET A CONSISTENT BUILD OF GCC ON MAC +# (SOMETIMES IT CRASHES IN libiberty with g++) AND SOMETIMES IT FINISHES, +# SO, MORE TESTS ARE NEEDED ON MAC AND WE'LL USE THE HOST'S COMPILER UNTIL +# THEN. + if [ "x$(on_mac_os)" = xyesno ]; then \ $(call makelink,gfortran); \ $(call makelink,g++); \ - $(call makelink,gcc); \ + $(call makelink,gcc,copy); \ else \ - \ rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\ rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \ rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\ \ + ln -fs $(ildir) $(idir)/lib64; \ + \ cd $(ddir); \ rm -rf gcc-build gcc-$(gcc-version); \ - tar xf $< && \ - mkdir $(ddir)/gcc-build && \ - cd $(ddir)/gcc-build && \ - ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ + tar xf $< \ + && mkdir $(ddir)/gcc-build \ + && cd $(ddir)/gcc-build \ + && ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ --prefix=$(idir) \ --with-mpc=$(idir) \ --with-mpfr=$(idir) \ @@ -787,9 +807,18 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ --enable-default-pie \ --enable-default-ssp \ --enable-cet=auto \ - --enable-decimal-float && \ - make SHELL=$(ibdir)/bash -j$$(nproc) && \ - make SHELL=$(ibdir)/bash install && \ - cd .. && \ - rm -rf gcc-build gcc-$(gcc-version); \ + --enable-decimal-float \ + && make SHELL=$(ibdir)/bash -j$$(nproc) \ + && make SHELL=$(ibdir)/bash install \ + && cd .. \ + && rm -rf gcc-build gcc-$(gcc-version) \ + \ + && if [ "x$(on_mac_os)" != xyes ]; then \ + for f in $$(find $(idir)/libexec/gcc); do \ + if ldd $$f &> /dev/null; then \ + patchelf --set-rpath $(ildir) $$f; \ + fi; \ + done; \ + fi; \ fi + diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 946c047..6711d17 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -29,51 +29,6 @@ -# Top level environment -include reproduce/config/pipeline/LOCAL.mk -include reproduce/src/make/dependencies-build-rules.mk -include reproduce/config/pipeline/dependency-versions.mk - -ddir = $(BDIR)/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib -ilidir = $(BDIR)/dependencies/installed/lib/built -ipydir = $(BDIR)/dependencies/installed/lib/built/python - -# Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astroquery matplotlib #scipy -all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) - -# Other basic environment settings: We are only including the host -# operating system's PATH environment variable (after our own!) for the -# compiler and linker. For the library binaries and headers, we are only -# using our internally built libraries. -# -# To investigate: -# -# 1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all -# the parameters defined bellow as `NAME=VALUE' statements before -# calling Bash. This will enable us to completely ignore the user's -# native environment. -# -# 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the -# user's environment. -.ONESHELL: -.SHELLFLAGS := --noprofile --norc -ec -export CCACHE_DISABLE := 1 -export PATH := $(ibdir) -export LD_RUN_PATH := $(ildir) -export LD_LIBRARY_PATH := $(ildir) -export SHELL := $(ibdir)/bash -export CPPFLAGS := -I$(idir)/include -export PKG_CONFIG_PATH := $(ildir)/pkgconfig -export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export LDFLAGS := $(rpath_command) -L$(ildir) - - - # Python enviroment @@ -84,9 +39,9 @@ export LDFLAGS := $(rpath_command) -L$(ildir) # systems which might interfere. To be safe, we are removing all their # values. export PYTHONPATH := $(installdir)/lib/python/site-packages +export PYTHONPATH2 := $(PYTHONPATH) export PYTHONPATH3 := $(PYTHONPATH) export _LMFILES_ := -export PYTHONPATH2 := export LOADEDMODULES := export MPI_PYTHON_SITEARCH := export MPI_PYTHON2_SITEARCH := @@ -106,7 +61,7 @@ export MPI_PYTHON3_SITEARCH := # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. -tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ +pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ astroquery-$(astroquery-version).tar.gz \ astropy-$(astropy-version).tar.gz \ beautifulsoup4-$(beautifulsoup4-version).tar.gz \ @@ -116,6 +71,7 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ cryptography-$(cryptography-version).tar.gz \ cycler-$(cycler-version).tar.gz \ entrypoints-$(entrypoints-version).tar.gz \ + h5py-$(h5py-version).tar.gz \ html5lib-$(html5lib-version).tar.gz \ idna-$(idna-version).tar.gz \ jeepney-$(jeepney-version).tar.gz \ @@ -123,6 +79,7 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ keyring-$(keyring-version).tar.gz \ libffi-$(libffi-version).tar.gz \ matplotlib-$(matplotlib-version).tar.gz \ + mpi4py-$(mpi4py-version).tar.gz \ numpy-$(numpy-version).zip \ pip-$(pip-version).tar.gz \ pycparser-$(pycparser-version).tar.gz \ @@ -140,8 +97,8 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ webencodings-$(webencodings-version).tar.gz \ virtualenv-$(virtualenv-version).tar.gz \ , $(tdir)/$(t) ) -topurl=https://files.pythonhosted.org/packages -$(tarballs): $(tdir)/%: +pytopurl=https://files.pythonhosted.org/packages +$(pytarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* $@ else @@ -157,7 +114,10 @@ $(tarballs): $(tdir)/%: # because the tokenization above will produce `python' as the # first string. if [ $* = python-dateutil-$(python-dateutil-version).tar.gz ]; then - n=dateutil + n=dateutil + elif [ $* = h5py-$(h5py-version).tar.gz ]; then + n=h5py + # elif [ $* = strange-tarball5name-version.tar.gz ]; then # n=strange5-name else @@ -180,7 +140,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = secretstorage ]; then mergenames=0 hash=a6/89/df343dbc2957a317127e7ff2983230dc5336273be34f2e1911519d85aeb5 - h=$(topurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz + h=$(pytopurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz elif [ $$n = asn ]; then h=fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4 elif [ $$n = astroquery ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396 elif [ $$n = astropy ]; then h=eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 @@ -191,13 +151,15 @@ $(tarballs): $(tdir)/%: elif [ $$n = cryptography ]; then h=07/ca/bc827c5e55918ad223d59d299fff92f3563476c3b00d0a9157d9c0217449 elif [ $$n = cycler ]; then h=c2/4b/137dea450d6e1e3d474e1d873cd1d4f7d3beed7e0dc973b06e8e10d32488 elif [ $$n = entrypoints ]; then h=b4/ef/063484f1f9ba3081e920ec9972c96664e2edb9fdc3d8669b0e3b8fc0ad7c + elif [ $$n = h5py ]; then h=43/27/a6e7dcb8ae20a4dbf3725321058923fec262b6f7835179d78ccc8d98deec elif [ $$n = html ]; then h=85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f elif [ $$n = idna ]; then h=ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7 elif [ $$n = jeepney ]; then h=16/1d/74adf3b164a8d19a60d0fcf706a751ffa2a1eaa8e5bbb1b6705c92a05263 elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2 elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864 elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f - elif [ $$n = numpy ]; then h=2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c + elif [ $$n = mpi ]; then h=55/a2/c827b196070e161357b49287fa46d69f25641930fd5f854722319d431843 + elif [ $$n = numpy ]; then h=cf/8d/6345b4f32b37945fedc1e027e83970005fc9c699068d2f566b82826515f2 elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af elif [ $$n = pycparser ]; then h=68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a elif [ $$n = pyparsing ]; then h=b9/b8/6b32b3e84014148dcd60dd05795e35c2e7f4b72f918616c61fdce83d27fc @@ -212,7 +174,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = urllib ]; then h=b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1 elif [ $$n = virtualenv ]; then h=51/aa/c395a6e6eaaedfa5a04723b6446a1df783b16cca6fec66e671cede514688 elif [ $$n = webencodings ]; then h=0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47 -# elif [ $$n = strange5-name ]; then h=XXXXX +# elif [ $$n = strange5-name ]; then h=XXXXX else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." @@ -227,7 +189,7 @@ $(tarballs): $(tdir)/%: # storing all the tarballs in one directory, we want it to have # the same naming convention, so we'll download it to a temporary # name, then rename that. - if [ $$mergenames = 1 ]; then tarballurl=$(topurl)/$$h/"$*" + if [ $$mergenames = 1 ]; then tarballurl=$(pytopurl)/$$h/"$*" else tarballurl=$$h fi @@ -249,18 +211,24 @@ $(tarballs): $(tdir)/%: # -------------------- # # To build Python packages with direct access to a `setup.py' (if no direct -# access to `setup.py' is needed, pip can be used) +# access to `setup.py' is needed, pip can be used). # Arguments of this function are the numbers # 1) Unpack command # 2) Package name # 3) Unpacked directory name after unpacking the tarball -pybuild = cd $(ddir); rm -rf $(3); \ - if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ - cd $(3); \ - python3 setup.py build && \ - python3 setup.py install && \ - cd .. && rm -rf $(3) && \ - echo "done!" > $@ +# 4) site.cfg file (optional) +pybuild = cd $(ddir); rm -rf $(3); \ + if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ + cd $(3); \ + if [ "x$(4)" != x ]; then \ + sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ + -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ + $(4) > site.cfg; \ + fi; \ + python3 setup.py build \ + && python3 setup.py install \ + && cd .. && rm -rf $(3) \ + && echo "done!" > $@ @@ -271,24 +239,27 @@ pybuild = cd $(ddir); rm -rf $(3); \ # # While this Makefile is for Python programs, in some cases, we need # certain programs (like Python itself), or libraries for the modules. -$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz +$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz + $(call gbuild, $<, libffi-$(libffi-version)) \ + echo "libffi is built" > $@ + +$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \ + $(ilidir)/libffi # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. -# if [ x$(on_mac_os) = xyes ]; then \ -# export CC=clang; \ -# export CXX=clang++; \ -# fi; \ - - $(call gbuild, $<, Python-$(python-version)) \ + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + export CXX=clang++; \ + fi; \ + $(call gbuild, $<, Python-$(python-version),, \ + --without-ensurepip \ + --with-system-ffi \ + --enable-shared) \ && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') \ && ln -s $(ildir)/python$$v $(ildir)/python \ && rm -rf $(ipydir) && mkdir $(ipydir) -$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz - $(call gbuild, $<, libffi-$(libffi-version)) \ - echo "libffi is built" > $@ - @@ -297,7 +268,7 @@ $(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz # # All the necessary Python modules go here. $(ipydir)/asn1crypto: $(tdir)/asn1crypto-$(asn1crypto-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, asn1crypto-$(asn1crypto-version)) $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ @@ -310,7 +281,9 @@ $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ $(call pybuild, tar xf, $<, astroquery-$(astroquery-version)) $(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ - $(ipydir)/numpy + $(ipydir)/h5py \ + $(ipydir)/numpy \ + $(ipydir)/scipy $(call pybuild, tar xf, $<, astropy-$(astropy-version)) $(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \ @@ -318,81 +291,107 @@ $(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.g $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version)) $(ipydir)/certifi: $(tdir)/certifi-$(certifi-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, certifi-$(certifi-version)) $(ipydir)/cffi: $(tdir)/cffi-$(cffi-version).tar.gz \ - $(ipydir)/pycparser \ - $(ilidir)/libffi + $(ilidir)/libffi \ + $(ipydir)/pycparser $(call pybuild, tar xf, $<, cffi-$(cffi-version)) $(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, chardet-$(chardet-version)) $(ipydir)/cryptography: $(tdir)/cryptography-$(cryptography-version).tar.gz \ - $(ipydir)/cffi \ - $(ipydir)/asn1crypto + $(ipydir)/asn1crypto \ + $(ipydir)/cffi $(call pybuild, tar xf, $<, cryptography-$(cryptography-version)) $(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \ - $(ipydir)/six + $(ipydir)/six $(call pybuild, tar xf, $<, cycler-$(cycler-version)) $(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version)) +$(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \ + $(ipydir)/setuptools \ + $(ilidir)/hdf5 + # $(ipydir)/mpi4py # AFTER its problem is fixed. + #export HDF5_MPI=ON; # AFTER its problem is fixed. + export HDF5_DIR=$(ildir); \ + $(call pybuild, tar xf, $<, h5py-$(h5py-version)) + $(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \ $(ipydir)/six \ $(ipydir)/webencodings $(call pybuild, tar xf, $<, html5lib-$(html5lib-version)) $(ipydir)/idna: $(tdir)/idna-$(idna-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, idna-$(idna-version)) $(ipydir)/jeepney: $(tdir)/jeepney-$(jeepney-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, jeepney-$(jeepney-version)) $(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \ - $(ipydir)/setuptools_scm \ + $(ipydir)/entrypoints \ $(ipydir)/secretstorage \ - $(ipydir)/entrypoints + $(ipydir)/setuptools_scm $(call pybuild, tar xf, $<, keyring-$(keyring-version)) $(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ - $(ipydir)/setuptools + $(ipydir)/setuptools $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version)) - $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/cycler \ $(ilidir)/freetype \ + $(ipydir)/kiwisolver \ $(ipydir)/numpy \ $(ipydir)/pyparsing \ - $(ipydir)/python-dateutil \ - $(ipydir)/kiwisolver + $(ipydir)/python-dateutil $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) +# Currently mpi4py doesn't build because of some conflict with OpenMPI: +# +# In file included from src/mpi4py.MPI.c:591, +# from src/MPI.c:4: +# src/mpi4py.MPI.c: In function '__pyx_f_6mpi4py_3MPI_del_Datatype': +# src/mpi4py.MPI.c:15094:36: error: expected expression before '_Static_assert' +# __pyx_t_1 = (((__pyx_v_ob[0]) == MPI_UB) != 0); +# +# But atleast on my system it fails. +$(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \ + $(ipydir)/setuptools \ + $(ilidir)/openmpi + $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version)) + $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ - $(ibdir)/python3 - export BLAS=None; \ - export ATLAS=None; \ - export LAPACK=None; \ - $(call pybuild, unzip, $<, numpy-$(numpy-version)) + $(ipydir)/setuptools \ + $(ilidir)/openblas \ + $(ilidir)/fftw + if [ x$(on_mac_os) = xyes ]; then \ + export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ + else \ + export LDFLAGS="$(LDFLAGS) -shared"; \ + fi; \ + conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf) $(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, pip-$(pip-version)) $(ipydir)/pycparser: $(tdir)/pycparser-$(pycparser-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, pycparser-$(pycparser-version)) $(ipydir)/pyparsing: $(tdir)/pyparsing-$(pyparsing-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version)) $(ipydir)/python-dateutil: $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz \ @@ -410,7 +409,13 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ $(ipydir)/numpy - $(call pybuild, tar xf, $<, scipy-$(scipy-version)) + if [ x$(on_mac_os) = xyes ]; then \ + export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ + else \ + export LDFLAGS="$(LDFLAGS) -shared"; \ + fi; \ + conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf) $(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \ $(ipydir)/cryptography \ @@ -426,17 +431,17 @@ $(ipydir)/setuptools_scm: $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.g $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version)) $(ipydir)/six: $(tdir)/six-$(six-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, six-$(six-version)) $(ipydir)/soupsieve: $(tdir)/soupsieve-$(soupsieve-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version)) $(ipydir)/urllib3: $(tdir)/urllib3-$(urllib3-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, urllib3-$(urllib3-version)) $(ipydir)/webencodings: $(tdir)/webencodings-$(webencodings-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, webencodings-$(webencodings-version)) diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 2b3b70b..fdbfd7f 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -35,19 +35,28 @@ include reproduce/src/make/dependencies-build-rules.mk include reproduce/config/pipeline/dependency-texlive.mk include reproduce/config/pipeline/dependency-versions.mk + ddir = $(BDIR)/dependencies tdir = $(BDIR)/dependencies/tarballs idir = $(BDIR)/dependencies/installed ibdir = $(BDIR)/dependencies/installed/bin ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built +ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). -top-level-programs = astnoisechisel metastore unzip zip -top-level-libraries = freetype -all: $(ddir)/texlive-versions.tex \ - $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ - $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) +# +# About ATLAS: currently the core pipeline does not depend on ATLAS but many +# high level software depend on it. The current rule for ATLAS is tested +# successfully on Mac (only static) and GNU/Linux (shared and static). But, +# since it takes a few hours to build, it is not currently a target. +top-level-libraries = # atlas +top-level-programs = astnoisechisel flock metastore unzip zip +top-level-python = astroquery matplotlib +all: $(ddir)/texlive-versions.tex \ + $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ + $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ + $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host # operating system's PATH environment variable (after our own!) for the @@ -67,16 +76,17 @@ all: $(ddir)/texlive-versions.tex \ .SHELLFLAGS := --noprofile --norc -ec export CCACHE_DISABLE := 1 export PATH := $(ibdir) -export LD_RUN_PATH := $(ildir) -export LD_LIBRARY_PATH := $(ildir) export SHELL := $(ibdir)/bash export CPPFLAGS := -I$(idir)/include export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +export LD_RUN_PATH := $(ildir):$(il64dir) +export LD_LIBRARY_PATH := $(ildir):$(il64dir) export LDFLAGS := $(rpath_command) -L$(ildir) - +# Python packages +include reproduce/src/make/dependencies-python.mk # Tarballs @@ -90,21 +100,27 @@ export LDFLAGS := $(rpath_command) -L$(ildir) # another format, we'll do the modification before the download so the # downloaded file has our desired format. tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ + atlas-$(atlas-version).tar.bz2 \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ freetype-$(freetype-version).tar.gz \ + fftw-$(fftw-version).tar.gz \ ghostscript-$(ghostscript-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ gsl-$(gsl-version).tar.gz \ + hdf5-$(hdf5-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ + lapack-$(lapack-version).tar.gz \ libbsd-$(libbsd-version).tar.xz \ libpng-$(libpng-version).tar.xz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ + openmpi-$(openmpi-version).tar.gz \ unzip-$(unzip-version).tar.gz \ + openblas-$(openblas-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ zip-$(zip-version).tar.gz \ @@ -113,12 +129,12 @@ $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* $@ else - # Remove all numbers, `-' and `.' from the tarball name so we can - # search more easily only with the program name. + # Remove all numbers, `-' and `.' from the tarball name so we can + # search more easily only with the program name. n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ | awk '{print $$1}' ) - # Set the top download link of the requested tarball. + # Set the top download link of the requested tarball. mergenames=1 if [ $$n = cfitsio ]; then mergenames=0 @@ -129,15 +145,24 @@ $(tarballs): $(tdir)/%: : (l==2 ? "%d00\n" \ : "%d000\n") ), $$1)}') w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz + elif [ $$n = atlas ]; then + mergenames=0 + w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download + elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype + elif [ $$n = hdf ]; then + mergenames=0 + majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') + w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$* elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://ftp.gnu.org/gnu/gnuastro elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files + elif [ $$n = lapack ]; then w=http://www.netlib.org/lapack elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng elif [ $$n = libtool ]; then w=http://ftp.gnu.org/gnu/libtool @@ -145,6 +170,13 @@ $(tarballs): $(tdir)/%: mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://akhlaghi.org/src + elif [ $$n = openblas ]; then + mergenames=0 + w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz + elif [ $$n = openmpi ]; then + mergenames=0 + majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') + w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') @@ -160,20 +192,20 @@ $(tarballs): $(tdir)/%: exit 1 fi - # Download the requested tarball. Note that some packages may not - # follow our naming convention (where the package name is merged - # with its version number). In such cases, `w' will be the full - # address, not just the top directory address. But since we are - # storing all the tarballs in one directory, we want it to have - # the same naming convention, so we'll download it to a temporary - # name, then rename that. + # Download the requested tarball. Note that some packages may not + # follow our naming convention (where the package name is merged + # with its version number). In such cases, `w' will be the full + # address, not just the top directory address. But since we are + # storing all the tarballs in one directory, we want it to have + # the same naming convention, so we'll download it to a temporary + # name, then rename that. if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*" else tarballurl=$$w fi - # If the download fails, Wget will write the error message in the - # target file, so Make will think that its done! To avoid this - # problem, we'll rename the output. + # If the download fails, Wget will write the error message in the + # target file, so Make will think that its done! To avoid this + # problem, we'll rename the output. echo "Downloading $$tarballurl" if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then rm -f $@ @@ -226,12 +258,26 @@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, $<, gsl-$(gsl-version), static) \ && echo "GNU Scientific Library is built" > $@ +$(ilidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz + $(call gbuild, $<, fftw-$(fftw-version), static, \ + --enable-shared) \ + && echo "FFTW is built" > $@ + # Freetype is necessary to install matplotlib $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ $(ilidir)/libpng $(call gbuild, $<, freetype-$(freetype-version), static) \ && echo "freetype is built" > $@ +$(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ + $(ilidir)/openmpi + export CC=mpicc; \ + export FC=mpif90; \ + $(call gbuild, $<, hdf5-$(hdf5-version), static, \ + --enable-parallel \ + --enable-fortran, V=1) \ + && echo "HDF5 library is built" > $@ + $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \ && echo "libbsd is built" > $@ @@ -249,6 +295,100 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ --disable-webp --disable-zstd) \ && echo "Libtiff is built" > $@ +$(ilidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz + $(call gbuild, $<, openmpi-$(openmpi-version), static, , V=1) \ + && echo "OpenMPI is built" > $@ + +$(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ + $(tdir)/lapack-$(lapack-version).tar.gz + + # Get the operating system specific features (how to get + # CPU frequency and the library suffixes). To make the steps + # more readable, the different library version suffixes are + # named with a single character: `s' for no version in the + # name, `m' for the major version suffix, and `f' for the + # full version suffix. + # GCC in Mac OS doesn't work. To work around this issue, on Mac + # systems we force ATLAS to use `clang' instead of `gcc'. + if [ x$(on_mac_os) = xyes ]; then + s=dylib + m=3.dylib + f=3.6.1.dylib + core=$$(sysctl hw.cpufrequency | awk '{print $$2/1000000}') + clangflag="--force-clang=$(ibdir)/clang" + else + s=so + m=so.3 + f=so.3.6.1 + clangflag= + core=$$(cat /proc/cpuinfo | grep "cpu MHz" \ + | head -n 1 \ + | sed "s/.*: \([0-9.]*\).*/\1/") + fi + + # See if the shared libraries should be build for a single CPU + # thread or multiple threads. + N=$$(nproc) + srcdir=$$(pwd)/reproduce/src/make + if [ $$N = 1 ]; then + sharedmk=$$srcdir/dependencies-atlas-single.mk + else + sharedmk=$$srcdir/dependencies-atlas-multiple.mk + fi + + # The linking step here doesn't recognize the `-Wl' in the + # `rpath_command'. + export LDFLAGS=-L$(ildir) + cd $(ddir) \ + && tar xf $< \ + && cd ATLAS \ + && rm -rf build \ + && mkdir build \ + && cd build \ + && ../configure -b 64 -D c -DPentiumCPS=$$core \ + --with-netlib-lapack-tarfile=$(word 2, $^) \ + --cripple-atlas-performance \ + -Fa alg -fPIC --shared $$clangflag \ + --prefix=$(idir) \ + && make \ + && if [ "x$(on_mac_os)" != xyes ]; then \ + cd lib && make -f $$sharedmk && cd .. \ + && for l in lib/*.$$s*; do \ + patchelf --set-rpath $(ildir) $$l; done \ + && cp -d lib/*.$$s* $(ildir) \ + && ln -fs $(ildir)/libblas.$$s $(ildir)/libblas.$$m \ + && ln -fs $(ildir)/libf77blas.$$s $(ildir)/libf77blas.$$m \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$s \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$m; \ + fi \ + && make install + + # We need to check the existance of `libptlapack.a', but we can't + # do this in the `&&' steps above (it will conflict). So we'll do + # the check after seeing if `libtatlas.so' is installed, then we'll + # finalize the build (delete the untarred directory). + if [ "x$(on_mac_os)" != xyes ]; then \ + [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \ + cd $(ddir); \ + rm -rf ATLAS; \ + fi + + # We'll check the full installation with the static library (not + # currently building shared library on Mac. + if [ -f $(ildir)/libatlas.a ]; then echo "Atlas is built" > $@; fi + +$(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + fi; \ + cd $(ddir) \ + && tar xf $< \ + && cd OpenBLAS-$(openblas-version) \ + && make \ + && make PREFIX=$(idir) install \ + && cd .. \ + && rm -rf OpenBLAS-$(openblas-version) \ + && echo "Libtiff is built" > $@ @@ -333,11 +473,6 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ make install && \ cd ..&& rm -rf cmake-$(cmake-version) -# Some programs that depend on cURL (in particular CMake) don't necessarily -# have easiy ways to explicity tell them to also link with libcurl's -# dependencies (libssl, libcrypto, and libz). So we won't force curl to -# only be static. -# # cURL (and its library, which is needed by several programs here) can # optionally link with many different network-related libraries on the host # system that we are not yet building in the pipeline. Many of these are @@ -363,7 +498,7 @@ $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz --without-axtls \ --disable-ldaps \ --disable-ldap \ - --without-nss ) + --without-nss, V=1) # On Mac OS, libtool does different things, so to avoid confusion, we'll # prefix GNU's libtool executables with `glibtool'. diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 545fcc1..66e7c77 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -369,10 +369,6 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\\grepversion}{$(grep-version)}" >> $@ echo "\newcommand{\\gzipversion}{$(gzip-version)}" >> $@ echo "\newcommand{\\islversion}{$(isl-version)}" >> $@ - echo "\newcommand{\\libbsdversion}{$(libbsd-version)}" >> $@ - echo "\newcommand{\\libffiversion}{$(libffi-version)}" >> $@ - echo "\newcommand{\\libpngversion}{$(libpng-version)}" >> $@ - echo "\newcommand{\\libtoolversion}{$(libtool-version)}" >> $@ echo "\newcommand{\\lzipversion}{$(lzip-version)}" >> $@ echo "\newcommand{\\makeversion}{$(make-version)}" >> $@ echo "\newcommand{\\metastoreversion}{$(metastore-version)}" >> $@ @@ -394,12 +390,18 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\\zipversion}{$(zip-version)}" >> $@ # Libraries. + echo "\newcommand{\\atlasversion}{$(cfitsio-version)}" >> $@ echo "\newcommand{\\cfitsioversion}{$(cfitsio-version)}" >> $@ echo "\newcommand{\\curlversion}{$(curl-version)}" >> $@ echo "\newcommand{\\gslversion}{$(gsl-version)}" >> $@ + echo "\newcommand{\\lapack}{$(lapack-version)}" >> $@ + echo "\newcommand{\\libbsdversion}{$(libbsd-version)}" >> $@ + echo "\newcommand{\\libffiversion}{$(libffi-version)}" >> $@ echo "\newcommand{\\libgittwoversion}{$(libgit2-version)}" >> $@ echo "\newcommand{\\libjpegversion}{$(libjpeg-version)}" >> $@ + echo "\newcommand{\\libpngversion}{$(libpng-version)}" >> $@ echo "\newcommand{\\libtiffversion}{$(libtiff-version)}" >> $@ + echo "\newcommand{\\libtoolversion}{$(libtool-version)}" >> $@ echo "\newcommand{\\wcslibversion}{$(wcslib-version)}" >> $@ echo "\newcommand{\\zlibversion}{$(zlib-version)}" >> $@ @@ -414,8 +416,9 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\\cryptographyversion}{$(cryptography-version)}" >> $@ echo "\newcommand{\\cyclerversion}{$(cycler-version)}" >> $@ echo "\newcommand{\\entrypointsversion}{$(entrypoints-version)}" >> $@ + echo "\newcommand{\\hpyversion}{$(h5py-version)}" >> $@ echo "\newcommand{\\htmlfivelibversion}{$(html5lib-version)}" >> $@ - echo "\newcommand{\\indaversion}{$(idna-version)}" >> $@ + echo "\newcommand{\\idaversion}{$(idna-version)}" >> $@ echo "\newcommand{\\jeepneyversion}{$(jeepney-version)}" >> $@ echo "\newcommand{\\kiwisolverversion}{$(kiwisolver-version)}" >> $@ echo "\newcommand{\\keyringversion}{$(keyring-version)}" >> $@ @@ -433,7 +436,7 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\\sixversion}{$(six-version)}" >> $@ echo "\newcommand{\\soupsieveversion}{$(soupsieve-version)}" >> $@ echo "\newcommand{\\urllibthreeversion}{$(urllib3-version)}" >> $@ - echo "\newcommand{\\virtualenvversion}{$(virtualenv-version)}" >> $@ +# echo "\newcommand{\\virtualenvversion}{$(virtualenv-version)}" >> $@ echo "\newcommand{\\webencodingsversion}{$(webencodings-version)}" >> $@ # TeX package versions |