aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xconfigure17
-rw-r--r--paper.tex69
-rw-r--r--reproduce/config/pipeline/dependency-numpy-scipy.cfg39
-rw-r--r--reproduce/config/pipeline/dependency-versions.mk30
-rwxr-xr-xreproduce/src/make/dependencies-atlas-multiple.mk47
-rwxr-xr-xreproduce/src/make/dependencies-atlas-single.mk29
-rw-r--r--reproduce/src/make/dependencies-basic.mk77
-rw-r--r--reproduce/src/make/dependencies-python.mk213
-rw-r--r--reproduce/src/make/dependencies.mk189
-rw-r--r--reproduce/src/make/initialize.mk15
10 files changed, 508 insertions, 217 deletions
diff --git a/configure b/configure
index 14be525..4f711e6 100755
--- a/configure
+++ b/configure
@@ -875,23 +875,6 @@ numthreads=$($instdir/bin/nproc)
-# Python dependencies
-# -------------------
-#
-# Python has its own installation program. To help in managing the
-# dependencies we make the installation of packages as a separate
-# Makefile.
-./.local/bin/make -f reproduce/src/make/dependencies-python.mk \
- rpath_command=$rpath_command \
- static_build=$static_build \
- on_mac_os=$on_mac_os \
- numthreads=$numthreads \
- -j$numthreads
-
-
-
-
-
# Make sure TeX Live installed successfully
# -----------------------------------------
diff --git a/paper.tex b/paper.tex
index e62ba46..ff03836 100644
--- a/paper.tex
+++ b/paper.tex
@@ -204,36 +204,49 @@ SUNDIAL ITN, and from the Spanish Ministry of Economy and Competitiveness
(MINECO) under grant number AYA2016-76219-P.
The following free software tools were also critical component of this
-research (in alphabetical order): Bzip2 \bziptwoversion, CFITSIO
-\cfitsioversion, CMake \cmakeversion, cURL \curlversion, Discoteq flock
-\flockversion, Git \gitversion, GNU Astronomy Utilities \gnuastroversion,
-GNU AWK \gawkversion, GNU Bash \bashversion, GNU Coreutils
-\coreutilsversion, GNU Diffutils \diffutilsversion, GNU Findutils
-\findutilsversion, GNU Grep \grepversion, GNU Gzip \gzipversion, GNU
-Libtool \libtoolversion, GNU Make \makeversion, GNU NCURSES
-\ncursesversion, GNU Readline \readlineversion, GNU Sed \sedversion, GNU
-Scientific Library (GSL) \gslversion, GNU Tar \tarversion, GNU Wget
-\wgetversion, GNU Which \whichversion, Lzip \lzipversion, GPL Ghostscript
-\ghostscriptversion, Libbsd \libbsdversion, Libgit2 \libgittwoversion,
-Libjpeg \libjpegversion, Libtiff \libtiffversion, Metastore (forked)
-\metastoreversion, OpenSSL \opensslversion, Pkg-config \pkgconfigversion,
+research (in alphabetical order): ATLAS \atlasversion, Bzip2
+\bziptwoversion, CFITSIO \cfitsioversion, CMake \cmakeversion, cURL
+\curlversion, Discoteq flock \flockversion, FreeType \freetypeversion, Git
+\gitversion, GNU Astronomy Utilities \gnuastroversion, GNU AWK \gawkversion,
+GNU Bash \bashversion, GNU Coreutils \coreutilsversion, GNU Diffutils
+\diffutilsversion, GNU Findutils \findutilsversion, GNU Grep \grepversion,
+GNU Gzip \gzipversion, GNU Libtool \libtoolversion, GNU Make \makeversion,
+GNU NCURSES \ncursesversion, GNU Readline \readlineversion, GNU Sed
+\sedversion, GNU Scientific Library (GSL) \gslversion, GNU Tar \tarversion,
+GNU Wget \wgetversion, GNU Which \whichversion, Lapack \lapackversion, Lzip
+\lzipversion, GPL Ghostscript \ghostscriptversion, Libbsd \libbsdversion,
+Libgit2 \libgitwoversion, Libjpeg \libjpegversion, Libpng \libpngversion,
+Libtiff \libtiffversion, Metastore (forked) \metastoreversion, OpenSSL
+\opensslversion, Patchelf \patchelfversion, Pkg-config \pkgconfigversion,
Unzip \unzipversion, WCSLIB \wcslibversion, XZ Utils \xzversion, Zip
\zipversion, and ZLib \zlibversion. We use Python {\pythonversion} with the
-following packages: Numpy {\numpyversion} and Astropy {\astropyversion}
-\citep{astropy2013, astropy2018}. The final paper was produced with \TeX{}
-Live \texliveversion, using the following packages: \TeX{} \textexversion,
-EC \texecversion, NewTX \texnewtxversion, Fontaxes \texfontaxesversion,
-Keyval, \texxkeyvalversion, Etoolbox \texetoolboxversion, Xcolor
-\texxcolorversion, Setspace \texsetspaceversion, Caption
-\texcaptionversion, Footmisc \texfootmiscversion, Datetime
-\texdatetimeversion, Fmtcount \texfmtcountversion, Titlesec
-\textitlesecversion, Preprint \texpreprintversion, Ulem \texulemversion,
-Bib\LaTeX{} \texbiblatexversion, Biber \texbiberversion, Logreq
-\texlogreqversion, PGF/TiKZ \texpgfversion, PGFPlots \texpgfplotsversion,
-FP \texfpversion, Courier \texcourierversion, \TeX-gyre \textexgyreversion,
-TXFonts \textxfontsversion, Times \textimesversion. We are very grateful to
-all their creators for freely providing this necessary infrastructure. This
-research would not be possible without them.
+following packages: Asn1crypto \asncryptoversion, Astroquery
+\astroqueryversion, Astropy {\astropyversion} \citep{astropy2013,
+astropy2018}, BeautifulSoup \beautifulsoupversion, Certifi \certifiversion,
+Cffi \cffiversion, Chardet \chardetversion, Cryptography
+\cryptographyversion, Cycler \cyclerversion, EntryPoints
+\entrypointsversion, h5py \hpyversion, html5ib \htmlfivelibversion, idna
+\idnaversion, Jeepney \jeepneyversion, Kiwisolver \kiwisolverversion,
+keyring \keyringversion, Matplotlib \matplotlibversion, Numpy \numpyversion,
+pycparser \pycparserversion, PyParsing \pyparsingversion, python-dateutil
+\pythondateutilversion, Requests \requestsversion, Scipy \scipyversion,
+ScreenStorage \screenstorageversion, Setuptools \setuptoolsversion,
+Setuptools-scm \setuptoolsscmversion, Six \sixversion, SoupSieve
+\soupsieveversion, urllib3 \urllibthreeversion and webencondings
+\webencondingsversion. The final paper was produced with \TeX{} Live
+\texliveversion, using the following packages: \TeX{} \textexversion, EC
+\texecversion, NewTX \texnewtxversion, Fontaxes \texfontaxesversion, Keyval,
+\texxkeyvalversion, Etoolbox \texetoolboxversion, Xcolor \texxcolorversion,
+Setspace \texsetspaceversion, Caption \texcaptionversion, Footmisc
+\texfootmiscversion, Datetime \texdatetimeversion, Fmtcount
+\texfmtcountversion, Titlesec \textitlesecversion, Preprint
+\texpreprintversion, Ulem \texulemversion, Bib\LaTeX{} \texbiblatexversion,
+Biber \texbiberversion, Logreq \texlogreqversion, PGF/TiKZ \texpgfversion,
+PGFPlots \texpgfplotsversion, FP \texfpversion, Courier \texcourierversion,
+\TeX-gyre \textexgyreversion, TXFonts \textxfontsversion, Times
+\textimesversion. We are very grateful to all their creators for freely
+providing this necessary infrastructure. This research would not be possible
+without them.
%% Tell BibLaTeX to put the bibliography list here.
\printbibliography
diff --git a/reproduce/config/pipeline/dependency-numpy-scipy.cfg b/reproduce/config/pipeline/dependency-numpy-scipy.cfg
new file mode 100644
index 0000000..0eecf3a
--- /dev/null
+++ b/reproduce/config/pipeline/dependency-numpy-scipy.cfg
@@ -0,0 +1,39 @@
+# THIS IS A COPY OF NUMPY'S site.cfg.example, CUSTOMIZED FOR THIS PIPELINE
+# ------------------------------------------------------------------------
+
+# This file provides configuration information about non-Python dependencies for
+# numpy.distutils-using packages. Create a file like this called "site.cfg" next
+# to your package's setup.py file and fill in the appropriate sections. Not all
+# packages will use all sections so you should leave out sections that your
+# package does not use.
+
+# IMPORTANT NOTE
+# --------------
+#
+# The `ALL' grouping does't apply to ATLAS!!!!!
+
+ [ALL]
+ library_dirs = @LIBDIR@
+ include_dirs = @INCDIR@
+
+
+# FFT libraries
+# -------------
+# There are two FFT libraries that we can configure here: FFTW (2 and 3) and djbfft.
+# Note that these libraries are not used by numpy or scipy.
+#
+# http://fftw.org/
+# https://cr.yp.to/djbfft.html
+#
+# Given only this section, numpy.distutils will try to figure out which version
+# of FFTW you are using.
+
+# Even with this section, Numpy apparently does not use fftw. But we will
+# keep it here for the future (if a solution is found).
+[fftw]
+libraries = fftw3
+#
+# For djbfft, numpy.distutils will look for either djbfft.a or libdjbfft.a .
+#[djbfft]
+#include_dirs = /usr/local/djbfft/include
+#library_dirs = /usr/local/djbfft/lib
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index 87b837c..2e74a45 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -25,6 +25,7 @@ binutils-version = 2.31.1
cmake-version = 3.12.4
coreutils-version = 8.30
diffutils-version = 3.7
+fftw-version = 3.3.8
findutils-version = 4.6.0.199-e3fc
flock-version = 0.2.3
freetype-version = 2.9
@@ -36,6 +37,7 @@ gmp-version = 6.1.2
gnuastro-version = 0.8
grep-version = 3.3
gzip-version = 1.10
+hdf5-version = 1.10.5
isl-version = 0.18
libbsd-version = 0.9.1
libffi-version = 3.2.1
@@ -47,6 +49,7 @@ metastore-version = 1.1.2-23-fa9170b
mpfr-version = 4.0.2
mpc-version = 1.1.0
ncurses-version = 6.1
+openmpi-version = 4.0.1
openssl-version = 1.1.1a
patchelf-version = 0.9
pkgconfig-version = 0.29.2
@@ -61,13 +64,26 @@ xz-version = 5.2.4
zip-version = 3.0
# Libraries
+atlas-version = 3.10.3
cfitsio-version = 3.45
curl-version = 7.63.0
gsl-version = 2.5
libjpeg-version = v9b
libtiff-version = 4.0.10
+openblas-version = 0.3.5
zlib-version = 1.2.11
+# Special libraries
+# -----------------
+#
+# When updating the version of these libraries, please look into the build
+# rule first: In one way or another, the version string becomes necessary
+# during their build and must be accounted for.
+bzip2-version = 1.0.6
+lapack-version = 3.8.0
+libgit2-version = 0.26.0
+wcslib-version = 6.2
+
# Python packages
# ---------------
#
@@ -83,13 +99,15 @@ chardet-version = 3.0.4
cryptography-version = 2.6.1
cycler-version = 0.10.0
entrypoints-version = 0.3
+h5py-version = 2.9.0
html5lib-version = 1.0.1
idna-version = 2.8
jeepney-version = 0.4
kiwisolver-version = 1.0.1
keyring-version = 18.0.0
matplotlib-version = 3.0.2
-numpy-version = 1.16.1
+mpi4py-version = 3.0.1
+numpy-version = 1.16.2
pip-version = 19.0.2
pycparser-version = 2.19
pyparsing-version = 2.3.1
@@ -104,13 +122,3 @@ soupsieve-version = 1.8
urllib3-version = 1.24.1
virtualenv-version = 16.4.0
webencodings-version = 0.5.1
-
-# Special libraries
-# -----------------
-#
-# When updating the version of these libraries, please look into the build
-# rule first: In one way or another, the version string becomes necessary
-# during their build and must be accounted for.
-bzip2-version = 1.0.6
-libgit2-version = 0.26.0
-wcslib-version = 6.2
diff --git a/reproduce/src/make/dependencies-atlas-multiple.mk b/reproduce/src/make/dependencies-atlas-multiple.mk
new file mode 100755
index 0000000..d66890f
--- /dev/null
+++ b/reproduce/src/make/dependencies-atlas-multiple.mk
@@ -0,0 +1,47 @@
+ORIGLDFLAGS := $(LDFLAGS)
+
+include Make.inc
+
+all: libatlas.so libf77blas.so libptf77blas.so libstcblas.so libptcblas.so \
+ libblas.so libcblas.so liblapack.so.3.6.1 libptlapack.so.3.6.1 liblapack.so.3
+
+libatlas.so: libatlas.a
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \
+ --whole-archive libatlas.a --no-whole-archive -lc $(LIBS)
+
+libf77blas.so : libf77blas.a libatlas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libf77blas.so.3 \
+ -o $@ --whole-archive libf77blas.a --no-whole-archive \
+ $(F77SYSLIB) -L. -latlas
+
+libptf77blas.so : libptf77blas.a libatlas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libblas.so.3 \
+ -o $@ --whole-archive libptf77blas.a --no-whole-archive \
+ $(F77SYSLIB) -L. -latlas
+
+libstcblas.so : libcblas.a libatlas.so libblas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libstcblas.so \
+ -o $@ --whole-archive libcblas.a -L. -latlas -lblas
+
+libptcblas.so : libptcblas.a libatlas.so libblas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libcblas.so \
+ -o $@ --whole-archive libptcblas.a -L. -latlas -lblas
+
+libblas.so: libptf77blas.so
+ ln -s $< $@
+
+libcblas.so: libptcblas.so
+ ln -s $< $@
+
+liblapack.so.3.6.1 : liblapack.a libstcblas.so libf77blas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libstlapack.so.3 \
+ -o $@ --whole-archive liblapack.a --no-whole-archive \
+ $(F77SYSLIB) -L. -lstcblas -lf77blas
+
+libptlapack.so.3.6.1 : libptlapack.a libcblas.so libblas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname liblapack.so.3 \
+ -o $@ --whole-archive libptlapack.a --no-whole-archive \
+ $(F77SYSLIB) -L. -lcblas -lblas
+
+liblapack.so.3: libptlapack.so.3.6.1
+ ln -s $< $@
diff --git a/reproduce/src/make/dependencies-atlas-single.mk b/reproduce/src/make/dependencies-atlas-single.mk
new file mode 100755
index 0000000..12dd19b
--- /dev/null
+++ b/reproduce/src/make/dependencies-atlas-single.mk
@@ -0,0 +1,29 @@
+ORIGLDFLAGS := $(LDFLAGS)
+
+include Make.inc
+
+all: libatlas.so libf77blas.so libcblas.so libblas.so liblapack.so.3.6.1
+
+libatlas.so: libatlas.a
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \
+ --whole-archive libatlas.a --no-whole-archive -lc $(LIBS)
+
+libf77blas.so : libf77blas.a libatlas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libblas.so.3 \
+ -o $@ --whole-archive libf77blas.a --no-whole-archive \
+ $(F77SYSLIB) -L. -latlas
+
+libcblas.so : libcblas.a libatlas.so libblas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \
+ --whole-archive libcblas.a -L. -latlas -lblas
+
+libblas.so: libf77blas.so
+ ln -s $< $@
+
+liblapack.so.3.6.1 : liblapack.a libcblas.so libblas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname liblapack.so.3 \
+ -o $@ --whole-archive liblapack.a --no-whole-archive \
+ $(F77SYSLIB) -L. -lcblas -lblas
+
+liblapack.so.3: liblapack.so.3.6.1
+ ln -s $< $@
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index cdb4a8c..52a41d9 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -204,12 +204,13 @@ $(tarballs): $(tdir)/%: $(lockdir)
# is very annoying and can cause many complications. We thus remove any
# part of PATH of that has `ccache' in it before making symbolic links to
# the programs we are not building ourselves.
-makelink = origpath="$$PATH"; \
+makelink = origpath="$$PATH"; \
export PATH=$$(echo $(syspath) | tr : '\n' | grep -v ccache \
- | paste -s -d:); \
+ | tr '\n' :); \
a=$$(which $(1) 2> /dev/null); \
- if [ -f $(ibdir)/$(1) ]; then rm $(ibdir)/$(1); fi; \
- if [ x$$a != x ]; then ln -s $$a $(ibdir)/$(1); fi; \
+ if [ -e $(ibdir)/$(1) ]; then rm $(ibdir)/$(1); fi; \
+ if [ x"$(2)" = xcopy ]; then c=cp; else c="ln -s"; fi; \
+ if [ x$$a != x ]; then $$c $$a $(ibdir)/$(1); fi; \
export PATH="$$origpath"
$(ibdir) $(ildir):; mkdir $@
$(ibdir)/low-level-links: | $(ibdir) $(ildir)
@@ -576,8 +577,15 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \
--with-zlib-lib=$(ildir) \
--with-zlib-include=$(idir)/include, , , \
./config ) && \
- cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem && \
- echo "OpenSSL is built and ready" > $@
+ cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem; \
+ if [ $$? = 0 ]; then \
+ if [ x$(on_mac_os) = xyes ]; then \
+ echo "No need to fix rpath in libssl"; \
+ else \
+ patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \
+ fi; \
+ echo "OpenSSL is built and ready" > $@; \
+ fi
# GNU Wget
#
@@ -734,13 +742,13 @@ $(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz
#
# We are currently having problems installing GCC on macOS, so for the time
# being, if the pipeline is being run on a macOS, we'll just set a link.
-ifeq ($(on_mac_os),yes)
-gcc-prerequisites =
-else
+#ifeq ($(on_mac_os),yes)
+#gcc-prerequisites =
+#else
gcc-prerequisites = $(tdir)/gcc-$(gcc-version).tar.xz \
$(ilidir)/isl \
$(ilidir)/mpc
-endif
+#endif
$(ibdir)/gcc: $(gcc-prerequisites) \
$(ibdir)/ls \
$(ibdir)/sed \
@@ -752,24 +760,36 @@ $(ibdir)/gcc: $(gcc-prerequisites) \
$(ibdir)/which
# On a macOS, we (currently!) won't build GCC because of some
- # errors we are still trying to fix. So, we'll just make a symbolic
- # link to the host's executables.
- if [ "x$(on_mac_os)" = xyes ]; then \
+ # errors we are still trying to find. So, we'll just make a
+ # symbolic link to the host's executables.
+ #
+ # GCC builds is own libraries in '$(idir)/lib64'. But all other
+ # libraries are in '$(idir)/lib'. Since this pipeline is only for a
+ # single architecture, we can trick GCC into building its libraries
+ # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic
+ # link to '$(idir)/lib'.
+
+# SO FAR WE HAVEN'T BEEN ABLE TO GET A CONSISTENT BUILD OF GCC ON MAC
+# (SOMETIMES IT CRASHES IN libiberty with g++) AND SOMETIMES IT FINISHES,
+# SO, MORE TESTS ARE NEEDED ON MAC AND WE'LL USE THE HOST'S COMPILER UNTIL
+# THEN.
+ if [ "x$(on_mac_os)" = xyesno ]; then \
$(call makelink,gfortran); \
$(call makelink,g++); \
- $(call makelink,gcc); \
+ $(call makelink,gcc,copy); \
else \
- \
rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\
rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \
rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\
\
+ ln -fs $(ildir) $(idir)/lib64; \
+ \
cd $(ddir); \
rm -rf gcc-build gcc-$(gcc-version); \
- tar xf $< && \
- mkdir $(ddir)/gcc-build && \
- cd $(ddir)/gcc-build && \
- ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \
+ tar xf $< \
+ && mkdir $(ddir)/gcc-build \
+ && cd $(ddir)/gcc-build \
+ && ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \
--prefix=$(idir) \
--with-mpc=$(idir) \
--with-mpfr=$(idir) \
@@ -787,9 +807,18 @@ $(ibdir)/gcc: $(gcc-prerequisites) \
--enable-default-pie \
--enable-default-ssp \
--enable-cet=auto \
- --enable-decimal-float && \
- make SHELL=$(ibdir)/bash -j$$(nproc) && \
- make SHELL=$(ibdir)/bash install && \
- cd .. && \
- rm -rf gcc-build gcc-$(gcc-version); \
+ --enable-decimal-float \
+ && make SHELL=$(ibdir)/bash -j$$(nproc) \
+ && make SHELL=$(ibdir)/bash install \
+ && cd .. \
+ && rm -rf gcc-build gcc-$(gcc-version) \
+ \
+ && if [ "x$(on_mac_os)" != xyes ]; then \
+ for f in $$(find $(idir)/libexec/gcc); do \
+ if ldd $$f &> /dev/null; then \
+ patchelf --set-rpath $(ildir) $$f; \
+ fi; \
+ done; \
+ fi; \
fi
+
diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk
index 946c047..6711d17 100644
--- a/reproduce/src/make/dependencies-python.mk
+++ b/reproduce/src/make/dependencies-python.mk
@@ -29,51 +29,6 @@
-# Top level environment
-include reproduce/config/pipeline/LOCAL.mk
-include reproduce/src/make/dependencies-build-rules.mk
-include reproduce/config/pipeline/dependency-versions.mk
-
-ddir = $(BDIR)/dependencies
-tdir = $(BDIR)/dependencies/tarballs
-idir = $(BDIR)/dependencies/installed
-ibdir = $(BDIR)/dependencies/installed/bin
-ildir = $(BDIR)/dependencies/installed/lib
-ilidir = $(BDIR)/dependencies/installed/lib/built
-ipydir = $(BDIR)/dependencies/installed/lib/built/python
-
-# Define the top-level programs to build (installed in `.local/bin').
-top-level-python = astroquery matplotlib #scipy
-all: $(foreach p, $(top-level-python), $(ipydir)/$(p))
-
-# Other basic environment settings: We are only including the host
-# operating system's PATH environment variable (after our own!) for the
-# compiler and linker. For the library binaries and headers, we are only
-# using our internally built libraries.
-#
-# To investigate:
-#
-# 1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all
-# the parameters defined bellow as `NAME=VALUE' statements before
-# calling Bash. This will enable us to completely ignore the user's
-# native environment.
-#
-# 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the
-# user's environment.
-.ONESHELL:
-.SHELLFLAGS := --noprofile --norc -ec
-export CCACHE_DISABLE := 1
-export PATH := $(ibdir)
-export LD_RUN_PATH := $(ildir)
-export LD_LIBRARY_PATH := $(ildir)
-export SHELL := $(ibdir)/bash
-export CPPFLAGS := -I$(idir)/include
-export PKG_CONFIG_PATH := $(ildir)/pkgconfig
-export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig
-export LDFLAGS := $(rpath_command) -L$(ildir)
-
-
-
# Python enviroment
@@ -84,9 +39,9 @@ export LDFLAGS := $(rpath_command) -L$(ildir)
# systems which might interfere. To be safe, we are removing all their
# values.
export PYTHONPATH := $(installdir)/lib/python/site-packages
+export PYTHONPATH2 := $(PYTHONPATH)
export PYTHONPATH3 := $(PYTHONPATH)
export _LMFILES_ :=
-export PYTHONPATH2 :=
export LOADEDMODULES :=
export MPI_PYTHON_SITEARCH :=
export MPI_PYTHON2_SITEARCH :=
@@ -106,7 +61,7 @@ export MPI_PYTHON3_SITEARCH :=
# convention, but include the name/version in their tarball names with
# another format, we'll do the modification before the download so the
# downloaded file has our desired format.
-tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \
+pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \
astroquery-$(astroquery-version).tar.gz \
astropy-$(astropy-version).tar.gz \
beautifulsoup4-$(beautifulsoup4-version).tar.gz \
@@ -116,6 +71,7 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \
cryptography-$(cryptography-version).tar.gz \
cycler-$(cycler-version).tar.gz \
entrypoints-$(entrypoints-version).tar.gz \
+ h5py-$(h5py-version).tar.gz \
html5lib-$(html5lib-version).tar.gz \
idna-$(idna-version).tar.gz \
jeepney-$(jeepney-version).tar.gz \
@@ -123,6 +79,7 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \
keyring-$(keyring-version).tar.gz \
libffi-$(libffi-version).tar.gz \
matplotlib-$(matplotlib-version).tar.gz \
+ mpi4py-$(mpi4py-version).tar.gz \
numpy-$(numpy-version).zip \
pip-$(pip-version).tar.gz \
pycparser-$(pycparser-version).tar.gz \
@@ -140,8 +97,8 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \
webencodings-$(webencodings-version).tar.gz \
virtualenv-$(virtualenv-version).tar.gz \
, $(tdir)/$(t) )
-topurl=https://files.pythonhosted.org/packages
-$(tarballs): $(tdir)/%:
+pytopurl=https://files.pythonhosted.org/packages
+$(pytarballs): $(tdir)/%:
if [ -f $(DEPENDENCIES-DIR)/$* ]; then
cp $(DEPENDENCIES-DIR)/$* $@
else
@@ -157,7 +114,10 @@ $(tarballs): $(tdir)/%:
# because the tokenization above will produce `python' as the
# first string.
if [ $* = python-dateutil-$(python-dateutil-version).tar.gz ]; then
- n=dateutil
+ n=dateutil
+ elif [ $* = h5py-$(h5py-version).tar.gz ]; then
+ n=h5py
+
# elif [ $* = strange-tarball5name-version.tar.gz ]; then
# n=strange5-name
else
@@ -180,7 +140,7 @@ $(tarballs): $(tdir)/%:
elif [ $$n = secretstorage ]; then
mergenames=0
hash=a6/89/df343dbc2957a317127e7ff2983230dc5336273be34f2e1911519d85aeb5
- h=$(topurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz
+ h=$(pytopurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz
elif [ $$n = asn ]; then h=fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4
elif [ $$n = astroquery ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396
elif [ $$n = astropy ]; then h=eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352
@@ -191,13 +151,15 @@ $(tarballs): $(tdir)/%:
elif [ $$n = cryptography ]; then h=07/ca/bc827c5e55918ad223d59d299fff92f3563476c3b00d0a9157d9c0217449
elif [ $$n = cycler ]; then h=c2/4b/137dea450d6e1e3d474e1d873cd1d4f7d3beed7e0dc973b06e8e10d32488
elif [ $$n = entrypoints ]; then h=b4/ef/063484f1f9ba3081e920ec9972c96664e2edb9fdc3d8669b0e3b8fc0ad7c
+ elif [ $$n = h5py ]; then h=43/27/a6e7dcb8ae20a4dbf3725321058923fec262b6f7835179d78ccc8d98deec
elif [ $$n = html ]; then h=85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f
elif [ $$n = idna ]; then h=ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7
elif [ $$n = jeepney ]; then h=16/1d/74adf3b164a8d19a60d0fcf706a751ffa2a1eaa8e5bbb1b6705c92a05263
elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2
elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864
elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f
- elif [ $$n = numpy ]; then h=2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c
+ elif [ $$n = mpi ]; then h=55/a2/c827b196070e161357b49287fa46d69f25641930fd5f854722319d431843
+ elif [ $$n = numpy ]; then h=cf/8d/6345b4f32b37945fedc1e027e83970005fc9c699068d2f566b82826515f2
elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af
elif [ $$n = pycparser ]; then h=68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a
elif [ $$n = pyparsing ]; then h=b9/b8/6b32b3e84014148dcd60dd05795e35c2e7f4b72f918616c61fdce83d27fc
@@ -212,7 +174,7 @@ $(tarballs): $(tdir)/%:
elif [ $$n = urllib ]; then h=b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1
elif [ $$n = virtualenv ]; then h=51/aa/c395a6e6eaaedfa5a04723b6446a1df783b16cca6fec66e671cede514688
elif [ $$n = webencodings ]; then h=0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47
-# elif [ $$n = strange5-name ]; then h=XXXXX
+# elif [ $$n = strange5-name ]; then h=XXXXX
else
echo; echo; echo;
echo "'$$n' not recognized as a dependency name to download."
@@ -227,7 +189,7 @@ $(tarballs): $(tdir)/%:
# storing all the tarballs in one directory, we want it to have
# the same naming convention, so we'll download it to a temporary
# name, then rename that.
- if [ $$mergenames = 1 ]; then tarballurl=$(topurl)/$$h/"$*"
+ if [ $$mergenames = 1 ]; then tarballurl=$(pytopurl)/$$h/"$*"
else tarballurl=$$h
fi
@@ -249,18 +211,24 @@ $(tarballs): $(tdir)/%:
# --------------------
#
# To build Python packages with direct access to a `setup.py' (if no direct
-# access to `setup.py' is needed, pip can be used)
+# access to `setup.py' is needed, pip can be used).
# Arguments of this function are the numbers
# 1) Unpack command
# 2) Package name
# 3) Unpacked directory name after unpacking the tarball
-pybuild = cd $(ddir); rm -rf $(3); \
- if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \
- cd $(3); \
- python3 setup.py build && \
- python3 setup.py install && \
- cd .. && rm -rf $(3) && \
- echo "done!" > $@
+# 4) site.cfg file (optional)
+pybuild = cd $(ddir); rm -rf $(3); \
+ if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \
+ cd $(3); \
+ if [ "x$(4)" != x ]; then \
+ sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \
+ -e 's|@INCDIR[@]|'"$(idir)/include"'|' \
+ $(4) > site.cfg; \
+ fi; \
+ python3 setup.py build \
+ && python3 setup.py install \
+ && cd .. && rm -rf $(3) \
+ && echo "done!" > $@
@@ -271,24 +239,27 @@ pybuild = cd $(ddir); rm -rf $(3); \
#
# While this Makefile is for Python programs, in some cases, we need
# certain programs (like Python itself), or libraries for the modules.
-$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz
+$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz
+ $(call gbuild, $<, libffi-$(libffi-version)) \
+ echo "libffi is built" > $@
+
+$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \
+ $(ilidir)/libffi
# On Mac systems, the build complains about `clang' specific
# features, so we can't use our own GCC build here.
-# if [ x$(on_mac_os) = xyes ]; then \
-# export CC=clang; \
-# export CXX=clang++; \
-# fi; \
-
- $(call gbuild, $<, Python-$(python-version)) \
+ if [ x$(on_mac_os) = xyes ]; then \
+ export CC=clang; \
+ export CXX=clang++; \
+ fi; \
+ $(call gbuild, $<, Python-$(python-version),, \
+ --without-ensurepip \
+ --with-system-ffi \
+ --enable-shared) \
&& v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \
{printf "%d.%d\n", $$1, $$2}') \
&& ln -s $(ildir)/python$$v $(ildir)/python \
&& rm -rf $(ipydir) && mkdir $(ipydir)
-$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz
- $(call gbuild, $<, libffi-$(libffi-version)) \
- echo "libffi is built" > $@
-
@@ -297,7 +268,7 @@ $(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz
#
# All the necessary Python modules go here.
$(ipydir)/asn1crypto: $(tdir)/asn1crypto-$(asn1crypto-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, asn1crypto-$(asn1crypto-version))
$(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \
@@ -310,7 +281,9 @@ $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \
$(call pybuild, tar xf, $<, astroquery-$(astroquery-version))
$(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \
- $(ipydir)/numpy
+ $(ipydir)/h5py \
+ $(ipydir)/numpy \
+ $(ipydir)/scipy
$(call pybuild, tar xf, $<, astropy-$(astropy-version))
$(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \
@@ -318,81 +291,107 @@ $(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.g
$(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version))
$(ipydir)/certifi: $(tdir)/certifi-$(certifi-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, certifi-$(certifi-version))
$(ipydir)/cffi: $(tdir)/cffi-$(cffi-version).tar.gz \
- $(ipydir)/pycparser \
- $(ilidir)/libffi
+ $(ilidir)/libffi \
+ $(ipydir)/pycparser
$(call pybuild, tar xf, $<, cffi-$(cffi-version))
$(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, chardet-$(chardet-version))
$(ipydir)/cryptography: $(tdir)/cryptography-$(cryptography-version).tar.gz \
- $(ipydir)/cffi \
- $(ipydir)/asn1crypto
+ $(ipydir)/asn1crypto \
+ $(ipydir)/cffi
$(call pybuild, tar xf, $<, cryptography-$(cryptography-version))
$(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \
- $(ipydir)/six
+ $(ipydir)/six
$(call pybuild, tar xf, $<, cycler-$(cycler-version))
$(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version))
+$(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \
+ $(ipydir)/setuptools \
+ $(ilidir)/hdf5
+ # $(ipydir)/mpi4py # AFTER its problem is fixed.
+ #export HDF5_MPI=ON; # AFTER its problem is fixed.
+ export HDF5_DIR=$(ildir); \
+ $(call pybuild, tar xf, $<, h5py-$(h5py-version))
+
$(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \
$(ipydir)/six \
$(ipydir)/webencodings
$(call pybuild, tar xf, $<, html5lib-$(html5lib-version))
$(ipydir)/idna: $(tdir)/idna-$(idna-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, idna-$(idna-version))
$(ipydir)/jeepney: $(tdir)/jeepney-$(jeepney-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, jeepney-$(jeepney-version))
$(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \
- $(ipydir)/setuptools_scm \
+ $(ipydir)/entrypoints \
$(ipydir)/secretstorage \
- $(ipydir)/entrypoints
+ $(ipydir)/setuptools_scm
$(call pybuild, tar xf, $<, keyring-$(keyring-version))
$(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \
- $(ipydir)/setuptools
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version))
-
$(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
$(ipydir)/cycler \
$(ilidir)/freetype \
+ $(ipydir)/kiwisolver \
$(ipydir)/numpy \
$(ipydir)/pyparsing \
- $(ipydir)/python-dateutil \
- $(ipydir)/kiwisolver
+ $(ipydir)/python-dateutil
$(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version))
+# Currently mpi4py doesn't build because of some conflict with OpenMPI:
+#
+# In file included from src/mpi4py.MPI.c:591,
+# from src/MPI.c:4:
+# src/mpi4py.MPI.c: In function '__pyx_f_6mpi4py_3MPI_del_Datatype':
+# src/mpi4py.MPI.c:15094:36: error: expected expression before '_Static_assert'
+# __pyx_t_1 = (((__pyx_v_ob[0]) == MPI_UB) != 0);
+#
+# But atleast on my system it fails.
+$(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \
+ $(ipydir)/setuptools \
+ $(ilidir)/openmpi
+ $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version))
+
$(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
- $(ibdir)/python3
- export BLAS=None; \
- export ATLAS=None; \
- export LAPACK=None; \
- $(call pybuild, unzip, $<, numpy-$(numpy-version))
+ $(ipydir)/setuptools \
+ $(ilidir)/openblas \
+ $(ilidir)/fftw
+ if [ x$(on_mac_os) = xyes ]; then \
+ export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \
+ else \
+ export LDFLAGS="$(LDFLAGS) -shared"; \
+ fi; \
+ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
+ $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf)
$(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, pip-$(pip-version))
$(ipydir)/pycparser: $(tdir)/pycparser-$(pycparser-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, pycparser-$(pycparser-version))
$(ipydir)/pyparsing: $(tdir)/pyparsing-$(pyparsing-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version))
$(ipydir)/python-dateutil: $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz \
@@ -410,7 +409,13 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \
$(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \
$(ipydir)/numpy
- $(call pybuild, tar xf, $<, scipy-$(scipy-version))
+ if [ x$(on_mac_os) = xyes ]; then \
+ export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \
+ else \
+ export LDFLAGS="$(LDFLAGS) -shared"; \
+ fi; \
+ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
+ $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf)
$(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \
$(ipydir)/cryptography \
@@ -426,17 +431,17 @@ $(ipydir)/setuptools_scm: $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.g
$(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version))
$(ipydir)/six: $(tdir)/six-$(six-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, six-$(six-version))
$(ipydir)/soupsieve: $(tdir)/soupsieve-$(soupsieve-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version))
$(ipydir)/urllib3: $(tdir)/urllib3-$(urllib3-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, urllib3-$(urllib3-version))
$(ipydir)/webencodings: $(tdir)/webencodings-$(webencodings-version).tar.gz \
- $(ibdir)/python3
+ $(ipydir)/setuptools
$(call pybuild, tar xf, $<, webencodings-$(webencodings-version))
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index 2b3b70b..fdbfd7f 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -35,19 +35,28 @@ include reproduce/src/make/dependencies-build-rules.mk
include reproduce/config/pipeline/dependency-texlive.mk
include reproduce/config/pipeline/dependency-versions.mk
+
ddir = $(BDIR)/dependencies
tdir = $(BDIR)/dependencies/tarballs
idir = $(BDIR)/dependencies/installed
ibdir = $(BDIR)/dependencies/installed/bin
ildir = $(BDIR)/dependencies/installed/lib
ilidir = $(BDIR)/dependencies/installed/lib/built
+ipydir = $(BDIR)/dependencies/installed/lib/built/python
# Define the top-level programs to build (installed in `.local/bin').
-top-level-programs = astnoisechisel metastore unzip zip
-top-level-libraries = freetype
-all: $(ddir)/texlive-versions.tex \
- $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \
- $(foreach p, $(top-level-libraries), $(ilidir)/$(p))
+#
+# About ATLAS: currently the core pipeline does not depend on ATLAS but many
+# high level software depend on it. The current rule for ATLAS is tested
+# successfully on Mac (only static) and GNU/Linux (shared and static). But,
+# since it takes a few hours to build, it is not currently a target.
+top-level-libraries = # atlas
+top-level-programs = astnoisechisel flock metastore unzip zip
+top-level-python = astroquery matplotlib
+all: $(ddir)/texlive-versions.tex \
+ $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \
+ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \
+ $(foreach p, $(top-level-python), $(ipydir)/$(p))
# Other basic environment settings: We are only including the host
# operating system's PATH environment variable (after our own!) for the
@@ -67,16 +76,17 @@ all: $(ddir)/texlive-versions.tex \
.SHELLFLAGS := --noprofile --norc -ec
export CCACHE_DISABLE := 1
export PATH := $(ibdir)
-export LD_RUN_PATH := $(ildir)
-export LD_LIBRARY_PATH := $(ildir)
export SHELL := $(ibdir)/bash
export CPPFLAGS := -I$(idir)/include
export PKG_CONFIG_PATH := $(ildir)/pkgconfig
export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig
+export LD_RUN_PATH := $(ildir):$(il64dir)
+export LD_LIBRARY_PATH := $(ildir):$(il64dir)
export LDFLAGS := $(rpath_command) -L$(ildir)
-
+# Python packages
+include reproduce/src/make/dependencies-python.mk
# Tarballs
@@ -90,21 +100,27 @@ export LDFLAGS := $(rpath_command) -L$(ildir)
# another format, we'll do the modification before the download so the
# downloaded file has our desired format.
tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
+ atlas-$(atlas-version).tar.bz2 \
cmake-$(cmake-version).tar.gz \
curl-$(curl-version).tar.gz \
freetype-$(freetype-version).tar.gz \
+ fftw-$(fftw-version).tar.gz \
ghostscript-$(ghostscript-version).tar.gz \
git-$(git-version).tar.xz \
gnuastro-$(gnuastro-version).tar.lz \
gsl-$(gsl-version).tar.gz \
+ hdf5-$(hdf5-version).tar.gz \
install-tl-unx.tar.gz \
jpegsrc.$(libjpeg-version).tar.gz \
+ lapack-$(lapack-version).tar.gz \
libbsd-$(libbsd-version).tar.xz \
libpng-$(libpng-version).tar.xz \
libtool-$(libtool-version).tar.xz \
libgit2-$(libgit2-version).tar.gz \
metastore-$(metastore-version).tar.gz \
+ openmpi-$(openmpi-version).tar.gz \
unzip-$(unzip-version).tar.gz \
+ openblas-$(openblas-version).tar.gz \
tiff-$(libtiff-version).tar.gz \
wcslib-$(wcslib-version).tar.bz2 \
zip-$(zip-version).tar.gz \
@@ -113,12 +129,12 @@ $(tarballs): $(tdir)/%:
if [ -f $(DEPENDENCIES-DIR)/$* ]; then
cp $(DEPENDENCIES-DIR)/$* $@
else
- # Remove all numbers, `-' and `.' from the tarball name so we can
- # search more easily only with the program name.
+ # Remove all numbers, `-' and `.' from the tarball name so we can
+ # search more easily only with the program name.
n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \
| awk '{print $$1}' )
- # Set the top download link of the requested tarball.
+ # Set the top download link of the requested tarball.
mergenames=1
if [ $$n = cfitsio ]; then
mergenames=0
@@ -129,15 +145,24 @@ $(tarballs): $(tdir)/%:
: (l==2 ? "%d00\n" \
: "%d000\n") ), $$1)}')
w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz
+ elif [ $$n = atlas ]; then
+ mergenames=0
+ w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download
elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12
elif [ $$n = curl ]; then w=https://curl.haxx.se/download
+ elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw
elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype
+ elif [ $$n = hdf ]; then
+ mergenames=0
+ majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}')
+ w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$*
elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926
elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git
elif [ $$n = gnuastro ]; then w=http://ftp.gnu.org/gnu/gnuastro
elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl
elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet
elif [ $$n = jpegsrc ]; then w=http://ijg.org/files
+ elif [ $$n = lapack ]; then w=http://www.netlib.org/lapack
elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases
elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng
elif [ $$n = libtool ]; then w=http://ftp.gnu.org/gnu/libtool
@@ -145,6 +170,13 @@ $(tarballs): $(tdir)/%:
mergenames=0
w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
elif [ $$n = metastore ]; then w=http://akhlaghi.org/src
+ elif [ $$n = openblas ]; then
+ mergenames=0
+ w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz
+ elif [ $$n = openmpi ]; then
+ mergenames=0
+ majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}')
+ w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$*
elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff
elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src
mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//')
@@ -160,20 +192,20 @@ $(tarballs): $(tdir)/%:
exit 1
fi
- # Download the requested tarball. Note that some packages may not
- # follow our naming convention (where the package name is merged
- # with its version number). In such cases, `w' will be the full
- # address, not just the top directory address. But since we are
- # storing all the tarballs in one directory, we want it to have
- # the same naming convention, so we'll download it to a temporary
- # name, then rename that.
+ # Download the requested tarball. Note that some packages may not
+ # follow our naming convention (where the package name is merged
+ # with its version number). In such cases, `w' will be the full
+ # address, not just the top directory address. But since we are
+ # storing all the tarballs in one directory, we want it to have
+ # the same naming convention, so we'll download it to a temporary
+ # name, then rename that.
if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*"
else tarballurl=$$w
fi
- # If the download fails, Wget will write the error message in the
- # target file, so Make will think that its done! To avoid this
- # problem, we'll rename the output.
+ # If the download fails, Wget will write the error message in the
+ # target file, so Make will think that its done! To avoid this
+ # problem, we'll rename the output.
echo "Downloading $$tarballurl"
if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then
rm -f $@
@@ -226,12 +258,26 @@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz
$(call gbuild, $<, gsl-$(gsl-version), static) \
&& echo "GNU Scientific Library is built" > $@
+$(ilidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz
+ $(call gbuild, $<, fftw-$(fftw-version), static, \
+ --enable-shared) \
+ && echo "FFTW is built" > $@
+
# Freetype is necessary to install matplotlib
$(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \
$(ilidir)/libpng
$(call gbuild, $<, freetype-$(freetype-version), static) \
&& echo "freetype is built" > $@
+$(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \
+ $(ilidir)/openmpi
+ export CC=mpicc; \
+ export FC=mpif90; \
+ $(call gbuild, $<, hdf5-$(hdf5-version), static, \
+ --enable-parallel \
+ --enable-fortran, V=1) \
+ && echo "HDF5 library is built" > $@
+
$(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz
$(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \
&& echo "libbsd is built" > $@
@@ -249,6 +295,100 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \
--disable-webp --disable-zstd) \
&& echo "Libtiff is built" > $@
+$(ilidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz
+ $(call gbuild, $<, openmpi-$(openmpi-version), static, , V=1) \
+ && echo "OpenMPI is built" > $@
+
+$(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
+ $(tdir)/lapack-$(lapack-version).tar.gz
+
+ # Get the operating system specific features (how to get
+ # CPU frequency and the library suffixes). To make the steps
+ # more readable, the different library version suffixes are
+ # named with a single character: `s' for no version in the
+ # name, `m' for the major version suffix, and `f' for the
+ # full version suffix.
+ # GCC in Mac OS doesn't work. To work around this issue, on Mac
+ # systems we force ATLAS to use `clang' instead of `gcc'.
+ if [ x$(on_mac_os) = xyes ]; then
+ s=dylib
+ m=3.dylib
+ f=3.6.1.dylib
+ core=$$(sysctl hw.cpufrequency | awk '{print $$2/1000000}')
+ clangflag="--force-clang=$(ibdir)/clang"
+ else
+ s=so
+ m=so.3
+ f=so.3.6.1
+ clangflag=
+ core=$$(cat /proc/cpuinfo | grep "cpu MHz" \
+ | head -n 1 \
+ | sed "s/.*: \([0-9.]*\).*/\1/")
+ fi
+
+ # See if the shared libraries should be build for a single CPU
+ # thread or multiple threads.
+ N=$$(nproc)
+ srcdir=$$(pwd)/reproduce/src/make
+ if [ $$N = 1 ]; then
+ sharedmk=$$srcdir/dependencies-atlas-single.mk
+ else
+ sharedmk=$$srcdir/dependencies-atlas-multiple.mk
+ fi
+
+ # The linking step here doesn't recognize the `-Wl' in the
+ # `rpath_command'.
+ export LDFLAGS=-L$(ildir)
+ cd $(ddir) \
+ && tar xf $< \
+ && cd ATLAS \
+ && rm -rf build \
+ && mkdir build \
+ && cd build \
+ && ../configure -b 64 -D c -DPentiumCPS=$$core \
+ --with-netlib-lapack-tarfile=$(word 2, $^) \
+ --cripple-atlas-performance \
+ -Fa alg -fPIC --shared $$clangflag \
+ --prefix=$(idir) \
+ && make \
+ && if [ "x$(on_mac_os)" != xyes ]; then \
+ cd lib && make -f $$sharedmk && cd .. \
+ && for l in lib/*.$$s*; do \
+ patchelf --set-rpath $(ildir) $$l; done \
+ && cp -d lib/*.$$s* $(ildir) \
+ && ln -fs $(ildir)/libblas.$$s $(ildir)/libblas.$$m \
+ && ln -fs $(ildir)/libf77blas.$$s $(ildir)/libf77blas.$$m \
+ && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$s \
+ && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$m; \
+ fi \
+ && make install
+
+ # We need to check the existance of `libptlapack.a', but we can't
+ # do this in the `&&' steps above (it will conflict). So we'll do
+ # the check after seeing if `libtatlas.so' is installed, then we'll
+ # finalize the build (delete the untarred directory).
+ if [ "x$(on_mac_os)" != xyes ]; then \
+ [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \
+ cd $(ddir); \
+ rm -rf ATLAS; \
+ fi
+
+ # We'll check the full installation with the static library (not
+ # currently building shared library on Mac.
+ if [ -f $(ildir)/libatlas.a ]; then echo "Atlas is built" > $@; fi
+
+$(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz
+ if [ x$(on_mac_os) = xyes ]; then \
+ export CC=clang; \
+ fi; \
+ cd $(ddir) \
+ && tar xf $< \
+ && cd OpenBLAS-$(openblas-version) \
+ && make \
+ && make PREFIX=$(idir) install \
+ && cd .. \
+ && rm -rf OpenBLAS-$(openblas-version) \
+ && echo "Libtiff is built" > $@
@@ -333,11 +473,6 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
make install && \
cd ..&& rm -rf cmake-$(cmake-version)
-# Some programs that depend on cURL (in particular CMake) don't necessarily
-# have easiy ways to explicity tell them to also link with libcurl's
-# dependencies (libssl, libcrypto, and libz). So we won't force curl to
-# only be static.
-#
# cURL (and its library, which is needed by several programs here) can
# optionally link with many different network-related libraries on the host
# system that we are not yet building in the pipeline. Many of these are
@@ -363,7 +498,7 @@ $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz
--without-axtls \
--disable-ldaps \
--disable-ldap \
- --without-nss )
+ --without-nss, V=1)
# On Mac OS, libtool does different things, so to avoid confusion, we'll
# prefix GNU's libtool executables with `glibtool'.
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index 545fcc1..66e7c77 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -369,10 +369,6 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
echo "\newcommand{\\grepversion}{$(grep-version)}" >> $@
echo "\newcommand{\\gzipversion}{$(gzip-version)}" >> $@
echo "\newcommand{\\islversion}{$(isl-version)}" >> $@
- echo "\newcommand{\\libbsdversion}{$(libbsd-version)}" >> $@
- echo "\newcommand{\\libffiversion}{$(libffi-version)}" >> $@
- echo "\newcommand{\\libpngversion}{$(libpng-version)}" >> $@
- echo "\newcommand{\\libtoolversion}{$(libtool-version)}" >> $@
echo "\newcommand{\\lzipversion}{$(lzip-version)}" >> $@
echo "\newcommand{\\makeversion}{$(make-version)}" >> $@
echo "\newcommand{\\metastoreversion}{$(metastore-version)}" >> $@
@@ -394,12 +390,18 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
echo "\newcommand{\\zipversion}{$(zip-version)}" >> $@
# Libraries.
+ echo "\newcommand{\\atlasversion}{$(cfitsio-version)}" >> $@
echo "\newcommand{\\cfitsioversion}{$(cfitsio-version)}" >> $@
echo "\newcommand{\\curlversion}{$(curl-version)}" >> $@
echo "\newcommand{\\gslversion}{$(gsl-version)}" >> $@
+ echo "\newcommand{\\lapack}{$(lapack-version)}" >> $@
+ echo "\newcommand{\\libbsdversion}{$(libbsd-version)}" >> $@
+ echo "\newcommand{\\libffiversion}{$(libffi-version)}" >> $@
echo "\newcommand{\\libgittwoversion}{$(libgit2-version)}" >> $@
echo "\newcommand{\\libjpegversion}{$(libjpeg-version)}" >> $@
+ echo "\newcommand{\\libpngversion}{$(libpng-version)}" >> $@
echo "\newcommand{\\libtiffversion}{$(libtiff-version)}" >> $@
+ echo "\newcommand{\\libtoolversion}{$(libtool-version)}" >> $@
echo "\newcommand{\\wcslibversion}{$(wcslib-version)}" >> $@
echo "\newcommand{\\zlibversion}{$(zlib-version)}" >> $@
@@ -414,8 +416,9 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
echo "\newcommand{\\cryptographyversion}{$(cryptography-version)}" >> $@
echo "\newcommand{\\cyclerversion}{$(cycler-version)}" >> $@
echo "\newcommand{\\entrypointsversion}{$(entrypoints-version)}" >> $@
+ echo "\newcommand{\\hpyversion}{$(h5py-version)}" >> $@
echo "\newcommand{\\htmlfivelibversion}{$(html5lib-version)}" >> $@
- echo "\newcommand{\\indaversion}{$(idna-version)}" >> $@
+ echo "\newcommand{\\idaversion}{$(idna-version)}" >> $@
echo "\newcommand{\\jeepneyversion}{$(jeepney-version)}" >> $@
echo "\newcommand{\\kiwisolverversion}{$(kiwisolver-version)}" >> $@
echo "\newcommand{\\keyringversion}{$(keyring-version)}" >> $@
@@ -433,7 +436,7 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
echo "\newcommand{\\sixversion}{$(six-version)}" >> $@
echo "\newcommand{\\soupsieveversion}{$(soupsieve-version)}" >> $@
echo "\newcommand{\\urllibthreeversion}{$(urllib3-version)}" >> $@
- echo "\newcommand{\\virtualenvversion}{$(virtualenv-version)}" >> $@
+# echo "\newcommand{\\virtualenvversion}{$(virtualenv-version)}" >> $@
echo "\newcommand{\\webencodingsversion}{$(webencodings-version)}" >> $@
# TeX package versions
generated by cgit v1.2.1 (git 2.18.0) at 2024-12-26 06:42:26 +0000