diff options
| -rw-r--r-- | .file-metadata | bin | 3956 -> 3927 bytes | |||
| -rw-r--r-- | paper.tex | 67 | ||||
| -rw-r--r-- | reproduce/config/pipeline/dependency-numpy-scipy.cfg | 47 | ||||
| -rw-r--r-- | reproduce/config/pipeline/dependency-versions.mk | 24 | ||||
| -rwxr-xr-x | reproduce/src/make/dependencies-atlas-multiple.mk | 47 | ||||
| -rwxr-xr-x | reproduce/src/make/dependencies-atlas-single.mk | 29 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies-basic.mk | 48 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies-python.mk | 117 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies.mk | 87 | ||||
| -rw-r--r-- | reproduce/src/make/initialize.mk | 15 |
10 files changed, 374 insertions, 107 deletions
diff --git a/.file-metadata b/.file-metadata Binary files differindex 0fec14c..5a4f942 100644 --- a/.file-metadata +++ b/.file-metadata @@ -202,36 +202,47 @@ SUNDIAL ITN, and from the Spanish Ministry of Economy and Competitiveness (MINECO) under grant number AYA2016-76219-P. The following free software tools were also critical component of this -research (in alphabetical order): Bzip2 \bziptwoversion, CFITSIO -\cfitsioversion, CMake \cmakeversion, cURL \curlversion, Discoteq flock -\flockversion, Git \gitversion, GNU Astronomy Utilities \gnuastroversion, -GNU AWK \gawkversion, GNU Bash \bashversion, GNU Coreutils -\coreutilsversion, GNU Diffutils \diffutilsversion, GNU Findutils -\findutilsversion, GNU Grep \grepversion, GNU Gzip \gzipversion, GNU -Libtool \libtoolversion, GNU Make \makeversion, GNU NCURSES -\ncursesversion, GNU Readline \readlineversion, GNU Sed \sedversion, GNU -Scientific Library (GSL) \gslversion, GNU Tar \tarversion, GNU Wget -\wgetversion, GNU Which \whichversion, Lzip \lzipversion, GPL Ghostscript -\ghostscriptversion, Libbsd \libbsdversion, Libgit2 \libgittwoversion, -Libjpeg \libjpegversion, Libtiff \libtiffversion, Metastore (forked) -\metastoreversion, OpenSSL \opensslversion, Pkg-config \pkgconfigversion, +esearch (in alphabetical order): ATLAS \atlasversion, Bzip2 +\bziptwoversion, CFITSIO \cfitsioversion, CMake \cmakeversion, cURL +\curlversion, Discoteq flock \flockversion, FreeType \freetypeversion, Git +\gitversion, GNU Astronomy Utilities \gnuastroversion, GNU AWK \gawkversion, +GNU Bash \bashversion, GNU Coreutils \coreutilsversion, GNU Diffutils +\diffutilsversion, GNU Findutils \findutilsversion, GNU Grep \grepversion, +GNU Gzip \gzipversion, GNU Libtool \libtoolversion, GNU Make \makeversion, +GNU NCURSES \ncursesversion, GNU Readline \readlineversion, GNU Sed +\sedversion, GNU Scientific Library (GSL) \gslversion, GNU Tar \tarversion, +GNU Wget \wgetversion, GNU Which \whichversion, Lapack \lapackversion, Lzip +\lzipversion, GPL Ghostscript \ghostscriptversion, Libbsd \libbsdversion, +Libgit2 \libgitwoversion, Libjpeg \libjpegversion, Libpng \libpngversion, +Libtiff \libtiffversion, Metastore (forked) \metastoreversion, OpenSSL +\opensslversion, Patchelf \patchelfversion, Pkg-config \pkgconfigversion, Unzip \unzipversion, WCSLIB \wcslibversion, XZ Utils \xzversion, Zip \zipversion, and ZLib \zlibversion. We use Python {\pythonversion} with the -following packages: Numpy {\numpyversion} and Astropy {\astropyversion} -\citep{astropy2013, astropy2018}. The final paper was produced with \TeX{} -Live \texliveversion, using the following packages: \TeX{} \textexversion, -EC \texecversion, NewTX \texnewtxversion, Fontaxes \texfontaxesversion, -Keyval, \texxkeyvalversion, Etoolbox \texetoolboxversion, Xcolor -\texxcolorversion, Setspace \texsetspaceversion, Caption -\texcaptionversion, Footmisc \texfootmiscversion, Datetime -\texdatetimeversion, Fmtcount \texfmtcountversion, Titlesec -\textitlesecversion, Preprint \texpreprintversion, Ulem \texulemversion, -Bib\LaTeX{} \texbiblatexversion, Biber \texbiberversion, Logreq -\texlogreqversion, PGF/TiKZ \texpgfversion, PGFPlots \texpgfplotsversion, -FP \texfpversion, Courier \texcourierversion, \TeX-gyre \textexgyreversion, -TXFonts \textxfontsversion, Times \textimesversion. We are very grateful to -all their creators for freely providing this necessary infrastructure. This -research would not be possible without them. +following packages: Asn1crypto \asncryptoversion, Astroquery +\astroqueryversion, Astropy {\astropyversion} \citep{astropy2013, +astropy2018}, BeautifulSoup \beautifulsoupversion, Certifi \certifiversion, +Cffi \cffiversion, Chardet \chardetversion, Cryptography +\cryptographyversion, Cycler \cyclerversion, EntryPoints +\entrypointsversion, h5py \hpyversion, html5ib \htmlfivelibversion, idna +\idnaversion, Jeepney \jeepneyversion, Kiwisolver \kiwisolverversion, +keyring \keyringversion, Matplotlib \matplotlibversion, Numpy \numpyversion + + + The final paper was produced with +\TeX{} Live \texliveversion, using the following packages: \TeX{} +\textexversion, EC \texecversion, NewTX \texnewtxversion, Fontaxes +\texfontaxesversion, Keyval, \texxkeyvalversion, Etoolbox +\texetoolboxversion, Xcolor \texxcolorversion, Setspace +\texsetspaceversion, Caption \texcaptionversion, Footmisc +\texfootmiscversion, Datetime \texdatetimeversion, Fmtcount +\texfmtcountversion, Titlesec \textitlesecversion, Preprint +\texpreprintversion, Ulem \texulemversion, Bib\LaTeX{} \texbiblatexversion, +Biber \texbiberversion, Logreq \texlogreqversion, PGF/TiKZ \texpgfversion, +PGFPlots \texpgfplotsversion, FP \texfpversion, Courier \texcourierversion, +\TeX-gyre \textexgyreversion, TXFonts \textxfontsversion, Times +\textimesversion. We are very grateful to all their creators for freely +providing this necessary infrastructure. This research would not be +possible without them. %% Tell BibLaTeX to put the bibliography list here. \printbibliography diff --git a/reproduce/config/pipeline/dependency-numpy-scipy.cfg b/reproduce/config/pipeline/dependency-numpy-scipy.cfg new file mode 100644 index 0000000..55e93bc --- /dev/null +++ b/reproduce/config/pipeline/dependency-numpy-scipy.cfg @@ -0,0 +1,47 @@ +# THIS IS A COPY OF NUMPY'S site.cfg.example, CUSTOMIZED FOR THIS PIPELINE +# ------------------------------------------------------------------------ + +# This file provides configuration information about non-Python dependencies for +# numpy.distutils-using packages. Create a file like this called "site.cfg" next +# to your package's setup.py file and fill in the appropriate sections. Not all +# packages will use all sections so you should leave out sections that your +# package does not use. + +# IMPORTANT NOTE +# -------------- +# +# The `ALL' grouping does't apply to ATLAS!!!!! + + [ALL] + library_dirs = @LIBDIR@ + include_dirs = @INCDIR@ + +# Atlas +# ----- +# Atlas is an open source optimized implementation of the BLAS and Lapack +# routines. NumPy will try to build against Atlas by default when available in +# the system library dirs. To build numpy against a custom installation of +# Atlas you can add an explicit section such as the following. Here we assume +# that Atlas was configured with ``prefix=/opt/atlas``. +# + [atlas] + library_dirs = @LIBDIR@ + include_dirs = @INCDIR@ + +# FFT libraries +# ------------- +# There are two FFT libraries that we can configure here: FFTW (2 and 3) and djbfft. +# Note that these libraries are not used by numpy or scipy. +# +# http://fftw.org/ +# https://cr.yp.to/djbfft.html +# +# Given only this section, numpy.distutils will try to figure out which version +# of FFTW you are using. +#[fftw] +#libraries = fftw3 +# +# For djbfft, numpy.distutils will look for either djbfft.a or libdjbfft.a . +#[djbfft] +#include_dirs = /usr/local/djbfft/include +#library_dirs = /usr/local/djbfft/lib diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 7cdddac..211ebbd 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -42,6 +42,7 @@ xz-version = 5.2.4 zip-version = 3.0 # Libraries +atlas-version = 3.10.3 cfitsio-version = 3.45 curl-version = 7.63.0 gsl-version = 2.5 @@ -49,6 +50,17 @@ libjpeg-version = v9b libtiff-version = 4.0.10 zlib-version = 1.2.11 +# Special libraries +# ----------------- +# +# When updating the version of these libraries, please look into the build +# rule first: In one way or another, the version string becomes necessary +# during their build and must be accounted for. +bzip2-version = 1.0.6 +lapack-version = 3.8.0 +libgit2-version = 0.26.0 +wcslib-version = 6.2 + # Python packages # --------------- # @@ -64,13 +76,14 @@ chardet-version = 3.0.4 cryptography-version = 2.6.1 cycler-version = 0.10.0 entrypoints-version = 0.3 +h5py-version = 2.9.0 html5lib-version = 1.0.1 idna-version = 2.8 jeepney-version = 0.4 kiwisolver-version = 1.0.1 keyring-version = 18.0.0 matplotlib-version = 3.0.2 -numpy-version = 1.16.1 +numpy-version = 1.16.2 pip-version = 19.0.2 pycparser-version = 2.19 pyparsing-version = 2.3.1 @@ -86,12 +99,3 @@ urllib3-version = 1.24.1 virtualenv-version = 16.4.0 webencodings-version = 0.5.1 -# Special libraries -# ----------------- -# -# When updating the version of these libraries, please look into the build -# rule first: In one way or another, the version string becomes necessary -# during their build and must be accounted for. -bzip2-version = 1.0.6 -libgit2-version = 0.26.0 -wcslib-version = 6.2 diff --git a/reproduce/src/make/dependencies-atlas-multiple.mk b/reproduce/src/make/dependencies-atlas-multiple.mk new file mode 100755 index 0000000..d66890f --- /dev/null +++ b/reproduce/src/make/dependencies-atlas-multiple.mk @@ -0,0 +1,47 @@ +ORIGLDFLAGS := $(LDFLAGS) + +include Make.inc + +all: libatlas.so libf77blas.so libptf77blas.so libstcblas.so libptcblas.so \ + libblas.so libcblas.so liblapack.so.3.6.1 libptlapack.so.3.6.1 liblapack.so.3 + +libatlas.so: libatlas.a + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \ + --whole-archive libatlas.a --no-whole-archive -lc $(LIBS) + +libf77blas.so : libf77blas.a libatlas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libf77blas.so.3 \ + -o $@ --whole-archive libf77blas.a --no-whole-archive \ + $(F77SYSLIB) -L. -latlas + +libptf77blas.so : libptf77blas.a libatlas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libblas.so.3 \ + -o $@ --whole-archive libptf77blas.a --no-whole-archive \ + $(F77SYSLIB) -L. -latlas + +libstcblas.so : libcblas.a libatlas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libstcblas.so \ + -o $@ --whole-archive libcblas.a -L. -latlas -lblas + +libptcblas.so : libptcblas.a libatlas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libcblas.so \ + -o $@ --whole-archive libptcblas.a -L. -latlas -lblas + +libblas.so: libptf77blas.so + ln -s $< $@ + +libcblas.so: libptcblas.so + ln -s $< $@ + +liblapack.so.3.6.1 : liblapack.a libstcblas.so libf77blas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libstlapack.so.3 \ + -o $@ --whole-archive liblapack.a --no-whole-archive \ + $(F77SYSLIB) -L. -lstcblas -lf77blas + +libptlapack.so.3.6.1 : libptlapack.a libcblas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname liblapack.so.3 \ + -o $@ --whole-archive libptlapack.a --no-whole-archive \ + $(F77SYSLIB) -L. -lcblas -lblas + +liblapack.so.3: libptlapack.so.3.6.1 + ln -s $< $@ diff --git a/reproduce/src/make/dependencies-atlas-single.mk b/reproduce/src/make/dependencies-atlas-single.mk new file mode 100755 index 0000000..12dd19b --- /dev/null +++ b/reproduce/src/make/dependencies-atlas-single.mk @@ -0,0 +1,29 @@ +ORIGLDFLAGS := $(LDFLAGS) + +include Make.inc + +all: libatlas.so libf77blas.so libcblas.so libblas.so liblapack.so.3.6.1 + +libatlas.so: libatlas.a + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \ + --whole-archive libatlas.a --no-whole-archive -lc $(LIBS) + +libf77blas.so : libf77blas.a libatlas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libblas.so.3 \ + -o $@ --whole-archive libf77blas.a --no-whole-archive \ + $(F77SYSLIB) -L. -latlas + +libcblas.so : libcblas.a libatlas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \ + --whole-archive libcblas.a -L. -latlas -lblas + +libblas.so: libf77blas.so + ln -s $< $@ + +liblapack.so.3.6.1 : liblapack.a libcblas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname liblapack.so.3 \ + -o $@ --whole-archive liblapack.a --no-whole-archive \ + $(F77SYSLIB) -L. -lcblas -lblas + +liblapack.so.3: liblapack.so.3.6.1 + ln -s $< $@ diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 2e6caf4..4d39141 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -574,8 +574,15 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ --with-zlib-lib=$(ildir) \ --with-zlib-include=$(idir)/include, , , \ ./config ) && \ - cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem && \ - echo "OpenSSL is built and ready" > $@ + cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem; \ + if [ $$? = 0 ]; then \ + if [ x$(on_mac_os) = xyes ]; then \ + echo "No need to fix rpath in libssl"; \ + else \ + patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \ + fi; \ + echo "OpenSSL is built and ready" > $@; \ + fi # GNU Wget # @@ -750,8 +757,14 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ $(ibdir)/which # On a macOS, we (currently!) won't build GCC because of some - # errors we are still trying to fix. So, we'll just make a symbolic - # link to the host's executables. + # errors we are still trying to find. So, we'll just make a + # symbolic link to the host's executables. + # + # GCC builds is own libraries in '$(idir)/lib64'. But all other + # libraries are in '$(idir)/lib'. Since this pipeline is only for a + # single architecture, we can trick GCC into building its libraries + # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic + # link to '$(idir)/lib'. if [ "x$(on_mac_os)" = xyes ]; then \ $(call makelink,gfortran); \ $(call makelink,g++); \ @@ -762,12 +775,14 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \ rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\ \ + ln -fs $(ildir) $(idir)/lib64; \ + \ cd $(ddir); \ rm -rf gcc-build gcc-$(gcc-version); \ - tar xf $< && \ - mkdir $(ddir)/gcc-build && \ - cd $(ddir)/gcc-build && \ - ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ + tar xf $< \ + && mkdir $(ddir)/gcc-build \ + && cd $(ddir)/gcc-build \ + && ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ --prefix=$(idir) \ --with-mpc=$(idir) \ --with-mpfr=$(idir) \ @@ -785,9 +800,16 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ --enable-default-pie \ --enable-default-ssp \ --enable-cet=auto \ - --enable-decimal-float && \ - make SHELL=$(ibdir)/bash -j$$(nproc) && \ - make SHELL=$(ibdir)/bash install && \ - cd .. && \ - rm -rf gcc-build gcc-$(gcc-version); \ + --enable-decimal-float \ + && make SHELL=$(ibdir)/bash -j$$(nproc) \ + && make SHELL=$(ibdir)/bash install \ + && cd .. \ + && rm -rf gcc-build gcc-$(gcc-version) \ + \ + && for f in $$(find $(idir)/libexec/gcc); do \ + if ldd $$f &> /dev/null; then \ + patchelf --set-rpath $(ildir) $$f; \ + fi; \ + done; \ fi + diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 946c047..54d3a56 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -43,7 +43,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built ipydir = $(BDIR)/dependencies/installed/lib/built/python # Define the top-level programs to build (installed in `.local/bin'). -top-level-python = astroquery matplotlib #scipy +top-level-python = astroquery matplotlib scipy all: $(foreach p, $(top-level-python), $(ipydir)/$(p)) # Other basic environment settings: We are only including the host @@ -116,6 +116,7 @@ tarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ cryptography-$(cryptography-version).tar.gz \ cycler-$(cycler-version).tar.gz \ entrypoints-$(entrypoints-version).tar.gz \ + h5py-$(h5py-version).tar.gz \ html5lib-$(html5lib-version).tar.gz \ idna-$(idna-version).tar.gz \ jeepney-$(jeepney-version).tar.gz \ @@ -157,7 +158,10 @@ $(tarballs): $(tdir)/%: # because the tokenization above will produce `python' as the # first string. if [ $* = python-dateutil-$(python-dateutil-version).tar.gz ]; then - n=dateutil + n=dateutil + elif [ $* = h5py-$(h5py-version).tar.gz ]; then + n=h5py + # elif [ $* = strange-tarball5name-version.tar.gz ]; then # n=strange5-name else @@ -191,13 +195,14 @@ $(tarballs): $(tdir)/%: elif [ $$n = cryptography ]; then h=07/ca/bc827c5e55918ad223d59d299fff92f3563476c3b00d0a9157d9c0217449 elif [ $$n = cycler ]; then h=c2/4b/137dea450d6e1e3d474e1d873cd1d4f7d3beed7e0dc973b06e8e10d32488 elif [ $$n = entrypoints ]; then h=b4/ef/063484f1f9ba3081e920ec9972c96664e2edb9fdc3d8669b0e3b8fc0ad7c + elif [ $$n = h5py ]; then h=43/27/a6e7dcb8ae20a4dbf3725321058923fec262b6f7835179d78ccc8d98deec elif [ $$n = html ]; then h=85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f elif [ $$n = idna ]; then h=ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7 elif [ $$n = jeepney ]; then h=16/1d/74adf3b164a8d19a60d0fcf706a751ffa2a1eaa8e5bbb1b6705c92a05263 elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2 elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864 elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f - elif [ $$n = numpy ]; then h=2b/26/07472b0de91851b6656cbc86e2f0d5d3a3128e7580f23295ef58b6862d6c + elif [ $$n = numpy ]; then h=cf/8d/6345b4f32b37945fedc1e027e83970005fc9c699068d2f566b82826515f2 elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af elif [ $$n = pycparser ]; then h=68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a elif [ $$n = pyparsing ]; then h=b9/b8/6b32b3e84014148dcd60dd05795e35c2e7f4b72f918616c61fdce83d27fc @@ -212,7 +217,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = urllib ]; then h=b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1 elif [ $$n = virtualenv ]; then h=51/aa/c395a6e6eaaedfa5a04723b6446a1df783b16cca6fec66e671cede514688 elif [ $$n = webencodings ]; then h=0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47 -# elif [ $$n = strange5-name ]; then h=XXXXX +# elif [ $$n = strange5-name ]; then h=XXXXX else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." @@ -249,18 +254,24 @@ $(tarballs): $(tdir)/%: # -------------------- # # To build Python packages with direct access to a `setup.py' (if no direct -# access to `setup.py' is needed, pip can be used) +# access to `setup.py' is needed, pip can be used). # Arguments of this function are the numbers # 1) Unpack command # 2) Package name # 3) Unpacked directory name after unpacking the tarball -pybuild = cd $(ddir); rm -rf $(3); \ - if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ - cd $(3); \ - python3 setup.py build && \ - python3 setup.py install && \ - cd .. && rm -rf $(3) && \ - echo "done!" > $@ +# 4) site.cfg file (optional) +pybuild = cd $(ddir); rm -rf $(3); \ + if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ + cd $(3); \ + if [ "x$(4)" != x ]; then \ + sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ + -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ + $(4) > site.cfg; \ + fi; \ + python3 setup.py build \ + && python3 setup.py install \ + && cd .. && rm -rf $(3) \ + && echo "done!" > $@ @@ -271,7 +282,12 @@ pybuild = cd $(ddir); rm -rf $(3); \ # # While this Makefile is for Python programs, in some cases, we need # certain programs (like Python itself), or libraries for the modules. -$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz +$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz + $(call gbuild, $<, libffi-$(libffi-version)) \ + echo "libffi is built" > $@ + +$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \ + $(ilidir)/libffi # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. # if [ x$(on_mac_os) = xyes ]; then \ @@ -279,16 +295,18 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz # export CXX=clang++; \ # fi; \ - $(call gbuild, $<, Python-$(python-version)) \ + $(call gbuild, $<, Python-$(python-version),, \ + --enable-optimizations \ + --without-ensurepip \ + --with-system-ffi \ + --enable-shared \ + --with-threads \ + --with-lto ) \ && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') \ && ln -s $(ildir)/python$$v $(ildir)/python \ && rm -rf $(ipydir) && mkdir $(ipydir) -$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz - $(call gbuild, $<, libffi-$(libffi-version)) \ - echo "libffi is built" > $@ - @@ -297,7 +315,7 @@ $(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz # # All the necessary Python modules go here. $(ipydir)/asn1crypto: $(tdir)/asn1crypto-$(asn1crypto-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, asn1crypto-$(asn1crypto-version)) $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ @@ -310,6 +328,7 @@ $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ $(call pybuild, tar xf, $<, astroquery-$(astroquery-version)) $(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ + $(ipydir)/h5py \ $(ipydir)/numpy $(call pybuild, tar xf, $<, astropy-$(astropy-version)) @@ -318,81 +337,83 @@ $(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.g $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version)) $(ipydir)/certifi: $(tdir)/certifi-$(certifi-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, certifi-$(certifi-version)) $(ipydir)/cffi: $(tdir)/cffi-$(cffi-version).tar.gz \ - $(ipydir)/pycparser \ - $(ilidir)/libffi + $(ilidir)/libffi \ + $(ipydir)/pycparser $(call pybuild, tar xf, $<, cffi-$(cffi-version)) $(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, chardet-$(chardet-version)) $(ipydir)/cryptography: $(tdir)/cryptography-$(cryptography-version).tar.gz \ - $(ipydir)/cffi \ - $(ipydir)/asn1crypto + $(ipydir)/asn1crypto \ + $(ipydir)/cffi $(call pybuild, tar xf, $<, cryptography-$(cryptography-version)) $(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \ - $(ipydir)/six + $(ipydir)/six $(call pybuild, tar xf, $<, cycler-$(cycler-version)) $(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version)) +$(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, h5py-$(h5py-version)) + $(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \ $(ipydir)/six \ $(ipydir)/webencodings $(call pybuild, tar xf, $<, html5lib-$(html5lib-version)) $(ipydir)/idna: $(tdir)/idna-$(idna-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, idna-$(idna-version)) $(ipydir)/jeepney: $(tdir)/jeepney-$(jeepney-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, jeepney-$(jeepney-version)) $(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \ - $(ipydir)/setuptools_scm \ + $(ipydir)/entrypoints \ $(ipydir)/secretstorage \ - $(ipydir)/entrypoints + $(ipydir)/setuptools_scm $(call pybuild, tar xf, $<, keyring-$(keyring-version)) $(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ $(ipydir)/setuptools $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version)) - $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/cycler \ $(ilidir)/freetype \ + $(ipydir)/kiwisolver \ $(ipydir)/numpy \ $(ipydir)/pyparsing \ - $(ipydir)/python-dateutil \ - $(ipydir)/kiwisolver + $(ipydir)/python-dateutil $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ - $(ibdir)/python3 - export BLAS=None; \ - export ATLAS=None; \ - export LAPACK=None; \ - $(call pybuild, unzip, $<, numpy-$(numpy-version)) + $(ipydir)/setuptools + export LDFLAGS="$$LDFLAGS -shared"; \ + conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf) $(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, pip-$(pip-version)) $(ipydir)/pycparser: $(tdir)/pycparser-$(pycparser-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, pycparser-$(pycparser-version)) $(ipydir)/pyparsing: $(tdir)/pyparsing-$(pyparsing-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version)) $(ipydir)/python-dateutil: $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz \ @@ -410,7 +431,9 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ $(ipydir)/numpy - $(call pybuild, tar xf, $<, scipy-$(scipy-version)) + export LDFLAGS="$$LDFLAGS -shared"; \ + conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf) $(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \ $(ipydir)/cryptography \ @@ -426,17 +449,17 @@ $(ipydir)/setuptools_scm: $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.g $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version)) $(ipydir)/six: $(tdir)/six-$(six-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, six-$(six-version)) $(ipydir)/soupsieve: $(tdir)/soupsieve-$(soupsieve-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version)) $(ipydir)/urllib3: $(tdir)/urllib3-$(urllib3-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, urllib3-$(urllib3-version)) $(ipydir)/webencodings: $(tdir)/webencodings-$(webencodings-version).tar.gz \ - $(ibdir)/python3 + $(ipydir)/setuptools $(call pybuild, tar xf, $<, webencodings-$(webencodings-version)) diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index deb9f39..f89c176 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -44,7 +44,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). top-level-programs = astnoisechisel flock metastore unzip zip -top-level-libraries = freetype +top-level-libraries = atlas freetype all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) @@ -67,12 +67,12 @@ all: $(ddir)/texlive-versions.tex \ .SHELLFLAGS := --noprofile --norc -ec export CCACHE_DISABLE := 1 export PATH := $(ibdir) -export LD_RUN_PATH := $(ildir) -export LD_LIBRARY_PATH := $(ildir) export SHELL := $(ibdir)/bash export CPPFLAGS := -I$(idir)/include export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +export LD_RUN_PATH := $(ildir):$(il64dir) +export LD_LIBRARY_PATH := $(ildir):$(il64dir) export LDFLAGS := $(rpath_command) -L$(ildir) @@ -90,6 +90,7 @@ export LDFLAGS := $(rpath_command) -L$(ildir) # another format, we'll do the modification before the download so the # downloaded file has our desired format. tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ + atlas-$(atlas-version).tar.bz2 \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ flock-$(flock-version).tar.xz \ @@ -100,6 +101,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ gsl-$(gsl-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ + lapack-$(lapack-version).tar.gz \ libbsd-$(libbsd-version).tar.xz \ libpng-$(libpng-version).tar.xz \ libtool-$(libtool-version).tar.xz \ @@ -130,6 +132,9 @@ $(tarballs): $(tdir)/%: : (l==2 ? "%d00\n" \ : "%d000\n") ), $$1)}') w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz + elif [ $$n = atlas ]; then + mergenames=0 + w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) @@ -140,6 +145,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = gsl ]; then w=http://ftpmirror.gnu.org/gnu/gsl elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files + elif [ $$n = lapack ]; then w=http://www.netlib.org/lapack elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool @@ -251,6 +257,81 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ --disable-webp --disable-zstd) \ && echo "Libtiff is built" > $@ +$(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ + $(tdir)/lapack-$(lapack-version).tar.gz + + + # Get the operating system specific features (how to get + # CPU frequency and the library suffixes). To make the steps + # more readable, the different library version suffixes are + # named with a single character: `s' for no version in the + # name, `m' for the major version suffix, and `f' for the + # full version suffix. + if [ x$(on_mac_os) = xyes ]; then + s=dylib + m=3.dylib + f=3.6.1.dylib + core=$$(sysctl hw.cpufrequency | awk '{print $$2/1000000}') + else + s=so + m=so.3 + f=so.3.6.1 + core=$$(cat /proc/cpuinfo | grep "cpu MHz" \ + | head -n 1 \ + | sed "s/.*: \([0-9.]*\).*/\1/") + fi + + # See if the shared libraries should be build for a single CPU + # thread or multiple threads. + N=$$(nproc) + srcdir=$$(pwd)/reproduce/src/make + if [ $$N = 1 ]; then + sharedmk=$$srcdir/dependencies-atlas-single.mk + else + sharedmk=$$srcdir/dependencies-atlas-multiple.mk + fi + + # The linking step here doesn't recognize the `-Wl' in the + # `rpath_command'. + export LDFLAGS=-L$(ildir) + + cd $(ddir) \ + && tar xf $< \ + && cd ATLAS \ + && rm -rf build \ + && mkdir build \ + && cd build \ + && ../configure -b 64 -D c -DPentiumCPS=$$core \ + --with-netlib-lapack-tarfile=$(word 2, $^) \ + --cripple-atlas-performance \ + -Fa alg -fPIC --shared \ + --prefix=$(idir) \ + && make \ + && cd lib && make -f $$sharedmk && cd .. \ + && if [ "x$(on_mac_os)" != xyes ]; then \ + for l in lib/*.$$s*; do \ + patchelf --set-rpath $(ildir) $$l; done \ + fi \ + && make install \ + && cp -d lib/*.$$s* $(ildir) \ + && ln -fs $(ildir)/libblas.$$s $(ildir)/libblas.$$m \ + && ln -fs $(ildir)/libf77blas.$$s $(ildir)/libf77blas.$$m \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$s \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$m; + + # We need to check the existance of `libptlapack.a', but we can't + # do this in the `&&' steps above (it will conflict). So we'll do + # the check after seeing if `libtatlas.so' is installed, then we'll + # finalize the build (delete the untarred directory). + if [ -e $(ildir)/libtatlas.$$s ]; then \ + [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \ + cd $(ddir); \ + rm -rf ATLAS; \ + echo "Atlas is built" > $@; \ + else \ + echo; echo "ATLAS wasn't installed!!!!"; exit 1; \ + fi + diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 545fcc1..66e7c77 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -369,10 +369,6 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\\grepversion}{$(grep-version)}" >> $@ echo "\newcommand{\\gzipversion}{$(gzip-version)}" >> $@ echo "\newcommand{\\islversion}{$(isl-version)}" >> $@ - echo "\newcommand{\\libbsdversion}{$(libbsd-version)}" >> $@ - echo "\newcommand{\\libffiversion}{$(libffi-version)}" >> $@ - echo "\newcommand{\\libpngversion}{$(libpng-version)}" >> $@ - echo "\newcommand{\\libtoolversion}{$(libtool-version)}" >> $@ echo "\newcommand{\\lzipversion}{$(lzip-version)}" >> $@ echo "\newcommand{\\makeversion}{$(make-version)}" >> $@ echo "\newcommand{\\metastoreversion}{$(metastore-version)}" >> $@ @@ -394,12 +390,18 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\\zipversion}{$(zip-version)}" >> $@ # Libraries. + echo "\newcommand{\\atlasversion}{$(cfitsio-version)}" >> $@ echo "\newcommand{\\cfitsioversion}{$(cfitsio-version)}" >> $@ echo "\newcommand{\\curlversion}{$(curl-version)}" >> $@ echo "\newcommand{\\gslversion}{$(gsl-version)}" >> $@ + echo "\newcommand{\\lapack}{$(lapack-version)}" >> $@ + echo "\newcommand{\\libbsdversion}{$(libbsd-version)}" >> $@ + echo "\newcommand{\\libffiversion}{$(libffi-version)}" >> $@ echo "\newcommand{\\libgittwoversion}{$(libgit2-version)}" >> $@ echo "\newcommand{\\libjpegversion}{$(libjpeg-version)}" >> $@ + echo "\newcommand{\\libpngversion}{$(libpng-version)}" >> $@ echo "\newcommand{\\libtiffversion}{$(libtiff-version)}" >> $@ + echo "\newcommand{\\libtoolversion}{$(libtool-version)}" >> $@ echo "\newcommand{\\wcslibversion}{$(wcslib-version)}" >> $@ echo "\newcommand{\\zlibversion}{$(zlib-version)}" >> $@ @@ -414,8 +416,9 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\\cryptographyversion}{$(cryptography-version)}" >> $@ echo "\newcommand{\\cyclerversion}{$(cycler-version)}" >> $@ echo "\newcommand{\\entrypointsversion}{$(entrypoints-version)}" >> $@ + echo "\newcommand{\\hpyversion}{$(h5py-version)}" >> $@ echo "\newcommand{\\htmlfivelibversion}{$(html5lib-version)}" >> $@ - echo "\newcommand{\\indaversion}{$(idna-version)}" >> $@ + echo "\newcommand{\\idaversion}{$(idna-version)}" >> $@ echo "\newcommand{\\jeepneyversion}{$(jeepney-version)}" >> $@ echo "\newcommand{\\kiwisolverversion}{$(kiwisolver-version)}" >> $@ echo "\newcommand{\\keyringversion}{$(keyring-version)}" >> $@ @@ -433,7 +436,7 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\\sixversion}{$(six-version)}" >> $@ echo "\newcommand{\\soupsieveversion}{$(soupsieve-version)}" >> $@ echo "\newcommand{\\urllibthreeversion}{$(urllib3-version)}" >> $@ - echo "\newcommand{\\virtualenvversion}{$(virtualenv-version)}" >> $@ +# echo "\newcommand{\\virtualenvversion}{$(virtualenv-version)}" >> $@ echo "\newcommand{\\webencodingsversion}{$(webencodings-version)}" >> $@ # TeX package versions |