aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRaul Infante-Sainz <infantesainz@gmail.com>2019-03-20 17:41:50 +0000
committerRaul Infante-Sainz <infantesainz@gmail.com>2019-03-20 17:41:50 +0000
commit75571a551f794223bea2995dc7775a49abd63654 (patch)
tree6cbba87197329edf504d08a7c48cd05301c516fb
parent5c51f92f2535a2153ae46fb6311ffd9a15c244b3 (diff)
Including ATLAS in the pipeline, not yet complete
An initial installation of atlas is now included in the pipeline, but we are still trying to make it compile and build smoothly. In the process, we found that GCC also needs some modifications (for example rpath issues).
-rw-r--r--.file-metadatabin3956 -> 4012 bytes
-rw-r--r--reproduce/config/pipeline/dependency-versions.mk21
-rwxr-xr-xreproduce/src/make/dependencies-atlas-multiple.mk47
-rwxr-xr-xreproduce/src/make/dependencies-atlas-single.mk29
-rw-r--r--reproduce/src/make/dependencies-basic.mk33
-rw-r--r--reproduce/src/make/dependencies-python.mk3
-rw-r--r--reproduce/src/make/dependencies.mk70
7 files changed, 179 insertions, 24 deletions
diff --git a/.file-metadata b/.file-metadata
index c44891d..5a39552 100644
--- a/.file-metadata
+++ b/.file-metadata
Binary files differ
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index 7cdddac..609d1cb 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -42,6 +42,7 @@ xz-version = 5.2.4
zip-version = 3.0
# Libraries
+atlas-version = 3.10.3
cfitsio-version = 3.45
curl-version = 7.63.0
gsl-version = 2.5
@@ -49,6 +50,17 @@ libjpeg-version = v9b
libtiff-version = 4.0.10
zlib-version = 1.2.11
+# Special libraries
+# -----------------
+#
+# When updating the version of these libraries, please look into the build
+# rule first: In one way or another, the version string becomes necessary
+# during their build and must be accounted for.
+bzip2-version = 1.0.6
+lapack-version = 3.8.0
+libgit2-version = 0.26.0
+wcslib-version = 6.2
+
# Python packages
# ---------------
#
@@ -86,12 +98,3 @@ urllib3-version = 1.24.1
virtualenv-version = 16.4.0
webencodings-version = 0.5.1
-# Special libraries
-# -----------------
-#
-# When updating the version of these libraries, please look into the build
-# rule first: In one way or another, the version string becomes necessary
-# during their build and must be accounted for.
-bzip2-version = 1.0.6
-libgit2-version = 0.26.0
-wcslib-version = 6.2
diff --git a/reproduce/src/make/dependencies-atlas-multiple.mk b/reproduce/src/make/dependencies-atlas-multiple.mk
new file mode 100755
index 0000000..d66890f
--- /dev/null
+++ b/reproduce/src/make/dependencies-atlas-multiple.mk
@@ -0,0 +1,47 @@
+ORIGLDFLAGS := $(LDFLAGS)
+
+include Make.inc
+
+all: libatlas.so libf77blas.so libptf77blas.so libstcblas.so libptcblas.so \
+ libblas.so libcblas.so liblapack.so.3.6.1 libptlapack.so.3.6.1 liblapack.so.3
+
+libatlas.so: libatlas.a
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \
+ --whole-archive libatlas.a --no-whole-archive -lc $(LIBS)
+
+libf77blas.so : libf77blas.a libatlas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libf77blas.so.3 \
+ -o $@ --whole-archive libf77blas.a --no-whole-archive \
+ $(F77SYSLIB) -L. -latlas
+
+libptf77blas.so : libptf77blas.a libatlas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libblas.so.3 \
+ -o $@ --whole-archive libptf77blas.a --no-whole-archive \
+ $(F77SYSLIB) -L. -latlas
+
+libstcblas.so : libcblas.a libatlas.so libblas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libstcblas.so \
+ -o $@ --whole-archive libcblas.a -L. -latlas -lblas
+
+libptcblas.so : libptcblas.a libatlas.so libblas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libcblas.so \
+ -o $@ --whole-archive libptcblas.a -L. -latlas -lblas
+
+libblas.so: libptf77blas.so
+ ln -s $< $@
+
+libcblas.so: libptcblas.so
+ ln -s $< $@
+
+liblapack.so.3.6.1 : liblapack.a libstcblas.so libf77blas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libstlapack.so.3 \
+ -o $@ --whole-archive liblapack.a --no-whole-archive \
+ $(F77SYSLIB) -L. -lstcblas -lf77blas
+
+libptlapack.so.3.6.1 : libptlapack.a libcblas.so libblas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname liblapack.so.3 \
+ -o $@ --whole-archive libptlapack.a --no-whole-archive \
+ $(F77SYSLIB) -L. -lcblas -lblas
+
+liblapack.so.3: libptlapack.so.3.6.1
+ ln -s $< $@
diff --git a/reproduce/src/make/dependencies-atlas-single.mk b/reproduce/src/make/dependencies-atlas-single.mk
new file mode 100755
index 0000000..12dd19b
--- /dev/null
+++ b/reproduce/src/make/dependencies-atlas-single.mk
@@ -0,0 +1,29 @@
+ORIGLDFLAGS := $(LDFLAGS)
+
+include Make.inc
+
+all: libatlas.so libf77blas.so libcblas.so libblas.so liblapack.so.3.6.1
+
+libatlas.so: libatlas.a
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \
+ --whole-archive libatlas.a --no-whole-archive -lc $(LIBS)
+
+libf77blas.so : libf77blas.a libatlas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libblas.so.3 \
+ -o $@ --whole-archive libf77blas.a --no-whole-archive \
+ $(F77SYSLIB) -L. -latlas
+
+libcblas.so : libcblas.a libatlas.so libblas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \
+ --whole-archive libcblas.a -L. -latlas -lblas
+
+libblas.so: libf77blas.so
+ ln -s $< $@
+
+liblapack.so.3.6.1 : liblapack.a libcblas.so libblas.so
+ ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname liblapack.so.3 \
+ -o $@ --whole-archive liblapack.a --no-whole-archive \
+ $(F77SYSLIB) -L. -lcblas -lblas
+
+liblapack.so.3: liblapack.so.3.6.1
+ ln -s $< $@
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index 5965f00..a5cda52 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -744,6 +744,12 @@ $(ibdir)/gcc: $(gcc-prerequisites) \
# On a macOS, we (currently!) won't build GCC because of some
# errors we are still trying to find. So, we'll just make a
# symbolic link to the host's executables.
+ #
+ # GCC builds is own libraries in '$(idir)/lib64'. But all other
+ # libraries are in '$(idir)/lib'. Since this pipeline is only for a
+ # single architecture, we can trick GCC into building its libraries
+ # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic
+ # link to '$(idir)/lib'.
if [ "x$(on_mac_os)" = xyes ]; then \
$(call makelink,gfortran); \
$(call makelink,gcc); \
@@ -754,12 +760,14 @@ $(ibdir)/gcc: $(gcc-prerequisites) \
rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \
rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\
\
+ ln -fs $(ildir) $(idir)/lib64; \
+ \
cd $(ddir); \
rm -rf gcc-build gcc-$(gcc-version); \
- tar xf $< && \
- mkdir $(ddir)/gcc-build && \
- cd $(ddir)/gcc-build && \
- ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \
+ tar xf $< \
+ && mkdir $(ddir)/gcc-build \
+ && cd $(ddir)/gcc-build \
+ && ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \
--prefix=$(idir) \
--with-mpc=$(idir) \
--with-mpfr=$(idir) \
@@ -777,9 +785,16 @@ $(ibdir)/gcc: $(gcc-prerequisites) \
--enable-default-pie \
--enable-default-ssp \
--enable-cet=auto \
- --enable-decimal-float && \
- make SHELL=$(ibdir)/bash -j$$(nproc) && \
- make SHELL=$(ibdir)/bash install && \
- cd .. && \
- rm -rf gcc-build gcc-$(gcc-version); \
+ --enable-decimal-float \
+ && make SHELL=$(ibdir)/bash -j$$(nproc) \
+ && make SHELL=$(ibdir)/bash install \
+ && cd .. \
+ && rm -rf gcc-build gcc-$(gcc-version) \
+ \
+ && for f in $$(find $(idir)/libexec/gcc); do \
+ if ldd $$f &> /dev/null; then \
+ patchelf --set-rpath $(ildir) $$f; \
+ fi; \
+ done; \
fi
+
diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk
index 946c047..3b92f7e 100644
--- a/reproduce/src/make/dependencies-python.mk
+++ b/reproduce/src/make/dependencies-python.mk
@@ -378,9 +378,6 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \
$(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
$(ibdir)/python3
- export BLAS=None; \
- export ATLAS=None; \
- export LAPACK=None; \
$(call pybuild, unzip, $<, numpy-$(numpy-version))
$(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index deb9f39..7b34b31 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -44,7 +44,7 @@ ilidir = $(BDIR)/dependencies/installed/lib/built
# Define the top-level programs to build (installed in `.local/bin').
top-level-programs = astnoisechisel flock metastore unzip zip
-top-level-libraries = freetype
+top-level-libraries = atlas freetype
all: $(ddir)/texlive-versions.tex \
$(foreach p, $(top-level-programs), $(ibdir)/$(p)) \
$(foreach p, $(top-level-libraries), $(ilidir)/$(p))
@@ -67,12 +67,12 @@ all: $(ddir)/texlive-versions.tex \
.SHELLFLAGS := --noprofile --norc -ec
export CCACHE_DISABLE := 1
export PATH := $(ibdir)
-export LD_RUN_PATH := $(ildir)
-export LD_LIBRARY_PATH := $(ildir)
export SHELL := $(ibdir)/bash
export CPPFLAGS := -I$(idir)/include
export PKG_CONFIG_PATH := $(ildir)/pkgconfig
export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig
+export LD_RUN_PATH := $(ildir):$(il64dir)
+export LD_LIBRARY_PATH := $(ildir):$(il64dir)
export LDFLAGS := $(rpath_command) -L$(ildir)
@@ -90,6 +90,7 @@ export LDFLAGS := $(rpath_command) -L$(ildir)
# another format, we'll do the modification before the download so the
# downloaded file has our desired format.
tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
+ atlas-$(atlas-version).tar.bz2 \
cmake-$(cmake-version).tar.gz \
curl-$(curl-version).tar.gz \
flock-$(flock-version).tar.xz \
@@ -100,6 +101,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
gsl-$(gsl-version).tar.gz \
install-tl-unx.tar.gz \
jpegsrc.$(libjpeg-version).tar.gz \
+ lapack-$(lapack-version).tar.gz \
libbsd-$(libbsd-version).tar.xz \
libpng-$(libpng-version).tar.xz \
libtool-$(libtool-version).tar.xz \
@@ -130,6 +132,9 @@ $(tarballs): $(tdir)/%:
: (l==2 ? "%d00\n" \
: "%d000\n") ), $$1)}')
w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz
+ elif [ $$n = atlas ]; then
+ mergenames=0
+ w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download
elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12
elif [ $$n = curl ]; then w=https://curl.haxx.se/download
elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version)
@@ -140,6 +145,7 @@ $(tarballs): $(tdir)/%:
elif [ $$n = gsl ]; then w=http://ftpmirror.gnu.org/gnu/gsl
elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet
elif [ $$n = jpegsrc ]; then w=http://ijg.org/files
+ elif [ $$n = lapack ]; then w=http://www.netlib.org/lapack
elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases
elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng
elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool
@@ -251,6 +257,64 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \
--disable-webp --disable-zstd) \
&& echo "Libtiff is built" > $@
+$(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \
+ $(tdir)/lapack-$(lapack-version).tar.gz
+
+
+ # Get the CPU frequency.
+ if [ x$(on_mac_os) = xyes ]; then
+ core=2400
+ else
+ core=$$(cat /proc/cpuinfo | grep "cpu MHz" \
+ | head -n 1 \
+ | sed "s/.*: \([0-9.]*\).*/\1/")
+ fi
+
+ # See if the shared libraries should be build for a single CPU
+ # thread or multiple threads.
+ N=$$(nproc)
+ srcdir=$$(pwd)/reproduce/src/make
+ if [ $$N = 1 ]; then
+ sharedmk=$$srcdir/dependencies-atlas-single.mk
+ else
+ sharedmk=$$srcdir/dependencies-atlas-multiple.mk
+ fi
+
+ # The linking step here doesn't recognize the `-Wl' in the
+ # `rpath_command'.
+ export LDFLAGS=-L$(ildir)
+
+ cd $(ddir)
+ tar xf $<
+ cd ATLAS
+ rm -rf build
+ mkdir build
+ cd build
+ ../configure -b 64 -D c -DPentiumCPS=$$core \
+ --with-netlib-lapack-tarfile=$(word 2, $^) \
+ --cripple-atlas-performance \
+ -Fa alg -fPIC --shared \
+ --prefix=$(idir)
+
+ # Do the basic build and the extra shared libraries.
+ make
+ cd lib && make -f $$sharedmk && cd ..
+
+ # Add RPATH to all the shared libraries.
+ for l in lib/*.so*; do patchelf --set-rpath $(ildir) $$l; done
+
+ # Install the libraires.
+ make install
+ cp -d lib/*.so* $(ildir)
+ [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir)
+ ln -fs $(ildir)/libblas.so $(ildir)/libblas.so.3
+ ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so
+ ln -fs $(ildir)/liblapack.so.3.6.1 $(ildir)/liblapack.so.3
+ exit 1
+ cd $(ddir)
+ rm -rf ATLAS
+ echo "Atlas is built" > $@
+