From 5950fb9d463b8258f2b1bbd633585eff1ff06002 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 17 Apr 2020 20:29:55 +0100 Subject: Patchelf is now built dynamically Until a few commits ago, PatchELF was built statically because it was used to patch `libstdc++' at the end of the GCC building phase, but PatchELF also depends on `libstdc++', so it would crash. However, recently when patching the GCC libraries, we don't directly apply Patchelf to the library, first we copy it to a temporary place, do the patching, then put it in its proper place. So the problem above won't happen any more. With this commit, I am thus removing the static flag from patchelf and letting it built dynamically all the time. The main problem was that some systems don't have a static C++ library, so PatchELF couldn't be built statically. Instead of adding more checks, we just fixed the core foundation of the problem. --- reproduce/software/make/basic.mk | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 17bb47a..e8fc105 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -567,18 +567,9 @@ $(ibidir)/readline: $(ibidir)/ncurses \ SHLIB_LIBS="-lncursesw" -j$(numthreads)) \ && echo "GNU Readline $(readline-version)" > $@ -# When we have a static C library, PatchELF will be built statically. This -# is because PatchELF links with the C++ standard library. But we need to -# run PatchELF later on `libstdc++'! This circular dependency can cause a -# crash, so when PatchELF can't be built statically, we won't build GCC -# either, see the `configure.sh' script where we define `good_static_libc' -# for more. $(ibidir)/patchelf: | $(ibidir)/make \ $(tdir)/patchelf-$(patchelf-version).tar.gz - if [ $(good_static_libc) = 1 ]; then \ - export LDFLAGS="$$LDFLAGS -static"; \ - fi; \ - $(call gbuild, patchelf-$(patchelf-version), static) \ + $(call gbuild, patchelf-$(patchelf-version)) \ && echo "PatchELF $(patchelf-version)" > $@ @@ -1342,7 +1333,7 @@ $(ibidir)/gcc: | $(ibidir)/binutils \ if [ "$$f" = $(ildir)/libstdc++.so ]; then \ patchelf --add-needed $(ildir)/libiconv.so $$tempname; \ fi; \ - mv $$tempname $$f; echo "corrected"; \ + mv $$tempname $$f; \ fi; \ done; \ fi \ -- cgit v1.2.1 From 5b72310aea8f33febc97a2f49f995917b716dcd0 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 17 Apr 2020 21:11:31 +0100 Subject: Removed confusing comment in configure.sh, and extra variable In the previous commit, we remove the `-static' flag from building PatchELF because it wasn't necessary any more. Howver, the comment for the check still included it and could be confusing. This is corrected with this commit. Also, we don't need the `good_static_libc' variable (that was only defined to pass onto PatchELF). This has also been corrected. --- reproduce/software/shell/configure.sh | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 919acde..9f6f3e6 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -331,15 +331,8 @@ fi # See if a link-able static C library exists # ------------------------------------------ # -# After building GCC, we must use PatchELF to correct its RPATHs. However, -# PatchELF links internally with `libstdc++'. So a dynamicly linked -# PatchELF cannot be used to correct the links to `libstdc++' in general -# (on some systems this causes no problem, but on others it doesn't!). -# -# However, to build a Static PatchELF, we need to be able to link with the -# static C library, which is not always available on some GNU/Linux -# systems. Therefore we need to check this here. If we can't build a static -# PatchELF, we won't build any GCC either. +# A static C library and the `sys/cdefs.h' header are necessary for +# building GCC. if [ x"$host_cc" = x0 ]; then echo; echo; echo "Checking if static C library is available..."; cat > $testsource < Date: Fri, 17 Apr 2020 23:55:26 +0100 Subject: Replaced name of directory under akhlaghi.org as backup server Until now, when a the raw tarball of some software wasn't usable, I would put it under my own webpage, or `akhlaghi.org/reproduce-software'. That same address was also used as a backup server. However, now the project has a proper name: Maneage. So I changed the directory on my own server to `akhlaghi.org/maneage-software'. With this commit, this new address has replaced the old one. But to avoid crashes in projects that haven't yet merged with the main Maneage branch, the old `reproduce-software' still works (its actually a symbolic link to the new directory now). --- reproduce/software/make/basic.mk | 14 +++++++------- reproduce/software/make/high-level.mk | 20 ++++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index e8fc105..19f5500 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -85,7 +85,7 @@ all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) # Servers to use as backup, later this should go in a file that is not # under version control (the actual server that the tarbal comes from is # irrelevant). -backupservers = http://akhlaghi.org/reproduce-software +backupservers = http://akhlaghi.org/maneage-software @@ -99,7 +99,7 @@ backupservers = http://akhlaghi.org/reproduce-software # we can preserve the variables). # # Software with main webpage at our backup repository -# (http://akhlaghi.org/reproduce-software): As of our latest check their +# (http://akhlaghi.org/maneage-software): As of our latest check their # major release tarballs either crash or don't build on some systems (for # example Make or Gzip), or they don't exist (for example Bzip2). So we are # building them from their Git history (which builds properly) or host them @@ -164,10 +164,10 @@ $(tarballs): $(tdir)/%: | $(lockdir) | awk '{print $$1}' ); \ \ mergenames=1; \ - if [ $$n = bash ]; then c=$(bash-checksum); w=http://akhlaghi.org/reproduce-software; \ + if [ $$n = bash ]; then c=$(bash-checksum); w=http://akhlaghi.org/maneage-software; \ elif [ $$n = binutils ]; then c=$(binutils-checksum); w=http://ftp.gnu.org/gnu/binutils; \ - elif [ $$n = bzip ]; then c=$(bzip2-checksum); w=http://akhlaghi.org/reproduce-software; \ - elif [ $$n = cert ]; then c=$(cert-checksum); w=http://akhlaghi.org/reproduce-software; \ + elif [ $$n = bzip ]; then c=$(bzip2-checksum); w=http://akhlaghi.org/maneage-software; \ + elif [ $$n = cert ]; then c=$(cert-checksum); w=http://akhlaghi.org/maneage-software; \ elif [ $$n = coreutils ]; then c=$(coreutils-checksum); w=http://ftp.gnu.org/gnu/coreutils;\ elif [ $$n = curl ]; then c=$(curl-checksum); w=https://curl.haxx.se/download; \ elif [ $$n = diffutils ]; then c=$(diffutils-checksum); w=http://ftp.gnu.org/gnu/diffutils;\ @@ -187,9 +187,9 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = m ]; then \ mergenames=0; \ c=$(m4-checksum); \ - w=http://akhlaghi.org/reproduce-software/m4-1.4.18-patched.tar.gz; \ + w=http://akhlaghi.org/maneage-software/m4-1.4.18-patched.tar.gz; \ elif [ $$n = make ]; then c=$(make-checksum); w=https://ftp.gnu.org/gnu/make; \ - elif [ $$n = metastore ]; then c=$(metastore-checksum); w=http://akhlaghi.org/reproduce-software; \ + elif [ $$n = metastore ]; then c=$(metastore-checksum); w=http://akhlaghi.org/maneage-software; \ elif [ $$n = mpc ]; then c=$(mpc-checksum); w=http://ftp.gnu.org/gnu/mpc; \ elif [ $$n = mpfr ]; then c=$(mpfr-checksum); w=http://www.mpfr.org/mpfr-current;\ elif [ $$n = ncurses ]; then c=$(ncurses-checksum); w=http://ftp.gnu.org/gnu/ncurses; \ diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index e62f02f..06ac332 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -103,7 +103,7 @@ export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh # Servers to use as backup, later this should go in a file that is not # under version control (the actual server that the tarbal comes from is # irrelevant). -backupservers = http://akhlaghi.org/reproduce-software +backupservers = http://akhlaghi.org/maneage-software # Building flags: # @@ -213,7 +213,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) # Set the top download link of the requested tarball. mergenames=1 - if [ $$n = apachelog ]; then c=$(apachelog4cxx-checksum); w=http://akhlaghi.org/reproduce-software + if [ $$n = apachelog ]; then c=$(apachelog4cxx-checksum); w=http://akhlaghi.org/maneage-software elif [ $* = apr-util-$(apr-util-version).tar.gz ]; then c=$(apr-util-checksum); w=https://www-us.apache.org/dist/apr elif [ $$n = apr ]; then c=$(apr-checksum); w=https://www-us.apache.org/dist/apr @@ -222,7 +222,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) mergenames=0 c=$(atlas-checksum) w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download - elif [ $$n = autoconf ]; then c=$(autoconf-checksum); w=http://akhlaghi.org/reproduce-software + elif [ $$n = autoconf ]; then c=$(autoconf-checksum); w=http://akhlaghi.org/maneage-software elif [ $$n = automake ]; then c=$(automake-checksum); w=http://ftp.gnu.org/gnu/automake elif [ $$n = bison ]; then c=$(bison-checksum); w=http://ftp.gnu.org/gnu/bison elif [ $$n = boost ]; then @@ -264,9 +264,9 @@ $(tarballs): $(tdir)/%: | $(lockdir) c=$(hdf5-checksum) majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$* - elif [ $$n = healpix ]; then c=$(healpix-checksum); w=http://akhlaghi.org/reproduce-software + elif [ $$n = healpix ]; then c=$(healpix-checksum); w=http://akhlaghi.org/maneage-software elif [ $$n = help ]; then c=$(help2man-checksum); w=http://ftp.gnu.org/gnu/help2man - elif [ $$n = imagemagick ]; then c=$(imagemagick-checksum); w=http://akhlaghi.org/reproduce-software + elif [ $$n = imagemagick ]; then c=$(imagemagick-checksum); w=http://akhlaghi.org/maneage-software elif [ $$n = imfit ]; then mergenames=0 c=$(imfit-checksum) @@ -274,7 +274,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = install ]; then c=NO-CHECK-SUM; w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then c=$(libjpeg-checksum); w=http://ijg.org/files elif [ $$n = lapack ]; then c=$(lapack-checksum); w=http://www.netlib.org/lapack - elif [ $$n = libnsl ]; then c=$(libnsl-checksum); w=http://akhlaghi.org/reproduce-software + elif [ $$n = libnsl ]; then c=$(libnsl-checksum); w=http://akhlaghi.org/maneage-software elif [ $$n = libpng ]; then c=$(libpng-checksum); w=https://download.sourceforge.net/libpng elif [ $$n = libgit ]; then mergenames=0 @@ -283,7 +283,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = libtirpc ]; then c=$(libtirpc-checksum); w=https://downloads.sourceforge.net/libtirpc elif [ $$n = libxml ]; then c=$(libxml2-checksum); w=ftp://xmlsoft.org/libxml2 elif [ $$n = missfits ]; then c=$(missfits-checksum); w=https://www.astromatic.net/download/missfits - elif [ $$n = netpbm ]; then c=$(netpbm-checksum); w=http://akhlaghi.org/reproduce-software + elif [ $$n = netpbm ]; then c=$(netpbm-checksum); w=http://akhlaghi.org/maneage-software elif [ $$n = openblas ]; then mergenames=0 c=$(openblas-checksum) @@ -299,15 +299,15 @@ $(tarballs): $(tdir)/%: | $(lockdir) majver=$$(echo $(R-version) | sed -e's/\./ /g' | awk '{print $$1}') w=https://cran.r-project.org/src/base/R-$$majver elif [ $$n = rpcsvc ]; then c=$(rpcsvc-proto-checksum); w=https://github.com/thkukuk/rpcsvc-proto/releases/download/v$(rpcsvc-proto-version) - elif [ $$n = scamp ]; then c=$(scamp-checksum); w=http://akhlaghi.org/reproduce-software + elif [ $$n = scamp ]; then c=$(scamp-checksum); w=http://akhlaghi.org/maneage-software elif [ $$n = scons ]; then mergenames=0 c=$(scons-checksum) w=https://sourceforge.net/projects/scons/files/scons/$(scons-version)/scons-$(scons-version).tar.gz/download - elif [ $$n = sextractor ]; then c=$(sextractor-checksum); w=http://akhlaghi.org/reproduce-software + elif [ $$n = sextractor ]; then c=$(sextractor-checksum); w=http://akhlaghi.org/maneage-software elif [ $$n = swarp ]; then c=$(swarp-checksum); w=https://www.astromatic.net/download/swarp elif [ $$n = swig ]; then c=$(swig-checksum); w=https://sourceforge.net/projects/swig/files/swig/swig-$(swig-version) - elif [ $$n = tides ]; then c=$(tides-checksum); w=http://akhlaghi.org/reproduce-software + elif [ $$n = tides ]; then c=$(tides-checksum); w=http://akhlaghi.org/maneage-software elif [ $$n = tiff ]; then c=$(libtiff-checksum); w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then c=$(wcslib-checksum); w=ftp://ftp.atnf.csiro.au/pub/software/wcslib elif [ $$n = xlsxio ]; then -- cgit v1.2.1 From 3a49e2c1a627a1240919439bb1b52005e260e099 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sat, 18 Apr 2020 00:34:20 +0100 Subject: Properly adding libiconv to the libraries that libstdc++ links with Of the GCC dynamically linked libraries we need to manually add RPATH to all and for `libstdc++' we also need to tell it to link with `libiconv'. Until now, the conditional to check for libstdc++ was not working and thus libiconv wasn't been added to it. With this commit the conditional has been corrected and is now working. Also, to help in reading the logs, an echo statement was added after every call to PatchELF. --- reproduce/software/make/basic.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 19f5500..d1fbc3f 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -1330,8 +1330,11 @@ $(ibidir)/gcc: | $(ibidir)/binutils \ if [ x"$$isdynamic" != x ]; then \ cp $$f $$tempname; \ patchelf --set-rpath $(ildir) $$tempname; \ - if [ "$$f" = $(ildir)/libstdc++.so ]; then \ + echo "$$f: added rpath"; \ + islibcpp=$$(echo $$f | grep "libstdc++"); \ + if [ x"$$islibcpp" != x ]; then \ patchelf --add-needed $(ildir)/libiconv.so $$tempname; \ + echo "$$f: linked with libiconv"; \ fi; \ mv $$tempname $$f; \ fi; \ -- cgit v1.2.1