From 3c9bf3aff30f02c7d31bd86f36c4db2520f8ffa4 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 3 May 2024 13:07:41 +0200 Subject: Configuration: no dependency on /bin/sh and useful run-time options SUMMARY: no change necessary in your project, this commit only involves changes in how already-existing software are built. Some handy options have also been added to the top-level project script and the copyright years have been updated. Until now, if the host's '/bin/sh' had conflicts with the Maneage environment, the configuration of Maneage would crash as soon as we entered the building of high-level software. The full scenario is described in the comments of the newly added 'reproduce/software/shell/prep-source.sh'. This is most relevant when building older Maneage'd project in newer environments. With this commit, the following changes were made to avoid the problem above: - Maneage edits the source code of all installed software to replace '/bin/sh' with Maneage's own shell before the programs are built. Through this, we were able to solve the problem described above. - The portable '#!/usr/bin/env sh' shebangs are now used at the start of the scripts that run during configure time so it uses the first available shell that it finds in its PATH (the system's before Dash is built), then Dash, and after Dash is built, Bash. - For TeXLive, since we don't install it from source, it was necessary to add the libraries necessary for the local '/bin/sh' in LD_LIBRARY_PATH. Some high-level options have been added to the './project' script to simplify certain operations: --keep-going: do not stop upon the first crash, but keep going on to build targets until all build-able targets have been built. This is very useful for debugging large pipelines and allows you to isolate the problematic part of your project. --highlight-all: equivalent to calling both '--highlight-new' & '--highlight-notes'. --- reproduce/software/shell/configure.sh | 78 ++++++++++++++++++++++++++++------- 1 file changed, 64 insertions(+), 14 deletions(-) (limited to 'reproduce/software/shell/configure.sh') diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 4f71bee..857f3b5 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -2,9 +2,9 @@ # # Necessary preparations/configurations for the reproducible project. # -# Copyright (C) 2018-2023 Mohammad Akhlaghi -# Copyright (C) 2021-2023 Raul Infante-Sainz -# Copyright (C) 2022-2023 Pedram Ashofteh Ardakani +# Copyright (C) 2018-2025 Mohammad Akhlaghi +# Copyright (C) 2021-2025 Raul Infante-Sainz +# Copyright (C) 2022-2025 Pedram Ashofteh Ardakani # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -1252,6 +1252,14 @@ instlibdir="$instdir"/lib if ! [ -d "$instlibdir" ]; then mkdir "$instlibdir"; fi ln -fs "$instlibdir" "$instdir"/lib64 +# Wrapper over Make as a single command so it does not default to '/bin/sh' +# during installation (needed by some programs like CMake). +instbindir=$instdir/bin +if ! [ -d $instbindir ]; then mkdir $instbindir; fi +makewshell="$instbindir/make-with-shell" +echo "$instbindir/make SHELL=$instbindir/bash \$@" > $makewshell +chmod +x $makewshell + @@ -1558,6 +1566,35 @@ fi +# Libraries necessary for the system's shell +# ------------------------------------------ +# +# In some cases (mostly the programs that Maneage doesn't yet build by +# itself), the programs may call the system's shell, not Maneage's +# shell. After we close-off the system environment from Maneage, this will +# cause a crash! To avoid such cases, we need to find the locations of the +# libraries that the shell needs and temporarily add them to the library +# search path. +if [ x"$$on_mac_os" != xyes ]; then + sys_library_sh_path=$(otool -L /bin/sh \ + | awk '/\/lib/{print $1}' \ + | sed 's#/[^/]*$##' \ + | sort \ + | uniq \ + | awk '{printf "%s:", $1}END{printf "\b"}') +else + sys_library_sh_path=$(ldd /bin/sh \ + | awk '{if($3!="") print $3}' \ + | sed 's#/[^/]*$##' \ + | sort \ + | uniq \ + | awk '{printf "%s:", $1}END{printf "\b"}') +fi + + + + + # Find Zenodo URL for software downloading # ---------------------------------------- # @@ -1646,6 +1683,7 @@ fi # tools, but we have to be very portable (and use minimal features in all). echo; echo "Building necessary software (if necessary)..." .local/bin/make $keepgoing -f reproduce/software/make/basic.mk \ + sys_library_sh_path=$sys_library_sh_path \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ @@ -1673,17 +1711,29 @@ else numthreads=$jobs fi .local/bin/env -i HOME=$bdir \ - .local/bin/make $keepgoing -f reproduce/software/make/high-level.mk \ - user_backup_urls="$user_backup_urls" \ - sys_library_path=$sys_library_path \ - rpath_command=$rpath_command \ - all_highlevel=$all_highlevel \ - static_build=$static_build \ - numthreads=$numthreads \ - on_mac_os=$on_mac_os \ - sys_cpath=$sys_cpath \ - host_cc=$host_cc \ - -j$numthreads + .local/bin/make $keepgoing \ + -f reproduce/software/make/high-level.mk \ + sys_library_sh_path=$sys_library_sh_path \ + user_backup_urls="$user_backup_urls" \ + sys_library_path=$sys_library_path \ + rpath_command=$rpath_command \ + all_highlevel=$all_highlevel \ + static_build=$static_build \ + numthreads=$numthreads \ + on_mac_os=$on_mac_os \ + sys_cpath=$sys_cpath \ + host_cc=$host_cc \ + -j$numthreads + + + + + +# Delete the temporary Make wrapper +# --------------------------------- +# +# See above for its description. +rm $makewshell -- cgit v1.2.1 From 890858795fd6e0c8c1f3050adec7f4bc78e9e47c Mon Sep 17 00:00:00 2001 From: Giacomo Lorenzetti Date: Tue, 21 Jan 2025 18:57:10 +0100 Subject: IMPORTANT: 73 software upgrades and added offline mode Summary: This is a software update to make Maneage more portable and up to date. It does not involve any Maneage infrastructure changes. You should just re-build your project to make sure the updated software haven't removed/changed any of their features that you were using. Until now, the software in Maneage had not been updated for almost 2.5 years (since August 2022) and were starting to get dated! Also, when no internet is available (for example in a HPC), the Zenodo check would always cause a crash and needed to be commented manually. With this commit, all the basic software that had been updated as well as the dependencies of Gnuastro (which we commit to maintaining) have been updated after a check of their webpage (see the full list below). To fix the offline issue, a new '--offline' option has been added to the project script. The other following changes were made: - tarball-prepare: has become more resilient (for dealing with file name with spaces or Lzip, which should not be compressed). - Readme: improve documentation when used in Docker. Below you can see the list of basic software that have been updated. The only basic software that have not been updated upstream since the previous Maneage software update are 'bison', 'flock', 'm4', 'pkgconfig', 'unzip' and 'zip'. bash 5.2-rc2 5.2.37 binutils 2.39 2.43.1 coreutils 9.1 9.6 curl 7.84.0 8.11.1 dash 0.5.11-057cd65 0.5.12 diffutils 3.8 3.10 file 5.42 5.46 findutils 4.9.0 4.10.0 gawk 5.1.1 5.3.1 gcc 12.1.0 14.2.0 gettext 0.21 0.23.1 git 2.37.1 2.48.1 gmp 6.2.1 6.3.0 grep 3.7 3.11 gzip 1.12 1.13 help2man 1.49.2 1.49.3 isl 0.24 0.27 less 590 668 libiconv 1.17 1.18 libtool 2.4.7 2.5.4 libunistring 1.0 1.3 libxml2 2.9.12 2.13.5 lzip 1.23 1.25 make 4.3 4.4.1 mpc 1.2.1 1.3.1 mpfr 4.1.0 4.2.1 nano 6.4 8.3 ncurses 6.3 6.5 openssl 3.0.5 3.4.0 perl 5.36.0 5.40.1 podlators 4.14 6.0.2 readline 8.2-rc2 8.2.13 sed 4.8 4.9 tar 1.34 1.35 texinfo 6.8 7.2 wget 1.21.2 1.25.0 which 2.21 2.23 xz 5.2.5 5.6.3 zlib 1.2.11 1.3.1 The high-level software that have been updated. autoconf 2.71 2.72 automake 1.16.5 1.17 cfitsio 4.1.0 4.5.0 cmake 3.24.0 3.31.5 expat 2.4.1 2.6.4 fontconfig 2.14.0 2.16.0 freetype 2.11.0 2.13.3 ghostscript 9.56.1 10.04.0 gnuastro 0.18 0.23 gsl 2.7 2.8 libffi 3.4.2 3.4.7 libgit2 1.3.0 1.9.0 libice 1.0.10 1.1.2 libidn 1.38 1.42 libjpeg 9e 9f libpaper 1.1.28 1.1.29 libpng 1.6.37 1.6.46 libpthread-stubs 0.4 0.5 libsm 1.2.3 1.2.5 libtiff 4.4.0 4.7.0 libxau 1.0.9 1.0.12 libxcb 1.15 1.17 libxdmcp 1.1.3 1.1.5 libxext 1.3.4 1.3.6 libxt 1.2.1 1.3.1 python 3.10.6 3.13.12 util-macros 1.19.3 1.20.2 wcslib 7.11 8.4 xcb-proto 1.15 1.17 xorgproto 2022.1 2024.1 xtrans 1.4.0 1.5.2 --- reproduce/software/shell/configure.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'reproduce/software/shell/configure.sh') diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 857f3b5..f226ced 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -1612,11 +1612,11 @@ fi # which will download the DOI-resolved webpage, and extract the Zenodo-URL # of the most recent version from there (using the 'coreutils' tarball as # an example, the directory part of the URL for all the other software are -# the same). This is not done if the option '--debug' is used. +# the same). This is not done if the options '--debug' or `--offline` are used. zenodourl="" user_backup_urls="" zenodocheck=.build/software/zenodo-check.html -if [ x$debug = x ]; then +if [ x$debug = x ] && [ x$offline = x ]; then if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ | sed -n -e'/http/p' \ @@ -1723,6 +1723,7 @@ fi on_mac_os=$on_mac_os \ sys_cpath=$sys_cpath \ host_cc=$host_cc \ + offline=$offline \ -j$numthreads -- cgit v1.2.1 From a55a407c1f2a1b280be78f24abd1fe6d4a8032e2 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Thu, 13 Feb 2025 19:27:57 +0100 Subject: Configuration: new server for tarballs and portability fixed SUMMARY: no change is necessary for your pipeline after upgrading to this commit. Until this commit, the following noteworthy portability problems existed that have been fixed as described in each item. - In commit 8908587 (titled "IMPORTANT: 73 software upgrades and added offline mode"; committed on 2025-02-11); the versions of all (except those that were not updated upstream) basic software as well as the high-level software necessary for Gnuastro were updated. This included Python. However, the update of Python could cause crashes in the Python-related software that are also in Maneage but not tested for that commit. -- With this commit, Python has been revered back to 3.10.6 (from 3.13.2) and it has been taken to a dedicated part of 'versions.conf' (reminding the reader that the version of Python should only be updated by the Python maintainer: to ensure it doesn't conflict with Python-based builds). - Following the update of XZ Utils in 8908587, the custom build recipe that was necessary is no longer necessary. This was reported by Boud Roukema (see https://savannah.nongnu.org/bugs/index.php?62700). -- With this commit, XZ Utils uses our basic high-level build scripts. - CMake could use out-of-Maneage libraries for those that we already had in Maneage on some systems. This was reported by Boud Roukema (see https://savannah.nongnu.org/bugs/?63043) -- With this commit, CMake is built with '--no-system-libs'; so it builds and statically links to all its necessary libraries internally (not confusing between Maneage and the host). - Building Binutils 2.39 was likely to fail on older GCC versions due to special features (such as the 'gprofng' feature when building with GCC 6.2.0). This was reported by Boud Roukema in https://savannah.nongnu.org/bugs/index.php?63242. -- With this commit, the 'gprofng' feature of Binutils has been disabled by default to allow reproducibility on older systems. If you need it, remove the '--enable-gprofng=no' option in the build recipe of Binutils. - The Maneage tarball repository is now also kept in 'gitlab.cefca.es'. -- With this commit, a new server URL has been added for this. - The list of TeXLive packages included some packages that get installed in the basic scheme that we do for the initial setup of TeXLive. -- With this commit, the extra packages have been removed. --- reproduce/software/shell/configure.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'reproduce/software/shell/configure.sh') diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index f226ced..d58a829 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -1439,6 +1439,15 @@ fi +# Make sure the temporary build directory is empty (un-finished +# source/build files from previous builds can remain there during debugging +# or software updates). +rm -rf $tmpblddir/* + + + + + # Inform the user that the build process is starting # ------------------------------------------------- # @@ -1575,7 +1584,7 @@ fi # cause a crash! To avoid such cases, we need to find the locations of the # libraries that the shell needs and temporarily add them to the library # search path. -if [ x"$$on_mac_os" != xyes ]; then +if [ x"$on_mac_os" != xyes ]; then sys_library_sh_path=$(otool -L /bin/sh \ | awk '/\/lib/{print $1}' \ | sed 's#/[^/]*$##' \ -- cgit v1.2.1