From 3c9bf3aff30f02c7d31bd86f36c4db2520f8ffa4 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 3 May 2024 13:07:41 +0200 Subject: Configuration: no dependency on /bin/sh and useful run-time options SUMMARY: no change necessary in your project, this commit only involves changes in how already-existing software are built. Some handy options have also been added to the top-level project script and the copyright years have been updated. Until now, if the host's '/bin/sh' had conflicts with the Maneage environment, the configuration of Maneage would crash as soon as we entered the building of high-level software. The full scenario is described in the comments of the newly added 'reproduce/software/shell/prep-source.sh'. This is most relevant when building older Maneage'd project in newer environments. With this commit, the following changes were made to avoid the problem above: - Maneage edits the source code of all installed software to replace '/bin/sh' with Maneage's own shell before the programs are built. Through this, we were able to solve the problem described above. - The portable '#!/usr/bin/env sh' shebangs are now used at the start of the scripts that run during configure time so it uses the first available shell that it finds in its PATH (the system's before Dash is built), then Dash, and after Dash is built, Bash. - For TeXLive, since we don't install it from source, it was necessary to add the libraries necessary for the local '/bin/sh' in LD_LIBRARY_PATH. Some high-level options have been added to the './project' script to simplify certain operations: --keep-going: do not stop upon the first crash, but keep going on to build targets until all build-able targets have been built. This is very useful for debugging large pipelines and allows you to isolate the problematic part of your project. --highlight-all: equivalent to calling both '--highlight-new' & '--highlight-notes'. --- reproduce/software/shell/bashrc.sh | 2 +- reproduce/software/shell/configure.sh | 78 +++++++++++++++++++----- reproduce/software/shell/git-post-checkout | 2 +- reproduce/software/shell/git-pre-commit | 2 +- reproduce/software/shell/pre-make-build.sh | 24 +++++--- reproduce/software/shell/prep-source.sh | 94 +++++++++++++++++++++++++++++ reproduce/software/shell/run-parts.in | 4 +- reproduce/software/shell/tarball-prepare.sh | 4 +- 8 files changed, 181 insertions(+), 29 deletions(-) create mode 100755 reproduce/software/shell/prep-source.sh (limited to 'reproduce/software/shell') diff --git a/reproduce/software/shell/bashrc.sh b/reproduce/software/shell/bashrc.sh index 814de2c..6bbd774 100755 --- a/reproduce/software/shell/bashrc.sh +++ b/reproduce/software/shell/bashrc.sh @@ -34,7 +34,7 @@ # shell'). # # -# Copyright (C) 2019-2023 Mohammad Akhlaghi +# Copyright (C) 2019-2025 Mohammad Akhlaghi # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 4f71bee..857f3b5 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -2,9 +2,9 @@ # # Necessary preparations/configurations for the reproducible project. # -# Copyright (C) 2018-2023 Mohammad Akhlaghi -# Copyright (C) 2021-2023 Raul Infante-Sainz -# Copyright (C) 2022-2023 Pedram Ashofteh Ardakani +# Copyright (C) 2018-2025 Mohammad Akhlaghi +# Copyright (C) 2021-2025 Raul Infante-Sainz +# Copyright (C) 2022-2025 Pedram Ashofteh Ardakani # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -1252,6 +1252,14 @@ instlibdir="$instdir"/lib if ! [ -d "$instlibdir" ]; then mkdir "$instlibdir"; fi ln -fs "$instlibdir" "$instdir"/lib64 +# Wrapper over Make as a single command so it does not default to '/bin/sh' +# during installation (needed by some programs like CMake). +instbindir=$instdir/bin +if ! [ -d $instbindir ]; then mkdir $instbindir; fi +makewshell="$instbindir/make-with-shell" +echo "$instbindir/make SHELL=$instbindir/bash \$@" > $makewshell +chmod +x $makewshell + @@ -1558,6 +1566,35 @@ fi +# Libraries necessary for the system's shell +# ------------------------------------------ +# +# In some cases (mostly the programs that Maneage doesn't yet build by +# itself), the programs may call the system's shell, not Maneage's +# shell. After we close-off the system environment from Maneage, this will +# cause a crash! To avoid such cases, we need to find the locations of the +# libraries that the shell needs and temporarily add them to the library +# search path. +if [ x"$$on_mac_os" != xyes ]; then + sys_library_sh_path=$(otool -L /bin/sh \ + | awk '/\/lib/{print $1}' \ + | sed 's#/[^/]*$##' \ + | sort \ + | uniq \ + | awk '{printf "%s:", $1}END{printf "\b"}') +else + sys_library_sh_path=$(ldd /bin/sh \ + | awk '{if($3!="") print $3}' \ + | sed 's#/[^/]*$##' \ + | sort \ + | uniq \ + | awk '{printf "%s:", $1}END{printf "\b"}') +fi + + + + + # Find Zenodo URL for software downloading # ---------------------------------------- # @@ -1646,6 +1683,7 @@ fi # tools, but we have to be very portable (and use minimal features in all). echo; echo "Building necessary software (if necessary)..." .local/bin/make $keepgoing -f reproduce/software/make/basic.mk \ + sys_library_sh_path=$sys_library_sh_path \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ @@ -1673,17 +1711,29 @@ else numthreads=$jobs fi .local/bin/env -i HOME=$bdir \ - .local/bin/make $keepgoing -f reproduce/software/make/high-level.mk \ - user_backup_urls="$user_backup_urls" \ - sys_library_path=$sys_library_path \ - rpath_command=$rpath_command \ - all_highlevel=$all_highlevel \ - static_build=$static_build \ - numthreads=$numthreads \ - on_mac_os=$on_mac_os \ - sys_cpath=$sys_cpath \ - host_cc=$host_cc \ - -j$numthreads + .local/bin/make $keepgoing \ + -f reproduce/software/make/high-level.mk \ + sys_library_sh_path=$sys_library_sh_path \ + user_backup_urls="$user_backup_urls" \ + sys_library_path=$sys_library_path \ + rpath_command=$rpath_command \ + all_highlevel=$all_highlevel \ + static_build=$static_build \ + numthreads=$numthreads \ + on_mac_os=$on_mac_os \ + sys_cpath=$sys_cpath \ + host_cc=$host_cc \ + -j$numthreads + + + + + +# Delete the temporary Make wrapper +# --------------------------------- +# +# See above for its description. +rm $makewshell diff --git a/reproduce/software/shell/git-post-checkout b/reproduce/software/shell/git-post-checkout index a1340d7..7b521a3 100755 --- a/reproduce/software/shell/git-post-checkout +++ b/reproduce/software/shell/git-post-checkout @@ -4,7 +4,7 @@ # controlled files (with each commit) using the 'metastore' program. # # Copyright (C) 2016 Przemyslaw Pawelczyk -# Copyright (C) 2018-2023 Mohammad Akhlaghi +# Copyright (C) 2018-2025 Mohammad Akhlaghi # # This script is taken from the 'examples/hooks/pre-commit' file of the # 'metastore' package (installed within the project, with an MIT license diff --git a/reproduce/software/shell/git-pre-commit b/reproduce/software/shell/git-pre-commit index 02ba488..7b98ad0 100755 --- a/reproduce/software/shell/git-pre-commit +++ b/reproduce/software/shell/git-pre-commit @@ -4,7 +4,7 @@ # controlled files (with each commit) using the 'metastore' program. # # Copyright (C) 2016 Przemyslaw Pawelczyk -# Copyright (C) 2018-2023 Mohammad Akhlaghi +# Copyright (C) 2018-2025 Mohammad Akhlaghi # # WARNING: # diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh index 808429b..93d3266 100755 --- a/reproduce/software/shell/pre-make-build.sh +++ b/reproduce/software/shell/pre-make-build.sh @@ -2,7 +2,7 @@ # # Very basic tools necessary to start Maneage's default building. # -# Copyright (C) 2020-2023 Mohammad Akhlaghi +# Copyright (C) 2020-2025 Mohammad Akhlaghi # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -48,7 +48,7 @@ instdir="$sdir"/installed tmpblddir="$sdir"/build-tmp confdir=reproduce/software/config ibidir="$instdir"/version-info/proglib -downloadwrapper=reproduce/analysis/bash/download-multi-try +downloadwrapper=reproduce/analysis/bash/download-multi-try.sh # Derived directories bindir="$instdir"/bin @@ -123,7 +123,8 @@ download_tarball() { # Make sure this is the correct tarball. if type sha512sum > /dev/null 2> /dev/null; then checksum=$(sha512sum "$ucname" | awk '{print $1}') - expectedchecksum=$(awk '/^'$progname'-checksum/{print $3}' "$checksumsfile") + expectedchecksum=$(awk '/^'$progname'-checksum/{print $3}' \ + "$checksumsfile") if [ x$checksum = x$expectedchecksum ]; then mv "$ucname" "$maneagetar" else echo "ERROR: Non-matching checksum: $tarball" @@ -135,8 +136,10 @@ download_tarball() { fi fi - # If the tarball is newer than the (possibly existing) program (the version - # has changed), then delete the program. + # If the tarball is newer than the (possibly existing) program (the + # version has changed), then delete the program. When the LaTeX name is + # not given here, the software is re-built later (close to the end of + # 'basic.mk') and the name is properly placed there. if [ -f "$ibidir/$progname" ]; then if [ "$maneagetar" -nt "$ibidir/$progname" ]; then rm "$ibidir/$progname" @@ -210,10 +213,15 @@ build_program() { fi fi - # Clean up the source directory + # Clean up the source directory and write the LaTeX name of the + # software (if necessary: otherwise, just make an empty file). cd "$topdir" rm -rf "$tmpblddir/$unpackdir" - echo "$progname_tex $version" > "$ibidir/$progname" + if [ x"$progname_tex" = x ]; then + echo "" > "$ibidir/$progname" + else + echo "$progname_tex $version" > "$ibidir/$progname" + fi fi } @@ -256,7 +264,7 @@ build_program # Make that has a different executable name (using the '--program-prefix=' # configure option) from the "default" make (which is this one!). progname="make" -progname_tex="GNU Make" +progname_tex="" # Make re-built after GCC (empty string to avoid repetition) url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) tarball=$progname-$version.tar.lz diff --git a/reproduce/software/shell/prep-source.sh b/reproduce/software/shell/prep-source.sh new file mode 100755 index 0000000..dcdc472 --- /dev/null +++ b/reproduce/software/shell/prep-source.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env sh +# +# Necessary corrections in the un-packed source of programs to make them +# portable (for example to not use '/bin/sh'). +# +# Usage: Run in top source directory (will work on all files within the +# directory that it is run in ): +# ./prep-source.sh /FULL/ADDRESS/TO/DESIRED/BIN +# +# Copyright (C) 2024-2025 Mohammad Akhlaghi +# +# This script is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This script is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this script. If not, see . + + + + + +# Abort the script in case of an error +set -e + + + + +# Read the first argument. +bindir="$1" +if [ x"$bindir" = x ]; then + printf "$0: no argument (location of the 'bin/' directory " + printf "containing the 'bash' executable)\n" + exit 1 +elif ! [ -d "$bindir" ]; then + printf "$0: the directory given as the first argument ('$bindir')" + printf "does not exist" +fi + + + + + +# Find all the files that contain the '/bin/sh' string and correct them to +# Maneage's own Bash. We are using 'while read' to read the file names line +# by line. This is necessary to account file names that include the 'SPACE' +# character (happens in CMake for example!). +# +# Note that dates are important in the source directory (files depend on +# each other), so we should read the original date and after making. We are +# also not using GNU SED's '-i' ('--in-place') option because the host OS +# may not have GNU SED. +# +# Actual situation which prompted the addition of this step: a Maneage'd +# project (with GNU Bash 5.1.8 and Readline 8.1.1) was being built on a +# system where '/bin/sh' was GNU Bash 5.2.26 and had Readline 8.2.010. The +# newer version of Bash needed the newer Readline library function(s) that +# were not available in Maneage's Readline library. Therefore, as soon as +# the basic software were built and Maneage entered the creation of +# high-level software (where we completely close-off the host environment), +# Maneage crashed with the following '/bin/sh' error: +# +# /bin/sh: symbol lookup error: /bin/sh: undefined symbol: rl_trim_arg_from_keyseq +# +# This lead to the discovery that through '/bin/sh' the host operating +# system was leaking into our closed Maneage environment which needs to be +# closed. This needs a source-level correction because '/bin/sh' is +# hard-coded in the source code of almost all programs (their build +# scripts); and in special programs like GNU Make, GNU M4 or CMake it is +# actually hardcoded in the source code (not just build scripts). +if [ -f "$bindir/bash" ]; then shpath="$bindir"/bash +else shpath="$bindir"/dash +fi +grep -I -r -e'/bin/sh' $(pwd)/* \ + | sed -e's|:|\t|' \ + | awk 'BEGIN{FS="\t"}{print $1}' \ + | sort \ + | uniq \ + | while read filename; do \ + tmp="$filename".tmp; \ + origtime="$(date -R -r "$filename")"; \ + origperm=$(stat -c '%a' "$filename"); \ + sed -e's|/bin/sh|'"$shpath"'|g' "$filename" > "$tmp"; \ + mv "$tmp" "$filename"; \ + chmod $origperm "$filename"; \ + touch -d"$origtime" "$filename"; \ + echo "Corrected /bin/sh in $filename"; \ + done diff --git a/reproduce/software/shell/run-parts.in b/reproduce/software/shell/run-parts.in index a7b7705..053f5f3 100755 --- a/reproduce/software/shell/run-parts.in +++ b/reproduce/software/shell/run-parts.in @@ -10,8 +10,8 @@ # However, it didn't have a copyright statement. So one is being added # here. # -# Copyright (C) 2023 Authors mentioned above. -# Copyright (C) 2020-2023 Mohammad Akhlaghi +# Copyright (C) 2025 Authors mentioned above. +# Copyright (C) 2025-2024 Mohammad Akhlaghi # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/shell/tarball-prepare.sh b/reproduce/software/shell/tarball-prepare.sh index 472b4c0..e12ff98 100755 --- a/reproduce/software/shell/tarball-prepare.sh +++ b/reproduce/software/shell/tarball-prepare.sh @@ -15,8 +15,8 @@ # # Discussion: https://savannah.nongnu.org/task/?15699 # -# Copyright (C) 2022-2023 Mohammad Akhlaghi -# Copyright (C) 2022-2023 Pedram Ashofteh Ardakani +# Copyright (C) 2022-2025 Mohammad Akhlaghi +# Copyright (C) 2022-2025 Pedram Ashofteh Ardakani # Released under GNU GPLv3+ # Abort the script in case of an error. -- cgit v1.2.1