diff options
Diffstat (limited to 'reproduce/software/shell')
| -rw-r--r-- | reproduce/software/shell/apptainer-README.md | 6 | ||||
| -rwxr-xr-x | reproduce/software/shell/apptainer.sh | 83 | ||||
| -rwxr-xr-x | reproduce/software/shell/bashrc.sh | 8 | ||||
| -rwxr-xr-x | reproduce/software/shell/configure.sh | 311 | ||||
| -rw-r--r-- | reproduce/software/shell/docker-README.md | 23 | ||||
| -rwxr-xr-x | reproduce/software/shell/docker.sh | 35 | ||||
| -rwxr-xr-x | reproduce/software/shell/git-post-checkout | 2 | ||||
| -rwxr-xr-x | reproduce/software/shell/git-pre-commit | 2 | ||||
| -rwxr-xr-x | reproduce/software/shell/pre-make-build.sh | 20 | ||||
| -rwxr-xr-x | reproduce/software/shell/prep-source.sh | 47 | ||||
| -rwxr-xr-x | reproduce/software/shell/run-parts.in | 4 | ||||
| -rwxr-xr-x | reproduce/software/shell/tarball-prepare.sh | 9 |
12 files changed, 337 insertions, 213 deletions
diff --git a/reproduce/software/shell/apptainer-README.md b/reproduce/software/shell/apptainer-README.md index a7826ec..d2a14c8 100644 --- a/reproduce/software/shell/apptainer-README.md +++ b/reproduce/software/shell/apptainer-README.md @@ -1,7 +1,7 @@ # Maneage'd projects in Apptainer -Copyright (C) 2025-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org>\ -Copyright (C) 2025-2025 Giacomo Lorenzetti <glorenzetti@cefca.es>\ +Copyright (C) 2025-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org>\ +Copyright (C) 2025-2026 Giacomo Lorenzetti <glorenzetti@cefca.es>\ See the end of the file for license conditions. For an introduction on containers, see the "Building in containers" section @@ -47,7 +47,7 @@ software environment) to easily move it from one computer to another. project. This file is necessary for future runs of your project within the container. - 3. To execute your project remote the `--build-only` and use `./run.sh` to + 3. To execute your project remove the `--build-only` and use `./run.sh` to execute it. If you want to enter your Maneage'd project shell, add the `--project-shell` option to the call inside `./run.sh`. diff --git a/reproduce/software/shell/apptainer.sh b/reproduce/software/shell/apptainer.sh index c581ade..2aa3535 100755 --- a/reproduce/software/shell/apptainer.sh +++ b/reproduce/software/shell/apptainer.sh @@ -41,8 +41,8 @@ # # Known problems: # -# Copyright (C) 2025-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2025-2025 Giacomo Lorenzetti <glorenzetti@cefca.es> +# Copyright (C) 2025-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2025-2026 Giacomo Lorenzetti <glorenzetti@cefca.es> # # This script is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -71,14 +71,16 @@ set -e # Default option values +sif="" jobs=0 +debug=0 quiet=0 +host_cc=0 source_dir= build_only= -base_name="" +base_sif="" shm_size=20gb scriptname="$0" -project_name="" project_shell=0 container_shell=0 base_os=debian:stable-slim @@ -97,9 +99,13 @@ Top-level script to build and run a Maneage'd project within Apptainer. --source-dir=STR Directory of source code (default: 'pwd -P'). Apptainer images + --sif=STR Project's apptainer image (a '.sif' file). --base-os=STR Base OS name (default: '$base_os'). - --base-name=STR Base OS apptainer image (a '.sif' file). - --project-name=STR Project's apptainer image (a '.sif' file). + --base-sif=STR Base OS apptainer image (a '.sif' file). + + Configuration options + --host-cc Use containers's C compiler, don't build GCC. + -d, --debug[=FLAGS] Use -j1 and stop on first crash. Interactive shell --project-shell Open the project's shell within the container. @@ -155,10 +161,18 @@ do --source-dir=*) source_dir="${1#*=}"; check_v "$1" "$source_dir"; shift;; # Container options. - --base-name) base_name="$2"; check_v "$1" "$base_name"; shift;shift;; - --base-name=*) base_name="${1#*=}"; check_v "$1" "$base_name"; shift;; - --project-name) project_name="$2"; check_v "$1" "$project_name"; shift;shift;; - --project-name=*) project_name="${1#*=}"; check_v "$1" "$project_name"; shift;; + --sif) sif="$2"; check_v "$1" "$sif"; shift;shift;; + --sif=*) sif="${1#*=}"; check_v "$1" "$sif"; shift;; + --base-os) base_os="$2"; check_v "$1" "$base_os"; shift;shift;; + --base-os=*) base_os="${1#*=}"; check_v "$1" "$base_os"; shift;; + --base-sif) base_sif="$2"; check_v "$1" "$base_sif"; shift;shift;; + --base-sif=*) base_sif="${1#*=}"; check_v "$1" "$base_sif"; shift;; + + # Configuration options. + --host-cc) host_cc=1; shift;; + --host-cc=*) on_off_option_error --host-cc;; + -d|--debug) debug=1; shift;; + -d|--debug=*) on_off_option_error --debug;; # Interactive shell. --project-shell) project_shell=1; shift;; @@ -193,21 +207,22 @@ done # # Make sure that the build directory is given and that it exists. if [ x$build_dir = x ]; then - printf "$scriptname: '--build-dir' not provided, this is the location " - printf "that all built analysis files will be kept on the host OS\n" + printf "$scriptname: '--build-dir' not provided, this is the " + printf "location that all built analysis files will be kept on " + printf "the host OS\n" exit 1; else if ! [ -d $build_dir ]; then - printf "$scriptname: '$build_dir' (value to '--build-dir') doesn't " - printf "exist\n" + printf "$scriptname: '$build_dir' (value to '--build-dir') " + printf "doesn't exist\n" exit 1; fi fi # Set the default project and base-OS image names (inside the build # directory). -if [ x"$base_name" = x ]; then base_name=$build_dir/maneage-base.sif; fi -if [ x"$project_name" = x ]; then project_name=$build_dir/maneaged.sif; fi +if [ x"$base_sif" = x ]; then base_sif=$build_dir/maneage-base.sif; fi +if [ x"$sif" = x ]; then sif=$build_dir/maneaged.sif; fi @@ -278,9 +293,9 @@ fi # # Build the base operating system using Maneage's './project configure' # step. -if [ -f $project_name ]; then +if [ -f $sif ]; then if [ $quiet = 0 ]; then - printf "$scriptname: info: project's image ('$project_name') " + printf "$scriptname: info: project's image ('$sif') " printf "already exists and will be used. If you want to build a " printf "new project image, give a new name to '--project-name'. " printf "To remove this message run with '--quiet'\n" @@ -289,12 +304,13 @@ else # Build the basic definition, with just Debian-slim with minimal # necessary tools. - if [ -f $base_name ]; then + if [ -f $base_sif ]; then if [ $quiet = 0 ]; then - printf "$scriptname: info: base OS docker image ('$base_name') " - printf "already exists and will be used. If you want to build a " - printf "new base OS image, give a new name to '--base-name'. " - printf "To remove this message run with '--quiet'\n" + printf "$scriptname: info: base OS apptainer image " + printf "('$base_sif') already exists and will be used. " + printf "If you want to build a new base OS image, give " + printf "a new name to '--base-sif'. To remove this " + printf "message run with '--quiet'\n" fi else @@ -308,7 +324,7 @@ From: $base_os EOF # Build the base operating system container and delete the # temporary definition file. - apptainer build $base_name $base_def + apptainer build $base_sif $base_def rm $base_def fi @@ -325,11 +341,14 @@ EOF # software tarball directory, they will all be symbolic links that # aren't valid when the user runs the container (since we only # mount the software tarballs at build time). + confopts="" intbuild=/home/maneager/build maneage_def=$build_dir/maneage.def + if [ $host_cc = 1 ]; then confopts="--host-cc"; fi + if [ $debug = 1 ]; then confopts="$confopts --debug"; fi cat <<EOF > $maneage_def Bootstrap: localimage -From: $base_name +From: $base_sif %setup mkdir -p \${APPTAINER_ROOTFS}/home/maneager/input @@ -339,7 +358,7 @@ From: $base_name %post cd /home/maneager/source - ./project configure --jobs=$jobs \\ + ./project configure --jobs=$jobs --no-pause $confopts \\ --input-dir=/home/maneager/input \\ --build-dir=$intbuild \\ --software-dir=/home/maneager/tarballs-software @@ -347,7 +366,7 @@ From: $base_name %runscript cd /home/maneager/source - if ./project configure --build-dir=$intbuild \\ + if ./project configure --build-dir=$intbuild $confopts \\ --existing-conf --no-pause \\ --offline --quiet; then \\ if [ x"\$maneage_apptainer_stat" = xshell ]; then \\ @@ -361,7 +380,7 @@ From: $base_name else \\ printf "$scriptname: '\$maneage_apptainer_stat' (value "; \\ printf "to 'maneage_apptainer_stat' environment variable) "; \\ - printf "is not recognized: should be either 'shell' or 'run'"; \\ + printf "is not recognized: should be either 'shell' or 'run'\n"; \\ exit 1; \\ fi; \\ else \\ @@ -384,7 +403,7 @@ EOF $software_dir_mnt \ --ignore-fakeroot-command \ \ - $project_name \ + $sif \ $maneage_def # Clean up. @@ -396,7 +415,7 @@ fi if ! [ x"$build_only" = x ]; then if [ $quiet = 0 ]; then printf "$scriptname: info: Maneaged project has been configured " - printf "successfully in the '$project_name' image" + printf "successfully in the '$sif' image\n" fi exit 0 fi @@ -418,7 +437,7 @@ else fi # Build the hostname from the name of the SIF file of the project name. -hstname=$(echo "$project_name" \ +hstname=$(echo "$sif" \ | awk 'BEGIN{FS="/"}{print $NF}' \ | sed -e's|.sif$||') @@ -453,4 +472,4 @@ apptainer $aopt \ --hostname $hstname \ --cwd /home/maneager/source \ \ - $project_name + $sif diff --git a/reproduce/software/shell/bashrc.sh b/reproduce/software/shell/bashrc.sh index 6bbd774..5534f3d 100755 --- a/reproduce/software/shell/bashrc.sh +++ b/reproduce/software/shell/bashrc.sh @@ -34,7 +34,7 @@ # shell'). # # -# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -62,13 +62,13 @@ if [ x$PROJECT_STATUS = xshell ]; then # A small introductory message. echo "----------------------------------------------------------------------" echo "Welcome to the Maneage interactive shell for this project, running" - echo " $(sh --version | awk 'NR==1')" + echo " $(sh --version | awk 'NR==1')" echo echo "This shell's home directory is the project's build directory:" - echo " HOME: $HOME" + echo " HOME: $HOME" echo echo "This shell's startup file is in the project's source directory:" - echo " $PROJECT_RCFILE" + echo " $PROJECT_RCFILE" echo echo "To return to your host shell, run the 'exit' command." echo "----------------------------------------------------------------------" diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 4887816..19c14ac 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -2,9 +2,9 @@ # # Necessary preparations/configurations for the reproducible project. # -# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2021-2025 Raul Infante-Sainz <infantesainz@gmail.com> -# Copyright (C) 2022-2025 Pedram Ashofteh Ardakani <pedramardakani@pm.me> +# Copyright (C) 2018-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2021-2026 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2022-2026 Pedram Ashofteh Ardakani <pedramardakani@pm.me> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -228,7 +228,7 @@ empty_build_tmp() { # accidentally delete the whole root partition (or a least the '/home' # partition of the user). if ! [ x"$( ls -A $tmpblddir )" = x ]; then - rm -r "$tmpblddir"/* + rm -rf "$tmpblddir"/* fi rm -r "$tmpblddir" } @@ -375,16 +375,23 @@ if [ $built_container = 0 ]; then fi # On macOS, the way of obtaining the number of cores is different - # between Intel or Apple M1 CPUs. Here we disinguish between Apple - # M1 or others. - maccputype=$(sysctl -n machdep.cpu.brand_string) - if [ x"$maccputype" = x"Apple M1" ]; then - address_size_physical=$(sysctl -n machdep.cpu.thread_count) - address_size_virtual=$(sysctl -n machdep.cpu.logical_per_package) - else - address_size_physical=$(sysctl -n machdep.cpu.address_bits.physical) - address_size_virtual=$(sysctl -n machdep.cpu.address_bits.virtual) - fi + # between Intel or Apple Silicon CPUs. Here we distinguish between + # Apple Silicon (any "Apple M*" brand string, e.g. M1, M1 Pro, M1 + # Max, M2, M2 Pro, ...) and Intel-based Macs. Note that the Apple + # Silicon group can be in the formats of "Apple M1" or "Apple M1 + # Pro", so we use a prefix match via a 'case' pattern to include + # both in one check. + maccputype=$(sysctl -n machdep.cpu.brand_string 2>/dev/null) + case "$maccputype" in + "Apple M"*) + address_size_physical=$(sysctl -n machdep.cpu.thread_count) + address_size_virtual=$(sysctl -n machdep.cpu.logical_per_package) + ;; + *) + address_size_physical=$(sysctl -n machdep.cpu.address_bits.physical) + address_size_virtual=$(sysctl -n machdep.cpu.address_bits.virtual) + ;; + esac address_sizes="$address_size_physical bits physical, " address_sizes+="$address_size_virtual bits virtual" else @@ -781,6 +788,25 @@ fi +# Older C standard versions +# ------------------------- +# +# Some basic packages require an old standard of C compilation; for their +# list, see the 'Not working with C23' list in '../config/versions.conf'. +# Here, we first try 'gnu17', but if that fails on the host compiler, we +# fall back to 'gnu99'. +if [ $built_container = 0 ]; then + printf "int main(void){; return 0;}\n" > $testsource + if gcc -std=gnu17 $testsource -o $testprog 2> /dev/null; then + std_c_old="gnu17" + else std_c_old="gnu99" + fi + rm $testsource $testprog +fi + + + + # Fortran compiler # ---------------- # @@ -856,33 +882,6 @@ fi -# See if the linker accepts -Wl,-rpath-link -# ----------------------------------------- -# -# '-rpath-link' is used to write the information of the linked shared -# library into the shared object (library or program). But some versions of -# LLVM's linker don't accept it an can cause problems. -# -# IMPORTANT NOTE: This test has to be done **AFTER** the definition of -# 'instdir', otherwise, it is going to be used as an empty string. -if [ $built_container = 0 ]; then - cat > $testsource <<EOF -#include <stdio.h> -#include <stdlib.h> -int main(void) {return EXIT_SUCCESS;} -EOF - if $CC $testsource -o$testprog -Wl,-rpath-link 2>/dev/null \ - > /dev/null; then - export rpath_command="-Wl,-rpath-link=$instdir/lib" - else - export rpath_command="" - fi - - # Delete the temporary directory for compiler checking. - rm -f $testprog $testsource - rm -r $compilertestdir - elapsed_time_from_prev_step compiler-rpath -fi @@ -1146,6 +1145,20 @@ EOF echo " ** Please select another directory." echo "" else + # Set the '.build' and '.local' symbolic links (and delete + # possibly existing symbolic links). These commands are also + # present in the top-level 'project' script, but they are only + # invoked when '--build-dir' is called. When it is not called + # (the user wants to insert the directories interactively: the + # scenario here), the links need to be created from + # scratch. Furthermore, in case the given directory to + # '--build-dir' has problems (fails to pass the sanity checks + # above), the symbolic links also need to be recreated. + rm -f .build .local + ln -s $bdir .build + ln -s $bdir/software/installed .local + + # Inform the user echo " -- Build directory set to ($instring): '$bdir'" fi done @@ -1329,11 +1342,17 @@ elapsed_time_from_prev_step software-dir # complete the configuration, we may also need to download the source code # of some necessary software packages (including the downloaders). So we # need to check the host's available tool for downloading at this step. +# +# NOTE: it is important that the name of the programs are not recorded to +# be absolute (for example 'wget' instead of '/usr/bin/wget'). This is +# because we build both 'wget' and 'curl' within 'basic.mk'. After they are +# built, Maneage's built software have higher precedence in 'PATH'. So +# without an absolute address, our own built software will be used as soon +# as they are installed/available. if [ $rewritelconfig = yes ]; then - if type wget > /dev/null 2>/dev/null; then - # 'which' isn't in POSIX, so we are using 'command -v' instead. - name=$(command -v wget) + # First look for wget on the host. + if type wget > /dev/null 2>/dev/null; then # See if the host wget has the '--no-use-server-timestamps' option # (for example wget 1.12 doesn't have it). If not, we'll have to @@ -1350,13 +1369,16 @@ if [ $rewritelconfig = yes ]; then # By default Wget keeps the remote file's timestamp, so we'll have # to disable it manually. - downloader="$name $wgetts -O"; + downloader="wget $wgetts -O"; + + # In case Wget doesn't exist, try cURL. elif type curl > /dev/null 2>/dev/null; then - name=$(command -v curl) # - cURL doesn't keep the remote file's timestamp by default. # - With the '-L' option, we tell cURL to follow redirects. - downloader="$name -L -o" + downloader="curl -L -o" + + # Neither Wget or cURL could be found. else cat <<EOF @@ -1396,6 +1418,55 @@ elapsed_time_from_prev_step downloader +# Libraries necessary for the system's shell +# ------------------------------------------ +# +# In some cases (mostly the programs that Maneage doesn't yet build by +# itself), the programs may call the system's shell, not Maneage's +# shell. After we close-off the system environment from Maneage, this will +# cause a crash! To avoid such cases, we need to find the locations of the +# libraries that the shell needs and temporarily add them to the library +# search path. +# +# About the 'grep -v "(0x[^)]*)"' term (from bug 66847, see [1]): On some +# systems [2], the output of 'ldd /bin/sh' includes a line for the vDSO [3] +# that is different to the formats that are assumed, prior to this commit, +# by the algorithm in 'configure.sh' when evaluating the variable +# 'sys_library_sh_path'. This leads to a fatal syntax error in (at least) +# 'ncurses', because the option using 'sys_library_sh_path' contains an +# unquoted RAM address in parentheses. Even if the address were quoted, it +# would still be incorrect. This 'grep' command excludes candidate host +# path strings that look like RAM addresses to address the problem. +# +# [1] https://savannah.nongnu.org/bugs/index.php?66847 +# [2] https://stackoverflow.com/questions/34428037/how-to-interpret-the-output-of-the-ldd-program +# [3] man vdso +if [ $built_container = 0 ]; then + if [ x"$on_mac_os" = xyes ]; then + sys_library_sh_path=$(otool -L /bin/sh \ + | awk '/\/lib/{print $1}' \ + | sed 's#/[^/]*$##' \ + | sort \ + | uniq \ + | awk '{if (NR==1) printf "%s", $1; \ + else printf ":%s", $1}') + else + sys_library_sh_path=$(ldd /bin/sh \ + | awk '{if($3!="") print $3}' \ + | sed 's#/[^/]*$##' \ + | grep -v "(0x[^)]*)" \ + | sort \ + | uniq \ + | awk '{if (NR==1) printf "%s", $1; \ + else printf ":%s", $1}') + fi + elapsed_time_from_prev_step sys-library-sh-path +fi + + + + + # When no local configuration existed, write the parameters into the local # configuration file. sdir=$bdir/software @@ -1415,6 +1486,7 @@ if [ $rewritelconfig = yes ]; then -e's|@sys_cpath[@]|'"$sys_cpath"'|' \ -e's|@downloader[@]|'"$downloader"'|' \ -e's|@groupname[@]|'"$maneage_group_name"'|' \ + -e's|@sys_library_sh_path[@]|'"$sys_library_sh_path"'|' \ $lconfin >> $lconf fi elapsed_time_from_prev_step LOCAL-write @@ -1486,6 +1558,41 @@ fi +# See if the linker accepts -Wl,-rpath-link +# ----------------------------------------- +# +# '-rpath-link' is used to write the information of the linked shared +# library into the shared object (library or program). But some versions of +# LLVM's linker don't accept it an can cause problems. +# +# IMPORTANT NOTE: This test has to be done **AFTER** the definition of +# 'instdir'. Otherwise, the rpath-link value set within rpath_command in +# the case of a successful test compile (if $CC ...) will be "/lib", +# i.e. the host system root /lib directory, instead of the maneage library +# directory. +if [ $built_container = 0 ]; then + cat > $testsource <<EOF +#include <stdio.h> +#include <stdlib.h> +int main(void) {return EXIT_SUCCESS;} +EOF + if $CC $testsource -o$testprog -Wl,-rpath-link 2>/dev/null \ + > /dev/null; then + export rpath_command="-Wl,-rpath-link=$instdir/lib" + else + export rpath_command="" + fi + + # Delete the temporary directory for compiler checking. + rm -f $testprog $testsource + rm -r $compilertestdir + elapsed_time_from_prev_step compiler-rpath +fi + + + + + # Software building directory (possibly in RAM) # --------------------------------------------- # @@ -1666,53 +1773,6 @@ fi -# Libraries necessary for the system's shell -# ------------------------------------------ -# -# In some cases (mostly the programs that Maneage doesn't yet build by -# itself), the programs may call the system's shell, not Maneage's -# shell. After we close-off the system environment from Maneage, this will -# cause a crash! To avoid such cases, we need to find the locations of the -# libraries that the shell needs and temporarily add them to the library -# search path. -# -# About the 'grep -v "(0x[^)]*)"' term (from bug 66847, see [1]): On some -# systems [2], the output of 'ldd /bin/sh' includes a line for the vDSO [3] -# that is different to the formats that are assumed, prior to this commit, -# by the algorithm in 'configure.sh' when evaluating the variable -# 'sys_library_sh_path'. This leads to a fatal syntax error in (at least) -# 'ncurses', because the option using 'sys_library_sh_path' contains an -# unquoted RAM address in parentheses. Even if the address were quoted, it -# would still be incorrect. This 'grep command excludes candidate host path -# strings that look like RAM addresses to address the problem. -# -# [1] https://savannah.nongnu.org/bugs/index.php?66847 -# [2] https://stackoverflow.com/questions/34428037/how-to-interpret-the-output-of-the-ldd-program -# [3] man vdso -if [ $built_container = 0 ]; then - if [ x"$on_mac_os" = xyes ]; then - sys_library_sh_path=$(otool -L /bin/sh \ - | awk '/\/lib/{print $1}' \ - | sed 's#/[^/]*$##' \ - | sort \ - | uniq \ - | awk '{if (NR==1) printf "%s", $1; \ - else printf ":%s", $1}') - else - sys_library_sh_path=$(ldd /bin/sh \ - | awk '{if($3!="") print $3}' \ - | sed 's#/[^/]*$##' \ - | grep -v "(0x[^)]*)" \ - | sort \ - | uniq \ - | awk '{if (NR==1) printf "%s", $1; \ - else printf ":%s", $1}') - fi - elapsed_time_from_prev_step sys-library-sh-path -fi - - - # Find Zenodo URL for software downloading # ---------------------------------------- @@ -1733,30 +1793,37 @@ fi # an example, the directory part of the URL for all the other software are # the same). This is not done if the options '--debug' or `--offline` are # used. -zenodourl="" -user_backup_urls="" -zenodocheck="$bdir"/software/zenodo-check.html -if [ $built_container = 0 ]; then - if [ x$debug = x ] && [ x$offline = x ]; then - if $downloader $zenodocheck \ - https://doi.org/10.5281/zenodo.3883409; then - zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ - | sed -n -e'/http/p' \ - | tr ' ' '\n' \ - | grep http \ - | sed -e 's/href="//' -e 's|/coreutils| |' \ - | awk 'NR==1{print $1}') - fi - fi - rm -f $zenodocheck - - # Add the Zenodo URL to the user's given back software URLs. Since the - # user can specify 'user_backup_urls' (not yet implemented as an option - # in './project'), we'll give preference to their specified servers, - # then add the Zenodo URL afterwards. - user_backup_urls="$user_backup_urls $zenodourl" - elapsed_time_from_prev_step zenodo-url -fi +# +# NOTE on commented region below (starting with '##'): As described [1], +# Zenodo's upload strategy does not allow a repository with more than one +# hundred files. So until the solution there has been implemented, this +# step has been commented. +# +# [1] https://savannah.nongnu.org/task/?16621 +## zenodourl="" +## user_backup_urls="" +## zenodocheck="$bdir"/software/zenodo-check.html +## if [ $built_container = 0 ]; then +## if [ x$debug = x ] && [ x$offline = x ]; then +## if $downloader $zenodocheck \ +## https://doi.org/10.5281/zenodo.3883409; then +## zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ +## | sed -n -e'/http/p' \ +## | tr ' ' '\n' \ +## | grep http \ +## | sed -e 's/href="//' -e 's|/coreutils| |' \ +## | awk 'NR==1{print $1}') +## fi +## fi +## rm -f $zenodocheck +## +## # Add the Zenodo URL to the user's given back software URLs. Since the +## # user can specify 'user_backup_urls' (not yet implemented as an option +## # in './project'), we'll give preference to their specified servers, +## # then add the Zenodo URL afterwards. +## user_backup_urls="$user_backup_urls $zenodourl" +## elapsed_time_from_prev_step zenodo-url +## fi @@ -1816,6 +1883,7 @@ if [ $quiet = 0 ]; then echo "Building/validating software: basic"; fi numthreads=$numthreads \ needs_ldl=$needs_ldl \ on_mac_os=$on_mac_os \ + std_c_old=$std_c_old \ host_cc=$host_cc \ -j$numthreads elapsed_time_from_prev_step make-software-basic @@ -1874,9 +1942,10 @@ ______________________________________________________ !!!!!!!!!!!!!!! Warning !!!!!!!!!!!!!!!! TeX Live couldn't be installed during the configuration (probably because -there were downloading problems). TeX Live is only necessary in making the -final PDF (which is only done after all the analysis has been complete). It -is not used at all during the analysis. +there were downloading problems, or you used the '--offline' option). TeX +Live is only necessary in making the final PDF (which is only done after +all the analysis has been complete). It is not used at all during the +analysis. Therefore, if you don't need the final PDF, and just want to do the analysis, you can safely ignore this warning and continue. diff --git a/reproduce/software/shell/docker-README.md b/reproduce/software/shell/docker-README.md index d651e22..dd46f4f 100644 --- a/reproduce/software/shell/docker-README.md +++ b/reproduce/software/shell/docker-README.md @@ -1,6 +1,6 @@ # Maneage'd projects in Docker -Copyright (C) 2021-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org>\ +Copyright (C) 2021-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org>\ See the end of the file for license conditions. For an introduction on containers, see the "Building in containers" section @@ -35,28 +35,25 @@ software environment) to easily move it from one computer to another. systemctl start docker ``` - 2. Using your favorite text editor, create a `run.sh` in your top Maneage - directory (as described in the comments at the start of the `docker.sh` - script in this directory). Just activate `--build-only` on the first - run so it doesn't go onto doing the analysis and just sets up the - software environment. Set the respective directory(s) based on your - filesystem (the software directory is optional). The `run.sh` file name - is already in `.gitignore` (because it contains local directories), so - Git will ignore it and it won't be committed by mistake. + 2. At the start of the `docker.sh` script in this directory, you can see + the necessary command to run. Just activate `--build-only` on the first + run so it doesn't go onto doing the analysis (that can be done at any + later time). Set the respective directory(s) based on your filesystem + (the software directory is optional). You can put the command in a + `run.sh` at the top of the project (it is already in `.gitignore`: + because it contains local directories). 3. After the setup is complete, remove the `--build-only` and run the command below to confirm that `maneage-base` (the OS of the container) and `maneaged` (your project's full Maneage'd environment) images are available. If you want different names for these images, add the - `--project-name` and `--base-name` options to the `docker.sh` call. + `--project-name` and `--base-name` options to the `docker.sh` call. You + can see the list of available images with this command: ```shell docker image list ``` - 4. You are now ready to do your analysis by removing the `--build-only` - option. - diff --git a/reproduce/software/shell/docker.sh b/reproduce/software/shell/docker.sh index 714c75f..5cd8f11 100755 --- a/reproduce/software/shell/docker.sh +++ b/reproduce/software/shell/docker.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # # Create a Docker container from an existing image of the built software # environment, but with the source, data and build (analysis) directories @@ -47,7 +47,7 @@ # clipped, log limit 2MiB reached]" message. We need to find a way to # fix this (so nothing gets clipped: useful for debugging). # -# Copyright (C) 2021-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2021-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -409,22 +409,25 @@ else # that the user gave there. mkdir $tmp_dir/$tsdir cp -r "$software_dir"/* $tmp_dir/$tsdir/ - printf " --mount=type=bind,source=$tsdir,target=$dts \x5C\n" >> $df + printf " --mount=type=bind,source=$tsdir,target=$dts \x5C\n" \ + >> $df fi # Construct the rest of the 'RUN' command. printf " cp -r $dsr /home/maneager/source; \x5C\n" >> $df printf " cd /home/maneager/source; \x5C\n" >> $df - printf " ./project configure --jobs=$jobs \x5C\n" >> $df + printf " ./project configure \x5C\n" >> $df + printf " --no-pause \x5C\n" >> $df + printf " --jobs=$jobs \x5C\n" >> $df printf " --build-dir=$intbuild \x5C\n" >> $df printf " --input-dir=/home/maneager/input \x5C\n" >> $df - printf " --software-dir=$dts; \x5C\n" >> $df + printf " --software-dir=$dts; \x5C\n" >> $df - # We are deleting the '.build/software/tarballs' directory because this - # directory is not relevant for the analysis of the project. But in - # case any tarball was downloaded, it will consume space within the - # container. - printf " rm -rf .build/software/tarballs; \x5C\n" >> $df + # We are deleting the contents of the '.build/software/tarballs' + # directory (tarballs of the sources of the software) because they are + # not relevant to the execution of the project and will consume space + # within the container. + printf " rm -rf .build/software/tarballs/*; \x5C\n" >> $df # We are deleting the source directory becaues later (at 'docker run' # time), the 'source' will be mounted directly from the host operating @@ -432,12 +435,9 @@ else printf " cd /home/maneager; \x5C\n" >> $df printf " rm -rf source\n" >> $df - # Build the Maneage container and delete the temporary directory. The - # '--progress plain' option is for Docker to print all the outputs - # (otherwise, it will only print a very small part!). + # Build the Maneage container and delete the temporary directory. cd $tmp_dir docker build ./ -t $project_name \ - --progress=plain \ --shm-size=$shm_size \ --no-cache \ 2>&1 | tee build.log @@ -453,14 +453,15 @@ if ! [ x"$image_file" = x ] && ! [ -f "$image_file" ]; then # Save the image into a tarball tarname=$(echo $image_file | sed -e's|.gz$||') if [ $quiet = 0 ]; then - printf "$scriptname: info: saving docker image to '$tarname'" + printf "$scriptname: info: saving docker image to '$tarname'\n" fi docker save -o $tarname $project_name # Compress the saved image if [ $quiet = 0 ]; then printf "$scriptname: info: compressing to '$image_file' (can " - printf "take +10 minutes, but volume decreases by more than half!)" + printf "take +10 minutes, but volume decreases by more than " + printf "half!)\n" fi gzip --best $tarname fi @@ -470,7 +471,7 @@ fi if ! [ x"$build_only" = x ]; then if [ $quiet = 0 ]; then printf "$scriptname: info: Maneaged project has been configured " - printf "successfully in the '$project_name' image" + printf "successfully in the '$project_name' image\n" fi exit 0 fi diff --git a/reproduce/software/shell/git-post-checkout b/reproduce/software/shell/git-post-checkout index 7b521a3..7bf747a 100755 --- a/reproduce/software/shell/git-post-checkout +++ b/reproduce/software/shell/git-post-checkout @@ -4,7 +4,7 @@ # controlled files (with each commit) using the 'metastore' program. # # Copyright (C) 2016 Przemyslaw Pawelczyk <przemoc@gmail.com> -# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is taken from the 'examples/hooks/pre-commit' file of the # 'metastore' package (installed within the project, with an MIT license diff --git a/reproduce/software/shell/git-pre-commit b/reproduce/software/shell/git-pre-commit index 7b98ad0..e570346 100755 --- a/reproduce/software/shell/git-pre-commit +++ b/reproduce/software/shell/git-pre-commit @@ -4,7 +4,7 @@ # controlled files (with each commit) using the 'metastore' program. # # Copyright (C) 2016 Przemyslaw Pawelczyk <przemoc@gmail.com> -# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # WARNING: # diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh index 172bdb6..3163f74 100755 --- a/reproduce/software/shell/pre-make-build.sh +++ b/reproduce/software/shell/pre-make-build.sh @@ -2,7 +2,7 @@ # # Very basic tools necessary to start Maneage's default building. # -# Copyright (C) 2020-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2020-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -120,16 +120,24 @@ download_tarball() { "$bservers" fi + # Make sure this is the correct tarball. if type sha512sum > /dev/null 2> /dev/null; then - checksum=$(sha512sum "$ucname" | awk '{print $1}') + + # On macOS, sha512sum wraps the hash in two lines. The 'tr -d '\n'' + # part joins them into one before awk extracts the first field. On + # Linux (GNU coreutils) the output is already a single line, so the + # 'tr' command has no effect. + checksum=$(sha512sum "$ucname" | tr -d '\n' | awk '{print $1}') expectedchecksum=$(awk '/^'$progname'-checksum/{print $3}' \ "$checksumsfile") - if [ x$checksum = x$expectedchecksum ]; then mv "$ucname" "$maneagetar" + if [ x$checksum = x$expectedchecksum ]; then + mv "$ucname" "$maneagetar" else - echo "ERROR: Non-matching checksum: $tarball" - echo "Checksum should be: $expectedchecksum" - echo "Checksum is: $checksum" + echo "ERROR: Non-matching checksum in $tarball" + echo " Checksum should be: $expectedchecksum" + echo " Checksum is: $checksum" + rm -f "$ucname" exit 1 fi; else mv "$ucname" "$maneagetar" diff --git a/reproduce/software/shell/prep-source.sh b/reproduce/software/shell/prep-source.sh index dcdc472..7f49b5c 100755 --- a/reproduce/software/shell/prep-source.sh +++ b/reproduce/software/shell/prep-source.sh @@ -7,7 +7,7 @@ # directory that it is run in ): # ./prep-source.sh /FULL/ADDRESS/TO/DESIRED/BIN # -# Copyright (C) 2024-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2024-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -32,6 +32,15 @@ set -e + +# Set the locale to C to make sure local language settings do not +# interefere (especially with the SED command). +export LC_ALL=C + + + + + # Read the first argument. bindir="$1" if [ x"$bindir" = x ]; then @@ -47,17 +56,18 @@ fi -# Find all the files that contain the '/bin/sh' string and correct them to -# Maneage's own Bash. We are using 'while read' to read the file names line -# by line. This is necessary to account file names that include the 'SPACE' -# character (happens in CMake for example!). +# Find all the files that contain the '/bin/sh' or '/bin/bash' strings and +# correct them to Maneage's own shell (Dash or Bash). We are using 'while +# read' to read the file names line by line. This is necessary to account +# file names that include the 'SPACE' character (happens in CMake for +# example!). # # Note that dates are important in the source directory (files depend on # each other), so we should read the original date and after making. We are # also not using GNU SED's '-i' ('--in-place') option because the host OS # may not have GNU SED. # -# Actual situation which prompted the addition of this step: a Maneage'd +# The situation which prompted the addition of this step: a Maneage'd # project (with GNU Bash 5.1.8 and Readline 8.1.1) was being built on a # system where '/bin/sh' was GNU Bash 5.2.26 and had Readline 8.2.010. The # newer version of Bash needed the newer Readline library function(s) that @@ -74,18 +84,35 @@ fi # hard-coded in the source code of almost all programs (their build # scripts); and in special programs like GNU Make, GNU M4 or CMake it is # actually hardcoded in the source code (not just build scripts). -if [ -f "$bindir/bash" ]; then shpath="$bindir"/bash +bashpath="$bindir"/bash +if [ -f "$bindir/bash" ]; then shpath="$bashpath" else shpath="$bindir"/dash fi -grep -I -r -e'/bin/sh' $(pwd)/* \ + +# On MacOS the syntax for 'stat' is a bit different, so if we are using the +# system one on mac we need a specific syntax. The one installed by maneage +# instead uses the ordinary Linux syntax. +if [ -f "$bindir/stat" ] || [ x"$on_mac_os" = xno ]; then + format="--format %a" +else + format="-f %OLp" +fi + +# On MacOS 'touch' wants the time expressed according to ISO8601 with a +# precision up to the seconds. We then use 'sed' to remove the information +# regarding the timezone, as the format is not accepted by 'touch'. +# LCTYPE and LANG are also required on macos systems by sed. +grep -I -r -e'/bin/sh' -e'/bin/bash' $(pwd)/* \ | sed -e's|:|\t|' \ | awk 'BEGIN{FS="\t"}{print $1}' \ | sort \ | uniq \ | while read filename; do \ tmp="$filename".tmp; \ - origtime="$(date -R -r "$filename")"; \ - origperm=$(stat -c '%a' "$filename"); \ + origtimex="$(date -Iseconds -r "$filename")"; \ + origtime=$(echo $origtimex | sed 's/.\{6\}$//'); \ + origperm=$(stat $format "$filename"); \ + LC_CTYPE=C; LANG=C; \ sed -e's|/bin/sh|'"$shpath"'|g' "$filename" > "$tmp"; \ mv "$tmp" "$filename"; \ chmod $origperm "$filename"; \ diff --git a/reproduce/software/shell/run-parts.in b/reproduce/software/shell/run-parts.in index 053f5f3..21c347f 100755 --- a/reproduce/software/shell/run-parts.in +++ b/reproduce/software/shell/run-parts.in @@ -10,8 +10,8 @@ # However, it didn't have a copyright statement. So one is being added # here. # -# Copyright (C) 2025 Authors mentioned above. -# Copyright (C) 2025-2024 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2026 Authors mentioned above. +# Copyright (C) 2026-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/shell/tarball-prepare.sh b/reproduce/software/shell/tarball-prepare.sh index 8e30931..ed1cfd0 100755 --- a/reproduce/software/shell/tarball-prepare.sh +++ b/reproduce/software/shell/tarball-prepare.sh @@ -15,9 +15,9 @@ # # Discussion: https://savannah.nongnu.org/task/?15699 # -# Copyright (C) 2022-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2022-2025 Pedram Ashofteh Ardakani <pedramardakani@pm.me> -# Copyright (C) 2025-2025 Giacomo Lorenzetti <glorenzetti@cefca.es> +# Copyright (C) 2022-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2022-2026 Pedram Ashofteh Ardakani <pedramardakani@pm.me> +# Copyright (C) 2025-2026 Giacomo Lorenzetti <glorenzetti@cefca.es> # # This script is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -51,6 +51,9 @@ basedir=$PWD scriptname=$0 + + + # The --help output print_help() { cat <<EOF |