From 3c9bf3aff30f02c7d31bd86f36c4db2520f8ffa4 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 3 May 2024 13:07:41 +0200 Subject: Configuration: no dependency on /bin/sh and useful run-time options SUMMARY: no change necessary in your project, this commit only involves changes in how already-existing software are built. Some handy options have also been added to the top-level project script and the copyright years have been updated. Until now, if the host's '/bin/sh' had conflicts with the Maneage environment, the configuration of Maneage would crash as soon as we entered the building of high-level software. The full scenario is described in the comments of the newly added 'reproduce/software/shell/prep-source.sh'. This is most relevant when building older Maneage'd project in newer environments. With this commit, the following changes were made to avoid the problem above: - Maneage edits the source code of all installed software to replace '/bin/sh' with Maneage's own shell before the programs are built. Through this, we were able to solve the problem described above. - The portable '#!/usr/bin/env sh' shebangs are now used at the start of the scripts that run during configure time so it uses the first available shell that it finds in its PATH (the system's before Dash is built), then Dash, and after Dash is built, Bash. - For TeXLive, since we don't install it from source, it was necessary to add the libraries necessary for the local '/bin/sh' in LD_LIBRARY_PATH. Some high-level options have been added to the './project' script to simplify certain operations: --keep-going: do not stop upon the first crash, but keep going on to build targets until all build-able targets have been built. This is very useful for debugging large pipelines and allows you to isolate the problematic part of your project. --highlight-all: equivalent to calling both '--highlight-new' & '--highlight-notes'. --- reproduce/analysis/bash/download-multi-try | 155 ------------------------- reproduce/analysis/bash/download-multi-try.sh | 160 ++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 155 deletions(-) delete mode 100755 reproduce/analysis/bash/download-multi-try create mode 100755 reproduce/analysis/bash/download-multi-try.sh (limited to 'reproduce/analysis/bash') diff --git a/reproduce/analysis/bash/download-multi-try b/reproduce/analysis/bash/download-multi-try deleted file mode 100755 index d7e9be2..0000000 --- a/reproduce/analysis/bash/download-multi-try +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/sh -# -# Attempt downloading multiple times before crashing whole project. From -# the top project directory (for the shebang above), this script must be -# run like this: -# -# $ /path/to/download-multi-try downloader lockfile input-url downloaded-name -# -# NOTE: The 'downloader' must contain the option to specify the output name -# in its end. For example "wget -O". Any other option can also be placed in -# the middle. -# -# Due to temporary network problems, a download may fail suddenly, but -# succeed in a second try a few seconds later. Without this script that -# temporary glitch in the network will permanently crash the project and -# it can't continue. The job of this script is to be patient and try the -# download multiple times before crashing the whole project. -# -# LOCK FILE: Since there is usually only one network port to the outside -# world, downloading is done much faster in serial, not in parallel. But -# the project's processing may be done in parallel (with multiple threads -# needing to download different files at the same time). Therefore, this -# script uses the 'flock' program to only do one download at a time. To -# benefit from it, any call to this script must be given the same lock -# file. If your system has multiple ports to the internet, or for any -# reason, you don't want to use a lock file, set the 'lockfile' name to -# 'nolock'. -# -# Copyright (C) 2019-2023 Mohammad Akhlaghi -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - - - - -# Script settings -# --------------- -# Stop the script if there are any errors. -set -e - - - - - -# Input arguments and necessary sanity checks. Note that the 5th argument -# (backup servers) isn't mandatory. -inurl="$3" -outname="$4" -lockfile="$2" -downloader="$1" -backupservers="$5" -if [ "x$downloader" = x ]; then - echo "$0: downloader (first argument) not given."; exit 1; -fi -if [ "x$lockfile" = x ]; then - echo "$0: lock file (second argument) not given."; exit 1; -fi -if [ "x$inurl" = x ]; then - echo "$0: full input URL (third argument) not given."; exit 1; -fi -if [ "x$outname" = x ]; then - echo "$0: output name (fourth argument) not given."; exit 1; -fi - - - - - -# Separate the actual filename, to possibly use backup server. -urlfile=$(echo "$inurl" | awk -F "/" '{print $NF}') - - - - - -# Try downloading multiple times before crashing. -counter=0 -maxcounter=10 -while [ ! -f "$outname" ]; do - - # Increment the counter. - counter=$(echo $counter | awk '{print $1+1}') - - # If we have passed a maximum number of trials, just exit with - # a failed code. - reachedmax=$(echo $counter \ - | awk '{if($1>'$maxcounter') print "yes"; else print "no";}') - if [ x$reachedmax = xyes ]; then - echo "" - echo "Failed $maxcounter download attempts: $outname" - echo "" - exit 1 - fi - - # If this isn't the first attempt print a notice and wait a little for - # the next trail. - if [ x$counter = x1 ]; then - just_a_place_holder=1 - else - tstep=$(echo $counter | awk '{print $1*5}') - echo "Download trial $counter for '$outname' in $tstep seconds." - sleep $tstep - fi - - # Attempt downloading the file. Note that the 'downloader' ends with - # the respective option to specify the output name. For example "wget - # -O" (so 'outname', that comes after it) will be the name of the - # downloaded file. - if [ x"$lockfile" = xnolock ]; then - if ! $downloader $outname $inurl; then rm -f $outname; fi - else - # Try downloading from the requested URL. - flock "$lockfile" sh -c \ - "if ! $downloader $outname \"$inurl\"; then rm -f $outname; fi" - fi - - # If the download failed, try the backup server(s). - if [ ! -f "$outname" ]; then - if [ x"$backupservers" != x ]; then - for bs in $backupservers; do - - # Use this backup server. - if [ x"$lockfile" = xnolock ]; then - if ! $downloader $outname $bs/$urlfile; then rm -f $outname; fi - else - flock "$lockfile" sh -c \ - "if ! $downloader $outname $bs/$urlfile; then rm -f $outname; fi" - fi - - # If the file was downloaded, break out of the loop that - # parses over the backup servers. - if [ -f "$outname" ]; then break; fi - done - fi - fi -done - - - - - -# Return successfully -exit 0 diff --git a/reproduce/analysis/bash/download-multi-try.sh b/reproduce/analysis/bash/download-multi-try.sh new file mode 100755 index 0000000..bea88d5 --- /dev/null +++ b/reproduce/analysis/bash/download-multi-try.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env sh +# +# Attempt downloading multiple times before crashing whole project. From +# the top project directory (for the shebang above), this script must be +# run like this: +# +# $ $SHELL /path/to/download-multi-try.sh downloader lockfile \ +# input-url downloaded-name +# +# NOTE: +# - This script doesn't have a Shebang because in different stages it +# should be built with different shells ('/bin/sh' before Maneage +# installs its own shell and afterwards with Maneage's own shell). +# - The 'downloader' must contain the option to specify the output name +# in its end. For example "wget -O". Any other option can also be placed in +# the middle. +# +# Due to temporary network problems, a download may fail suddenly, but +# succeed in a second try a few seconds later. Without this script that +# temporary glitch in the network will permanently crash the project and +# it can't continue. The job of this script is to be patient and try the +# download multiple times before crashing the whole project. +# +# LOCK FILE: Since there is usually only one network port to the outside +# world, downloading is done much faster in serial, not in parallel. But +# the project's processing may be done in parallel (with multiple threads +# needing to download different files at the same time). Therefore, this +# script uses the 'flock' program to only do one download at a time. To +# benefit from it, any call to this script must be given the same lock +# file. If your system has multiple ports to the internet, or for any +# reason, you don't want to use a lock file, set the 'lockfile' name to +# 'nolock'. +# +# Copyright (C) 2019-2025 Mohammad Akhlaghi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + + + + +# Script settings +# --------------- +# Stop the script if there are any errors. +set -e + + + + + +# Input arguments and necessary sanity checks. Note that the 5th argument +# (backup servers) isn't mandatory. +inurl="$3" +outname="$4" +lockfile="$2" +downloader="$1" +backupservers="$5" +if [ "x$downloader" = x ]; then + echo "$0: downloader (first argument) not given."; exit 1; +fi +if [ "x$lockfile" = x ]; then + echo "$0: lock file (second argument) not given."; exit 1; +fi +if [ "x$inurl" = x ]; then + echo "$0: full input URL (third argument) not given."; exit 1; +fi +if [ "x$outname" = x ]; then + echo "$0: output name (fourth argument) not given."; exit 1; +fi + + + + + +# Separate the actual filename, to possibly use backup server. +urlfile=$(echo "$inurl" | awk -F "/" '{print $NF}') + + + + + +# Try downloading multiple times before crashing. +counter=0 +maxcounter=10 +while [ ! -f "$outname" ]; do + + # Increment the counter. + counter=$(echo $counter | awk '{print $1+1}') + + # If we have passed a maximum number of trials, just exit with + # a failed code. + reachedmax=$(echo $counter \ + | awk '{if($1>'$maxcounter') print "yes"; else print "no";}') + if [ x$reachedmax = xyes ]; then + echo "" + echo "Failed $maxcounter download attempts: $outname" + echo "" + exit 1 + fi + + # If this isn't the first attempt print a notice and wait a little for + # the next trail. + if [ x$counter = x1 ]; then + just_a_place_holder=1 + else + tstep=$(echo $counter | awk '{print $1*5}') + echo "Download trial $counter for '$outname' in $tstep seconds." + sleep $tstep + fi + + # Attempt downloading the file. Note that the 'downloader' ends with + # the respective option to specify the output name. For example "wget + # -O" (so 'outname', that comes after it) will be the name of the + # downloaded file. + if [ x"$lockfile" = xnolock ]; then + if ! $downloader $outname $inurl; then rm -f $outname; fi + else + # Try downloading from the requested URL. + flock "$lockfile" sh -c \ + "if ! $downloader $outname \"$inurl\"; then rm -f $outname; fi" + fi + + # If the download failed, try the backup server(s). + if [ ! -f "$outname" ]; then + if [ x"$backupservers" != x ]; then + for bs in $backupservers; do + + # Use this backup server. + if [ x"$lockfile" = xnolock ]; then + if ! $downloader $outname $bs/$urlfile; then rm -f $outname; fi + else + flock "$lockfile" sh -c \ + "if ! $downloader $outname $bs/$urlfile; then rm -f $outname; fi" + fi + + # If the file was downloaded, break out of the loop that + # parses over the backup servers. + if [ -f "$outname" ]; then break; fi + done + fi + fi +done + + + + + +# Return successfully +exit 0 -- cgit v1.2.1