diff options
Diffstat (limited to 'reproduce/software/shell')
-rwxr-xr-x | reproduce/software/shell/bashrc.sh | 56 | ||||
-rwxr-xr-x | reproduce/software/shell/configure.sh | 168 | ||||
-rwxr-xr-x | reproduce/software/shell/git-post-checkout | 10 | ||||
-rwxr-xr-x | reproduce/software/shell/git-pre-commit | 10 | ||||
-rwxr-xr-x | reproduce/software/shell/pre-make-build.sh | 32 | ||||
-rwxr-xr-x | reproduce/software/shell/run-parts.in | 4 | ||||
-rwxr-xr-x | reproduce/software/shell/tarball-prepare.sh | 225 |
7 files changed, 437 insertions, 68 deletions
diff --git a/reproduce/software/shell/bashrc.sh b/reproduce/software/shell/bashrc.sh index 23845d6..814de2c 100755 --- a/reproduce/software/shell/bashrc.sh +++ b/reproduce/software/shell/bashrc.sh @@ -3,10 +3,10 @@ # To have better control over the environment of each analysis step (Make # recipe), besides having environment variables (directly included from # Make), it may also be useful to have a Bash startup file (this file). All -# of the Makefiles set this file as the `BASH_ENV' environment variable, so +# of the Makefiles set this file as the 'BASH_ENV' environment variable, so # it is loaded into all the Make recipes within the project. # -# The special `PROJECT_STATUS' environment variable is defined in every +# The special 'PROJECT_STATUS' environment variable is defined in every # top-level Makefile of the project. It defines the the state of the Make # that is calling this script. It can have three values: # @@ -28,8 +28,13 @@ # When doing the project's analysis: all software have known # versions. # +# shell +# ----- +# When the user has activated the interactive shell (with './project +# shell'). # -# Copyright (C) 2019-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# Copyright (C) 2019-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -43,3 +48,48 @@ # # You should have received a copy of the GNU General Public License # along with this script. If not, see <http://www.gnu.org/licenses/>. + + + + + +# Interactive mode settings. We don't want these within the pipeline +# because they are useless there (for example the introduction message or +# prompt) and can un-necessarily slow down the running jobs (recall that +# the shell is executed at the start of each recipe). +if [ x$PROJECT_STATUS = xshell ]; then + + # A small introductory message. + echo "----------------------------------------------------------------------" + echo "Welcome to the Maneage interactive shell for this project, running" + echo " $(sh --version | awk 'NR==1')" + echo + echo "This shell's home directory is the project's build directory:" + echo " HOME: $HOME" + echo + echo "This shell's startup file is in the project's source directory:" + echo " $PROJECT_RCFILE" + echo + echo "To return to your host shell, run the 'exit' command." + echo "----------------------------------------------------------------------" + + # To activate colors in generic commands. + alias ls='ls --color=auto' + alias grep='grep --color=auto' + + # Add the Git branch information to the command prompt only when Git is + # present. Also set the command-prompt color to purple for normal users + # and red when the root is running it. + if git --version &> /dev/null; then + parse_git_branch() { + git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/ (\1)/' + } + else + parse_git_branch() { echo &> /dev/null; } + fi + if [ x$(whoami) = xroot ]; then + export PS1="[\[\033[01;31m\]\u@\h \W\[\033[32m\]\$(parse_git_branch)\[\033[00m\]]# " + else + export PS1="[\[\033[01;35m\]maneage@\h \W\[\033[32m\]\$(parse_git_branch)\[\033[00m\]]$ " + fi +fi diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 0f7278f..4f71bee 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -2,8 +2,9 @@ # # Necessary preparations/configurations for the reproducible project. # -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2021 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2018-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2021-2023 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2022-2023 Pedram Ashofteh Ardakani <pedramardakani@pm.me> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -112,7 +113,7 @@ absolute_dir () # Check file permission handling (POSIX-compatibility) # ---------------------------------------------------- # -# Check if a `given' directory handles permissions as expected. +# Check if a 'given' directory handles permissions as expected. # # This is to prevent a known bug in the NTFS filesystem that prevents # proper installation of Perl, and probably some other packages. This @@ -120,15 +121,15 @@ absolute_dir () # file, and examines whether the given directory handles the file # permissions as expected. # -# Returns `0' if everything is fine, and `255' otherwise. Choosing `0' is -# to mimic the `$ echo $?' behavior, while choosing `255' is to prevent +# Returns '0' if everything is fine, and '255' otherwise. Choosing '0' is +# to mimic the '$ echo $?' behavior, while choosing '255' is to prevent # misunderstanding 0 and 1 as true and false. # # ===== CAUTION! ===== # # -# Since there is a `set -e' before running this function, the whole script -# stops and exits IF the `check_permission' (or any other function) returns -# anything OTHER than `0'! So, only use this function as a test. Here's a +# Since there is a 'set -e' before running this function, the whole script +# stops and exits IF the 'check_permission' (or any other function) returns +# anything OTHER than '0'! So, only use this function as a test. Here's a # minimal example: # # if $(check_permission $some_directory) ; then @@ -136,7 +137,7 @@ absolute_dir () # fi ; check_permission () { - # Make a `junk' file, activate its executable flag and record its + # Make a 'junk' file, activate its executable flag and record its # permissions generally. local junkfile="$1"/check_permission_tmp_file rm -f "$junkfile" @@ -190,7 +191,7 @@ free_space_warning() { fs_threshold=$1 fs_destpath="$2" - return $(df "$fs_destpath" \ + return $(df -P "$fs_destpath" \ | awk 'FNR==2 {if($4>'$fs_threshold') print 1; \ else print 0; }') } @@ -273,8 +274,17 @@ elif [ x$on_mac_os = xyes ]; then if [ x$hw_byteorder = x1234 ]; then byte_order="Little Endian"; elif [ x$hw_byteorder = x4321 ]; then byte_order="Big Endian"; fi - address_size_physical=$(sysctl -n machdep.cpu.address_bits.physical) - address_size_virtual=$(sysctl -n machdep.cpu.address_bits.virtual) + # On macOS, the way of obtaining the number of cores is different + # between Intel or Apple M1 CPUs. Here we disinguish between Apple M1 + # or others. + maccputype=$(sysctl -n machdep.cpu.brand_string) + if [ x"$maccputype" = x"Apple M1" ]; then + address_size_physical=$(sysctl -n machdep.cpu.thread_count) + address_size_virtual=$(sysctl -n machdep.cpu.logical_per_package) + else + address_size_physical=$(sysctl -n machdep.cpu.address_bits.physical) + address_size_virtual=$(sysctl -n machdep.cpu.address_bits.virtual) + fi address_sizes="$address_size_physical bits physical, " address_sizes+="$address_size_virtual bits virtual" else @@ -309,7 +319,9 @@ fi # system. Here, it is checked that this is the case, and if not, warn the user # about not having Xcode already installed. if [ x$on_mac_os = xyes ]; then - xcode=$(which xcodebuild) + + # 'which' isn't in POSIX, so we are using 'command -v' instead. + xcode=$(command -v xcodebuild) if [ x$xcode != x ]; then xcode_version=$(xcodebuild -version | grep Xcode) echo " " @@ -434,7 +446,9 @@ if ! [ -d $compilertestdir ]; then mkdir $compilertestdir; fi # also harmless for our test here, so it is generally added. testprog=$compilertestdir/test testsource=$compilertestdir/test.c -noccwarnings="-Wno-nullability-completeness" +if [ x$on_mac_os = xyes ]; then + noccwarnings="-Wno-nullability-completeness" +fi echo; echo; echo "Checking host C compiler ('$CC')..."; cat > $testsource <<EOF #include <stdio.h> @@ -475,8 +489,8 @@ fi # See if we need the dynamic-linker (-ldl) # ---------------------------------------- # -# Some programs (like Wget) need dynamic loading (using `libdl'). On -# GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But +# Some programs (like Wget) need dynamic loading (using 'libdl'). On +# GNU/Linux systems, we'll need the '-ldl' flag to link such programs. But # Mac OS doesn't need any explicit linking. So we'll check here to see if # it is present (thus necessary) or not. cat > $testsource <<EOF @@ -508,7 +522,7 @@ fi # programs will go and find their necessary libraries on the host system. # # Another good advantage of shared libraries is that we can actually use -# the shared library tool of the system (`ldd' with GNU C Library) and see +# the shared library tool of the system ('ldd' with GNU C Library) and see # exactly where each linked library comes from. But in static building, # unless you follow the build closely, its not easy to see if the source of # the library came from the system or our build. @@ -545,7 +559,7 @@ fi # # On some systems (in particular Debian-based OSs), the static C library # and necessary headers in a non-standard place, and we can't build GCC. So -# we need to find them first. The `sys/cdefs.h' header is also in a +# we need to find them first. The 'sys/cdefs.h' header is also in a # similarly different location. sys_cpath="" sys_library_path="" @@ -576,7 +590,7 @@ fi # See if a link-able static C library exists # ------------------------------------------ # -# A static C library and the `sys/cdefs.h' header are necessary for +# A static C library and the 'sys/cdefs.h' header are necessary for # building GCC. if [ x"$host_cc" = x0 ]; then echo; echo; echo "Checking if static C library is available..."; @@ -752,9 +766,9 @@ EOF # What to do with possibly existing configuration file # ---------------------------------------------------- # -# `LOCAL.conf' is the top-most local configuration for the project. If it +# 'LOCAL.conf' is the top-most local configuration for the project. If it # already exists when this script is run, we'll make a copy of it as backup -# (for example the user might have ran `./project configure' by mistake). +# (for example the user might have ran './project configure' by mistake). printnotice=yes rewritepconfig=yes if [ -f $pconf ]; then @@ -807,16 +821,31 @@ fi # need to check the host's available tool for downloading at this step. if [ $rewritepconfig = yes ]; then if type wget > /dev/null 2>/dev/null; then - name=$(which wget) + + # 'which' isn't in POSIX, so we are using 'command -v' instead. + name=$(command -v wget) + + # See if the host wget has the '--no-use-server-timestamps' option + # (for example wget 1.12 doesn't have it). If not, we'll have to + # remove it. This won't affect the analysis of Maneage in anyway, + # its just to avoid re-downloading if the server timestamps are + # bad; at the worst case, it will just cause a re-download of an + # input software source code (for data inputs, we will use our own + # wget that has this option). + tsname="no-use-server-timestamps" + tscheck=$(wget --help | grep $tsname || true) + if [ x"$tscheck" = x ]; then wgetts="" + else wgetts="--$tsname"; + fi # By default Wget keeps the remote file's timestamp, so we'll have # to disable it manually. - downloader="$name --no-use-server-timestamps -O"; + downloader="$name $wgetts -O"; elif type curl > /dev/null 2>/dev/null; then - name=$(which curl) + name=$(command -v curl) # - cURL doesn't keep the remote file's timestamp by default. - # - With the `-L' option, we tell cURL to follow redirects. + # - With the '-L' option, we tell cURL to follow redirects. downloader="$name -L -o" else cat <<EOF @@ -913,7 +942,7 @@ EOF # If it was newly created, it will be empty, so delete it. if ! [ "$(ls -A $bdir)" ]; then rm --dir "$bdir"; fi - # Inform the user that this is not acceptable and reset `bdir'. + # Inform the user that this is not acceptable and reset 'bdir'. bdir= echo " ** The build-directory cannot be under the source-directory." fi @@ -960,7 +989,7 @@ EOF fi # If the build directory was good, the loop will stop, if not, - # reset `build_dir' to blank, so it continues asking for another + # reset 'build_dir' to blank, so it continues asking for another # directory and let the user know that they must select a new # directory. if [ x"$bdir" = x ]; then @@ -1118,7 +1147,7 @@ else if [ x"$downloader" = x ]; then novalue="$novalue"DOWNLOADER; fi if [ x"$novalue" != x ]; then verr=1; err=1; fi - # Make sure `bdir' is an absolute path and it exists. + # Make sure 'bdir' is an absolute path and it exists. berr=0 ierr=0 bdir="$(absolute_dir "$inbdir")" @@ -1202,6 +1231,10 @@ if ! [ -d "$ibidir" ]; then mkdir "$ibidir"; fi ipydir="$verdir"/python if ! [ -d "$ipydir" ]; then mkdir "$ipydir"; fi +# R module versions and citation. +ircrandir="$verdir"/r-cran +if ! [ -d "$ircrandir" ]; then mkdir "$ircrandir"; fi + # Used software BibTeX entries. ictdir="$verdir"/cite if ! [ -d "$ictdir" ]; then mkdir "$ictdir"; fi @@ -1210,6 +1243,15 @@ if ! [ -d "$ictdir" ]; then mkdir "$ictdir"; fi itidir="$verdir"/tex if ! [ -d "$itidir" ]; then mkdir "$itidir"; fi +# Some software install their libraries in '$(idir)/lib64'. But all other +# libraries are in '$(idir)/lib'. Since Maneage's build is only for a +# single architecture, we can set the '$(idir)/lib64' as a symbolic link to +# '$(idir)/lib' so all the libraries are always available in the same +# place. +instlibdir="$instdir"/lib +if ! [ -d "$instlibdir" ]; then mkdir "$instlibdir"; fi +ln -fs "$instlibdir" "$instdir"/lib64 + @@ -1312,38 +1354,69 @@ if [ x"$shmdir" != x ]; then # Set the Maneage-specific directory within the shared # memory. We'll use the names of the two parent directories to the - # current/running directory, separated by a `-' instead of - # `/'. We'll then appended that with the user's name (in case + # current/running directory, separated by a '-' instead of + # '/'. We'll then appended that with the user's name (in case # multiple users may be working on similar project names). # - # Maybe later, we can use something like `mktemp' to add random + # Maybe later, we can use something like 'mktemp' to add random # characters to this name and make it unique to every run (even for # a single user). dirname=$(pwd | sed -e's/\// /g' \ | awk '{l=NF-1; printf("%s-%s", $l, $NF)}') tbshmdir="$shmdir"/"$dirname"-$(whoami) - if ! [ -d "$tbshmdir" ]; then mkdir "$tbshmdir"; fi + + # Try to make the directory if it does not yet exist. A failed + # directory creation will be tested for a few lines later, when + # testing for the existence and executability of a test file. + if ! [ -d "$tbshmdir" ]; then (mkdir "$tbshmdir" || true); fi # Some systems may protect '/dev/shm' against the right to execute # programs by ordinary users. We thus need to check that the device # allows execution within this directory by this user. shmexecfile="$tbshmdir"/shm-execution-check.sh rm -f $shmexecfile # We also don't want any existing flags. - cat > "$shmexecfile" <<EOF + + # Create the file to be executed, but do not fail fatally if it + # cannot be created. We will check a few lines later if the file + # really exists. + (cat > "$shmexecfile" <<EOF || true) #!/bin/sh -printf "This file successfully executed.\n" +echo "This file successfully executed." EOF - # Make the file executable and see if it runs. If not, set - # 'tbshmdir' to an empty string so it is not used in later steps. - # In any case, delete the temporary file afterwards. - chmod u+x "$shmexecfile" - if ! "$shmexecfile" &> /dev/null; then tbshmdir=""; fi - rm "$shmexecfile" + + # If the file was successfully created, then make the file + # executable and see if it runs. If not, set 'tbshmdir' to an empty + # string so it is not used in later steps. In any case, delete the + # temporary file afterwards. + # + # We aren't adding '&> /dev/null' after the execution command + # because it can produce false failures randomly on some systems. + if [ -e "$shmexecfile" ]; then + + # Add the executable flag. + chmod +x "$shmexecfile" + + # The following line tries to execute the file. + if "$shmexecfile"; then + # Successful execution. The colon is a "no-op" (no + # operation) shell command. + : + else + tbshmdir="" + fi + rm "$shmexecfile" + else + tbshmdir="" + fi fi else tbshmdir="" fi + + + + # If a shared memory directory was created, set the software building # directory to be a symbolic link to it. Otherwise, just build the # temporary build directory under the project's build directory. @@ -1426,7 +1499,7 @@ fi # See if the linker accepts -Wl,-rpath-link # ----------------------------------------- # -# `-rpath-link' is used to write the information of the linked shared +# '-rpath-link' is used to write the information of the linked shared # library into the shared object (library or program). But some versions of # LLVM's linker don't accept it an can cause problems. # @@ -1459,13 +1532,13 @@ rm -rf $compilertestdir -# Paths needed by the host compiler (only for `basic.mk') +# Paths needed by the host compiler (only for 'basic.mk') # ------------------------------------------------------- # # At the end of the basic build, we need to build GCC. But GCC will build # in multiple phases, making its own simple compiler in order to build # itself completely. The intermediate/simple compiler doesn't recognize -# some system specific locations like `/usr/lib/ARCHITECTURE' that some +# some system specific locations like '/usr/lib/ARCHITECTURE' that some # operating systems use. We thus need to tell the intermediate compiler # where its necessary libraries and headers are. if [ x"$sys_library_path" != x ]; then @@ -1502,7 +1575,7 @@ fi # which will download the DOI-resolved webpage, and extract the Zenodo-URL # of the most recent version from there (using the 'coreutils' tarball as # an example, the directory part of the URL for all the other software are -# the same). This is not done if the option `--debug' is used. +# the same). This is not done if the option '--debug' is used. zenodourl="" user_backup_urls="" zenodocheck=.build/software/zenodo-check.html @@ -1535,6 +1608,7 @@ user_backup_urls="$user_backup_urls $zenodourl" # (compression program), GNU Make (that 'basic.mk' is written in), Dash # (minimal Bash-like shell) and Flock (to lock files and enable serial # download). +export on_mac_os ./reproduce/software/shell/pre-make-build.sh \ "$bdir" "$ddir" "$downloader" "$user_backup_urls" @@ -1672,7 +1746,7 @@ fi # software. prepare_name_version () { - # First see if the (possible) `*' in the input arguments corresponds to + # First see if the (possible) '*' in the input arguments corresponds to # anything. Note that some of the given directories may be empty (no # software installed). hasfiles=0 @@ -1773,9 +1847,9 @@ hw_class_fixed="$(echo $hw_class | sed -e 's/_/\\_/')" # --------------------------------- # # By the time the script reaches here the temporary software build -# directory should be empty, so just delete it. Note `tmpblddir' may be a +# directory should be empty, so just delete it. Note 'tmpblddir' may be a # symbolic link to shared memory. So, to work in any scenario, first delete -# the contents of the directory (if it has any), then delete `tmpblddir'. +# the contents of the directory (if it has any), then delete 'tmpblddir'. .local/bin/rm -rf $tmpblddir/* $tmpblddir diff --git a/reproduce/software/shell/git-post-checkout b/reproduce/software/shell/git-post-checkout index 7a90108..a1340d7 100755 --- a/reproduce/software/shell/git-post-checkout +++ b/reproduce/software/shell/git-post-checkout @@ -1,14 +1,14 @@ #!@BINDIR@/bash # # The example hook script to store the metadata information of version -# controlled files (with each commit) using the `metastore' program. +# controlled files (with each commit) using the 'metastore' program. # # Copyright (C) 2016 Przemyslaw Pawelczyk <przemoc@gmail.com> -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # -# This script is taken from the `examples/hooks/pre-commit' file of the -# `metastore' package (installed within the project, with an MIT license -# for copyright). We have just changed the name of the `MSFILE' and also +# This script is taken from the 'examples/hooks/pre-commit' file of the +# 'metastore' package (installed within the project, with an MIT license +# for copyright). We have just changed the name of the 'MSFILE' and also # set special characters for the installation location of meta-store so our # own installation is found by Git. # diff --git a/reproduce/software/shell/git-pre-commit b/reproduce/software/shell/git-pre-commit index 85d3474..02ba488 100755 --- a/reproduce/software/shell/git-pre-commit +++ b/reproduce/software/shell/git-pre-commit @@ -1,10 +1,10 @@ #!@BINDIR@/bash # # The example hook script to store the metadata information of version -# controlled files (with each commit) using the `metastore' program. +# controlled files (with each commit) using the 'metastore' program. # # Copyright (C) 2016 Przemyslaw Pawelczyk <przemoc@gmail.com> -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # WARNING: # @@ -17,9 +17,9 @@ # git reset HEAD -- .metadata # git checkout HEAD -- .metadata # -# This script is taken from the `examples/hooks/pre-commit' file of the -# `metastore' package (installed within the project, with an MIT license -# for copyright). Here, the name of the `MSFILE' and also set special +# This script is taken from the 'examples/hooks/pre-commit' file of the +# 'metastore' package (installed within the project, with an MIT license +# for copyright). Here, the name of the 'MSFILE' and also set special # characters for the installation location of meta-store so our own # installation is found by Git. # diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh index a033963..808429b 100755 --- a/reproduce/software/shell/pre-make-build.sh +++ b/reproduce/software/shell/pre-make-build.sh @@ -2,7 +2,7 @@ # # Very basic tools necessary to start Maneage's default building. # -# Copyright (C) 2020-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2020-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -185,6 +185,19 @@ build_program() { # All others accept the configure script. ./configure --prefix="$instdir" $configoptions + # In Flock 0.4.0 there is a crash that can be fixed by simply + # replacing '%1u' with '%ld' on GNU/Linux and '%d' on macOS. This + # has been reported to flock maintainers: + # https://github.com/discoteq/flock/issues/33 + if [ x$progname = xflock ]; then + case $on_mac_os in + yes) sed -e's/\%1u/\%d/' src/flock.c > src/flock-new.c;; + no) sed -e's/\%1u/\%ld/' src/flock.c > src/flock-new.c;; + *) echo "pre-make-build.sh: '$on_mac_os' unrecognized value for on_mac_os";; + esac + mv src/flock-new.c src/flock.c + fi + # To build GNU Make, we don't want to assume the existance of a # Make program, so we use its 'build.sh' script and its own built # 'make' program to install itself. @@ -192,7 +205,7 @@ build_program() { /bin/sh build.sh ./make install else - make + make V=1 make install fi fi @@ -217,7 +230,7 @@ build_program() { # (without compression it is just ~400Kb). So we use its '.tar' file and # won't rely on the host's compression tools at all. progname="lzip" -progname_tex="Lzip" +progname_tex="" # Lzip re-built after GCC (empty string to avoid repetition) url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' "$versionsfile") tarball=$progname-$version.tar @@ -235,13 +248,20 @@ build_program # '--disable-dependency-tracking' configure-time option is necessary so # Make doesn't check for an existing 'make' implementation (recall that we # aren't assuming any 'make' on the host). +# +# If GNU Guile is already present on the host system, Make will try to link +# with it, and this will cause dependency problems later. So we have +# distabled Guile. If a project needs the Guile extensions of Make, we need +# to add a build rule for Guile in Maneage, with a special Guile-enabled +# Make that has a different executable name (using the '--program-prefix=' +# configure option) from the "default" make (which is this one!). progname="make" progname_tex="GNU Make" url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) tarball=$progname-$version.tar.lz download_tarball -build_program --disable-dependency-tracking +build_program "--disable-dependency-tracking --without-guile" @@ -274,11 +294,11 @@ fi # ----- # # Flock (or file-lock) is necessary to serialize operations when -# necessary. GNU/Linux machines have it as part of their `util-linux' +# necessary. GNU/Linux machines have it as part of their 'util-linux' # programs. But to be consistent in non-GNU/Linux systems, we will be using # our own build. # -# The reason that `flock' is built here is that generally the building of +# The reason that 'flock' is built here is that generally the building of # software is done in parallel, but we need it to serialize the download # process of the software tarballs to avoid network complications when too # many simultaneous download commands are called. diff --git a/reproduce/software/shell/run-parts.in b/reproduce/software/shell/run-parts.in index 7e649b1..a7b7705 100755 --- a/reproduce/software/shell/run-parts.in +++ b/reproduce/software/shell/run-parts.in @@ -10,8 +10,8 @@ # However, it didn't have a copyright statement. So one is being added # here. # -# Copyright (C) 2021 Authors mentioned above. -# Copyright (C) 2020-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2023 Authors mentioned above. +# Copyright (C) 2020-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/shell/tarball-prepare.sh b/reproduce/software/shell/tarball-prepare.sh new file mode 100755 index 0000000..472b4c0 --- /dev/null +++ b/reproduce/software/shell/tarball-prepare.sh @@ -0,0 +1,225 @@ +#!/bin/bash + +# Script to convert all files (tarballs in any format; just recognized +# by 'tar') within an 'odir' to a unified '.tar.lz' format. +# +# The inputs are assumed to be formatted with 'NAME_VERSION', and only for +# the names, we are currently assuming '.tar.*' (for the 'sed' +# command). Please modify/generalize accordingly. +# +# It will unpack the source in a certain empty directory with the +# 'tmpunpack' suffix, and rename the top directory to the requested format +# of NAME-VERSION also. So irrespective of the name of the top original +# tarball directory, the resulting tarball's top directory will have a name +# formatting of NAME-VERSION. +# +# Discussion: https://savannah.nongnu.org/task/?15699 +# +# Copyright (C) 2022-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2022-2023 Pedram Ashofteh Ardakani <pedramardakani@pm.me> +# Released under GNU GPLv3+ + +# Abort the script in case of an error. +set -e + + + + + +# Default arguments +odir= +idir= +quiet= +basedir=$PWD +scriptname=$0 + + +# The --help output +print_help() { + cat <<EOF +Usage: $scriptname [OPTIONS] + +Low-level script to create maneage-standard tarballs. + -o, --output-dir Target directory to write the packed tarballs. + Current: $odir + -i, --input-dir Directory containing original tarballs. + Current: $idir + -q, --quiet Suppress logging information. Only print the + final packed file and its sha512sum. +Maneage URL: https://maneage.org + +Report bugs: https://savannah.nongnu.org/bugs/?group=reproduce +EOF +} + + + + + +# Functions to check option values and complain if necessary. +on_off_option_error() { + if [ x"$2" = x ]; then + echo "$scriptname: '$1' doesn't take any values" + else + echo "$scriptname: '$1' (or '$2') doesn't take any values" + fi + exit 1 +} + +check_v() { + if [ x"$2" = x ]; then + cat <<EOF +$scriptname: option '$1' requires an argument. Try '$scriptname --help' for more information +EOF + exit 1; + fi +} + +option_given_and_valid() { + dirname="$1" + optionlong="$2" + optionshort="$3" + if [ x"$dirname" = x ]; then + cat <<EOF +$scriptname: no '--$optionlong' (or '-$optionshort') given: use this for identifying the directory containing the input tarballs +EOF + exit 1 + else + dirname=$(echo "$dirname" | sed 's|/$||'); # Remove possible trailing slash + if [ ! -d "$dirname" ]; then + cat <<EOF +$scriptname: '$dirname' that is given to '--$optionlong' (or '-$optionshort') couldn't be opened +EOF + exit 1 + else + outdir=$(realpath $dirname) + fi + fi + ogvout=$outdir +} + + + + + +# Parse the arguments +while [ $# -gt 0 ] +do + case $1 in + # Input and Output directories + -i|--input-dir) idir="$2"; check_v "$1" "$idir"; shift;shift;; + -i=*|--input-dir=*) idir="${1#*=}"; check_v "$1" "$idir"; shift;; + -i*) idir=$(echo "$1" | sed -e's/-i//'); check_v "$1" "$idir"; shift;; + -o|--output-dir) odir="$2"; check_v "$1" "$odir"; shift;shift;; + -o=*|--output-dir=*) odir="${1#*=}"; check_v "$1" "$odir"; shift;; + -o*) odir=$(echo "$1" | sed -e's/-o//'); check_v "$1" "$odir"; shift;; + + # Operating mode options + -?|--help) print_help; exit 0;; + -'?'*|--help=*) on_off_option_error --help -?;; + -q|--quiet) quiet=1; shift;; + -q*|--quiet=*) on_off_option_error --quiet -q;; + *) echo "$scriptname: unknown option '$1'"; exit 1;; + esac +done + + + + + +# Basic sanity checks +# +# Make sure the input and output directories are given. Also extract +# the absolute path to input and output directories and remove any +# possible trailing '/'. Working with a relative path is a great +# source of confusion and unwanted side-effects like moving/removing +# files by accident. +option_given_and_valid "$idir" "input-dir" "i" && idir=$ogvout +option_given_and_valid "$odir" "output-dir" "o" && odir=$ogvout + + + + + +# Unpack and pack all files in the '$idir' +# ---------------------------------------- +allfiles=$(ls $idir | sort) + +# Let user know number of tarballs if its not in quiet mode +if [ -z $quiet ]; then + nfiles=$(ls $idir | wc -l) + echo "Found $nfiles file(s) in '$idir/'" +fi + +# Process all files +for f in $allfiles; do + + # Extract the name and version (while replacing any possible '_' with + # '-' because some software separate name and version with '_'). + name=$(echo $(basename $f) \ + | sed -e 's/.tar.*//' -e's/_/-/') + + # Skip previously packed files + if [ -f $odir/$name.tar.lz ]; then + + # Print the info message if not in quiet mode + if [ -z $quiet ]; then + echo "$scriptname: $odir/$name.tar.lz: already present in output directory" + fi + + # skip this file + continue + else + + # Print the info message if not in quiet mode + if [ -z $quiet ]; then + echo "$scriptname: processing '$idir/$f'" + fi + fi + + # Create a temporary directory name + tmpdir=$odir/$name-tmpunpack + + # If the temporary directory exists, mkdir will throw an error. The + # developer needs to intervene manually to fix the issue. + mkdir $tmpdir + + + + + + # Move into the temporary directory + # --------------------------------- + # + # The default output directory for all the following commands: $tmpdir + cd $tmpdir + + # Unpack + tar -xf $idir/$f + + # Make sure the unpacked tarball is contained within a directory with + # the clean program name + if [ ! -d "$name" ]; then + mv * $name/ + fi + + # Put the current date on all the files because some packagers will not + # add dates to their release tarballs, resulting in dates of the + # Unix-time zero'th second (1970-01-01 at 00:00:00)! + touch $(find "$name"/ -type f) + + # Pack with recommended options + tar -c -Hustar --owner=root --group=root \ + -f $name.tar $name/ + lzip -9 $name.tar + + # Move the compressed file from the temporary directory to the target + # output directory + mv $name.tar.lz $odir/ + + # Print the sha512sum along with the filename for a quick reference + echo $(sha512sum $odir/$name.tar.lz) + + # Clean up the temporary directory + rm -rf $tmpdir +done |