diff options
Diffstat (limited to 'reproduce/software/shell')
-rwxr-xr-x | reproduce/software/shell/bashrc.sh | 56 | ||||
-rwxr-xr-x | reproduce/software/shell/configure.sh | 315 | ||||
-rwxr-xr-x | reproduce/software/shell/git-post-checkout | 10 | ||||
-rwxr-xr-x | reproduce/software/shell/git-pre-commit | 10 | ||||
-rwxr-xr-x | reproduce/software/shell/pre-make-build.sh | 32 | ||||
-rwxr-xr-x | reproduce/software/shell/run-parts.in | 4 | ||||
-rwxr-xr-x | reproduce/software/shell/tarball-prepare.sh | 225 |
7 files changed, 546 insertions, 106 deletions
diff --git a/reproduce/software/shell/bashrc.sh b/reproduce/software/shell/bashrc.sh index 23845d6..814de2c 100755 --- a/reproduce/software/shell/bashrc.sh +++ b/reproduce/software/shell/bashrc.sh @@ -3,10 +3,10 @@ # To have better control over the environment of each analysis step (Make # recipe), besides having environment variables (directly included from # Make), it may also be useful to have a Bash startup file (this file). All -# of the Makefiles set this file as the `BASH_ENV' environment variable, so +# of the Makefiles set this file as the 'BASH_ENV' environment variable, so # it is loaded into all the Make recipes within the project. # -# The special `PROJECT_STATUS' environment variable is defined in every +# The special 'PROJECT_STATUS' environment variable is defined in every # top-level Makefile of the project. It defines the the state of the Make # that is calling this script. It can have three values: # @@ -28,8 +28,13 @@ # When doing the project's analysis: all software have known # versions. # +# shell +# ----- +# When the user has activated the interactive shell (with './project +# shell'). # -# Copyright (C) 2019-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# Copyright (C) 2019-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -43,3 +48,48 @@ # # You should have received a copy of the GNU General Public License # along with this script. If not, see <http://www.gnu.org/licenses/>. + + + + + +# Interactive mode settings. We don't want these within the pipeline +# because they are useless there (for example the introduction message or +# prompt) and can un-necessarily slow down the running jobs (recall that +# the shell is executed at the start of each recipe). +if [ x$PROJECT_STATUS = xshell ]; then + + # A small introductory message. + echo "----------------------------------------------------------------------" + echo "Welcome to the Maneage interactive shell for this project, running" + echo " $(sh --version | awk 'NR==1')" + echo + echo "This shell's home directory is the project's build directory:" + echo " HOME: $HOME" + echo + echo "This shell's startup file is in the project's source directory:" + echo " $PROJECT_RCFILE" + echo + echo "To return to your host shell, run the 'exit' command." + echo "----------------------------------------------------------------------" + + # To activate colors in generic commands. + alias ls='ls --color=auto' + alias grep='grep --color=auto' + + # Add the Git branch information to the command prompt only when Git is + # present. Also set the command-prompt color to purple for normal users + # and red when the root is running it. + if git --version &> /dev/null; then + parse_git_branch() { + git branch 2> /dev/null | sed -e '/^[^*]/d' -e 's/* \(.*\)/ (\1)/' + } + else + parse_git_branch() { echo &> /dev/null; } + fi + if [ x$(whoami) = xroot ]; then + export PS1="[\[\033[01;31m\]\u@\h \W\[\033[32m\]\$(parse_git_branch)\[\033[00m\]]# " + else + export PS1="[\[\033[01;35m\]maneage@\h \W\[\033[32m\]\$(parse_git_branch)\[\033[00m\]]$ " + fi +fi diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 812f3d3..4f71bee 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -2,7 +2,9 @@ # # Necessary preparations/configurations for the reproducible project. # -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2021-2023 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2022-2023 Pedram Ashofteh Ardakani <pedramardakani@pm.me> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -111,7 +113,7 @@ absolute_dir () # Check file permission handling (POSIX-compatibility) # ---------------------------------------------------- # -# Check if a `given' directory handles permissions as expected. +# Check if a 'given' directory handles permissions as expected. # # This is to prevent a known bug in the NTFS filesystem that prevents # proper installation of Perl, and probably some other packages. This @@ -119,15 +121,15 @@ absolute_dir () # file, and examines whether the given directory handles the file # permissions as expected. # -# Returns `0' if everything is fine, and `255' otherwise. Choosing `0' is -# to mimic the `$ echo $?' behavior, while choosing `255' is to prevent +# Returns '0' if everything is fine, and '255' otherwise. Choosing '0' is +# to mimic the '$ echo $?' behavior, while choosing '255' is to prevent # misunderstanding 0 and 1 as true and false. # # ===== CAUTION! ===== # # -# Since there is a `set -e' before running this function, the whole script -# stops and exits IF the `check_permission' (or any other function) returns -# anything OTHER than `0'! So, only use this function as a test. Here's a +# Since there is a 'set -e' before running this function, the whole script +# stops and exits IF the 'check_permission' (or any other function) returns +# anything OTHER than '0'! So, only use this function as a test. Here's a # minimal example: # # if $(check_permission $some_directory) ; then @@ -135,7 +137,7 @@ absolute_dir () # fi ; check_permission () { - # Make a `junk' file, activate its executable flag and record its + # Make a 'junk' file, activate its executable flag and record its # permissions generally. local junkfile="$1"/check_permission_tmp_file rm -f "$junkfile" @@ -157,7 +159,7 @@ check_permission () return 1 else # Setting permission SUCCESSFUL - return 0 + return 0 fi } @@ -189,7 +191,7 @@ free_space_warning() { fs_threshold=$1 fs_destpath="$2" - return $(df "$fs_destpath" \ + return $(df -P "$fs_destpath" \ | awk 'FNR==2 {if($4>'$fs_threshold') print 1; \ else print 0; }') } @@ -272,8 +274,17 @@ elif [ x$on_mac_os = xyes ]; then if [ x$hw_byteorder = x1234 ]; then byte_order="Little Endian"; elif [ x$hw_byteorder = x4321 ]; then byte_order="Big Endian"; fi - address_size_physical=$(sysctl -n machdep.cpu.address_bits.physical) - address_size_virtual=$(sysctl -n machdep.cpu.address_bits.virtual) + # On macOS, the way of obtaining the number of cores is different + # between Intel or Apple M1 CPUs. Here we disinguish between Apple M1 + # or others. + maccputype=$(sysctl -n machdep.cpu.brand_string) + if [ x"$maccputype" = x"Apple M1" ]; then + address_size_physical=$(sysctl -n machdep.cpu.thread_count) + address_size_virtual=$(sysctl -n machdep.cpu.logical_per_package) + else + address_size_physical=$(sysctl -n machdep.cpu.address_bits.physical) + address_size_virtual=$(sysctl -n machdep.cpu.address_bits.virtual) + fi address_sizes="$address_size_physical bits physical, " address_sizes+="$address_size_virtual bits virtual" else @@ -308,7 +319,9 @@ fi # system. Here, it is checked that this is the case, and if not, warn the user # about not having Xcode already installed. if [ x$on_mac_os = xyes ]; then - xcode=$(which xcodebuild) + + # 'which' isn't in POSIX, so we are using 'command -v' instead. + xcode=$(command -v xcodebuild) if [ x$xcode != x ]; then xcode_version=$(xcodebuild -version | grep Xcode) echo " " @@ -423,8 +436,19 @@ if ! [ -d $compilertestdir ]; then mkdir $compilertestdir; fi # Check C compiler # ---------------- +# +# Here we check if the C compiler works properly. About the "no warning" +# variable ('nowarnings'): +# +# -Wno-nullability-completeness: on macOS Big Sur 11.2.3 and Xcode 12.4, +# hundreds of 'nullability-completeness' warnings are printed which can +# be very annoying and even hide important errors or warnings. It is +# also harmless for our test here, so it is generally added. testprog=$compilertestdir/test testsource=$compilertestdir/test.c +if [ x$on_mac_os = xyes ]; then + noccwarnings="-Wno-nullability-completeness" +fi echo; echo; echo "Checking host C compiler ('$CC')..."; cat > $testsource <<EOF #include <stdio.h> @@ -432,7 +456,7 @@ cat > $testsource <<EOF int main(void){printf("...C compiler works.\n"); return EXIT_SUCCESS;} EOF -if $CC $testsource -o$testprog && $testprog; then +if $CC $noccwarnings $testsource -o$testprog && $testprog; then rm $testsource $testprog else rm $testsource @@ -465,8 +489,8 @@ fi # See if we need the dynamic-linker (-ldl) # ---------------------------------------- # -# Some programs (like Wget) need dynamic loading (using `libdl'). On -# GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But +# Some programs (like Wget) need dynamic loading (using 'libdl'). On +# GNU/Linux systems, we'll need the '-ldl' flag to link such programs. But # Mac OS doesn't need any explicit linking. So we'll check here to see if # it is present (thus necessary) or not. cat > $testsource <<EOF @@ -498,7 +522,7 @@ fi # programs will go and find their necessary libraries on the host system. # # Another good advantage of shared libraries is that we can actually use -# the shared library tool of the system (`ldd' with GNU C Library) and see +# the shared library tool of the system ('ldd' with GNU C Library) and see # exactly where each linked library comes from. But in static building, # unless you follow the build closely, its not easy to see if the source of # the library came from the system or our build. @@ -535,7 +559,7 @@ fi # # On some systems (in particular Debian-based OSs), the static C library # and necessary headers in a non-standard place, and we can't build GCC. So -# we need to find them first. The `sys/cdefs.h' header is also in a +# we need to find them first. The 'sys/cdefs.h' header is also in a # similarly different location. sys_cpath="" sys_library_path="" @@ -566,7 +590,7 @@ fi # See if a link-able static C library exists # ------------------------------------------ # -# A static C library and the `sys/cdefs.h' header are necessary for +# A static C library and the 'sys/cdefs.h' header are necessary for # building GCC. if [ x"$host_cc" = x0 ]; then echo; echo; echo "Checking if static C library is available..."; @@ -742,9 +766,9 @@ EOF # What to do with possibly existing configuration file # ---------------------------------------------------- # -# `LOCAL.conf' is the top-most local configuration for the project. If it +# 'LOCAL.conf' is the top-most local configuration for the project. If it # already exists when this script is run, we'll make a copy of it as backup -# (for example the user might have ran `./project configure' by mistake). +# (for example the user might have ran './project configure' by mistake). printnotice=yes rewritepconfig=yes if [ -f $pconf ]; then @@ -761,7 +785,7 @@ fi # exists and we don't want to re-write it). if [ $rewritepconfig = no ]; then oldgroupname=$(awk '/GROUP-NAME/ {print $3; exit 0}' $pconf) - if [ "x$oldgroupname" = "x$reproducible_paper_group_name" ]; then + if [ "x$oldgroupname" = "x$maneage_group_name" ]; then just_a_place_holder_to_avoid_not_equal_test=1; else echo "-----------------------------" @@ -797,16 +821,31 @@ fi # need to check the host's available tool for downloading at this step. if [ $rewritepconfig = yes ]; then if type wget > /dev/null 2>/dev/null; then - name=$(which wget) + + # 'which' isn't in POSIX, so we are using 'command -v' instead. + name=$(command -v wget) + + # See if the host wget has the '--no-use-server-timestamps' option + # (for example wget 1.12 doesn't have it). If not, we'll have to + # remove it. This won't affect the analysis of Maneage in anyway, + # its just to avoid re-downloading if the server timestamps are + # bad; at the worst case, it will just cause a re-download of an + # input software source code (for data inputs, we will use our own + # wget that has this option). + tsname="no-use-server-timestamps" + tscheck=$(wget --help | grep $tsname || true) + if [ x"$tscheck" = x ]; then wgetts="" + else wgetts="--$tsname"; + fi # By default Wget keeps the remote file's timestamp, so we'll have # to disable it manually. - downloader="$name --no-use-server-timestamps -O"; + downloader="$name $wgetts -O"; elif type curl > /dev/null 2>/dev/null; then - name=$(which curl) + name=$(command -v curl) # - cURL doesn't keep the remote file's timestamp by default. - # - With the `-L' option, we tell cURL to follow redirects. + # - With the '-L' option, we tell cURL to follow redirects. downloader="$name -L -o" else cat <<EOF @@ -903,7 +942,7 @@ EOF # If it was newly created, it will be empty, so delete it. if ! [ "$(ls -A $bdir)" ]; then rm --dir "$bdir"; fi - # Inform the user that this is not acceptable and reset `bdir'. + # Inform the user that this is not acceptable and reset 'bdir'. bdir= echo " ** The build-directory cannot be under the source-directory." fi @@ -928,10 +967,10 @@ EOF fi fi - # If everything is still fine so far, see if we're able to - # manipulate file permissions in the directory's filesystem and if - # so, see if there is atleast 5GB free space. - if ! [ x"$bdir" = x ]; then + # If everything is still fine so far, see if we're able to + # manipulate file permissions in the directory's filesystem and if + # so, see if there is atleast 5GB free space. + if ! [ x"$bdir" = x ]; then if ! $(check_permission "$bdir"); then # Unable to handle permissions well bdir= @@ -950,7 +989,7 @@ EOF fi # If the build directory was good, the loop will stop, if not, - # reset `build_dir' to blank, so it continues asking for another + # reset 'build_dir' to blank, so it continues asking for another # directory and let the user know that they must select a new # directory. if [ x"$bdir" = x ]; then @@ -1079,7 +1118,7 @@ if [ $rewritepconfig = yes ]; then -e's|@ddir[@]|'"$ddir"'|' \ -e's|@sys_cpath[@]|'"$sys_cpath"'|' \ -e's|@downloader[@]|'"$downloader"'|' \ - -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \ + -e's|@groupname[@]|'"$maneage_group_name"'|' \ $pconf.in >> $pconf else # Read the values from existing configuration file. Note that the build @@ -1108,7 +1147,7 @@ else if [ x"$downloader" = x ]; then novalue="$novalue"DOWNLOADER; fi if [ x"$novalue" != x ]; then verr=1; err=1; fi - # Make sure `bdir' is an absolute path and it exists. + # Make sure 'bdir' is an absolute path and it exists. berr=0 ierr=0 bdir="$(absolute_dir "$inbdir")" @@ -1192,6 +1231,10 @@ if ! [ -d "$ibidir" ]; then mkdir "$ibidir"; fi ipydir="$verdir"/python if ! [ -d "$ipydir" ]; then mkdir "$ipydir"; fi +# R module versions and citation. +ircrandir="$verdir"/r-cran +if ! [ -d "$ircrandir" ]; then mkdir "$ircrandir"; fi + # Used software BibTeX entries. ictdir="$verdir"/cite if ! [ -d "$ictdir" ]; then mkdir "$ictdir"; fi @@ -1200,21 +1243,14 @@ if ! [ -d "$ictdir" ]; then mkdir "$ictdir"; fi itidir="$verdir"/tex if ! [ -d "$itidir" ]; then mkdir "$itidir"; fi -# Temporary software un-packing/build directory: if the host has the -# standard `/dev/shm' mounting-point, we'll do it in shared memory (on the -# RAM), to avoid harming/over-using the HDDs/SSDs. The RAM of most systems -# today (>8GB) is large enough for the parallel building of the software. -# -# For the name of the directory under `/dev/shm' (for this project), we'll -# use the names of the two parent directories to the current/running -# directory, separated by a `-' instead of `/'. We'll then appended that -# with the user's name (in case multiple users may be working on similar -# project names). Maybe later, we can use something like `mktemp' to add -# random characters to this name and make it unique to every run (even for -# a single user). -tmpblddir="$sdir"/build-tmp -rm -rf "$tmpblddir"/* "$tmpblddir" # If its a link, we need to empty its - # contents first, then itself. +# Some software install their libraries in '$(idir)/lib64'. But all other +# libraries are in '$(idir)/lib'. Since Maneage's build is only for a +# single architecture, we can set the '$(idir)/lib64' as a symbolic link to +# '$(idir)/lib' so all the libraries are always available in the same +# place. +instlibdir="$instdir"/lib +if ! [ -d "$instlibdir" ]; then mkdir "$instlibdir"; fi +ln -fs "$instlibdir" "$instdir"/lib64 @@ -1238,7 +1274,7 @@ if ! [ -d "$mtexdir" ]; then mkdir "$mtexdir"; fi # TeX build directory. If built in a group scenario, the TeX build # directory must be separate for each member (so they can work on their # relevant parts of the paper without conflicting with each other). -if [ "x$reproducible_paper_group_name" = x ]; then +if [ "x$maneage_group_name" = x ]; then texbdir="$texdir"/build else user=$(whoami) @@ -1287,29 +1323,106 @@ rm -f .gnuastro -# Set the top-level shared memory location. + + +# Software building directory (possibly in RAM) +# --------------------------------------------- +# +# Building the software for the project will need the creation of many +# small temporary files that will ultimately be deleted. To avoid harming +# HDDs/SSDs and improve speed, it is therefore better to build them in the +# RAM when possible. The RAM of most systems today (>8GB) is large enough +# for the parallel building of the software. + +# Set the top-level shared memory location. Currently there is only one +# standard location (for GNU/Linux OSs), so doing this check here and the +# main job below may seem redundant. However, it is written separately from +# the main code below because later, we expect to add more possible +# mounting locations (for other OSs). if [ -d /dev/shm ]; then shmdir=/dev/shm else shmdir="" fi -# If a shared memory mounted directory exists and there is enough space -# there (in RAM), build a temporary directory for this project. -needed_space=2000000 +# If a shared memory mounted directory exists and has the necessary +# conditions, set that directory to build software. if [ x"$shmdir" != x ]; then + + # Make sure it has enough space. + needed_space=2000000 available_space=$(df "$shmdir" | awk 'NR==2{print $4}') if [ $available_space -gt $needed_space ]; then + + # Set the Maneage-specific directory within the shared + # memory. We'll use the names of the two parent directories to the + # current/running directory, separated by a '-' instead of + # '/'. We'll then appended that with the user's name (in case + # multiple users may be working on similar project names). + # + # Maybe later, we can use something like 'mktemp' to add random + # characters to this name and make it unique to every run (even for + # a single user). dirname=$(pwd | sed -e's/\// /g' \ - | awk '{l=NF-1; printf("%s-%s",$l, $NF)}') + | awk '{l=NF-1; printf("%s-%s", $l, $NF)}') tbshmdir="$shmdir"/"$dirname"-$(whoami) - if ! [ -d "$tbshmdir" ]; then mkdir "$tbshmdir"; fi + + # Try to make the directory if it does not yet exist. A failed + # directory creation will be tested for a few lines later, when + # testing for the existence and executability of a test file. + if ! [ -d "$tbshmdir" ]; then (mkdir "$tbshmdir" || true); fi + + # Some systems may protect '/dev/shm' against the right to execute + # programs by ordinary users. We thus need to check that the device + # allows execution within this directory by this user. + shmexecfile="$tbshmdir"/shm-execution-check.sh + rm -f $shmexecfile # We also don't want any existing flags. + + # Create the file to be executed, but do not fail fatally if it + # cannot be created. We will check a few lines later if the file + # really exists. + (cat > "$shmexecfile" <<EOF || true) +#!/bin/sh +echo "This file successfully executed." +EOF + + # If the file was successfully created, then make the file + # executable and see if it runs. If not, set 'tbshmdir' to an empty + # string so it is not used in later steps. In any case, delete the + # temporary file afterwards. + # + # We aren't adding '&> /dev/null' after the execution command + # because it can produce false failures randomly on some systems. + if [ -e "$shmexecfile" ]; then + + # Add the executable flag. + chmod +x "$shmexecfile" + + # The following line tries to execute the file. + if "$shmexecfile"; then + # Successful execution. The colon is a "no-op" (no + # operation) shell command. + : + else + tbshmdir="" + fi + rm "$shmexecfile" + else + tbshmdir="" + fi fi else tbshmdir="" fi -# If a shared memory directory was created set `build-tmp' to be a -# symbolic link to it. Otherwise, just build the temporary build -# directory under the project build directory. + + + + +# If a shared memory directory was created, set the software building +# directory to be a symbolic link to it. Otherwise, just build the +# temporary build directory under the project's build directory. +tmpblddir="$sdir"/build-tmp +rm -rf "$tmpblddir"/* "$tmpblddir" # If it is a link, we need to empty + # its contents first, then itself. if [ x"$tbshmdir" = x ]; then mkdir "$tmpblddir"; else ln -s "$tbshmdir" "$tmpblddir"; fi @@ -1320,6 +1433,9 @@ fi # Inform the user that the build process is starting # ------------------------------------------------- +# +# Everything is ready, let the user know that the building is going to +# start. if [ $printnotice = yes ]; then tsec=10 cat <<EOF @@ -1328,12 +1444,13 @@ if [ $printnotice = yes ]; then Building dependencies ... ------------------------- -Necessary dependency programs and libraries will be built in +Necessary dependency programs and libraries will be installed in $sdir/installed -NOTE: the built software will NOT BE INSTALLED on your system (no root -access is required). They are only for local usage by this project. +NOTE: the built software will NOT BE INSTALLED in standard places of your +OS (so no root access is required). They are only for local usage by this +project. **TIP**: you can see which software are being installed at every moment with the following command. See "Inspecting status" section of @@ -1382,7 +1499,7 @@ fi # See if the linker accepts -Wl,-rpath-link # ----------------------------------------- # -# `-rpath-link' is used to write the information of the linked shared +# '-rpath-link' is used to write the information of the linked shared # library into the shared object (library or program). But some versions of # LLVM's linker don't accept it an can cause problems. # @@ -1415,13 +1532,13 @@ rm -rf $compilertestdir -# Paths needed by the host compiler (only for `basic.mk') +# Paths needed by the host compiler (only for 'basic.mk') # ------------------------------------------------------- # # At the end of the basic build, we need to build GCC. But GCC will build # in multiple phases, making its own simple compiler in order to build # itself completely. The intermediate/simple compiler doesn't recognize -# some system specific locations like `/usr/lib/ARCHITECTURE' that some +# some system specific locations like '/usr/lib/ARCHITECTURE' that some # operating systems use. We thus need to tell the intermediate compiler # where its necessary libraries and headers are. if [ x"$sys_library_path" != x ]; then @@ -1458,18 +1575,19 @@ fi # which will download the DOI-resolved webpage, and extract the Zenodo-URL # of the most recent version from there (using the 'coreutils' tarball as # an example, the directory part of the URL for all the other software are -# the same). +# the same). This is not done if the option '--debug' is used. +zenodourl="" user_backup_urls="" zenodocheck=.build/software/zenodo-check.html -if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then - zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ - | sed -n -e'/http/p' \ - | tr ' ' '\n' \ - | grep http \ - | sed -e 's/href="//' -e 's|/coreutils| |' \ - | awk 'NR==1{print $1}') -else - zenodourl="" +if [ x$debug = x ]; then + if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then + zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ + | sed -n -e'/http/p' \ + | tr ' ' '\n' \ + | grep http \ + | sed -e 's/href="//' -e 's|/coreutils| |' \ + | awk 'NR==1{print $1}') + fi fi rm -f $zenodocheck @@ -1490,6 +1608,7 @@ user_backup_urls="$user_backup_urls $zenodourl" # (compression program), GNU Make (that 'basic.mk' is written in), Dash # (minimal Bash-like shell) and Flock (to lock files and enable serial # download). +export on_mac_os ./reproduce/software/shell/pre-make-build.sh \ "$bdir" "$ddir" "$downloader" "$user_backup_urls" @@ -1497,6 +1616,28 @@ user_backup_urls="$user_backup_urls $zenodourl" +# Corrections for debugging mode +# ------------------------------ +# +# If the user wants to debug the software configuration, they are usually +# focused on the building of the single problematic software. Therefore, +# the default multi-threaded execution of Make with the '--keep-going' +# option are very annoying and can even hide important warnings. Recall +# that with '--keep-going', Make will continue building other targets, even +# if one target fails. When the user runs './project configure --debug', +# the 'debug' variable will not be empty and this mode will be activated. +if [ x$debug = x ]; then + keepgoing="--keep-going" +else + jobs=1 + numthreads=1 + keepgoing="" +fi + + + + + # Build other basic tools our own GNU Make # ---------------------------------------- # @@ -1504,7 +1645,7 @@ user_backup_urls="$user_backup_urls $zenodourl" # Bash, Make, or AWK. In this step, we'll install such low-level basic # tools, but we have to be very portable (and use minimal features in all). echo; echo "Building necessary software (if necessary)..." -.local/bin/make -k -f reproduce/software/make/basic.mk \ +.local/bin/make $keepgoing -f reproduce/software/make/basic.mk \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ @@ -1532,7 +1673,7 @@ else numthreads=$jobs fi .local/bin/env -i HOME=$bdir \ - .local/bin/make -k -f reproduce/software/make/high-level.mk \ + .local/bin/make $keepgoing -f reproduce/software/make/high-level.mk \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ @@ -1577,17 +1718,21 @@ is not used at all during the analysis. Therefore, if you don't need the final PDF, and just want to do the analysis, you can safely ignore this warning and continue. -If you later have internet access and would like to add TeX live to your -project, please delete the respective files, then re-run configure as shown -below. +If you later have internet access and would like to add TeX Live to your +project, then please delete the following two files: rm .local/version-info/tex/texlive-ready-tlmgr + rm .build/software/tarballs/install-tl-unx.tar.gz + +and re-run configure: + ./project configure -e !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EOF + sleep 10 # increase the chance that an interactive user reads this message fi @@ -1601,7 +1746,7 @@ fi # software. prepare_name_version () { - # First see if the (possible) `*' in the input arguments corresponds to + # First see if the (possible) '*' in the input arguments corresponds to # anything. Note that some of the given directories may be empty (no # software installed). hasfiles=0 @@ -1702,9 +1847,9 @@ hw_class_fixed="$(echo $hw_class | sed -e 's/_/\\_/')" # --------------------------------- # # By the time the script reaches here the temporary software build -# directory should be empty, so just delete it. Note `tmpblddir' may be a +# directory should be empty, so just delete it. Note 'tmpblddir' may be a # symbolic link to shared memory. So, to work in any scenario, first delete -# the contents of the directory (if it has any), then delete `tmpblddir'. +# the contents of the directory (if it has any), then delete 'tmpblddir'. .local/bin/rm -rf $tmpblddir/* $tmpblddir @@ -1725,10 +1870,10 @@ echo `.local/bin/date` > $finaltarget # # The configuration is now complete, we can inform the user on the next # step(s) to take. -if [ x$reproducible_paper_group_name = x ]; then +if [ x$maneage_group_name = x ]; then buildcommand="./project make -j8" else - buildcommand="./project make --group=$reproducible_paper_group_name -j8" + buildcommand="./project make --group=$maneage_group_name -j8" fi cat <<EOF diff --git a/reproduce/software/shell/git-post-checkout b/reproduce/software/shell/git-post-checkout index 7a90108..a1340d7 100755 --- a/reproduce/software/shell/git-post-checkout +++ b/reproduce/software/shell/git-post-checkout @@ -1,14 +1,14 @@ #!@BINDIR@/bash # # The example hook script to store the metadata information of version -# controlled files (with each commit) using the `metastore' program. +# controlled files (with each commit) using the 'metastore' program. # # Copyright (C) 2016 Przemyslaw Pawelczyk <przemoc@gmail.com> -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # -# This script is taken from the `examples/hooks/pre-commit' file of the -# `metastore' package (installed within the project, with an MIT license -# for copyright). We have just changed the name of the `MSFILE' and also +# This script is taken from the 'examples/hooks/pre-commit' file of the +# 'metastore' package (installed within the project, with an MIT license +# for copyright). We have just changed the name of the 'MSFILE' and also # set special characters for the installation location of meta-store so our # own installation is found by Git. # diff --git a/reproduce/software/shell/git-pre-commit b/reproduce/software/shell/git-pre-commit index 85d3474..02ba488 100755 --- a/reproduce/software/shell/git-pre-commit +++ b/reproduce/software/shell/git-pre-commit @@ -1,10 +1,10 @@ #!@BINDIR@/bash # # The example hook script to store the metadata information of version -# controlled files (with each commit) using the `metastore' program. +# controlled files (with each commit) using the 'metastore' program. # # Copyright (C) 2016 Przemyslaw Pawelczyk <przemoc@gmail.com> -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # WARNING: # @@ -17,9 +17,9 @@ # git reset HEAD -- .metadata # git checkout HEAD -- .metadata # -# This script is taken from the `examples/hooks/pre-commit' file of the -# `metastore' package (installed within the project, with an MIT license -# for copyright). Here, the name of the `MSFILE' and also set special +# This script is taken from the 'examples/hooks/pre-commit' file of the +# 'metastore' package (installed within the project, with an MIT license +# for copyright). Here, the name of the 'MSFILE' and also set special # characters for the installation location of meta-store so our own # installation is found by Git. # diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh index a033963..808429b 100755 --- a/reproduce/software/shell/pre-make-build.sh +++ b/reproduce/software/shell/pre-make-build.sh @@ -2,7 +2,7 @@ # # Very basic tools necessary to start Maneage's default building. # -# Copyright (C) 2020-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2020-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -185,6 +185,19 @@ build_program() { # All others accept the configure script. ./configure --prefix="$instdir" $configoptions + # In Flock 0.4.0 there is a crash that can be fixed by simply + # replacing '%1u' with '%ld' on GNU/Linux and '%d' on macOS. This + # has been reported to flock maintainers: + # https://github.com/discoteq/flock/issues/33 + if [ x$progname = xflock ]; then + case $on_mac_os in + yes) sed -e's/\%1u/\%d/' src/flock.c > src/flock-new.c;; + no) sed -e's/\%1u/\%ld/' src/flock.c > src/flock-new.c;; + *) echo "pre-make-build.sh: '$on_mac_os' unrecognized value for on_mac_os";; + esac + mv src/flock-new.c src/flock.c + fi + # To build GNU Make, we don't want to assume the existance of a # Make program, so we use its 'build.sh' script and its own built # 'make' program to install itself. @@ -192,7 +205,7 @@ build_program() { /bin/sh build.sh ./make install else - make + make V=1 make install fi fi @@ -217,7 +230,7 @@ build_program() { # (without compression it is just ~400Kb). So we use its '.tar' file and # won't rely on the host's compression tools at all. progname="lzip" -progname_tex="Lzip" +progname_tex="" # Lzip re-built after GCC (empty string to avoid repetition) url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' "$versionsfile") tarball=$progname-$version.tar @@ -235,13 +248,20 @@ build_program # '--disable-dependency-tracking' configure-time option is necessary so # Make doesn't check for an existing 'make' implementation (recall that we # aren't assuming any 'make' on the host). +# +# If GNU Guile is already present on the host system, Make will try to link +# with it, and this will cause dependency problems later. So we have +# distabled Guile. If a project needs the Guile extensions of Make, we need +# to add a build rule for Guile in Maneage, with a special Guile-enabled +# Make that has a different executable name (using the '--program-prefix=' +# configure option) from the "default" make (which is this one!). progname="make" progname_tex="GNU Make" url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) tarball=$progname-$version.tar.lz download_tarball -build_program --disable-dependency-tracking +build_program "--disable-dependency-tracking --without-guile" @@ -274,11 +294,11 @@ fi # ----- # # Flock (or file-lock) is necessary to serialize operations when -# necessary. GNU/Linux machines have it as part of their `util-linux' +# necessary. GNU/Linux machines have it as part of their 'util-linux' # programs. But to be consistent in non-GNU/Linux systems, we will be using # our own build. # -# The reason that `flock' is built here is that generally the building of +# The reason that 'flock' is built here is that generally the building of # software is done in parallel, but we need it to serialize the download # process of the software tarballs to avoid network complications when too # many simultaneous download commands are called. diff --git a/reproduce/software/shell/run-parts.in b/reproduce/software/shell/run-parts.in index 7e649b1..a7b7705 100755 --- a/reproduce/software/shell/run-parts.in +++ b/reproduce/software/shell/run-parts.in @@ -10,8 +10,8 @@ # However, it didn't have a copyright statement. So one is being added # here. # -# Copyright (C) 2021 Authors mentioned above. -# Copyright (C) 2020-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2023 Authors mentioned above. +# Copyright (C) 2020-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/shell/tarball-prepare.sh b/reproduce/software/shell/tarball-prepare.sh new file mode 100755 index 0000000..472b4c0 --- /dev/null +++ b/reproduce/software/shell/tarball-prepare.sh @@ -0,0 +1,225 @@ +#!/bin/bash + +# Script to convert all files (tarballs in any format; just recognized +# by 'tar') within an 'odir' to a unified '.tar.lz' format. +# +# The inputs are assumed to be formatted with 'NAME_VERSION', and only for +# the names, we are currently assuming '.tar.*' (for the 'sed' +# command). Please modify/generalize accordingly. +# +# It will unpack the source in a certain empty directory with the +# 'tmpunpack' suffix, and rename the top directory to the requested format +# of NAME-VERSION also. So irrespective of the name of the top original +# tarball directory, the resulting tarball's top directory will have a name +# formatting of NAME-VERSION. +# +# Discussion: https://savannah.nongnu.org/task/?15699 +# +# Copyright (C) 2022-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2022-2023 Pedram Ashofteh Ardakani <pedramardakani@pm.me> +# Released under GNU GPLv3+ + +# Abort the script in case of an error. +set -e + + + + + +# Default arguments +odir= +idir= +quiet= +basedir=$PWD +scriptname=$0 + + +# The --help output +print_help() { + cat <<EOF +Usage: $scriptname [OPTIONS] + +Low-level script to create maneage-standard tarballs. + -o, --output-dir Target directory to write the packed tarballs. + Current: $odir + -i, --input-dir Directory containing original tarballs. + Current: $idir + -q, --quiet Suppress logging information. Only print the + final packed file and its sha512sum. +Maneage URL: https://maneage.org + +Report bugs: https://savannah.nongnu.org/bugs/?group=reproduce +EOF +} + + + + + +# Functions to check option values and complain if necessary. +on_off_option_error() { + if [ x"$2" = x ]; then + echo "$scriptname: '$1' doesn't take any values" + else + echo "$scriptname: '$1' (or '$2') doesn't take any values" + fi + exit 1 +} + +check_v() { + if [ x"$2" = x ]; then + cat <<EOF +$scriptname: option '$1' requires an argument. Try '$scriptname --help' for more information +EOF + exit 1; + fi +} + +option_given_and_valid() { + dirname="$1" + optionlong="$2" + optionshort="$3" + if [ x"$dirname" = x ]; then + cat <<EOF +$scriptname: no '--$optionlong' (or '-$optionshort') given: use this for identifying the directory containing the input tarballs +EOF + exit 1 + else + dirname=$(echo "$dirname" | sed 's|/$||'); # Remove possible trailing slash + if [ ! -d "$dirname" ]; then + cat <<EOF +$scriptname: '$dirname' that is given to '--$optionlong' (or '-$optionshort') couldn't be opened +EOF + exit 1 + else + outdir=$(realpath $dirname) + fi + fi + ogvout=$outdir +} + + + + + +# Parse the arguments +while [ $# -gt 0 ] +do + case $1 in + # Input and Output directories + -i|--input-dir) idir="$2"; check_v "$1" "$idir"; shift;shift;; + -i=*|--input-dir=*) idir="${1#*=}"; check_v "$1" "$idir"; shift;; + -i*) idir=$(echo "$1" | sed -e's/-i//'); check_v "$1" "$idir"; shift;; + -o|--output-dir) odir="$2"; check_v "$1" "$odir"; shift;shift;; + -o=*|--output-dir=*) odir="${1#*=}"; check_v "$1" "$odir"; shift;; + -o*) odir=$(echo "$1" | sed -e's/-o//'); check_v "$1" "$odir"; shift;; + + # Operating mode options + -?|--help) print_help; exit 0;; + -'?'*|--help=*) on_off_option_error --help -?;; + -q|--quiet) quiet=1; shift;; + -q*|--quiet=*) on_off_option_error --quiet -q;; + *) echo "$scriptname: unknown option '$1'"; exit 1;; + esac +done + + + + + +# Basic sanity checks +# +# Make sure the input and output directories are given. Also extract +# the absolute path to input and output directories and remove any +# possible trailing '/'. Working with a relative path is a great +# source of confusion and unwanted side-effects like moving/removing +# files by accident. +option_given_and_valid "$idir" "input-dir" "i" && idir=$ogvout +option_given_and_valid "$odir" "output-dir" "o" && odir=$ogvout + + + + + +# Unpack and pack all files in the '$idir' +# ---------------------------------------- +allfiles=$(ls $idir | sort) + +# Let user know number of tarballs if its not in quiet mode +if [ -z $quiet ]; then + nfiles=$(ls $idir | wc -l) + echo "Found $nfiles file(s) in '$idir/'" +fi + +# Process all files +for f in $allfiles; do + + # Extract the name and version (while replacing any possible '_' with + # '-' because some software separate name and version with '_'). + name=$(echo $(basename $f) \ + | sed -e 's/.tar.*//' -e's/_/-/') + + # Skip previously packed files + if [ -f $odir/$name.tar.lz ]; then + + # Print the info message if not in quiet mode + if [ -z $quiet ]; then + echo "$scriptname: $odir/$name.tar.lz: already present in output directory" + fi + + # skip this file + continue + else + + # Print the info message if not in quiet mode + if [ -z $quiet ]; then + echo "$scriptname: processing '$idir/$f'" + fi + fi + + # Create a temporary directory name + tmpdir=$odir/$name-tmpunpack + + # If the temporary directory exists, mkdir will throw an error. The + # developer needs to intervene manually to fix the issue. + mkdir $tmpdir + + + + + + # Move into the temporary directory + # --------------------------------- + # + # The default output directory for all the following commands: $tmpdir + cd $tmpdir + + # Unpack + tar -xf $idir/$f + + # Make sure the unpacked tarball is contained within a directory with + # the clean program name + if [ ! -d "$name" ]; then + mv * $name/ + fi + + # Put the current date on all the files because some packagers will not + # add dates to their release tarballs, resulting in dates of the + # Unix-time zero'th second (1970-01-01 at 00:00:00)! + touch $(find "$name"/ -type f) + + # Pack with recommended options + tar -c -Hustar --owner=root --group=root \ + -f $name.tar $name/ + lzip -9 $name.tar + + # Move the compressed file from the temporary directory to the target + # output directory + mv $name.tar.lz $odir/ + + # Print the sha512sum along with the filename for a quick reference + echo $(sha512sum $odir/$name.tar.lz) + + # Clean up the temporary directory + rm -rf $tmpdir +done |