diff options
Diffstat (limited to 'reproduce/software/shell')
| -rwxr-xr-x | reproduce/software/shell/bashrc.sh | 6 | ||||
| -rwxr-xr-x | reproduce/software/shell/configure.sh | 218 | ||||
| -rwxr-xr-x | reproduce/software/shell/git-post-checkout | 10 | ||||
| -rwxr-xr-x | reproduce/software/shell/git-pre-commit | 10 | ||||
| -rwxr-xr-x | reproduce/software/shell/pre-make-build.sh | 30 | ||||
| -rwxr-xr-x | reproduce/software/shell/run-parts.in | 4 | ||||
| -rwxr-xr-x | reproduce/software/shell/tarball-prepare.sh | 181 |
7 files changed, 374 insertions, 85 deletions
diff --git a/reproduce/software/shell/bashrc.sh b/reproduce/software/shell/bashrc.sh index 23845d6..6bb871b 100755 --- a/reproduce/software/shell/bashrc.sh +++ b/reproduce/software/shell/bashrc.sh @@ -3,10 +3,10 @@ # To have better control over the environment of each analysis step (Make # recipe), besides having environment variables (directly included from # Make), it may also be useful to have a Bash startup file (this file). All -# of the Makefiles set this file as the `BASH_ENV' environment variable, so +# of the Makefiles set this file as the 'BASH_ENV' environment variable, so # it is loaded into all the Make recipes within the project. # -# The special `PROJECT_STATUS' environment variable is defined in every +# The special 'PROJECT_STATUS' environment variable is defined in every # top-level Makefile of the project. It defines the the state of the Make # that is calling this script. It can have three values: # @@ -29,7 +29,7 @@ # versions. # # -# Copyright (C) 2019-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index e15a8e9..07381e6 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -2,8 +2,9 @@ # # Necessary preparations/configurations for the reproducible project. # -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2021 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2021-2022 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2022 Pedram Ashofteh Ardakani <pedramardakani@pm.me> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -112,7 +113,7 @@ absolute_dir () # Check file permission handling (POSIX-compatibility) # ---------------------------------------------------- # -# Check if a `given' directory handles permissions as expected. +# Check if a 'given' directory handles permissions as expected. # # This is to prevent a known bug in the NTFS filesystem that prevents # proper installation of Perl, and probably some other packages. This @@ -120,15 +121,15 @@ absolute_dir () # file, and examines whether the given directory handles the file # permissions as expected. # -# Returns `0' if everything is fine, and `255' otherwise. Choosing `0' is -# to mimic the `$ echo $?' behavior, while choosing `255' is to prevent +# Returns '0' if everything is fine, and '255' otherwise. Choosing '0' is +# to mimic the '$ echo $?' behavior, while choosing '255' is to prevent # misunderstanding 0 and 1 as true and false. # # ===== CAUTION! ===== # # -# Since there is a `set -e' before running this function, the whole script -# stops and exits IF the `check_permission' (or any other function) returns -# anything OTHER than `0'! So, only use this function as a test. Here's a +# Since there is a 'set -e' before running this function, the whole script +# stops and exits IF the 'check_permission' (or any other function) returns +# anything OTHER than '0'! So, only use this function as a test. Here's a # minimal example: # # if $(check_permission $some_directory) ; then @@ -136,7 +137,7 @@ absolute_dir () # fi ; check_permission () { - # Make a `junk' file, activate its executable flag and record its + # Make a 'junk' file, activate its executable flag and record its # permissions generally. local junkfile="$1"/check_permission_tmp_file rm -f "$junkfile" @@ -158,7 +159,7 @@ check_permission () return 1 else # Setting permission SUCCESSFUL - return 0 + return 0 fi } @@ -309,7 +310,9 @@ fi # system. Here, it is checked that this is the case, and if not, warn the user # about not having Xcode already installed. if [ x$on_mac_os = xyes ]; then - xcode=$(which xcodebuild) + + # 'which' isn't in POSIX, so we are using 'command -v' instead. + xcode=$(command -v xcodebuild) if [ x$xcode != x ]; then xcode_version=$(xcodebuild -version | grep Xcode) echo " " @@ -475,8 +478,8 @@ fi # See if we need the dynamic-linker (-ldl) # ---------------------------------------- # -# Some programs (like Wget) need dynamic loading (using `libdl'). On -# GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But +# Some programs (like Wget) need dynamic loading (using 'libdl'). On +# GNU/Linux systems, we'll need the '-ldl' flag to link such programs. But # Mac OS doesn't need any explicit linking. So we'll check here to see if # it is present (thus necessary) or not. cat > $testsource <<EOF @@ -508,7 +511,7 @@ fi # programs will go and find their necessary libraries on the host system. # # Another good advantage of shared libraries is that we can actually use -# the shared library tool of the system (`ldd' with GNU C Library) and see +# the shared library tool of the system ('ldd' with GNU C Library) and see # exactly where each linked library comes from. But in static building, # unless you follow the build closely, its not easy to see if the source of # the library came from the system or our build. @@ -545,7 +548,7 @@ fi # # On some systems (in particular Debian-based OSs), the static C library # and necessary headers in a non-standard place, and we can't build GCC. So -# we need to find them first. The `sys/cdefs.h' header is also in a +# we need to find them first. The 'sys/cdefs.h' header is also in a # similarly different location. sys_cpath="" sys_library_path="" @@ -576,7 +579,7 @@ fi # See if a link-able static C library exists # ------------------------------------------ # -# A static C library and the `sys/cdefs.h' header are necessary for +# A static C library and the 'sys/cdefs.h' header are necessary for # building GCC. if [ x"$host_cc" = x0 ]; then echo; echo; echo "Checking if static C library is available..."; @@ -752,9 +755,9 @@ EOF # What to do with possibly existing configuration file # ---------------------------------------------------- # -# `LOCAL.conf' is the top-most local configuration for the project. If it +# 'LOCAL.conf' is the top-most local configuration for the project. If it # already exists when this script is run, we'll make a copy of it as backup -# (for example the user might have ran `./project configure' by mistake). +# (for example the user might have ran './project configure' by mistake). printnotice=yes rewritepconfig=yes if [ -f $pconf ]; then @@ -807,16 +810,18 @@ fi # need to check the host's available tool for downloading at this step. if [ $rewritepconfig = yes ]; then if type wget > /dev/null 2>/dev/null; then - name=$(which wget) + + # 'which' isn't in POSIX, so we are using 'command -v' instead. + name=$(command -v wget) # By default Wget keeps the remote file's timestamp, so we'll have # to disable it manually. downloader="$name --no-use-server-timestamps -O"; elif type curl > /dev/null 2>/dev/null; then - name=$(which curl) + name=$(command -v curl) # - cURL doesn't keep the remote file's timestamp by default. - # - With the `-L' option, we tell cURL to follow redirects. + # - With the '-L' option, we tell cURL to follow redirects. downloader="$name -L -o" else cat <<EOF @@ -913,7 +918,7 @@ EOF # If it was newly created, it will be empty, so delete it. if ! [ "$(ls -A $bdir)" ]; then rm --dir "$bdir"; fi - # Inform the user that this is not acceptable and reset `bdir'. + # Inform the user that this is not acceptable and reset 'bdir'. bdir= echo " ** The build-directory cannot be under the source-directory." fi @@ -938,10 +943,10 @@ EOF fi fi - # If everything is still fine so far, see if we're able to - # manipulate file permissions in the directory's filesystem and if - # so, see if there is atleast 5GB free space. - if ! [ x"$bdir" = x ]; then + # If everything is still fine so far, see if we're able to + # manipulate file permissions in the directory's filesystem and if + # so, see if there is atleast 5GB free space. + if ! [ x"$bdir" = x ]; then if ! $(check_permission "$bdir"); then # Unable to handle permissions well bdir= @@ -960,7 +965,7 @@ EOF fi # If the build directory was good, the loop will stop, if not, - # reset `build_dir' to blank, so it continues asking for another + # reset 'build_dir' to blank, so it continues asking for another # directory and let the user know that they must select a new # directory. if [ x"$bdir" = x ]; then @@ -1118,7 +1123,7 @@ else if [ x"$downloader" = x ]; then novalue="$novalue"DOWNLOADER; fi if [ x"$novalue" != x ]; then verr=1; err=1; fi - # Make sure `bdir' is an absolute path and it exists. + # Make sure 'bdir' is an absolute path and it exists. berr=0 ierr=0 bdir="$(absolute_dir "$inbdir")" @@ -1202,6 +1207,10 @@ if ! [ -d "$ibidir" ]; then mkdir "$ibidir"; fi ipydir="$verdir"/python if ! [ -d "$ipydir" ]; then mkdir "$ipydir"; fi +# R module versions and citation. +ircrandir="$verdir"/r-cran +if ! [ -d "$ircrandir" ]; then mkdir "$ircrandir"; fi + # Used software BibTeX entries. ictdir="$verdir"/cite if ! [ -d "$ictdir" ]; then mkdir "$ictdir"; fi @@ -1210,21 +1219,14 @@ if ! [ -d "$ictdir" ]; then mkdir "$ictdir"; fi itidir="$verdir"/tex if ! [ -d "$itidir" ]; then mkdir "$itidir"; fi -# Temporary software un-packing/build directory: if the host has the -# standard `/dev/shm' mounting-point, we'll do it in shared memory (on the -# RAM), to avoid harming/over-using the HDDs/SSDs. The RAM of most systems -# today (>8GB) is large enough for the parallel building of the software. -# -# For the name of the directory under `/dev/shm' (for this project), we'll -# use the names of the two parent directories to the current/running -# directory, separated by a `-' instead of `/'. We'll then appended that -# with the user's name (in case multiple users may be working on similar -# project names). Maybe later, we can use something like `mktemp' to add -# random characters to this name and make it unique to every run (even for -# a single user). -tmpblddir="$sdir"/build-tmp -rm -rf "$tmpblddir"/* "$tmpblddir" # If its a link, we need to empty its - # contents first, then itself. +# Some software install their libraries in '$(idir)/lib64'. But all other +# libraries are in '$(idir)/lib'. Since Maneage's build is only for a +# single architecture, we can set the '$(idir)/lib64' as a symbolic link to +# '$(idir)/lib' so all the libraries are always available in the same +# place. +instlibdir="$instdir"/lib +if ! [ -d "$instlibdir" ]; then mkdir "$instlibdir"; fi +ln -fs "$instlibdir" "$instdir"/lib64 @@ -1297,29 +1299,106 @@ rm -f .gnuastro -# Set the top-level shared memory location. + + +# Software building directory (possibly in RAM) +# --------------------------------------------- +# +# Building the software for the project will need the creation of many +# small temporary files that will ultimately be deleted. To avoid harming +# HDDs/SSDs and improve speed, it is therefore better to build them in the +# RAM when possible. The RAM of most systems today (>8GB) is large enough +# for the parallel building of the software. + +# Set the top-level shared memory location. Currently there is only one +# standard location (for GNU/Linux OSs), so doing this check here and the +# main job below may seem redundant. However, it is written separately from +# the main code below because later, we expect to add more possible +# mounting locations (for other OSs). if [ -d /dev/shm ]; then shmdir=/dev/shm else shmdir="" fi -# If a shared memory mounted directory exists and there is enough space -# there (in RAM), build a temporary directory for this project. -needed_space=2000000 +# If a shared memory mounted directory exists and has the necessary +# conditions, set that directory to build software. if [ x"$shmdir" != x ]; then + + # Make sure it has enough space. + needed_space=2000000 available_space=$(df "$shmdir" | awk 'NR==2{print $4}') if [ $available_space -gt $needed_space ]; then + + # Set the Maneage-specific directory within the shared + # memory. We'll use the names of the two parent directories to the + # current/running directory, separated by a '-' instead of + # '/'. We'll then appended that with the user's name (in case + # multiple users may be working on similar project names). + # + # Maybe later, we can use something like 'mktemp' to add random + # characters to this name and make it unique to every run (even for + # a single user). dirname=$(pwd | sed -e's/\// /g' \ - | awk '{l=NF-1; printf("%s-%s",$l, $NF)}') + | awk '{l=NF-1; printf("%s-%s", $l, $NF)}') tbshmdir="$shmdir"/"$dirname"-$(whoami) - if ! [ -d "$tbshmdir" ]; then mkdir "$tbshmdir"; fi + + # Try to make the directory if it does not yet exist. A failed + # directory creation will be tested for a few lines later, when + # testing for the existence and executability of a test file. + if ! [ -d "$tbshmdir" ]; then (mkdir "$tbshmdir" || true); fi + + # Some systems may protect '/dev/shm' against the right to execute + # programs by ordinary users. We thus need to check that the device + # allows execution within this directory by this user. + shmexecfile="$tbshmdir"/shm-execution-check.sh + rm -f $shmexecfile # We also don't want any existing flags. + + # Create the file to be executed, but do not fail fatally if it + # cannot be created. We will check a few lines later if the file + # really exists. + (cat > "$shmexecfile" <<EOF || true) +#!/bin/sh +echo "This file successfully executed." +EOF + + # If the file was successfully created, then make the file + # executable and see if it runs. If not, set 'tbshmdir' to an empty + # string so it is not used in later steps. In any case, delete the + # temporary file afterwards. + # + # We aren't adding '&> /dev/null' after the execution command + # because it can produce false failures randomly on some systems. + if [ -e "$shmexecfile" ]; then + + # Add the executable flag. + chmod +x "$shmexecfile" + + # The following line tries to execute the file. + if "$shmexecfile"; then + # Successful execution. The colon is a "no-op" (no + # operation) shell command. + : + else + tbshmdir="" + fi + rm "$shmexecfile" + else + tbshmdir="" + fi fi else tbshmdir="" fi -# If a shared memory directory was created set `build-tmp' to be a -# symbolic link to it. Otherwise, just build the temporary build -# directory under the project build directory. + + + + +# If a shared memory directory was created, set the software building +# directory to be a symbolic link to it. Otherwise, just build the +# temporary build directory under the project's build directory. +tmpblddir="$sdir"/build-tmp +rm -rf "$tmpblddir"/* "$tmpblddir" # If it is a link, we need to empty + # its contents first, then itself. if [ x"$tbshmdir" = x ]; then mkdir "$tmpblddir"; else ln -s "$tbshmdir" "$tmpblddir"; fi @@ -1330,6 +1409,9 @@ fi # Inform the user that the build process is starting # ------------------------------------------------- +# +# Everything is ready, let the user know that the building is going to +# start. if [ $printnotice = yes ]; then tsec=10 cat <<EOF @@ -1338,12 +1420,13 @@ if [ $printnotice = yes ]; then Building dependencies ... ------------------------- -Necessary dependency programs and libraries will be built in +Necessary dependency programs and libraries will be installed in $sdir/installed -NOTE: the built software will NOT BE INSTALLED on your system (no root -access is required). They are only for local usage by this project. +NOTE: the built software will NOT BE INSTALLED in standard places of your +OS (so no root access is required). They are only for local usage by this +project. **TIP**: you can see which software are being installed at every moment with the following command. See "Inspecting status" section of @@ -1392,7 +1475,7 @@ fi # See if the linker accepts -Wl,-rpath-link # ----------------------------------------- # -# `-rpath-link' is used to write the information of the linked shared +# '-rpath-link' is used to write the information of the linked shared # library into the shared object (library or program). But some versions of # LLVM's linker don't accept it an can cause problems. # @@ -1425,13 +1508,13 @@ rm -rf $compilertestdir -# Paths needed by the host compiler (only for `basic.mk') +# Paths needed by the host compiler (only for 'basic.mk') # ------------------------------------------------------- # # At the end of the basic build, we need to build GCC. But GCC will build # in multiple phases, making its own simple compiler in order to build # itself completely. The intermediate/simple compiler doesn't recognize -# some system specific locations like `/usr/lib/ARCHITECTURE' that some +# some system specific locations like '/usr/lib/ARCHITECTURE' that some # operating systems use. We thus need to tell the intermediate compiler # where its necessary libraries and headers are. if [ x"$sys_library_path" != x ]; then @@ -1468,7 +1551,7 @@ fi # which will download the DOI-resolved webpage, and extract the Zenodo-URL # of the most recent version from there (using the 'coreutils' tarball as # an example, the directory part of the URL for all the other software are -# the same). This is not done if the option `--debug' is used. +# the same). This is not done if the option '--debug' is used. zenodourl="" user_backup_urls="" zenodocheck=.build/software/zenodo-check.html @@ -1501,6 +1584,7 @@ user_backup_urls="$user_backup_urls $zenodourl" # (compression program), GNU Make (that 'basic.mk' is written in), Dash # (minimal Bash-like shell) and Flock (to lock files and enable serial # download). +export on_mac_os ./reproduce/software/shell/pre-make-build.sh \ "$bdir" "$ddir" "$downloader" "$user_backup_urls" @@ -1610,17 +1694,21 @@ is not used at all during the analysis. Therefore, if you don't need the final PDF, and just want to do the analysis, you can safely ignore this warning and continue. -If you later have internet access and would like to add TeX live to your -project, please delete the respective files, then re-run configure as shown -below. +If you later have internet access and would like to add TeX Live to your +project, then please delete the following two files: rm .local/version-info/tex/texlive-ready-tlmgr + rm .build/software/tarballs/install-tl-unx.tar.gz + +and re-run configure: + ./project configure -e !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! EOF + sleep 10 # increase the chance that an interactive user reads this message fi @@ -1634,7 +1722,7 @@ fi # software. prepare_name_version () { - # First see if the (possible) `*' in the input arguments corresponds to + # First see if the (possible) '*' in the input arguments corresponds to # anything. Note that some of the given directories may be empty (no # software installed). hasfiles=0 @@ -1735,9 +1823,9 @@ hw_class_fixed="$(echo $hw_class | sed -e 's/_/\\_/')" # --------------------------------- # # By the time the script reaches here the temporary software build -# directory should be empty, so just delete it. Note `tmpblddir' may be a +# directory should be empty, so just delete it. Note 'tmpblddir' may be a # symbolic link to shared memory. So, to work in any scenario, first delete -# the contents of the directory (if it has any), then delete `tmpblddir'. +# the contents of the directory (if it has any), then delete 'tmpblddir'. .local/bin/rm -rf $tmpblddir/* $tmpblddir diff --git a/reproduce/software/shell/git-post-checkout b/reproduce/software/shell/git-post-checkout index 7a90108..d49504d 100755 --- a/reproduce/software/shell/git-post-checkout +++ b/reproduce/software/shell/git-post-checkout @@ -1,14 +1,14 @@ #!@BINDIR@/bash # # The example hook script to store the metadata information of version -# controlled files (with each commit) using the `metastore' program. +# controlled files (with each commit) using the 'metastore' program. # # Copyright (C) 2016 Przemyslaw Pawelczyk <przemoc@gmail.com> -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> # -# This script is taken from the `examples/hooks/pre-commit' file of the -# `metastore' package (installed within the project, with an MIT license -# for copyright). We have just changed the name of the `MSFILE' and also +# This script is taken from the 'examples/hooks/pre-commit' file of the +# 'metastore' package (installed within the project, with an MIT license +# for copyright). We have just changed the name of the 'MSFILE' and also # set special characters for the installation location of meta-store so our # own installation is found by Git. # diff --git a/reproduce/software/shell/git-pre-commit b/reproduce/software/shell/git-pre-commit index 85d3474..10ad710 100755 --- a/reproduce/software/shell/git-pre-commit +++ b/reproduce/software/shell/git-pre-commit @@ -1,10 +1,10 @@ #!@BINDIR@/bash # # The example hook script to store the metadata information of version -# controlled files (with each commit) using the `metastore' program. +# controlled files (with each commit) using the 'metastore' program. # # Copyright (C) 2016 Przemyslaw Pawelczyk <przemoc@gmail.com> -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # WARNING: # @@ -17,9 +17,9 @@ # git reset HEAD -- .metadata # git checkout HEAD -- .metadata # -# This script is taken from the `examples/hooks/pre-commit' file of the -# `metastore' package (installed within the project, with an MIT license -# for copyright). Here, the name of the `MSFILE' and also set special +# This script is taken from the 'examples/hooks/pre-commit' file of the +# 'metastore' package (installed within the project, with an MIT license +# for copyright). Here, the name of the 'MSFILE' and also set special # characters for the installation location of meta-store so our own # installation is found by Git. # diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh index a033963..e7de93d 100755 --- a/reproduce/software/shell/pre-make-build.sh +++ b/reproduce/software/shell/pre-make-build.sh @@ -2,7 +2,7 @@ # # Very basic tools necessary to start Maneage's default building. # -# Copyright (C) 2020-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2020-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -185,6 +185,19 @@ build_program() { # All others accept the configure script. ./configure --prefix="$instdir" $configoptions + # In Flock 0.4.0 there is a crash that can be fixed by simply + # replacing '%1u' with '%ld' on GNU/Linux and '%d' on macOS. This + # has been reported to flock maintainers: + # https://github.com/discoteq/flock/issues/33 + if [ x$progname = xflock ]; then + case $on_mac_os in + yes) sed -e's/\%1u/\%d/' src/flock.c > src/flock-new.c;; + no) sed -e's/\%1u/\%ld/' src/flock.c > src/flock-new.c;; + *) echo "pre-make-build.sh: '$on_mac_os' unrecognized value for on_mac_os";; + esac + mv src/flock-new.c src/flock.c + fi + # To build GNU Make, we don't want to assume the existance of a # Make program, so we use its 'build.sh' script and its own built # 'make' program to install itself. @@ -192,7 +205,7 @@ build_program() { /bin/sh build.sh ./make install else - make + make V=1 make install fi fi @@ -235,13 +248,20 @@ build_program # '--disable-dependency-tracking' configure-time option is necessary so # Make doesn't check for an existing 'make' implementation (recall that we # aren't assuming any 'make' on the host). +# +# If GNU Guile is already present on the host system, Make will try to link +# with it, and this will cause dependency problems later. So we have +# distabled Guile. If a project needs the Guile extensions of Make, we need +# to add a build rule for Guile in Maneage, with a special Guile-enabled +# Make that has a different executable name (using the '--program-prefix=' +# configure option) from the "default" make (which is this one!). progname="make" progname_tex="GNU Make" url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) tarball=$progname-$version.tar.lz download_tarball -build_program --disable-dependency-tracking +build_program "--disable-dependency-tracking --without-guile" @@ -274,11 +294,11 @@ fi # ----- # # Flock (or file-lock) is necessary to serialize operations when -# necessary. GNU/Linux machines have it as part of their `util-linux' +# necessary. GNU/Linux machines have it as part of their 'util-linux' # programs. But to be consistent in non-GNU/Linux systems, we will be using # our own build. # -# The reason that `flock' is built here is that generally the building of +# The reason that 'flock' is built here is that generally the building of # software is done in parallel, but we need it to serialize the download # process of the software tarballs to avoid network complications when too # many simultaneous download commands are called. diff --git a/reproduce/software/shell/run-parts.in b/reproduce/software/shell/run-parts.in index 7e649b1..a6db9e0 100755 --- a/reproduce/software/shell/run-parts.in +++ b/reproduce/software/shell/run-parts.in @@ -10,8 +10,8 @@ # However, it didn't have a copyright statement. So one is being added # here. # -# Copyright (C) 2021 Authors mentioned above. -# Copyright (C) 2020-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2022 Authors mentioned above. +# Copyright (C) 2020-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/shell/tarball-prepare.sh b/reproduce/software/shell/tarball-prepare.sh new file mode 100755 index 0000000..ccc9318 --- /dev/null +++ b/reproduce/software/shell/tarball-prepare.sh @@ -0,0 +1,181 @@ +#!/bin/bash + +# Script to convert all files (tarballs in any format; just recognized +# by 'tar') within an 'odir' to a unified '.tar.lz' format. +# +# The inputs are assumed to be formatted with 'NAME_VERSION', and only for +# the names, we are currently assuming '.tar.*' (for the 'sed' +# command). Please modify/generalize accordingly. +# +# It will unpack the source in a certain empty directory with the +# 'tmpunpack' suffix, and rename the top directory to the requested format +# of NAME-VERSION also. So irrespective of the name of the top original +# tarball directory, the resulting tarball's top directory will have a name +# formatting of NAME-VERSION. +# +# Discussion: https://savannah.nongnu.org/task/?15699 +# +# Copyright (C) 2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2022 Pedram Ashofteh Ardakani <pedramardakani@pm.me> +# Released under GNU GPLv3+ + +# Abort the script in case of an error. +set -e + + + + + +# Default arguments +odir= +idir= +quiet= +basedir=$PWD + + +# The --help output +print_help() { + cat <<EOF +Usage: $0 [OPTIONS] + +Low-level script to create maneage-standard tarballs. + + -o, --output-dir Target directory to write the packed tarballs. + Current: $odir + + + -i, --input-dir Directory containing original tarballs. + Current: $idir + + -q, --quiet Suppress logging information. Only print the + final packed file and its sha512sum. + +Maneage URL: https://maneage.org + +Report bugs: https://savannah.nongnu.org/bugs/?group=reproduce +EOF +} + + + + +# Parse the arguments +while [ $# -gt 0 ] +do + case $1 in + -q|--quiet) quiet=1; shift;; + -h|--help|-'?') print_help; exit 0;; + -i|--input-dir) + # Remove the trailing '/' introduced by autocomplete + idir=$(echo "$2" | sed 's|/$||'); + shift; # past argument + shift;; # past value + -o|--output-dir) + # Remove the trailing '/' introduced by autocomplete + odir=$(echo "$2" | sed 's|/$||'); + shift; # past argument + shift;; # past value + *) echo "$0: unknown option '$1'"; exit 1;; + esac +done + + + + +# Extract the 'absolute path' to input and output directories. Working with +# relative path is a great source of confusion and unwanted side-effects +# like moving/removing files by accident. +if [ ! -d "$idir" ]; then + echo "$0: please pass the input directory (option --input-dir or -i)." + exit 1 +else + idir=$(realpath $idir) +fi + +if [ ! -d "$odir" ]; then + echo "$0: please pass the output directory (option --output-dir or -o)." + exit 1 +else + odir=$(realpath $odir) +fi + + + + + +# Unpack and pack all files in the '$idir' +# ---------------------------------------- +allfiles=$(ls $idir | sort) + +# Let user know number of tarballs if its not in quiet mode +if [ -z $quiet ]; then + nfiles=$(ls $idir | wc -l) + echo "Found $nfiles file(s) in '$idir/'" +fi + +# Process all files +for f in $allfiles; do + + # Seperate name and version number + name=$(echo $f | sed -e 's/.tar.*//' | \ + awk 'BEGIN { FS = "[-_ ]" } {print $1 "-" $2}') + + # Skip previously packed files + if [ -f $odir/$name.tar.lz ]; then + + # Print the info message if not in quiet mode + if [ -z $quiet ]; then + echo "$0: skipping '$odir/$name.tar.lz'" + fi + + # skip this file + continue + else + + # Print the info message if not in quiet mode + if [ -z $quiet ]; then + echo "$0: processing '$idir/$f'" + fi + fi + + # Create a temporary directory name + tmpdir=$odir/$name-tmpunpack + + # If the temporary directory exists, mkdir will throw an error. The + # developer needs to intervene manually to fix the issue. + mkdir $tmpdir + + + + + + # Move into the temporary directory + # --------------------------------- + # + # The default output directory for all the following commands: $tmpdir + cd $tmpdir + + # Unpack + tar -xf $idir/$f + + # Make sure the unpacked tarball is contained within a directory with + # the clean program name + if [ ! -d "$name" ]; then + mv * $name/ + fi + + # Pack with recommended options + tar -c -Hustar --owner=root --group=root \ + -f $name.tar $name/ + lzip -9 $name.tar + + # Move the compressed file from the temporary directory to the target + # output directory + mv $name.tar.lz $odir/ + + # Print the sha512sum along with the filename for a quick reference + echo $(sha512sum $odir/$name.tar.lz) + + # Clean up the temporary directory + rm -r $tmpdir +done |