Imported recent work in master, minor conflict fixed in paper.mk

Only two conflicts came up in the newly added comments of 'paper.mk' in the
Maneage branch. It happened because in this project we don't use
'pdflatex', but 'latex' alone.

3 files changed, 246 insertions, 45 deletions

diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh
index b2e3979..38dcf7f 100755
--- a/reproduce/software/shell/configure.sh
+++ b/reproduce/software/shell/configure.sh
@@ -153,6 +153,38 @@ check_permission ()
 
 
 
+# Check if there is enough free space available in the build directory
+# --------------------------------------------------------------------
+#
+# Use this function to check if there is enough free space in a
+# directory. It is meant to be passed to the 'if' statement in the
+# shell. So if there is enough space, it returns 0 (which translates to
+# TRUE), otherwise, the funcion returns 1 (which translates to FALSE).
+#
+# Expects to be called with two arguments, the first is the threshold and
+# the second is the desired directory. The 'df' function checks the given
+# path to see where it is mounted on, and how much free space there is on
+# that partition (in units of 1024 bytes).
+#
+# synopsis:
+# $ free_space_warning <acceptable_threshold> <path-to-check>
+#
+# example:
+# To check if there is 5MB of space available in /path/to/check
+# call the command with arguments as shown below:
+# $ free_space_warning 5000 /path/to/check/free/space
+free_space_warning()
+{
+    fs_threshold=$1
+    fs_destpath=$2
+    return $(df $fs_destpath \
+                | awk 'FNR==2 {if($4>'$fs_threshold') print 1; \
+                               else                   print 0; }')
+}
+
+
+
+
 
 # Check for C/C++ compilers
 # -------------------------
@@ -729,14 +761,14 @@ EOF
                 bdir=$(absolute_dir $build_dir)
                 rm -rf $build_dir/$junkname
             else
-                echo " ** Can't write in '$build_dir'"; echo
+                echo " ** Can't write in '$build_dir'";
             fi
         else
             if mkdir $build_dir 2> /dev/null; then
                 instring="the newly created"
                 bdir=$(absolute_dir $build_dir)
             else
-                echo " ** Can't create '$build_dir'"; echo
+                echo " ** Can't create '$build_dir'";
             fi
         fi
 
@@ -759,8 +791,19 @@ EOF
 	# file permissions.
 	if ! [ x"$bdir" = x ]; then
             if ! $(check_permission $bdir); then
+                # Unable to handle permissions well
                 bdir=
                 echo " ** File permissions can't be modified in this directory"
+            else
+                # Able to handle permissions, now check for 5GB free space
+                # in the given partition (note that the number is in units
+                # of 1024 bytes). If this is not the case, print a warning.
+                if $(free_space_warning 5000000 $bdir); then
+                    echo " !! LESS THAN 5GB FREE SPACE IN: $bdir"
+                    echo " !! We recommend choosing another partition."
+                    echo " !! Build will continue in 5 seconds..."
+                    sleep 5
+                fi
             fi
         fi
 
@@ -1113,7 +1156,7 @@ ln -s $instdir .local
 ln -s $texdir tex/build
 ln -s $tikzdir tex/tikz
 # --------- Delete for no Gnuastro ---------
-ln -s $topdir/reproduce/software/config/gnuastro .gnuastro
+ln -s $topdir/reproduce/analysis/config/gnuastro .gnuastro
 # ------------------------------------------
 
 
@@ -1250,6 +1293,48 @@ fi
 
 
 
+# Find Zenodo URL for software downloading
+# ----------------------------------------
+#
+# All free-software source tarballs that are potentially used in Maneage
+# are also archived in Zenodo with a certain concept-DOI. A concept-DOI is
+# a Zenodo terminology, meaning a fixed DOI of the project (that can have
+# many sub-DOIs for different versions). By default, the concept-DOI points
+# to the most recently uploaded version. However, the concept-DOI itself is
+# not directly usable for downloading files. The concept-DOI will just take
+# us to the top webpage of the most recent version of the upload.
+#
+# The problem is that as more software are added (as new Zenodo versions),
+# the most recent Zenodo-URL that the concept-DOI points to, also
+# changes. The most reliable solution was found to be the tiny script below
+# which will download the DOI-resolved webpage, and extract the Zenodo-URL
+# of the most recent version from there (using the 'coreutils' tarball as
+# an example, the directory part of the URL for all the other software are
+# the same).
+user_backup_urls=""
+zenodocheck=.build/software/zenodo-check.html
+if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then
+    zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \
+                    | sed -n -e'/http/p' \
+                    | tr ' ' '\n' \
+                    | grep http \
+                    | sed -e 's/href="//' -e 's|/coreutils| |' \
+                    | awk 'NR==1{print $1}')
+else
+    zenodourl=""
+fi
+rm -f $zenodocheck
+
+# Add the Zenodo URL to the user's given back software URLs. Since the user
+# can specify 'user_backup_urls' (not yet implemented as an option in
+# './project'), we'll give preference to their specified servers, then add
+# the Zenodo URL afterwards.
+user_backup_urls="$user_backup_urls $zenodourl"
+
+
+
+
+
 # Build core tools for project
 # ----------------------------
 #
@@ -1258,7 +1343,7 @@ fi
 # (minimal Bash-like shell) and Flock (to lock files and enable serial
 # download).
 ./reproduce/software/shell/pre-make-build.sh \
-    "$bdir" "$ddir" "$downloader"
+    "$bdir" "$ddir" "$downloader" "$user_backup_urls"
 
 
 
@@ -1272,6 +1357,7 @@ fi
 # tools, but we have to be very portable (and use minimal features in all).
 echo; echo "Building necessary software (if necessary)..."
 .local/bin/make -k -f reproduce/software/make/basic.mk \
+     user_backup_urls="$user_backup_urls" \
      sys_library_path=$sys_library_path \
      rpath_command=$rpath_command \
      static_build=$static_build \
@@ -1293,20 +1379,22 @@ echo; echo "Building necessary software (if necessary)..."
 # script. Bash and Make were the tools we need to run Makefiles, so we had
 # to build them in this script. But after this, we can rely on Makefiles.
 if [ $jobs = 0 ]; then
-    numthreads=$($instdir/bin/nproc --all)
+    numthreads=$(.local/bin/nproc --all)
 else
     numthreads=$jobs
 fi
 .local/bin/env -i HOME=$bdir \
-    .local/bin/make -k -f reproduce/software/make/high-level.mk \
-                    sys_library_path=$sys_library_path \
-                    rpath_command=$rpath_command \
-                    static_build=$static_build \
-                    numthreads=$numthreads \
-                    on_mac_os=$on_mac_os \
-                    sys_cpath=$sys_cpath \
-                    host_cc=$host_cc \
-                    -j$numthreads
+     .local/bin/make -k -f reproduce/software/make/high-level.mk \
+          user_backup_urls="$user_backup_urls" \
+          sys_library_path=$sys_library_path \
+          rpath_command=$rpath_command \
+          all_highlevel=$all_highlevel \
+          static_build=$static_build \
+          numthreads=$numthreads \
+          on_mac_os=$on_mac_os \
+          sys_cpath=$sys_cpath \
+          host_cc=$host_cc \
+          -j$numthreads
 
 
 
@@ -1400,26 +1488,27 @@ prepare_name_version ()
     fi
 }
 
+# Import the context/sentences for placing between the list of software
+# names during their acknowledgment.
+. $cdir/software_acknowledge_context.sh
+
 # Report the different software in separate contexts (separating Python and
 # TeX packages from the C/C++ programs and libraries).
 proglibs=$(prepare_name_version $verdir/proglib/*)
 pymodules=$(prepare_name_version $verdir/python/*)
 texpkg=$(prepare_name_version $verdir/tex/texlive)
 
-# Write them as one paragraph for LaTeX.
+# Acknowledge these software packages in a LaTeX paragraph.
 pkgver=$mtexdir/dependencies.tex
-.local/bin/echo "This research was done with the following free" > $pkgver
-.local/bin/echo "software programs and libraries: $proglibs."   >> $pkgver
+
+# Add the text to the ${pkgver} file.
+.local/bin/echo "$thank_software_introduce " > $pkgver
+.local/bin/echo "$thank_progs_libs $proglibs. "   >> $pkgver
 if [ x"$pymodules" != x ]; then
-    .local/bin/echo "Within Python, the following modules"      >> $pkgver
-    echo "were used: $pymodules."                               >> $pkgver
+    .local/bin/echo "$thank_python $pymodules. "   >> $pkgver
 fi
-.local/bin/echo "The \LaTeX{} source of the paper was compiled" >> $pkgver
-.local/bin/echo "to make the PDF using the following packages:" >> $pkgver
-.local/bin/echo "$texpkg. We are very grateful to all their"    >> $pkgver
-.local/bin/echo "creators for freely providing this necessary"  >> $pkgver
-.local/bin/echo "infrastructure. This research (and many "      >> $pkgver
-.local/bin/echo "others) would not be possible without them."   >> $pkgver
+.local/bin/echo "$thank_latex $texpkg. " >> $pkgver
+.local/bin/echo "$thank_software_conclude" >> $pkgver
 
 # Prepare the BibTeX entries for the used software (if there are any).
 hasentry=0
diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh
index e2ac789..05a4143 100755
--- a/reproduce/software/shell/pre-make-build.sh
+++ b/reproduce/software/shell/pre-make-build.sh
@@ -34,6 +34,7 @@ set -e
 bdir=$1
 ddir=$2
 downloader="$3"
+user_backup_urls="$4"
 
 
 
@@ -51,6 +52,7 @@ downloadwrapper=reproduce/analysis/bash/download-multi-try
 
 # Derived directories
 bindir=$instdir/bin
+urlfile=$confdir/urls.conf
 versionsfile=$confdir/versions.conf
 checksumsfile=$confdir/checksums.conf
 backupfile=$confdir/servers-backup.conf
@@ -65,8 +67,18 @@ export PATH="$bindir:$PATH"
 
 
 
-# Load the backup servers
-backupservers=$(awk '!/^#/{printf "%s ", $1}' $backupfile)
+# Load the backup servers, but separate the first one.
+backupservers=""
+topbackupserver=""
+maneage_backup_urls=$(awk '!/^#/{printf "%s ", $1}' $backupfile)
+backupservers_all="$user_backup_urls $maneage_backup_urls"
+for b in $backupservers_all; do
+    if [ x$topbackupserver = x ]; then
+        topbackupserver=$b
+    else
+        backupservers="$backupservers $b"
+    fi
+done
 
 
 
@@ -83,12 +95,21 @@ download_tarball() {
   else
     ucname=$tardir/$tarball.unchecked
 
+    # If the URL is empty, use the top backup server
+    if [ x$w = x ]; then
+        bservers="$backupservers"
+        tarballurl=$topbackupserver/$tarball
+    else
+        bservers="$backupservers_all"
+        tarballurl=$url/$tarball
+    fi
+
     # See if it is in the input software directory.
     if [ -f "$ddir/$tarball" ]; then
       cp $ddir/$tarball $ucname
     else
-      $downloadwrapper "$downloader" nolock $url/$tarball $ucname \
-                       "$backupservers"
+      $downloadwrapper "$downloader" nolock $tarballurl $ucname \
+                       "$bservers"
     fi
 
     # Make sure this is the correct tarball.
@@ -119,10 +140,14 @@ download_tarball() {
 
 
 
-# Build the program from the tarball
+# Build the program from the tarball. This function takes one argument
+# which is the configure-time options.
 build_program() {
   if ! [ -f $ibidir/$progname ]; then
 
+    # Options
+    configoptions=$1
+
     # Go into the temporary building directory.
     cd $tmpblddir
     unpackdir="$progname"-"$version"
@@ -140,13 +165,31 @@ build_program() {
       intar=$tardir/$tarball
     fi
 
-    # Unpack the tarball and build the program.
+    # Unpack the tarball and go into it.
     tar xf $intar
     if [ x$intarrm = x1 ]; then rm $intar; fi
     cd $unpackdir
-    ./configure --prefix=$instdir
-    make
-    make install
+
+    # build the project, either with Make and either without it.
+    if [ x$progname = xlzip ]; then
+        ./configure --build --check --installdir=$instdir/bin $configoptions
+    else
+        # All others accept the configure script.
+        ./configure --prefix=$instdir $configoptions
+
+        # To build GNU Make, we don't want to assume the existance of a
+        # Make program, so we use its 'build.sh' script and its own built
+        # 'make' program to install itself.
+        if [ x$progname = xmake ]; then
+            /bin/sh build.sh
+            ./make install
+        else
+            make
+            make install
+        fi
+    fi
+
+    # Clean up the source directory
     cd $topdir
     rm -rf $tmpblddir/$unpackdir
     echo "$progname_tex $version" > $ibidir/$progname
@@ -167,7 +210,7 @@ build_program() {
 # won't rely on the host's compression tools at all.
 progname="lzip"
 progname_tex="Lzip"
-url=http://akhlaghi.org/src
+url=$(awk '/^'$progname'-url/{print $3}' $urlfile)
 version=$(awk '/^'$progname'-version/{print $3}' $versionsfile)
 tarball=$progname-$version.tar
 download_tarball
@@ -180,19 +223,17 @@ build_program
 # GNU Make
 # --------
 #
-# The job orchestrator of Maneage is GNU Make. Although it is not
-# impossible to account for all the differences between various Make
-# implementations, its much easier (for reading the code and
-# writing/debugging it) if we can count on a special implementation. So
-# before going into the complex job orchestration in building high-level
-# software, we start by building GNU Make.
+# The job orchestrator of Maneage is GNU Make. The
+# '--disable-dependency-tracking' configure-time option is necessary so
+# Make doesn't check for an existing 'make' implementation (recall that we
+# aren't assuming any 'make' on the host).
 progname="make"
 progname_tex="GNU Make"
-url=http://akhlaghi.org/src
+url=$(awk '/^'$progname'-url/{print $3}' $urlfile)
 version=$(awk '/^'$progname'-version/{print $3}' $versionsfile)
 tarball=$progname-$version.tar.lz
 download_tarball
-build_program
+build_program --disable-dependency-tracking
 
 
 
@@ -206,7 +247,7 @@ build_program
 # (which builds GNU Bash).
 progname="dash"
 progname_tex="Dash"
-url=http://akhlaghi.org/src
+url=$(awk '/^'$progname'-url/{print $3}' $urlfile)
 version=$(awk '/^'$progname'-version/{print $3}' $versionsfile)
 tarball=$progname-$version.tar.lz
 download_tarball
@@ -235,7 +276,7 @@ fi
 # many simultaneous download commands are called.
 progname="flock"
 progname_tex="Discoteq flock"
-url=http://akhlaghi.org/src
+url=$(awk '/^'$progname'-url/{print $3}' $urlfile)
 version=$(awk '/^'$progname'-version/{print $3}' $versionsfile)
 tarball=$progname-$version.tar.lz
 download_tarball
diff --git a/reproduce/software/shell/run-parts.in b/reproduce/software/shell/run-parts.in
new file mode 100755
index 0000000..9213585
--- /dev/null
+++ b/reproduce/software/shell/run-parts.in
@@ -0,0 +1,71 @@
+#!MANEAGESHELL
+# run-parts:  Runs all the scripts found in a directory.
+# from Slackware, by Patrick J. Volkerding with ideas borrowed
+# from the Red Hat and Debian versions of this utility.
+#
+# USAGE IN MANEAGE: this script is built with the 'libpaper' package.
+#
+# The original file was taken from Linux From Scratch:
+# http://www.linuxfromscratch.org/blfs/view/svn/general/libpaper.html
+# However, it didn't have a copyright statement. So one is being added
+# here.
+#
+# Copyright (C) 2020 Authors mentioned above.
+# Copyright (C) 2020 Mohammad Akhlaghi <mohammad@akhlaghi.org>
+#
+# This script is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This script is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this script.  If not, see <http://www.gnu.org/licenses/>.
+
+# keep going when something fails
+set +e
+
+if [ $# -lt 1 ]; then
+  echo "Usage: run-parts <directory>"
+  exit 1
+fi
+
+if [ ! -d $1 ]; then
+  echo "Not a directory: $1"
+  echo "Usage: run-parts <directory>"
+  exit 1
+fi
+
+# There are several types of files that we would like to
+# ignore automatically, as they are likely to be backups
+# of other scripts:
+IGNORE_SUFFIXES="~ ^ , .bak .new .rpmsave .rpmorig .rpmnew .swp"
+
+# Main loop:
+for SCRIPT in $1/* ; do
+  # If this is not a regular file, skip it:
+  if [ ! -f $SCRIPT ]; then
+    continue
+  fi
+  # Determine if this file should be skipped by suffix:
+  SKIP=false
+  for SUFFIX in $IGNORE_SUFFIXES ; do
+    if [ ! "$(basename $SCRIPT $SUFFIX)" = "$(basename $SCRIPT)" ]; then
+      SKIP=true
+      break
+    fi
+  done
+  if [ "$SKIP" = "true" ]; then
+    continue
+  fi
+  # If we've made it this far, then run the script if it's executable:
+  if [ -x $SCRIPT ]; then
+    $SCRIPT || echo "$SCRIPT failed."
+  fi
+done
+
+exit 0