diff options
Diffstat (limited to 'reproduce/software/shell')
| -rwxr-xr-x | reproduce/software/shell/configure.sh | 139 | ||||
| -rwxr-xr-x | reproduce/software/shell/pre-make-build.sh | 81 | ||||
| -rwxr-xr-x | reproduce/software/shell/run-parts.in | 71 |
3 files changed, 246 insertions, 45 deletions
diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index b2e3979..38dcf7f 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -153,6 +153,38 @@ check_permission () +# Check if there is enough free space available in the build directory +# -------------------------------------------------------------------- +# +# Use this function to check if there is enough free space in a +# directory. It is meant to be passed to the 'if' statement in the +# shell. So if there is enough space, it returns 0 (which translates to +# TRUE), otherwise, the funcion returns 1 (which translates to FALSE). +# +# Expects to be called with two arguments, the first is the threshold and +# the second is the desired directory. The 'df' function checks the given +# path to see where it is mounted on, and how much free space there is on +# that partition (in units of 1024 bytes). +# +# synopsis: +# $ free_space_warning <acceptable_threshold> <path-to-check> +# +# example: +# To check if there is 5MB of space available in /path/to/check +# call the command with arguments as shown below: +# $ free_space_warning 5000 /path/to/check/free/space +free_space_warning() +{ + fs_threshold=$1 + fs_destpath=$2 + return $(df $fs_destpath \ + | awk 'FNR==2 {if($4>'$fs_threshold') print 1; \ + else print 0; }') +} + + + + # Check for C/C++ compilers # ------------------------- @@ -729,14 +761,14 @@ EOF bdir=$(absolute_dir $build_dir) rm -rf $build_dir/$junkname else - echo " ** Can't write in '$build_dir'"; echo + echo " ** Can't write in '$build_dir'"; fi else if mkdir $build_dir 2> /dev/null; then instring="the newly created" bdir=$(absolute_dir $build_dir) else - echo " ** Can't create '$build_dir'"; echo + echo " ** Can't create '$build_dir'"; fi fi @@ -759,8 +791,19 @@ EOF # file permissions. if ! [ x"$bdir" = x ]; then if ! $(check_permission $bdir); then + # Unable to handle permissions well bdir= echo " ** File permissions can't be modified in this directory" + else + # Able to handle permissions, now check for 5GB free space + # in the given partition (note that the number is in units + # of 1024 bytes). If this is not the case, print a warning. + if $(free_space_warning 5000000 $bdir); then + echo " !! LESS THAN 5GB FREE SPACE IN: $bdir" + echo " !! We recommend choosing another partition." + echo " !! Build will continue in 5 seconds..." + sleep 5 + fi fi fi @@ -1113,7 +1156,7 @@ ln -s $instdir .local ln -s $texdir tex/build ln -s $tikzdir tex/tikz # --------- Delete for no Gnuastro --------- -ln -s $topdir/reproduce/software/config/gnuastro .gnuastro +ln -s $topdir/reproduce/analysis/config/gnuastro .gnuastro # ------------------------------------------ @@ -1250,6 +1293,48 @@ fi +# Find Zenodo URL for software downloading +# ---------------------------------------- +# +# All free-software source tarballs that are potentially used in Maneage +# are also archived in Zenodo with a certain concept-DOI. A concept-DOI is +# a Zenodo terminology, meaning a fixed DOI of the project (that can have +# many sub-DOIs for different versions). By default, the concept-DOI points +# to the most recently uploaded version. However, the concept-DOI itself is +# not directly usable for downloading files. The concept-DOI will just take +# us to the top webpage of the most recent version of the upload. +# +# The problem is that as more software are added (as new Zenodo versions), +# the most recent Zenodo-URL that the concept-DOI points to, also +# changes. The most reliable solution was found to be the tiny script below +# which will download the DOI-resolved webpage, and extract the Zenodo-URL +# of the most recent version from there (using the 'coreutils' tarball as +# an example, the directory part of the URL for all the other software are +# the same). +user_backup_urls="" +zenodocheck=.build/software/zenodo-check.html +if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then + zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ + | sed -n -e'/http/p' \ + | tr ' ' '\n' \ + | grep http \ + | sed -e 's/href="//' -e 's|/coreutils| |' \ + | awk 'NR==1{print $1}') +else + zenodourl="" +fi +rm -f $zenodocheck + +# Add the Zenodo URL to the user's given back software URLs. Since the user +# can specify 'user_backup_urls' (not yet implemented as an option in +# './project'), we'll give preference to their specified servers, then add +# the Zenodo URL afterwards. +user_backup_urls="$user_backup_urls $zenodourl" + + + + + # Build core tools for project # ---------------------------- # @@ -1258,7 +1343,7 @@ fi # (minimal Bash-like shell) and Flock (to lock files and enable serial # download). ./reproduce/software/shell/pre-make-build.sh \ - "$bdir" "$ddir" "$downloader" + "$bdir" "$ddir" "$downloader" "$user_backup_urls" @@ -1272,6 +1357,7 @@ fi # tools, but we have to be very portable (and use minimal features in all). echo; echo "Building necessary software (if necessary)..." .local/bin/make -k -f reproduce/software/make/basic.mk \ + user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ static_build=$static_build \ @@ -1293,20 +1379,22 @@ echo; echo "Building necessary software (if necessary)..." # script. Bash and Make were the tools we need to run Makefiles, so we had # to build them in this script. But after this, we can rely on Makefiles. if [ $jobs = 0 ]; then - numthreads=$($instdir/bin/nproc --all) + numthreads=$(.local/bin/nproc --all) else numthreads=$jobs fi .local/bin/env -i HOME=$bdir \ - .local/bin/make -k -f reproduce/software/make/high-level.mk \ - sys_library_path=$sys_library_path \ - rpath_command=$rpath_command \ - static_build=$static_build \ - numthreads=$numthreads \ - on_mac_os=$on_mac_os \ - sys_cpath=$sys_cpath \ - host_cc=$host_cc \ - -j$numthreads + .local/bin/make -k -f reproduce/software/make/high-level.mk \ + user_backup_urls="$user_backup_urls" \ + sys_library_path=$sys_library_path \ + rpath_command=$rpath_command \ + all_highlevel=$all_highlevel \ + static_build=$static_build \ + numthreads=$numthreads \ + on_mac_os=$on_mac_os \ + sys_cpath=$sys_cpath \ + host_cc=$host_cc \ + -j$numthreads @@ -1400,26 +1488,27 @@ prepare_name_version () fi } +# Import the context/sentences for placing between the list of software +# names during their acknowledgment. +. $cdir/software_acknowledge_context.sh + # Report the different software in separate contexts (separating Python and # TeX packages from the C/C++ programs and libraries). proglibs=$(prepare_name_version $verdir/proglib/*) pymodules=$(prepare_name_version $verdir/python/*) texpkg=$(prepare_name_version $verdir/tex/texlive) -# Write them as one paragraph for LaTeX. +# Acknowledge these software packages in a LaTeX paragraph. pkgver=$mtexdir/dependencies.tex -.local/bin/echo "This research was done with the following free" > $pkgver -.local/bin/echo "software programs and libraries: $proglibs." >> $pkgver + +# Add the text to the ${pkgver} file. +.local/bin/echo "$thank_software_introduce " > $pkgver +.local/bin/echo "$thank_progs_libs $proglibs. " >> $pkgver if [ x"$pymodules" != x ]; then - .local/bin/echo "Within Python, the following modules" >> $pkgver - echo "were used: $pymodules." >> $pkgver + .local/bin/echo "$thank_python $pymodules. " >> $pkgver fi -.local/bin/echo "The \LaTeX{} source of the paper was compiled" >> $pkgver -.local/bin/echo "to make the PDF using the following packages:" >> $pkgver -.local/bin/echo "$texpkg. We are very grateful to all their" >> $pkgver -.local/bin/echo "creators for freely providing this necessary" >> $pkgver -.local/bin/echo "infrastructure. This research (and many " >> $pkgver -.local/bin/echo "others) would not be possible without them." >> $pkgver +.local/bin/echo "$thank_latex $texpkg. " >> $pkgver +.local/bin/echo "$thank_software_conclude" >> $pkgver # Prepare the BibTeX entries for the used software (if there are any). hasentry=0 diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh index e2ac789..05a4143 100755 --- a/reproduce/software/shell/pre-make-build.sh +++ b/reproduce/software/shell/pre-make-build.sh @@ -34,6 +34,7 @@ set -e bdir=$1 ddir=$2 downloader="$3" +user_backup_urls="$4" @@ -51,6 +52,7 @@ downloadwrapper=reproduce/analysis/bash/download-multi-try # Derived directories bindir=$instdir/bin +urlfile=$confdir/urls.conf versionsfile=$confdir/versions.conf checksumsfile=$confdir/checksums.conf backupfile=$confdir/servers-backup.conf @@ -65,8 +67,18 @@ export PATH="$bindir:$PATH" -# Load the backup servers -backupservers=$(awk '!/^#/{printf "%s ", $1}' $backupfile) +# Load the backup servers, but separate the first one. +backupservers="" +topbackupserver="" +maneage_backup_urls=$(awk '!/^#/{printf "%s ", $1}' $backupfile) +backupservers_all="$user_backup_urls $maneage_backup_urls" +for b in $backupservers_all; do + if [ x$topbackupserver = x ]; then + topbackupserver=$b + else + backupservers="$backupservers $b" + fi +done @@ -83,12 +95,21 @@ download_tarball() { else ucname=$tardir/$tarball.unchecked + # If the URL is empty, use the top backup server + if [ x$w = x ]; then + bservers="$backupservers" + tarballurl=$topbackupserver/$tarball + else + bservers="$backupservers_all" + tarballurl=$url/$tarball + fi + # See if it is in the input software directory. if [ -f "$ddir/$tarball" ]; then cp $ddir/$tarball $ucname else - $downloadwrapper "$downloader" nolock $url/$tarball $ucname \ - "$backupservers" + $downloadwrapper "$downloader" nolock $tarballurl $ucname \ + "$bservers" fi # Make sure this is the correct tarball. @@ -119,10 +140,14 @@ download_tarball() { -# Build the program from the tarball +# Build the program from the tarball. This function takes one argument +# which is the configure-time options. build_program() { if ! [ -f $ibidir/$progname ]; then + # Options + configoptions=$1 + # Go into the temporary building directory. cd $tmpblddir unpackdir="$progname"-"$version" @@ -140,13 +165,31 @@ build_program() { intar=$tardir/$tarball fi - # Unpack the tarball and build the program. + # Unpack the tarball and go into it. tar xf $intar if [ x$intarrm = x1 ]; then rm $intar; fi cd $unpackdir - ./configure --prefix=$instdir - make - make install + + # build the project, either with Make and either without it. + if [ x$progname = xlzip ]; then + ./configure --build --check --installdir=$instdir/bin $configoptions + else + # All others accept the configure script. + ./configure --prefix=$instdir $configoptions + + # To build GNU Make, we don't want to assume the existance of a + # Make program, so we use its 'build.sh' script and its own built + # 'make' program to install itself. + if [ x$progname = xmake ]; then + /bin/sh build.sh + ./make install + else + make + make install + fi + fi + + # Clean up the source directory cd $topdir rm -rf $tmpblddir/$unpackdir echo "$progname_tex $version" > $ibidir/$progname @@ -167,7 +210,7 @@ build_program() { # won't rely on the host's compression tools at all. progname="lzip" progname_tex="Lzip" -url=http://akhlaghi.org/src +url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) tarball=$progname-$version.tar download_tarball @@ -180,19 +223,17 @@ build_program # GNU Make # -------- # -# The job orchestrator of Maneage is GNU Make. Although it is not -# impossible to account for all the differences between various Make -# implementations, its much easier (for reading the code and -# writing/debugging it) if we can count on a special implementation. So -# before going into the complex job orchestration in building high-level -# software, we start by building GNU Make. +# The job orchestrator of Maneage is GNU Make. The +# '--disable-dependency-tracking' configure-time option is necessary so +# Make doesn't check for an existing 'make' implementation (recall that we +# aren't assuming any 'make' on the host). progname="make" progname_tex="GNU Make" -url=http://akhlaghi.org/src +url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) tarball=$progname-$version.tar.lz download_tarball -build_program +build_program --disable-dependency-tracking @@ -206,7 +247,7 @@ build_program # (which builds GNU Bash). progname="dash" progname_tex="Dash" -url=http://akhlaghi.org/src +url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) tarball=$progname-$version.tar.lz download_tarball @@ -235,7 +276,7 @@ fi # many simultaneous download commands are called. progname="flock" progname_tex="Discoteq flock" -url=http://akhlaghi.org/src +url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) tarball=$progname-$version.tar.lz download_tarball diff --git a/reproduce/software/shell/run-parts.in b/reproduce/software/shell/run-parts.in new file mode 100755 index 0000000..9213585 --- /dev/null +++ b/reproduce/software/shell/run-parts.in @@ -0,0 +1,71 @@ +#!MANEAGESHELL +# run-parts: Runs all the scripts found in a directory. +# from Slackware, by Patrick J. Volkerding with ideas borrowed +# from the Red Hat and Debian versions of this utility. +# +# USAGE IN MANEAGE: this script is built with the 'libpaper' package. +# +# The original file was taken from Linux From Scratch: +# http://www.linuxfromscratch.org/blfs/view/svn/general/libpaper.html +# However, it didn't have a copyright statement. So one is being added +# here. +# +# Copyright (C) 2020 Authors mentioned above. +# Copyright (C) 2020 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This script is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This script is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this script. If not, see <http://www.gnu.org/licenses/>. + +# keep going when something fails +set +e + +if [ $# -lt 1 ]; then + echo "Usage: run-parts <directory>" + exit 1 +fi + +if [ ! -d $1 ]; then + echo "Not a directory: $1" + echo "Usage: run-parts <directory>" + exit 1 +fi + +# There are several types of files that we would like to +# ignore automatically, as they are likely to be backups +# of other scripts: +IGNORE_SUFFIXES="~ ^ , .bak .new .rpmsave .rpmorig .rpmnew .swp" + +# Main loop: +for SCRIPT in $1/* ; do + # If this is not a regular file, skip it: + if [ ! -f $SCRIPT ]; then + continue + fi + # Determine if this file should be skipped by suffix: + SKIP=false + for SUFFIX in $IGNORE_SUFFIXES ; do + if [ ! "$(basename $SCRIPT $SUFFIX)" = "$(basename $SCRIPT)" ]; then + SKIP=true + break + fi + done + if [ "$SKIP" = "true" ]; then + continue + fi + # If we've made it this far, then run the script if it's executable: + if [ -x $SCRIPT ]; then + $SCRIPT || echo "$SCRIPT failed." + fi +done + +exit 0 |