#!/bin/sh # # Necessary preparations/configurations for the reproducible project. # # Copyright (C) 2018-2025 Mohammad Akhlaghi # Copyright (C) 2021-2025 Raul Infante-Sainz # Copyright (C) 2022-2025 Pedram Ashofteh Ardakani # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This script is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this script. If not, see . # Script settings # --------------- # Stop the script if there are any errors. set -e # Project-specific settings # ------------------------- # # The variables defined here may be different between different # projects. Ideally, they should be detected automatically, but we haven't # had the chance to implement it yet (please help if you can!). Until then, # please set them based on your project (if they differ from the core # branch). # If equals 1, a message will be printed, showing the nano-seconds since # previous step: useful with '-e --offline --nopause --quiet' to find # bottlenecks for speed optimization. Speed is important because this # script is called automatically every time by the container scripts. check_elapsed=0 # In case a fortran compiler is necessary to check. need_gfortran=0 # Internal source directories # --------------------------- # # These are defined to help make this script more readable. topdir="$(pwd)" optionaldir="/optional/path" cdir=reproduce/software/config # Notice for top of generated files # --------------------------------- # # In case someone opens the files output from the configuration scripts in # a text editor and wants to edit them, it is important to let them know # that their changes are not going to be permenant. create_file_with_notice () { if printf "# IMPORTANT: " > "$1" then # These commands may look messy, but the produced comments in the # file are the main goal and they are readable. (without having to # break our source-code line length). printf "file can be RE-WRITTEN after './project " >> "$1" printf "configure'.\n" >> "$1" printf "#\n" >> "$1" printf "# This file was created during configuration " >> "$1" printf "('./project configure').\n" >> "$1" printf "# Therefore, it is not under version control " >> "$1" printf "and any manual changes\n" >> "$1" printf "# to it will be over-written when the " >> "$1" printf "project is re-configured.\n" >> "$1" printf "#\n" >> "$1" else echo; echo "Can't write to $1"; echo; exit 1 fi } # Get absolute address # -------------------- # # Since the build directory will go into a symbolic link, we want it to be # an absolute address. With this function we can make sure of that. absolute_dir () { address="$1" if stat "$address" 1> /dev/null; then echo "$(cd "$(dirname "$1")" && pwd )/$(basename "$1")" else echo "$optionaldir" fi } # Check file permission handling (POSIX-compatibility) # ---------------------------------------------------- # # Check if a 'given' directory handles permissions as expected. # # This is to prevent a known bug in the NTFS filesystem that prevents # proper installation of Perl, and probably some other packages. This # function receives the directory as an argument and then, creates a dummy # file, and examines whether the given directory handles the file # permissions as expected. # # Returns '0' if everything is fine, and '255' otherwise. Choosing '0' is # to mimic the '$ echo $?' behavior, while choosing '255' is to prevent # misunderstanding 0 and 1 as true and false. # # ===== CAUTION! ===== # # # Since there is a 'set -e' before running this function, the whole script # stops and exits IF the 'check_permission' (or any other function) returns # anything OTHER than '0'! So, only use this function as a test. Here's a # minimal example: # # if $(check_permission $some_directory) ; then # echo "yay"; else "nay"; # fi ; check_permission () { # Make a 'junk' file, activate its executable flag and record its # permissions generally. local junkfile="$1"/check_permission_tmp_file rm -f "$junkfile" echo "Don't let my short life go to waste" > "$junkfile" chmod +x "$junkfile" local perm_before=$(ls -l "$junkfile" | awk '{print $1}') # Now, remove the executable flag and record the permissions. chmod -x "$junkfile" local perm_after=$(ls -l "$junkfile" | awk '{print $1}') # Clean up before leaving the function rm -f "$junkfile" # If the permissions are equal, the filesystem doesn't allow # permissions. if [ $perm_before = $perm_after ]; then # Setting permission FAILED return 1 else # Setting permission SUCCESSFUL return 0 fi } # Check if there is enough free space available in the build directory # -------------------------------------------------------------------- # # Use this function to check if there is enough free space in a # directory. It is meant to be passed to the 'if' statement in the # shell. So if there is enough space, it returns 0 (which translates to # TRUE), otherwise, the funcion returns 1 (which translates to FALSE). # # Expects to be called with two arguments, the first is the threshold and # the second is the desired directory. The 'df' function checks the given # path to see where it is mounted on, and how much free space there is on # that partition (in units of 1024 bytes). # # synopsis: # $ free_space_warning # # example: # To check if there is 5MB of space available in /path/to/check # call the command with arguments as shown below: # $ free_space_warning 5000 /path/to/check/free/space free_space_warning() { fs_threshold=$1 fs_destpath="$2" return $(df -P "$fs_destpath" \ | awk 'FNR==2 {if($4>'$fs_threshold') print 1; \ else print 0; }') } # Function to empty the temporary software building directory. This can # either be a symbolic link (to RAM) or an actual directory, so we can't # simply use 'rm -r' (because a symbolic link is not a directory for 'rm'). empty_build_tmp() { # 'ls -A' does not print the '.' and '..' and the '-z' option of '[' # checks if the string is empty or not. This allows us to only attempt # deleting the directory's contents if it actually has anything inside # of it. Otherwise, '*' will not expand and we'll get an 'rm' error # complaining that '$tmpblddir/*' doesn't exist. We also don't want to # use 'rm -rf $tmpblddir/*' because in case of a typo or while # debugging (if '$tmpblddir' becomes an empty string), this can # accidentally delete the whole root partition (or a least the '/home' # partition of the user). if ! [ x"$( ls -A $tmpblddir )" = x ]; then rm -r "$tmpblddir"/* fi rm -r "$tmpblddir" } # Function to report the elapsed time between steps (if it was activated # above with 'check_elapsed'). elapsed_time_from_prev_step() { if [ $check_elapsed = 1 ]; then chel_now=$(date +"%N"); chel_delta=$(echo $chel_prev $chel_now \ | awk '{ delta=($2-$1)/1e6; \ if(delta>0) d=delta; else d=0; \ print d}') chel_dsum=$(echo $chel_dsum $chel_delta | awk '{print $1+$2}') echo $chel_counter $chel_delta "$1" \ | awk '{ printf "Step %02d: %-6.2f [millisec]; %s\n", \ $1, $2, $3}' chel_counter=$((chel_counter+1)) chel_prev=$(date +"%N") fi } # In already-built container # -------------------------- # # We need to run './project configure' at the start of every run of Maneage # within a container (with 'shell' or 'make'). This is because we need to # ensure the versions of all software are correct. However, the container # filesystem (where the build/software directory is located) should be run # as read-only when doing the analysis. So we will not be able to run some # of the tests that require writing files or are generally not relevant # when the container is already built (we want the configure command to be # as fast as possible). # # The project source in Maneage'd containers is '/home/maneager/source'. built_container=0 if [ "$topdir" = /home/maneager/source ] \ && [ -f .build/software/config/hardware-parameters.tex ]; then built_container=1; fi # Initialize the elapsed time measurement parameters. if [ $check_elapsed = 1 ]; then chel_dsum=0.00 chel_counter=1 chel_prev=$(date +"%N") chel_start=$(date +"%N") fi # Identify the running OS # ----------------------- # # Some features are tailored to GNU/Linux systems, while the BSD-based # behavior is different. Initially we only tested macOS (hence the name of # the variable), but as FreeBSD is also being inlucded in our tests. As # more systems get used, we need to tailor these kinds of things better. if [ $built_container = 0 ]; then kernelname=$(uname -s) if [ $pauseformsg = 1 ]; then pausesec=10; else pausesec=0; fi if [ x$kernelname = xLinux ]; then on_mac_os=no # Don't forget to add the respective C++ compiler below (leave 'cc' in # the end). c_compiler_list="gcc clang cc" elif [ x$kernelname = xDarwin ]; then host_cc=1 on_mac_os=yes # Don't forget to add the respective C++ compiler below (leave 'cc' in # the end). c_compiler_list="clang gcc cc" else on_mac_os=no cat < /dev/null 2>/dev/null; then export CC=$c; if type $cplus > /dev/null 2>/dev/null; then export CXX=$cplus has_compilers=yes break fi fi done if [ x$has_compilers = xno ]; then cat < $testsource < #include int main(void){printf("Good!\n"); return EXIT_SUCCESS;} EOF if $CC $noccwarnings $testsource -o$testprog && $testprog > /dev/null; then if [ $quiet = 0 ]; then echo "... yes"; fi rm $testsource $testprog else rm $testsource cat < $testsource < #include int main(void) { void *handle=dlopen ("/lib/CEDD_LIB.so.6", RTLD_LAZY); return 0; } EOF if $CC $testsource -o$testprog 2>/dev/null > /dev/null; then needs_ldl=no; else needs_ldl=yes; fi elapsed_time_from_prev_step compiler-needs-dynamic-linker fi # See if the C compiler can build static libraries # ------------------------------------------------ # # We are manually only working with shared libraries: because some # high-level programs like Wget and cURL need dynamic linking and if we # build the libraries statically, our own builds will be ignored and these # programs will go and find their necessary libraries on the host system. # # Another good advantage of shared libraries is that we can actually use # the shared library tool of the system ('ldd' with GNU C Library) and see # exactly where each linked library comes from. But in static building, # unless you follow the build closely, its not easy to see if the source of # the library came from the system or our build. static_build=no # Print warning if the host CC is to be used. if [ $built_container = 0 ] && [ x$host_cc = x1 ]; then cat <&1 \ | tr ' ' '\n' \ | awk '/\-\-target/' \ | sed -e's/\-\-target=//') if [ x"$gcctarget" != x ]; then if [ -f /usr/lib/$gcctarget/libc.a ]; then export sys_library_path=/usr/lib/$gcctarget export sys_cpath=/usr/include/$gcctarget fi fi # For a check: #echo "sys_library_path: $sys_library_path" #echo "sys_cpath: $sys_cpath" elapsed_time_from_prev_step compiler-sys-cpath fi # See if a link-able static C library exists # ------------------------------------------ # # A static C library and the 'sys/cdefs.h' header are necessary for # building GCC. if [ $built_container = 0 ]; then if [ x"$host_cc" = x0 ]; then if [ $quiet = 0 ]; then echo; echo "Checking if static C library is available..."; fi cat > $testsource < #include #include int main(void){printf("...yes\n"); return EXIT_SUCCESS;} EOF cc_call="$CC $testsource $CPPFLAGS $LDFLAGS -o$testprog -static -lc" if $cc_call && $testprog > /dev/null; then gccwarning=0 rm $testsource $testprog if [ $quiet = 0 ]; then echo "... yes"; fi else echo; echo "Compilation command:"; echo "$cc_call" rm $testsource gccwarning=1 host_cc=1 cat < /dev/null 2>/dev/null; then hasfc=1; fi if [ $hasfc = 0 ]; then cat < $testsourcef echo " END" >> $testsourcef if gfortran $testsourcef -o$testprog && $testprog; then rm $testsourcef $testprog else rm $testsourcef cat < $testsource < #include int main(void) {return EXIT_SUCCESS;} EOF if $CC $testsource -o$testprog -Wl,-rpath-link 2>/dev/null \ > /dev/null; then export rpath_command="-Wl,-rpath-link=$instdir/lib" else export rpath_command="" fi # Delete the temporary directory for compiler checking. rm -f $testprog $testsource rm -r $compilertestdir elapsed_time_from_prev_step compiler-rpath fi # Paths needed by the host compiler (only for 'basic.mk') # ------------------------------------------------------- # # At the end of the basic build, we need to build GCC. But GCC will build # in multiple phases, making its own simple compiler in order to build # itself completely. The intermediate/simple compiler doesn't recognize # some system specific locations like '/usr/lib/ARCHITECTURE' that some # operating systems use. We thus need to tell the intermediate compiler # where its necessary libraries and headers are. if [ $built_container = 0 ]; then if [ x"$sys_library_path" != x ]; then if [ x"$LIBRARY_PATH" = x ]; then export LIBRARY_PATH="$sys_library_path" else export LIBRARY_PATH="$LIBRARY_PATH:$sys_library_path" fi if [ x"$CPATH" = x ]; then export CPATH="$sys_cpath" else export CPATH="$CPATH:$sys_cpath" fi fi elapsed_time_from_prev_step compiler-paths fi # Inform the user # --------------- # # Print some basic information so the user gets a feeling of what is going # on and is prepared on what will happen next. if [ $quiet = 0 ]; then cat < "$build_dir"/$junkname ; then rm -f "$build_dir"/$junkname instring="the already existing" bdir="$(absolute_dir "$build_dir")" else echo " ** Can't write in '$build_dir'"; fi else if mkdir "$build_dir" 2> /dev/null; then instring="the newly created" bdir="$(absolute_dir "$build_dir")" else echo " ** Can't create '$build_dir'"; fi fi # If it is given, make sure it isn't a subdirectory of the source # directory. if ! [ x"$bdir" = x ]; then if echo "$bdir/" \ | grep '^'"$currentdir/" 2> /dev/null > /dev/null; then # If it was newly created, it will be empty, so delete it. if ! [ "$(ls -A $bdir)" ]; then rm --dir "$bdir"; fi # Inform the user that this is not acceptable and reset # 'bdir'. bdir= printf " ** The build-directory cannot be under the " printf "source-directory." fi fi # If things are fine so far, make sure it does not contain a space # or other meta-characters which can cause problems during software # building. if ! [ x"$bdir" = x ]; then hasmeta=0; case $bdir in *['!'\@\#\$\%\^\&\*\(\)\+\;\ ]* ) hasmeta=1 ;; esac if [ $hasmeta = 1 ]; then # If it was newly created, it will be empty, so delete it. if ! [ "$(ls -A "$bdir")" ]; then rm --dir "$bdir"; fi # Inform the user and set 'bdir' to empty again. bdir= printf " ** Build directory should not contain " printf "meta-characters (like SPACE, %, \$, !, ;, or " printf "parenthesis, among others): they can interrup " printf "the build for some software." fi fi # If everything is still fine so far, see if we're able to # manipulate file permissions in the directory's filesystem and if # so, see if there is atleast 5GB free space. if ! [ x"$bdir" = x ]; then if ! $(check_permission "$bdir"); then # Unable to handle permissions well bdir= printf " ** File permissions can not be modified in " printf "this directory" else # Able to handle permissions, now check for 5GB free space # in the given partition (note that the number is in units # of 1024 bytes). If this is not the case, print a warning. if $(free_space_warning 5000000 "$bdir"); then cat < /dev/null 2>/dev/null; then # 'which' isn't in POSIX, so we are using 'command -v' instead. name=$(command -v wget) # See if the host wget has the '--no-use-server-timestamps' option # (for example wget 1.12 doesn't have it). If not, we'll have to # remove it. This won't affect the analysis of Maneage in anyway, # its just to avoid re-downloading if the server timestamps are # bad; at the worst case, it will just cause a re-download of an # input software source code (for data inputs, we will use our own # wget that has this option). tsname="no-use-server-timestamps" tscheck=$(wget --help | grep $tsname || true) if [ x"$tscheck" = x ]; then wgetts="" else wgetts="--$tsname"; fi # By default Wget keeps the remote file's timestamp, so we'll have # to disable it manually. downloader="$name $wgetts -O"; elif type curl > /dev/null 2>/dev/null; then name=$(command -v curl) # - cURL doesn't keep the remote file's timestamp by default. # - With the '-L' option, we tell cURL to follow redirects. downloader="$name -L -o" else cat <> $lconf fi elapsed_time_from_prev_step LOCAL-write # Project's top-level built software directories # ---------------------------------------------- # # These directories are possibly needed by many steps of process, so to # avoid too many directory dependencies throughout the software and # analysis Makefiles (thus making them hard to read), we are just building # them here tardir="$sdir"/tarballs instdir="$sdir"/installed tmpblddir="$sdir"/build-tmp # Second-level directories. instlibdir="$instdir"/lib instbindir="$instdir"/bin verdir="$instdir"/version-info # Sub-directories of version-info itidir="$verdir"/tex ictdir="$verdir"/cite ipydir="$verdir"/python ibidir="$verdir"/proglib ircrandir="$verdir"/r-cran if [ $built_container = 0 ]; then # Top-level directories. if ! [ -d "$tardir" ]; then mkdir "$tardir"; fi if ! [ -d "$instdir" ]; then mkdir "$instdir"; fi # Second-level directories. if ! [ -d "$verdir" ]; then mkdir "$verdir"; fi if ! [ -d "$instbindir" ]; then mkdir "$instbindir"; fi # Sub-directories of version-info if ! [ -d "$itidir" ]; then mkdir "$itidir"; fi if ! [ -d "$ictdir" ]; then mkdir "$ictdir"; fi if ! [ -d "$ipydir" ]; then mkdir "$ipydir"; fi if ! [ -d "$ibidir" ]; then mkdir "$ibidir"; fi if ! [ -d "$ircrandir" ]; then mkdir "$ircrandir"; fi # Some software install their libraries in '$(idir)/lib64'. But all # other libraries are in '$(idir)/lib'. Since Maneage's build is only # for a single architecture, we can set the '$(idir)/lib64' as a # symbolic link to '$(idir)/lib' so all the libraries are always # available in the same place. if ! [ -d "$instlibdir" ]; then mkdir "$instlibdir"; fi ln -fs "$instlibdir" "$instdir"/lib64 # Wrapper over Make as a single command so it does not default to # '/bin/sh' during installation (needed by some programs like CMake). makewshell="$instbindir/make-with-shell" if ! [ -f "$makewshell" ]; then echo "$instbindir/make SHELL=$instbindir/bash \$@" > $makewshell chmod +x $makewshell fi # Report the execution time of this step. elapsed_time_from_prev_step subdirectories-of-build fi # Software building directory (possibly in RAM) # --------------------------------------------- # # Building the software for the project will need the creation of many # small temporary files that will ultimately be deleted. To avoid harming # HDDs/SSDs and improve speed, it is therefore better to build them in the # RAM when possible. The RAM of most systems today (>8GB) is large enough # for the parallel building of the software. # # Set the top-level shared memory location. Currently there is only one # standard location (for GNU/Linux OSs), so doing this check here and the # main job below may seem redundant. However, it is written separately from # the main code below because later, we expect to add more possible # mounting locations (for other OSs). if [ $built_container = 0 ]; then if [ -d /dev/shm ]; then shmdir=/dev/shm else shmdir="" fi # If a shared memory mounted directory exists and has the necessary # conditions, set that directory to build software. if [ x"$shmdir" != x ]; then # Make sure it has enough space. needed_space=2000000 available_space=$(df "$shmdir" | awk 'NR==2{print $4}') if [ $available_space -gt $needed_space ]; then # Set the Maneage-specific directory within the shared # memory. We'll use the names of the two parent directories to # the current/running directory, separated by a '-' instead of # '/'. We'll then appended that with the user's name (in case # multiple users may be working on similar project names). # # Maybe later, we can use something like 'mktemp' to add random # characters to this name and make it unique to every run (even # for a single user). dirname=$(pwd | sed -e's/\// /g' \ | awk '{l=NF-1; printf("%s-%s", $l, $NF)}') tbshmdir="$shmdir"/"$dirname"-$(whoami) # Try to make the directory if it does not yet exist. A failed # directory creation will be tested for a few lines later, when # testing for the existence and executability of a test file. if ! [ -d "$tbshmdir" ]; then (mkdir "$tbshmdir" || true); fi # Some systems may protect '/dev/shm' against the right to # execute programs by ordinary users. We thus need to check # that the device allows execution within this directory by # this user. shmexecfile="$tbshmdir"/shm-execution-check.sh rm -f $shmexecfile # We also don't want any existing flags. # Create the file to be executed, but do not fail fatally if it # cannot be created. We will check a few lines later if the # file really exists. (cat > "$shmexecfile" < /dev/null' after the execution command # because it can produce false failures randomly on some # systems. if [ -e "$shmexecfile" ]; then # Add the executable flag. chmod +x "$shmexecfile" # The following line tries to execute the file. if "$shmexecfile"; then # Successful execution. The colon is a "no-op" (no # operation) shell command. : else tbshmdir="" fi rm "$shmexecfile" else tbshmdir="" fi fi else tbshmdir="" fi # If a shared memory directory was created, set the software building # directory to be a symbolic link to it. Otherwise, just build the # temporary build directory under the project's build directory. # # If it is a link, we need to empty its contents first, then itself. if [ -d "$tmpblddir" ]; then empty_build_tmp; fi # Now that we are sure it doesn't exist, we'll make it (either as a # directory or as a symbolic link). if [ x"$tbshmdir" = x ]; then mkdir "$tmpblddir"; else ln -s "$tbshmdir" "$tmpblddir"; fi # Report the time this step took. elapsed_time_from_prev_step temporary-software-building-dir fi # Inform the user that the build process is starting # ------------------------------------------------- # # Everything is ready, let the user know that the building is going to # start. if [ $quiet = 0 ]; then cat < /dev/null 2> /dev/null; then numthreads=$(nproc --all); else numthreads=$(sysctl -a | awk '/^hw\.ncpu/{print $2}') if [ x"$numthreads" = x ]; then numthreads=1; fi fi else numthreads=$jobs fi elapsed_time_from_prev_step num-threads fi # Libraries necessary for the system's shell # ------------------------------------------ # # In some cases (mostly the programs that Maneage doesn't yet build by # itself), the programs may call the system's shell, not Maneage's # shell. After we close-off the system environment from Maneage, this will # cause a crash! To avoid such cases, we need to find the locations of the # libraries that the shell needs and temporarily add them to the library # search path. # # About the 'grep -v "(0x[^)]*)"' term (from bug 66847, see [1]): On some # systems [2], the output of 'ldd /bin/sh' includes a line for the vDSO [3] # that is different to the formats that are assumed, prior to this commit, # by the algorithm in 'configure.sh' when evaluating the variable # 'sys_library_sh_path'. This leads to a fatal syntax error in (at least) # 'ncurses', because the option using 'sys_library_sh_path' contains an # unquoted RAM address in parentheses. Even if the address were quoted, it # would still be incorrect. This 'grep command excludes candidate host path # strings that look like RAM addresses to address the problem. # # [1] https://savannah.nongnu.org/bugs/index.php?66847 # [2] https://stackoverflow.com/questions/34428037/how-to-interpret-the-output-of-the-ldd-program # [3] man vdso if [ $built_container = 0 ]; then if [ x"$on_mac_os" = xyes ]; then sys_library_sh_path=$(otool -L /bin/sh \ | awk '/\/lib/{print $1}' \ | sed 's#/[^/]*$##' \ | sort \ | uniq \ | awk '{if (NR==1) printf "%s", $1; \ else printf ":%s", $1}') else sys_library_sh_path=$(ldd /bin/sh \ | awk '{if($3!="") print $3}' \ | sed 's#/[^/]*$##' \ | grep -v "(0x[^)]*)" \ | sort \ | uniq \ | awk '{if (NR==1) printf "%s", $1; \ else printf ":%s", $1}') fi elapsed_time_from_prev_step sys-library-sh-path fi # Find Zenodo URL for software downloading # ---------------------------------------- # # All free-software source tarballs that are potentially used in Maneage # are also archived in Zenodo with a certain concept-DOI. A concept-DOI is # a Zenodo terminology, meaning a fixed DOI of the project (that can have # many sub-DOIs for different versions). By default, the concept-DOI points # to the most recently uploaded version. However, the concept-DOI itself is # not directly usable for downloading files. The concept-DOI will just take # us to the top webpage of the most recent version of the upload. # # The problem is that as more software are added (as new Zenodo versions), # the most recent Zenodo-URL that the concept-DOI points to, also # changes. The most reliable solution was found to be the tiny script below # which will download the DOI-resolved webpage, and extract the Zenodo-URL # of the most recent version from there (using the 'coreutils' tarball as # an example, the directory part of the URL for all the other software are # the same). This is not done if the options '--debug' or `--offline` are # used. zenodourl="" user_backup_urls="" zenodocheck="$bdir"/software/zenodo-check.html if [ $built_container = 0 ]; then if [ x$debug = x ] && [ x$offline = x ]; then if $downloader $zenodocheck \ https://doi.org/10.5281/zenodo.3883409; then zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ | sed -n -e'/http/p' \ | tr ' ' '\n' \ | grep http \ | sed -e 's/href="//' -e 's|/coreutils| |' \ | awk 'NR==1{print $1}') fi fi rm -f $zenodocheck # Add the Zenodo URL to the user's given back software URLs. Since the # user can specify 'user_backup_urls' (not yet implemented as an option # in './project'), we'll give preference to their specified servers, # then add the Zenodo URL afterwards. user_backup_urls="$user_backup_urls $zenodourl" elapsed_time_from_prev_step zenodo-url fi # Corrections for debugging mode # ------------------------------ # # If the user wants to debug the software configuration, they are usually # focused on the building of the single problematic software. Therefore, # the default multi-threaded execution of Make with the '--keep-going' # option are very annoying and can even hide important warnings. Recall # that with '--keep-going', Make will continue building other targets, even # if one target fails. When the user runs './project configure --debug', # the 'debug' variable will not be empty and this mode will be activated. if [ x$debug = x ]; then keepgoing="--keep-going" else jobs=1 numthreads=1 keepgoing="" fi # Core software # ------------- # # Here we build the core tools that 'basic.mk' depends on: Lzip # (compression program), GNU Make (that 'basic.mk' is written in), Dash # (minimal Bash-like shell) and Flock (to lock files and enable serial # operations where necessary: mostly in download). export on_mac_os if [ $quiet = 0 ]; then echo "Building/validating software: pre-make"; fi ./reproduce/software/shell/pre-make-build.sh \ "$bdir" "$ddir" "$downloader" "$user_backup_urls" elapsed_time_from_prev_step make-software-pre-make # Basic software # -------------- # # Having built the core tools, we are now ready to build GCC and all its # dependencies (the "basic" software). if [ $quiet = 0 ]; then echo "Building/validating software: basic"; fi .local/bin/make $keepgoing -f reproduce/software/make/basic.mk \ sys_library_sh_path=$sys_library_sh_path \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ static_build=$static_build \ numthreads=$numthreads \ needs_ldl=$needs_ldl \ on_mac_os=$on_mac_os \ host_cc=$host_cc \ -j$numthreads elapsed_time_from_prev_step make-software-basic # High-level software # ------------------- # # Having our custom GCC in place, we can now build the high-level (science) # software: we are using our custom-built 'env' to ensure that nothing from # the host environment leaks into the high-level software environment. if [ $quiet = 0 ]; then echo "Building/validating software: high-level"; fi .local/bin/env -i HOME=$bdir \ .local/bin/make $keepgoing \ -f reproduce/software/make/high-level.mk \ sys_library_sh_path=$sys_library_sh_path \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ all_highlevel=$all_highlevel \ static_build=$static_build \ numthreads=$numthreads \ on_mac_os=$on_mac_os \ sys_cpath=$sys_cpath \ host_cc=$host_cc \ offline=$offline \ -j$numthreads elapsed_time_from_prev_step make-software-high-level # Make sure TeX Live installed successfully # ----------------------------------------- # # TeX Live is managed over the internet, so if there isn't any, or it # suddenly gets cut, it can't be built. However, when TeX Live isn't # installed, the project can do all its processing independent of it. It # will just stop at the stage when all the processing is complete and it is # only necessary to build the PDF. So we don't want to stop the project's # configuration and building if its not present. if [ $built_container = 0 ]; then if [ -f $itidir/texlive-ready-tlmgr ]; then texlive_result=$(cat $itidir/texlive-ready-tlmgr) else texlive_result="NOT!" fi if [ x"$texlive_result" = x"NOT!" ]; then cat <0 { \ c++; \ if(c==1) \ { \ if('$num'==1) printf("%s", $0); \ else printf("%s", $0); \ } \ else if(c=='$num') printf(" and %s\n", $0); \ else printf(", %s", $0) \ }' fi } # Relevant files pkgver=$sconfdir/dependencies.tex pkgbib=$sconfdir/dependencies-bib.tex # Build the software LaTeX source but only when not in a container. if [ $built_container = 0 ]; then # Import the context/sentences for placing between the list of software # names during their acknowledgment. . $cdir/software_acknowledge_context.sh # Report the different software in separate contexts (separating Python # and TeX packages from the C/C++ programs and libraries). proglibs=$(prepare_name_version $verdir/proglib/*) pymodules=$(prepare_name_version $verdir/python/*) texpkg=$(prepare_name_version $verdir/tex/texlive) # Acknowledge these software packages in a LaTeX paragraph. .local/bin/echo "$thank_software_introduce " > $pkgver .local/bin/echo "$thank_progs_libs $proglibs. " >> $pkgver if [ x"$pymodules" != x ]; then .local/bin/echo "$thank_python $pymodules. " >> $pkgver fi .local/bin/echo "$thank_latex $texpkg. " >> $pkgver .local/bin/echo "$thank_software_conclude" >> $pkgver # Prepare the BibTeX entries for the used software (if there are any). hasentry=0 bibfiles="$ictdir/*" for f in $bibfiles; do if [ -f $f ]; then hasentry=1; break; fi; done; # Fill it in with all the BibTeX entries in this directory. We'll just # avoid writing any comments (usually copyright notices) and also put an # empty line after each file's contents to make the output more readable. echo "" > $pkgbib # We don't want to inherit any pre-existing content. if [ $hasentry = 1 ]; then for f in $bibfiles; do awk '!/^%/{print} END{print ""}' $f >> $pkgbib done fi # Report the time that this operation took. elapsed_time_from_prev_step tex-macros fi # Report machine architecture (has to be final created file) # ---------------------------------------------------------- # # This is the final file that is created in the configuration phase: it is # used by the high-level project script to verify that configuration has # been completed. If any other files should be created in the final statges # of configuration, be sure to add them before this. # # Since harware class might include underscore, it must be replaced with # '\_', otherwise pdftex would complain and break the build process when # doing ./project make. if [ $built_container = 0 ]; then hw_class=$(uname -m) hwparam="$sconfdir/hardware-parameters.tex" hw_class_fixed="$(echo $hw_class | sed -e 's/_/\\_/')" .local/bin/echo "\\newcommand{\\machinearchitecture}{$hw_class_fixed}" \ > $hwparam .local/bin/echo "\\newcommand{\\machinebyteorder}{$byte_order}" \ >> $hwparam .local/bin/echo "\\newcommand{\\machineaddresssizes}{$address_sizes}" \ >> $hwparam elapsed_time_from_prev_step hardware-params fi # Clean up and final notice # ------------------------- # # The configuration is now complete. We just need to delete the temporary # build directory and inform the user (if '--quiet' wasn't called) on the # next step(s). if [ -d $tmpblddir ]; then empty_build_tmp; fi if [ $quiet = 0 ]; then # Suggest the command to use. if [ x$maneage_group_name = x ]; then buildcommand="./project make -j8" else buildcommand="./project make --group=$maneage_group_name -j8" fi # Print the message. cat <