diff options
Diffstat (limited to 'project')
| -rwxr-xr-x | project | 378 |
1 files changed, 261 insertions, 117 deletions
@@ -1,9 +1,10 @@ #!/bin/sh # # High-level script to manage the project. -# Run `./project --help' for a description of how to use it. +# Run './project --help' for a description of how to use it. # -# Copyright (C) 2019-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2021-2025 Raul Infante-Sainz <infantesainz@gmail.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,16 +30,21 @@ set -e # Default option values -jobs=0 # 0 is for the default for the `configure.sh' script. +jobs=0 # 0 is for the default for the 'configure.sh' script. group= debug= +quiet=0 +timing=0 host_cc=0 +offline= operation= build_dir= input_dir= +keep_going= check_config= make_targets= software_dir= +pauseformsg=1 clean_texdir=0 prepare_redo=0 highlightnew=0 @@ -46,7 +52,6 @@ all_highlevel=0 existing_conf=0 highlightnotes=0 scriptname="./project" -minmapsize=10000000000 @@ -59,9 +64,9 @@ minmapsize=10000000000 # into the respective variable. # # Each option has two lines because we want to process both these formats: -# `--name=value' and `--name value'. The former (with `=') is a single +# '--name=value' and '--name value'. The former (with '=') is a single # command-line argument, so we just need to shift the counter by one. The -# latter (without `=') is two arguments, so we'll need two shifts. +# latter (without '=') is two arguments, so we'll need two shifts. # # Note on the case strings: for every option, we need three lines: one when # the option name and value are separate. Another when there is an equal @@ -87,7 +92,7 @@ RECOMMENDATION: If this is the first time you are configuring this template, please don't use the options and let the script explain each parameter in full detail by simply running './project configure'. -Project 'make' special features. +Project 'make' special tagets ./project make Build the project on one thread ./project make -jN Built the project in parallel on N threads. ./project make clean Clean all files generated by 'make' (not software). @@ -104,28 +109,34 @@ Project 'make' special features. With the options below you can modify the default behavior. Configure options: + --all-highlevel Build all high-level software (for development). -b, --build-dir=STR Top directory to build the project in. - -e, --existing-conf Use (possibly existing) local configuration. - --host-cc Use host system's C compiler, don't build GCC. - -i, --input-dir=STR Directory containing input datasets (optional). - -m, --minmapsize=INT [Gnuastro] Minimum number of bytes to use RAM. - -s, --software-dir=STR Directory containing necessary software tarballs. --check-config During configuration, show what is being built. --clean-texdir Remove possibly existing build-time subdirectories under the project's 'tex/' directory (can happen when source is from arXiv for example). - --all-highlevel Build all high-level software (for development). + -e, --existing-conf Use (possibly existing) local configuration. + -i, --input-dir=STR Directory containing input datasets (optional). + --host-cc Use host system's C compiler, don't build GCC. + --quiet Do not print basic info messages (with '-e'). + --no-pause Do not sleep/pause after basic info messages. + -s, --software-dir=STR Directory containing necessary software tarballs. Configure and Make options: + -d, --debug[=FLAGS] In configure: use -j1, no -k, and no Zenodo check. + In make: 'FLAGS' will be directly passed to 'make'. + -k, --keep-going Keep going when some targets can't be made. -g, --group=STR Build and run with write permissions for a group. -j, --jobs=INT Number of threads to build/run the software. -?, --help Print this help list. Make (analysis) options: - -d, --debug=FLAGS Print various types of debugging information. -p, --prepare-redo Re-do preparation (only done automatically once). + -t, --timing Starting and ending times written in 'timing.txt'. Make (final PDF) options: + --refresh-bib Force refresh the bibliography. + --highlight-all Activate '--highlight-new' & '--highlight-notes'. --highlight-new Highlight '\new' parts of text as green. --highlight-notes Show '\tonote' regions as red text in PDF. @@ -172,56 +183,72 @@ do make) func_operation_set $1; shift;; shell) func_operation_set $1; shift;; - # Configure options: - -b|--build-dir) build_dir="$2"; check_v "$1" "$build_dir"; shift;shift;; - -b=*|--build-dir=*) build_dir="${1#*=}"; check_v "$1" "$build_dir"; shift;; - -b*) build_dir=$(echo "$1" | sed -e's/-b//'); check_v "$1" "$build_dir"; shift;; - -e|--existing-conf) existing_conf=1; shift;; + -e|--existing-conf) existing_conf=1; shift;; -e*|--existing-conf=*) on_off_option_error --existing-conf -e;; - --host-cc) host_cc=1; shift;; + --host-cc) host_cc=1; shift;; --host-cc=*) on_off_option_error --host-cc;; - -i|--input-dir) input_dir="$2"; check_v "$1" "$input_dir"; shift;shift;; - -i=*|--input-dir=*) input_dir="${1#*=}"; check_v "$1" "$input_dir"; shift;; - -i*) input_dir=$(echo "$1" | sed -e's/-i//'); check_v "$1" "$input_dir"; shift;; - -m|--minmapsize) minmapsize="$2"; check_v "$1" "$minmapsize"; shift;shift;; - -m=*|--minmapsize=*) minmapsize="${1#*=}"; check_v "$1" "$minmapsize"; shift;; - -m*) minmapsize=$(echo "$1" | sed -e's/-m//'); check_v "$1" "$minmapsize"; shift;; - -s|--software-dir) software_dir="$2"; check_v "$1" "$software_dir"; shift;shift;; - -s=*|--software-dir=*) software_dir="${1#*=}"; check_v "$1" "$software_dir"; shift;; - -s*) software_dir=$(echo "$1" | sed -e's/-s//'); check_v "$1" "$software_dir"; shift;; - --check-config) check_config=1; shift;; + --offline) offline=1; shift;; + --offline=*) on_off_option_error --offline;; + -i|--input-dir) input_dir="$2"; check_v "$1" "$input_dir"; shift;shift;; + -i=*|--input-dir=*) input_dir="${1#*=}"; check_v "$1" "$input_dir"; shift;; + -i*) input_dir=$(echo "$1" | sed -e's/-i//'); check_v "$1" "$input_dir"; shift;; + -s|--software-dir) software_dir="$2"; check_v "$1" "$software_dir"; shift;shift;; + -s=*|--software-dir=*) software_dir="${1#*=}"; check_v "$1" "$software_dir"; shift;; + -s*) software_dir=$(echo "$1" | sed -e's/-s//'); check_v "$1" "$software_dir"; shift;; + --check-config) check_config=1; shift;; --check-config=*) on_off_option_error --check-config;; - --clean-texdir) clean_texdir=1; shift;; + --clean-texdir) clean_texdir=1; shift;; --clean-texdir=*) on_off_option_error --clean-texdir;; - --all-highlevel) all_highlevel=1; shift;; + --all-highlevel) all_highlevel=1; shift;; --all-highlevel=*) on_off_option_error --all-highlevel;; + --no-pause) pauseformsg=0; shift;; + --no-pause=*) on_off_option_error --no-pause;; + --quiet) quiet=1; shift;; + --quiet=*) on_off_option_error --quiet;; # Configure and Make options: - -g|--group) group="$2"; check_v group "$group"; shift;shift;; - -g=*|--group=*) group="${1#*=}"; check_v group "$group"; shift;; - -g*) group=$(echo "$1" | sed -e's/-g//'); check_v group "$group"; shift;; - -j|--jobs) jobs="$2"; check_v jobs "$jobs"; shift;shift;; - -j=*|--jobs=*) jobs="${1#*=}"; check_v jobs "$jobs"; shift;; - -j*) jobs=$(echo "$1" | sed -e's/-j//'); check_v jobs "$jobs"; shift;; - -'?'|--help) print_help; exit 0;; - -'?'*|--help=*) on_off_option_error --help -?;; - - # Make options - # ------------ - # - # Note that Make's `debug' can take values, but when called without any - # value, it is like giving it a value of `a'): - --highlight-new) highlightnew=1; shift;; - --highlight-new=*) on_off_option_error --highlight-new;; - --highlight-notes) highlightnotes=1; shift;; - --highlight-notes=*) on_off_option_error --highlight-notes;; - -d|--debug) if [ x"$2" = x ]; then debug=a; shift; - else debug="$2"; check_v debug "$debug"; shift;shift; fi;; - -d=*|--debug=*) debug="${1#*=}"; check_v debug "$debug"; shift;; - -d*) debug=$(echo "$1" | sed -e's/-d//'); check_v debug "$debug"; shift;; - -p|--prepare-redo) prepare_redo=1; shift;; - -p=*|--prepare-redo=*) on_off_option_error --prepare-redo; shift;; + -b|--build-dir) build_dir="$2"; check_v "$1" "$build_dir";shift;shift;; + -b=*|--build-dir=*) build_dir="${1#*=}"; check_v "$1" "$build_dir";shift;; + -b*) build_dir=$(echo "$1" | sed -e's/-b//'); check_v "$1" "$build_dir";shift;; + -g|--group) group="$2"; check_v group "$group"; shift;shift;; + -g=*|--group=*) group="${1#*=}"; check_v group "$group"; shift;; + -g*) group=$(echo "$1" | sed -e's/-g//'); check_v group "$group"; shift;; + -j|--jobs) jobs="$2"; check_v jobs "$jobs"; shift;shift;; + -j=*|--jobs=*) jobs="${1#*=}"; check_v jobs "$jobs"; shift;; + -j*) jobs=$(echo "$1" | sed -e's/-j//'); check_v jobs "$jobs"; shift;; + -k|--keep-going) keep_going="--keep-going"; shift;; + -k=*|--keep-going=*) on_off_option_error --keep-going -k;; + -k*) on_off_option_error --keep-going -k;; + -'?'|--help) print_help; exit 0;; + -'?'*|--help=*) on_off_option_error --help -?;; + + # Make options (analysis): + -p|--prepare-redo) prepare_redo=1; shift;; + -p=*|--prepare-redo=*) on_off_option_error --prepare-redo; shift;; + -t|--timing) timing=1; shift;; + -t=*|--timing=*) on_off_option_error --timing; shift;; + + # Make options (final PDF): + --refresh-bib) [ -f tex/src/references.tex ] && touch tex/src/references.tex; shift;; + --highlight-all) highlightnew=1; highlightnotes=1; shift;; + --highlight-all=*) on_off_option_error --highlight-new;; + --highlight-new) highlightnew=1; shift;; + --highlight-new=*) on_off_option_error --highlight-new;; + --highlight-notes) highlightnotes=1; shift;; + --highlight-notes=*) on_off_option_error --highlight-notes;; + -d|--debug) if [ x$operation = x ]; then + echo "Please set the operation before calling '--debug'"; exit 1 + elif [ x$operation = xconfigure ]; then debug=a; shift; + elif [ x$operation = xmake ]; then + if [ x"$2" = x ]; then + echo "In make-mode, '--debug' needs a value; see GNU Make manual"; exit 1 + else debug="$2"; check_v debug "$debug"; shift;shift; fi + else + echo "Operation '$operation' not recognized, please use 'configure' or 'make'" + fi;; + -d=*|--debug=*) debug="${1#*=}"; check_v debug "$debug"; shift;; + -d*) debug=$(echo "$1" | sed -e's/-d//'); check_v debug "$debug"; shift;; # Unrecognized option: -*) echo "$scriptname: unknown option '$1'"; exit 1;; @@ -238,8 +265,8 @@ done # Check configuration status # -------------------------- if ! [ x$check_config = x ]; then - # Find the color option to pass to `ls'. Note that `--color' (for GNU - # Coreutils `ls') should be checked first because it also has `-G', but + # Find the color option to pass to 'ls'. Note that '--color' (for GNU + # Coreutils 'ls') should be checked first because it also has '-G', but # for something else. if ls --color 2> /dev/null > /dev/null; then coloropt="--color=auto" elif ls -G 2> /dev/null > /dev/null; then coloropt="-G" @@ -275,8 +302,8 @@ EOF ls $coloropt .build/software/build-tmp || junk=1; fi - # Make the temporary directory, delete its contents, then put new - # links of all built software. + # Make the temporary directory, delete its contents, then put + # new links of all built software. if ! [ -d $checkdir ]; then mkdir $checkdir; fi rm -f $checkdir/* @@ -285,7 +312,7 @@ EOF check=$(ls .local/version-info/python/) if ! [ "x$check" = x ]; then printresults=1 - ln -s .local/version-info/python/* $checkdir/ + ln -s "$(pwd)"/.local/version-info/python/* $checkdir/ fi check=$(ls .local/version-info/proglib/) if ! [ "x$check" = x ]; then @@ -297,10 +324,11 @@ EOF if [ $printresults = 1 ]; then echo "--- Last 5 packages that were built:" - # Then sort all the links based on the most recent dates of the - # files they link to (with '-L'). - ls -Llt $checkdir \ - | awk '/^-/ && c++<5 {printf "[at %s] %s\n", $(NF-1), $NF}' + # Then sort all the links based on the most recent dates of + # the files they link to (with '-L'). + ls -Llt $checkdir \ + | awk '/^-/ && c++<5 {printf "[at %s] %s\n", \ + $(NF-1), $NF}' fi else cat <<EOF @@ -324,28 +352,65 @@ fi -# Basic group settings -# -------------------- +# Group check +# ----------- if ! [ x$group = x ]; then # Check if group is usable. - if ! sg "$group" "echo test &> /dev/null" &> /dev/null; then - echo "$scriptname: '$group' is not a usable group name on this system."; - echo "(TIP: you can use the 'groups' command to see your groups)" + if ! sg "$group" "echo Group \'$group\' exists"; then + printf "$scriptname: '$group' is not a usable group name on " + printf "this system. TIP: you can use the 'groups' command " + printf "to see your groups)\n" exit 1 fi # Set the group option for running Make. - gopt="reproducible_paper_group_name=$group" + gopt="maneage_group_name=$group" fi -# Error when configuration isn't run -configuration_necessary() { - cat <<EOF +# Build directory symbolic links +# ------------------------------ +# +# The source directory will contain two symbolic links that point to the +# build directory: + +# - .build: the top build directory. +# +# - .local: the second to the top software installed directory. They are +# used during the configuration phase to simplify commands and are also +# very useful during the development of a maneage'd project (to easily +# get to the build directory or execut Maneage'd software). +# +# This needs to be done on every run because: +# - './project configure' can be run with a new build directory, and +# keeping the old '.build' conflicts with the new build directory +# that the user gave. +# - './project make' or './project shell' (within a newly cloned source +# directory from inside a container): the links do not exist but have +# to be set to the container's build directory. +# - This is not an expensive operation. +if ! [ x"$build_dir" = x ]; then + rm -f .build .local + ln -s $build_dir .build + ln -s $build_dir/software/installed .local +fi + + + + + +# Function to validate configuration +# ---------------------------------- +# +# Check if the configuration is missing/incomplete +configuration_check() { + confdone=software/config/hardware-parameters.tex + if ! [ -f .build/$confdone ]; then + cat <<EOF The project is either (1) not configured on this system, or (2) the configuration wasn't successful. @@ -365,28 +430,85 @@ If there was a problem, please let us know by filling this online form: http://savannah.nongnu.org/support/?func=additem&group=reproduce EOF - exit 1 + exit 1 + fi } -# Run operations in controlled environment -# ---------------------------------------- +# Function for TeX Preparations +# ----------------------------- +# +# Make sure that the necessary analysis directories directory exist in the +# build directory. These will be necessary in various phases of hte +# analysis and having them inside the lower-level Make steps will require +# setting them as prerequisites for many basic jobs (thus making the +# Makefiles harder to read and add potentials for bugs: forgetting to add +# them for example). Also, we don't want the configure phase to make any +# edits in the analysis directory, so they are not built there. +tex_preparations () { + + # Extract the location of the build directory. + bdir=$(.local/bin/realpath .build) + + # We are using our custom-built 'mkdir' which is guaranteed to have the + # '-p' option (that will also build intermediate directories) and won't + # complain if the directory already exists. + badir=$bdir/analysis + texdir=$badir/tex + btexdir=$texdir/build + tikzdir=$btexdir/tikz + mtexdir=$texdir/macros + .local/bin/mkdir -p $mtexdir $btexdir $tikzdir + + # If 'tex/build' and 'tex/tikz' are symbolic links then 'rm -f' + # will delete them and we can continue. However, when the project + # is being built from the tarball (from arXiv for example), these + # two are not symbolic links but actual directories with the + # necessary built-components to build the PDF in them. In this + # case, because 'tex/build' is a directory, 'rm -f' will fail, so + # we'll just rename the two directories (as backup) and let the + # project build the proper symbolic links afterwards. + if rm -f tex/build; then + rm -f tex/tikz + else + mv tex/tikz tex/tikz-from-tarball + mv tex/build tex/build-from-tarball + fi + + # Build the symbolic links. + if ! [ -L tex/tikz ]; then ln -s "$tikzdir" tex/tikz; fi + if ! [ -L tex/build ]; then ln -s "$texdir" tex/build; fi +} + + + + + +# Function to run in controlled environment +# ----------------------------------------- +# +# Controlling the environment is necessary for running the analysis. Like +# the other funcitons here, this is defined to simplify the high-level code +# within the 'make)' switch statement. +perms="u+r,u+w,g+r,g+w,o-r,o-w,o-x" controlled_env() { + # Get the full address of the build directory: bdir=`.local/bin/realpath .build` - # Remove all existing environment variables (with `env -i') and only + # Remove all existing environment variables (with 'env -i') and only # use some pre-defined environment variables, then build the project. envmake=".local/bin/env -i HOME=$bdir sys_rm=$(which rm) $gopt" envmake="$envmake highlightnew=$highlightnew" envmake="$envmake highlightnotes=$highlightnotes .local/bin/make" envmake="$envmake --no-builtin-rules --no-builtin-variables -f $1" + envmake="$envmake $keep_going" if ! [ x"$debug" = x ]; then envmake="$envmake --debug=$debug"; fi - # Set the number of jobs. Note that for the `configure.sh' script the + # Set the number of jobs. Note that for the 'configure.sh' script the # default value has to be 0, so the default is the maximum number of # threads. But here, the default value is 1. if ! [ x"$jobs" = x0 ]; then envmake="$envmake -j$jobs"; fi @@ -406,7 +528,6 @@ controlled_env() { # Do requested operation # ---------------------- -perms="u+r,u+w,g+r,g+w,o-r,o-w,o-x" configscript=./reproduce/software/shell/configure.sh case $operation in @@ -418,7 +539,7 @@ case $operation in # # In some scenarios (for example when using a tarball from arXiv), # it may happen that the host server has removed the executable - # flags of all the files. In `README.md' we instruct the readers on + # flags of all the files. In 'README.md' we instruct the readers on # setting the executable flag of this script. But we don't want the # user to have to worry about any other file that needs an # executable flag. @@ -427,8 +548,11 @@ case $operation in # to make sure they have them, we are activating the executable # flags by default here every time './project configure' is run. If # any other file in your project needs such flags, add them here. - chmod +x reproduce/software/shell/* reproduce/software/config/*.sh \ - reproduce/analysis/bash/* + if ! [ -x reproduce/software/shell/configure.sh ]; then + chmod +x reproduce/analysis/bash/* \ + reproduce/software/shell/* \ + reproduce/software/config/*.sh + fi # If the user requested, clean the TeX directory from the extra # (to-be-built) directories that may already be there (and will not @@ -439,15 +563,18 @@ case $operation in # Variables to pass to the configuration script. export jobs=$jobs + export debug=$debug + export quiet=$quiet export host_cc=$host_cc + export offline=$offline export build_dir=$build_dir export input_dir=$input_dir export scriptname=$scriptname - export minmapsize=$minmapsize + export pauseformsg=$pauseformsg + export maneage_group_name=$group export software_dir=$software_dir export existing_conf=$existing_conf export all_highlevel=$all_highlevel - export reproducible_paper_group_name=$group # Run the configuration script if [ x"$group" = x ]; then @@ -459,11 +586,12 @@ case $operation in # Set the group writing permission for everything in the # installed software directory. The common build process sets # the writing permissions of the installed programs/libraries - # to `755'. So group members can't write over a file. This + # to '755'. So group members can't write over a file. This # creates problems when another group member wants to update # the software for example. We thus need to manually add the # group writing flag to all installed software files. - echo "Enabling group writing permission on all installed software..." + printf "Enabling group writing permission on all installed " + printf "software...\n" .local/bin/chmod -R g+w .local/; fi ;; @@ -475,58 +603,74 @@ case $operation in # Batch execution of the project. make) - # Make sure the configure script has been completed properly - # (`configuration-done.txt' exists). - if ! [ -f .build/software/configuration-done.txt ]; then - configuration_necessary - fi + # Make sure the configure script is complete and necessary LaTeX + # directories are in place. + configuration_check + tex_preparations + + # Register the start of this run if requested (we are appending the + # new information so previous information is preserved until the + # user intentionally deletes/cleans it). + if [ $timing = 1 ]; then echo "start: $(date)" >> timing.txt; fi # Run data preparation phase (optionally build Makefiles with - # special values for optimizing the main `top-make.mk'). But note + # special values for optimizing the main 'top-make.mk'). But note # that data preparation is only done automatically the first time - # the project is built (when `.build/software/preparation-done.mk' + # the project is built (when '.build/analysis/preparation-done.mk' # doesn't yet exist). After that, if the user wants to re-do the - # preparation they have to use the `--prepare-redo' option. - if ! [ -f .build/software/preparation-done.mk ] \ + # preparation they have to use the '--prepare-redo' option. + if ! [ -f .build/analysis/preparation-done.mk ] \ || [ x"$prepare_redo" = x1 ]; then controlled_env reproduce/analysis/make/top-prepare.mk fi - # Run the actual project. + # Call top-make (highest level analysis Makefile). controlled_env reproduce/analysis/make/top-make.mk + + # Register the time of the project's ending. + if [ $timing = 1 ]; then echo "end: $(date)" >> timing.txt; fi ;; + + + + # Interactive shell of Maneage. shell) - # Make sure the configure script has been completed properly - # (`configuration-done.txt' exists). - if ! [ -f .build/software/configuration-done.txt ]; then - configuration_necessary - fi + # Make sure the configure script has been completed properly. + configuration_check # Run the project's own shell without inheriting any environment # from the host. The 'TERM' environment variable is necessary for # tools like some text editors. - bdir=`.local/bin/realpath .build` + bdir=$(.local/bin/realpath .build) instdir="$bdir"/software/installed + bindir="$bdir"/software/installed/bin + rcfile=$(pwd)/reproduce/software/shell/bashrc.sh .local/bin/env -i \ - HOME="$bdir" \ - TERM="$TERM" \ - CCACHE_DISABLE=1 \ - PATH="$instdir"/bin \ - LDFLAGS=-L"$instdir"/lib \ - SHELL="$instdir"/bin/bash \ - CPPFLAGS=-I"$instdir"/include \ - LD_LIBRARY_PATH="$instdir"/lib \ - OMPI_MCA_plm_rsh_agent=/bin/false \ - PYTHONPATH="$instdir"/lib/python/site-packages \ - PYTHONPATH3="$instdir"/lib/python/site-packages \ - PS1="[\[\033[32m\](maneage)\[\033[00m\] \u@\h \W]$ " \ - "$instdir"/bin/bash + HOME="$bdir" \ + TERM="$TERM" \ + PATH="$bindir" \ + CCACHE_DISABLE=1 \ + PROJECT_STATUS=shell \ + SHELL="$bindir"/bash \ + COLORTERM="$COLORTERM" \ + PROJECT_RCFILE="$rcfile" \ + LDFLAGS=-L"$instdir"/lib \ + CPPFLAGS=-I"$instdir"/include \ + LD_LIBRARY_PATH="$instdir"/lib \ + OMPI_MCA_plm_rsh_agent=/bin/false \ + PYTHONPATH="$instdir"/lib/python/site-packages \ + PYTHONPATH3="$instdir"/lib/python/site-packages \ + PS1="[\[\033[01;35m\]maneage@\h \W\[\033[32m\]\[\033[00m\]]$ " \ + "$bindir"/bash --noprofile --rcfile "$rcfile" ;; + + + # Operation not specified. *) cat <<EOF @@ -538,5 +682,5 @@ Please run with '--help' for more information. EOF exit 1 - ;; + ;; esac |