path: root/project

#!/bin/bash
#
# High-level script to manage the project.
# Run `./project --help' for a description of how to use it.
#
# Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org>
#
# This script is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This script is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details. See <http://www.gnu.org/licenses/>.


# Basic settings
# --------------
# Stop the script if there are any errors.
set -e





# Default option values
jobs=0     # 0 is for the default for the `configure.sh' script.
group=
debug=
host_cc=0
operation=
build_dir=
input_dir=
make_targets=
software_dir=
clean_texdir=0
existing_conf=0
scriptname="./project"
minmapsize=10000000000





# Parse the options
# -----------------
#
# Separate command-line arguments from options. Then put the option value
# into the respective variable.
#
# Each option has two lines because we want to process both these formats:
# `--name=value' and `--name value'. The former (with `=') is a single
# command-line argument, so we just need to shift the counter by one. The
# latter (without `=') is two arguments, so we'll need two shifts.
#
# Note on the case strings: for every option, we need three lines: one when
# the option name and value are separate. Another when there is an equal
# between them, and finally one where the value is immediately after the
# short-format. This exact order is important. Otherwise, there will be a
# conflict between them.

print_help() {
    # Print the output.
    cat <<EOF
Usage: $scriptname configure [OPTIONS]
       $scriptname prepare   [OPTIONS]
       $scriptname make      [OPTIONS]

Top-level script to manage the reproducible project. The high-level
operation is defined by the (mandatory) second argument:

  configure    - Configure project for this machine (e.g., build software).
  prepare      - Low-level preparations to optimize building with 'make'.
  make         - Run the project (do analysis and build outputs).

RECOMMENDATION: If this is the first time you are configuring this
template, please don't use the options and let the script explain each
parameter in full detail by simply running './project configure'.

Project 'make' special features.
  ./project make           Build the project on one thread
  ./project make -jN       Built the project in parallel on N threads.
  ./project make clean     Clean all files generated by 'make' (not software).
  ./project make distclean Clean everything (including compiled software).
  ./project make dist      Produce a LaTeX-ready-to-build distribution tarball
                           ('tar.gz') of the project. This is ready to be
                           uploaded to servers like 'arXiv.org'.
  ./project make dist-zip  Similar to 'dist', but compress with '.zip'.

With the options below you can modify the default behavior.
Configure options:
  -b, --build-dir=STR      Top directory to build the project in.
  -e, --existing-conf      Use (possibly existing) local configuration.
      --host-cc            Use host system's C compiler, don't build GCC.
  -i, --input-dir=STR      Directory containing input datasets (optional).
  -m, --minmapsize=INT     [Gnuastro] Minimum number of bytes to use RAM.
  -s, --software-dir=STR   Directory containing necessary software tarballs.
      --clean-texdir       Remove possibly existing build-time subdirectories
                           under the project's 'tex/' directory (can happen
                           when source is from arXiv for example).

Configure and Make options:
  -g, --group=STR          Build and run with write permissions for a group.
  -j, --jobs=INT           Number of threads to build/run the software.
  -?, --help               Print this help list.

Make options:
  -d, --debug=FLAGS        Print various types of debugging information.

Mandatory or optional arguments to long options are also mandatory or optional
for any corresponding short options.

Reproducible paper template: https://gitlab.com/makhlaghi/reproducible-paper

Report bugs to mohammad@akhlaghi.org
EOF
}

on_off_option_error() {
    if [ "x$2" = x ]; then
        echo "$scriptname: '$1' doesn't take any values."
    else
        echo "$scriptname: '$1' (or '$2') doesn't take any values."
    fi
    exit 1
}

check_v() {
    if [ x"$2" = x ]; then
        echo "$scriptname: option '$1' requires an argument."
        echo "Try '$scriptname --help' for more information."
        exit 1;
    fi
}

func_operation_set() {
    if [ x$operation = x ]; then
        operation=$1
    else
        echo "Only one operation ('configure' or 'make') may be given."
        exit 1
    fi
}

while [[ $# -gt 0 ]]
do
 case $1 in
  # Main operation.
  configure)    func_operation_set $1; shift;;
  prepare)      func_operation_set $1; shift;;
  make)         func_operation_set $1; shift;;


  # Configure options:
  -b|--builddir)         build_dir="$2";                             check_v "$1" "$build_dir";    shift;shift;;
  -b=*|--build-dir=*)    build_dir="${1#*=}";                        check_v "$1" "$build_dir";    shift;;
  -b*)                   build_dir=$(echo    "$1" | sed -e's/-b//'); check_v "$1" "$build_dir";    shift;;
  -e|--existing-conf)    existing_conf=1;                                                          shift;;
  -e*|--existing-conf=*) on_off_option_error --existing-conf -e;;
  --host-cc)             host_cc=1;                                                                shift;;
  --host-cc=*)           on_off_option_error --host-cc;;
  -i|--inputdir)         input_dir="$2";                             check_v "$1" "$input_dir";    shift;shift;;
  -i=*|--inputdir=*)     input_dir="${1#*=}";                        check_v "$1" "$input_dir";    shift;;
  -i*)                   input_dir=$(echo    "$1" | sed -e's/-i//'); check_v "$1" "$input_dir";    shift;;
  -m|--minmapsize)       minmapsize="$2";                            check_v "$1" "$minmapsize";   shift;shift;;
  -m=*|--minmapsize=*)   minmapsize="${1#*=}";                       check_v "$1" "$minmapsize";   shift;;
  -m*)                   minmapsize=$(echo   "$1" | sed -e's/-m//'); check_v "$1" "$minmapsize";   shift;;
  -s|--software-dir)     software_dir="$2";                          check_v "$1" "$software_dir"; shift;shift;;
  -s=*|--software-dir=*) software_dir="${1#*=}";                     check_v "$1" "$software_dir"; shift;;
  -s*)                   software_dir=$(echo "$1" | sed -e's/-s//'); check_v "$1" "$software_dir"; shift;;
  --clean-texdir)        clean_texdir=1;                                                           shift;;
  --clean-texdir=*)      on_off_option_error --clean-texdir;;

  # Configure and Make options:
  -g|--group)             group="$2";                            check_v group "$group";   shift;shift;;
  -g=*|--group=*)         group="${1#*=}";                       check_v group "$group";   shift;;
  -g*)                    group=$(echo   "$1" | sed -e's/-g//'); check_v group "$group";   shift;;
  -j|--jobs)              jobs="$2";                             check_v jobs  "$jobs";    shift;shift;;
  -j=*|--jobs=*)          jobs="${1#*=}";                        check_v jobs  "$jobs";    shift;;
  -j*)                    jobs=$(echo "$1" | sed -e's/-j//');    check_v jobs  "$jobs";    shift;;
  -'?'|--help)            print_help; exit 0;;
  -'?'*|--help=*)         on_off_option_error --help -?;;

  # Make options (note that Make's `debug' can take values, but when called
  # without any value, it is like giving it a value of `a'):
  -d|--debug)             if [ x"$2" = x ]; then debug=a;          shift;
                          else debug="$2"; check_v debug "$debug"; shift;shift; fi;;
  -d=*|--debug=*)         debug="${1#*=}";                       check_v debug "$debug";   shift;;
  -d*)                    debug=$(echo "$1" | sed -e's/-d//');   check_v debug "$debug";   shift;;

  # Unrecognized option:
  -*) echo "$scriptname: unknown option '$1'"; exit 1;;

  # Not an option, an argument (so its a Make target).
  *) make_targets="$make_targets $1"; shift;;
 esac
done





# Basic group settings
# --------------------
if ! [ x$group = x ]; then

    # Check if group is usable.
    if ! sg "$group" "echo test &> /dev/null" &> /dev/null; then
        echo "$scriptname: '$group' is not a usable group name on this system.";
        echo "(TIP: you can use the 'groups' command to see your groups)"
        exit 1
    fi

    # Set the group option for running Make.
    gopt="reproducible_paper_group_name=$group"
fi





# Run operations in controlled environment
# ----------------------------------------
controlled_env() {
    # Get the full address of the build directory:
    bdir=`.local/bin/realpath .build`

    # Remove all existing environment variables (with `env -i') and only
    # use some pre-defined environment variables, then build the project.
    envmake=".local/bin/env -i HOME=$bdir sys_rm=$(which rm) $gopt"
    envmake="$envmake .local/bin/make --no-builtin-rules"
    envmake="$envmake --no-builtin-variables -f $1"
    if ! [ x"$debug" = x  ]; then envmake="$envmake --debug=$debug"; fi

    # Set the number of jobs. Note that for the `configure.sh' script the
    # default value has to be 0, so the default is the maximum number of
    # threads. But here, the default value is 1.
    if ! [ x"$jobs"  = x0 ]; then envmake="$envmake -j$jobs";  fi

    # Run the project
    if [ x"$group" = x ]; then
        $envmake $make_targets
    else
        # Set the group and permission flags.
        sg "$group" "umask $perms && $envmake $make_targets"
    fi
}





# Error messages
# --------------
#
# Having the error messages here helps the over-all process be more
# readable.
print_error_abort() {
    case $1 in
        prepare)
            cat <<EOF

The project isn't configured for this system, or the configuration wasn't
successful. To configure the project, please use this command:

      $ ./project configure

(TIP: if you have already ran this command once, run it with '-e' to use
the previous configuration, run with '--help' for more info)

EOF
            exit 1;
        ;;
        make)
            cat <<EOF

The project preparation hasn't been completed, or it wasn't successful. To
prepare the project prior to building it, please use this command:

      $ ./project prepare

EOF
            exit 1;
        ;;
    esac
}




# Do requested operation
# ----------------------
perms="u+r,u+w,g+r,g+w,o-r,o-w,o-x"
configscript=./reproduce/software/bash/configure.sh
case $operation in

    # Build the project's software.
    configure)

        # Set executable flags
        # --------------------
        #
        # In some scenarios (for example when using a tarball from arXiv),
        # it may happen that the host server has removed the executable
        # flags of all the files. In `README.md' we instruct the readers on
        # setting the executable flag of this script. But we don't want the
        # user to have to worry about any other file that needs an
        # executable flag.
        #
        # Basically, all the files (shell scripts) in the two
        # `reproduce/*/bash' should need executable flags, so we are giving
        # them executable flags by default. If any other file in your project
        # needs such flags, add them here.
        chmod +x reproduce/software/bash/* reproduce/analysis/bash/*

        # If the user requested, clean the TeX directory from the extra
        # (to-be-built) directories that may already be there (and will not
        # allow the configuration to complete).
        if [ x"$clean_texdir" = x1 ]; then
            rm -rf tex/build tex/tikz
        fi

        # Variables to pass to the configuration script.
        export jobs=$jobs
        export host_cc=$host_cc
        export build_dir=$build_dir
        export input_dir=$input_dir
        export scriptname=$scriptname
        export minmapsize=$minmapsize
        export software_dir=$software_dir
        export existing_conf=$existing_conf
        export reproducible_paper_group_name=$group

        # Run the configuration script
        if [ x"$group" = x ]; then
            $configscript
        else
            # Set the group and permission flags.
            sg "$group" "umask $perms && $configscript"

            # Set the group writing permission for everything in the
            # installed software directory. The common build process sets
            # the writing permissions of the installed programs/libraries
            # to `755'. So group members can't write over a file. This
            # creates problems when another group member wants to update
            # the software for example. We thus need to manually add the
            # group writing flag to all installed software files.
            echo "Enabling group writing permission on all installed software..."
            .local/bin/chmod -R g+w .local/;
        fi
        ;;





    # Run the input management.
    prepare)

        # Make sure the configure script has been completed properly
        # (`configuration-done.txt' exists).
        if ! [ -f .build/software/configuration-done.txt ]; then
            print_error_abort $operation
        fi

        # Run input-preparations in control environment
        controlled_env reproduce/analysis/make/top-prepare.mk
        ;;





    # Run the project
    make)

        # Make sure the configure script has been completed properly
        # (`configuration-done.txt' exists).
        if ! [ -f .build/software/preparation-done.mk ]; then
            print_error_abort $operation
        fi

        # Run the actual project.
        controlled_env reproduce/analysis/make/top-make.mk
        ;;





    # Operation not specified.
    *)
        echo "No operation defined."
        echo "Please run with '--help' for more information."
        echo "Available operations are: 'configure', 'prepare', or 'make')."
        exit 1
    ;;
esac