From 8cfc728ecfb81775fd7bcfde887d1fccc4e4e8ad Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 19 Jan 2020 22:42:24 +0000 Subject: New --check-config option to ./project to check software build status Until now, it was necessry to run a long `while true' loop to see what is currently being built at configure time. So with this commit, a new `--checkconfig' option has been added to `./project' that can be called to run that loop and make it easier to check. --- project | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) (limited to 'project') diff --git a/project b/project index 51274eb..5d161be 100755 --- a/project +++ b/project @@ -33,6 +33,7 @@ host_cc=0 operation= build_dir= input_dir= +check_config= make_targets= software_dir= clean_texdir=0 @@ -97,6 +98,7 @@ Configure options: -i, --input-dir=STR Directory containing input datasets (optional). -m, --minmapsize=INT [Gnuastro] Minimum number of bytes to use RAM. -s, --software-dir=STR Directory containing necessary software tarballs. + --check-config During configuration, show what is being built. --clean-texdir Remove possibly existing build-time subdirectories under the project's 'tex/' directory (can happen when source is from arXiv for example). @@ -170,6 +172,8 @@ do -s|--software-dir) software_dir="$2"; check_v "$1" "$software_dir"; shift;shift;; -s=*|--software-dir=*) software_dir="${1#*=}"; check_v "$1" "$software_dir"; shift;; -s*) software_dir=$(echo "$1" | sed -e's/-s//'); check_v "$1" "$software_dir"; shift;; + --check-config) check_config=1; shift;; + --check-config=*) on_off_option_error --check-config;; --clean-texdir) clean_texdir=1; shift;; --clean-texdir=*) on_off_option_error --clean-texdir;; @@ -202,6 +206,44 @@ done +# Check configuration status +# -------------------------- +if ! [ x$check_config = x ]; then + # Find the color option to pass to `ls'. Note that `--color' (for GNU + # Coreutils `ls') should be checked first because it also has `-G', but + # for something else. + if ls --color &> /dev/null; then coloropt="--color=auto" + elif ls -G &> /dev/null; then coloropt="-G" + else coloropt="" + fi + + # Print a notice to let the user know what is happening. + cat < Date: Thu, 23 Jan 2020 18:54:29 +0000 Subject: IMPORTANT: Project preparation is now also done with project make Until now, the main commands to run the project were these: `./project configure' (to build the software), `./project prepare' (to possibly arrange input datasets and build special configuration Makefiles) and finally `./project make' to run the project. The main logic behind the "prepare" phase `top-prepare.mk' is to build configuration files that can be fed into the "make" step and optimize its operation. For example when the total number of necessary inputs for the majority of the analysis is not as large as the total number of inputs. With "prepare" (when necessary), you go through the raw inputs, select the ones that are necessary for the rest of the project. The output of `top-prepare.mk' is a configuration file (a Make variable) that keeps the IDs (numbers, names, etc). That configuration file would then be used in the `top-make.mk' to identify the lower level targets and allow optimal project organization and management. But the last two are both part of the analysis, and while they indeed need different calls to Make to be executed, many projects don't actually need a preparation phase: ultimately, its an implementation choice by the project developers and doesn't concern the project users (or the developers when they are running it). To avoid confusing the users, or simply annoying them when a projet doesn't need it, with this commit, the top-level `top-prepare.mk' and `top-make.mk' Makefiles are called with the single `./project make' command and `./project prepare' has been dropped. I noticed this while writing the paper on this system. --- project | 82 ++++++++++++++++++----------------------------------------------- 1 file changed, 22 insertions(+), 60 deletions(-) (limited to 'project') diff --git a/project b/project index 5d161be..d994b09 100755 --- a/project +++ b/project @@ -66,14 +66,12 @@ print_help() { # Print the output. cat < Date: Thu, 23 Jan 2020 19:17:16 +0000 Subject: Hashbangs of project and configure.sh set to /bin/sh Until now, the hashbang of these two shell scripts was set to `/bin/bash', hence assuming that GNU Bash exists on the host system! But this is an extra requirement on the host operating system and these two scripts should be written such that they operate on a POSIX shell (the generic `/bin/sh' which can point to any shell program). With this commit this has been implemented! We may confront some errors as the system is run on other systems, but we should fix such errors and work hard to make these two scripts as POSIX-compatible as possible (runnable on any shell, so as not to force users to install Bash before running the project). This completes Task #15525. --- project | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'project') diff --git a/project b/project index d994b09..264895a 100755 --- a/project +++ b/project @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # # High-level script to manage the project. # Run `./project --help' for a description of how to use it. -- cgit v1.2.1 From f37005b729065f0e4ff6bfa99e5410ebb210cd60 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 27 Jan 2020 15:15:40 +0000 Subject: Initial scripts compatible with Dash (minimalistic POSIX) Until now, the initial project scripts were primarily tested with GNU Bash. But Bash is not generally available on all systems (it has many features beyond POSIX). Because of this, effectively we were imposing the requirement on the user that they must have Bash installed. We recently started this with setting the shebang of `project' and `reproduce/software/bash/configure.sh' to `/bin/sh'. After doing so, Raul and Gaspar reported an error on their systems. To fix the problem, I installed Dash (a minimalist POSIX-compliant shell) on my computer and temporarily set the shebangs to `/bin/dash', ran the project configuration step and fixed all issues that came up. With this commit, it can go all the way to building GCC on my system's Dash. After this stage (when `high-level.mk' is called), there is no problem, because we have our own version of GNU Bash and that installed version is used. Probably some more issues still remain and will hopefully be found in the future. While doing this, I also noticed the following two minor issues: - The `./project configure' option `--input-dir' was not recognized because it was mistakenly checking `--inputdir'. It has been corrected. - The test C programs now use the `< /dev/null; then coloropt="--color=auto" - elif ls -G &> /dev/null; then coloropt="-G" + if ls --color 2> /dev/null > /dev/null; then coloropt="--color=auto" + elif ls -G 2> /dev/null > /dev/null; then coloropt="-G" else coloropt="" fi -- cgit v1.2.1 From 35ed6cf0df743175688b49a4559793cb7f6e9d66 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sat, 1 Feb 2020 21:24:00 +0100 Subject: IMPORTANT: reproduce/software/bash renamed to reproduce/software/shell Until now the shell scripts in the software building phase were in the `reproduce/software/bash' directory. But given our recent change to a POSIX-only start, the `configure.sh' shell script (which is the main component of this directory) is no longer written with Bash. With this commit, to fix that problem, that directory's name has been changed to `reproduce/software/shell'. --- project | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'project') diff --git a/project b/project index 89361ae..f9a7537 100755 --- a/project +++ b/project @@ -294,7 +294,7 @@ controlled_env() { # Do requested operation # ---------------------- perms="u+r,u+w,g+r,g+w,o-r,o-w,o-x" -configscript=./reproduce/software/bash/configure.sh +configscript=./reproduce/software/shell/configure.sh case $operation in # Build the project's software. @@ -314,7 +314,7 @@ case $operation in # `reproduce/*/bash' should need executable flags, so we are giving # them executable flags by default. If any other file in your project # needs such flags, add them here. - chmod +x reproduce/software/bash/* reproduce/analysis/bash/* + chmod +x reproduce/software/shell/* reproduce/analysis/bash/* # If the user requested, clean the TeX directory from the extra # (to-be-built) directories that may already be there (and will not -- cgit v1.2.1 From d180c5d8d5430f1e826ccf1ee9849a14a091b9b8 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sat, 1 Feb 2020 21:53:47 +0100 Subject: Better message for analysis when configuration wasn't complete Until now, when `./project make' was run after an insuccessful run of `./project configure', it would just say to run `./project configure'. But for a first time user, this could be confusing because when the configuration is done in parallel, the error message can be very high on the command-line outputs and not seen clearly. With this commit, the error message is more complete and describes the problem and what the users should do in which circumstance. --- project | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'project') diff --git a/project b/project index f9a7537..3241ede 100755 --- a/project +++ b/project @@ -365,13 +365,22 @@ case $operation in if ! [ -f .build/software/configuration-done.txt ]; then cat < Date: Sat, 29 Feb 2020 21:26:54 +0000 Subject: IMPORTANT: re-preparation can only be done with --prepare-redo Until now, the preparation phase was always executed before the final build phase when running `./project make'. But when it becomes necessary, project preparation can be slow and will un-necessarily slow down the project while the project is growing (focus is on the analysis that is done after preparation). With this commit, preparation will be done automatically the first time that the project is run (`.build/software/preparation-done.mk' doesn't exist). However, after preperation is complete once, future runs of `./project make' won't do preparation any more (by calling `top-prepare.mk'). They will directly call `top-make.mk' for the analysis. To manually invoke preparation after the first attempt, the `./project make' script should be run with the new `--prepare-redo' option. Also, since the preparation phase is now automatically done before the analysis phase, the long notice that describes running `./project make' at the end of the preparation phase has been removed in `top-prepare.mk'. It now just prints a short line, saying the preparation has been complete. Finally, when the project has not been run with the proper group configuration, it ends with an `exit 1' so the main `./project' script doesn't proceed any further. --- project | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) (limited to 'project') diff --git a/project b/project index 3241ede..e2d4c66 100755 --- a/project +++ b/project @@ -37,6 +37,7 @@ check_config= make_targets= software_dir= clean_texdir=0 +prepare_redo=0 existing_conf=0 scriptname="./project" minmapsize=10000000000 @@ -108,6 +109,7 @@ Configure and Make options: Make options: -d, --debug=FLAGS Print various types of debugging information. + -p, --prepare-redo Re-do preparation (only done automatically once). Mandatory or optional arguments to long options are also mandatory or optional for any corresponding short options. @@ -184,12 +186,17 @@ do -'?'|--help) print_help; exit 0;; -'?'*|--help=*) on_off_option_error --help -?;; - # Make options (note that Make's `debug' can take values, but when called - # without any value, it is like giving it a value of `a'): + # Make options + # ------------ + # + # Note that Make's `debug' can take values, but when called without any + # value, it is like giving it a value of `a'): -d|--debug) if [ x"$2" = x ]; then debug=a; shift; else debug="$2"; check_v debug "$debug"; shift;shift; fi;; -d=*|--debug=*) debug="${1#*=}"; check_v debug "$debug"; shift;; -d*) debug=$(echo "$1" | sed -e's/-d//'); check_v debug "$debug"; shift;; + -p|--prepare-redo) prepare_redo=1; shift;; + -p=*|--prepare-redo=*) on_off_option_error --prepare-redo; shift;; # Unrecognized option: -*) echo "$scriptname: unknown option '$1'"; exit 1;; @@ -386,9 +393,16 @@ EOF exit 1 fi - # Run input-preparations (optionally build Makefiles with special - # values for optimizing the main `top-make.mk'). - controlled_env reproduce/analysis/make/top-prepare.mk + # Run data preparation phase (optionally build Makefiles with + # special values for optimizing the main `top-make.mk'). But note + # that data preparation is only done automatically the first time + # the project is built (when `.build/software/preparation-done.mk' + # doesn't yet exist). After that, if the user wants to re-do the + # preparation they have to use the `--prepare-redo' option. + if ! [ -f .build/software/preparation-done.mk ] \ + || [ x"$prepare_redo" = x1 ]; then + controlled_env reproduce/analysis/make/top-prepare.mk + fi # Run the actual project. controlled_env reproduce/analysis/make/top-make.mk -- cgit v1.2.1