From c3e82b1c7b6c358719f3d1dd5cf4871cde3446a7 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 19 Mar 2021 17:52:08 +0000 Subject: Configuration: --debug option available in this phase also Until now, each time there was a problem in the configuration of Maneage'd projects and debugging was necessary, we had to take the following changes: - Run the configuration on a single thread ('-j1') to see the building of only the problematic software. - Disable the Zenodo check manually by commenting those parts of 'reproduce/software/shell/configure.sh'. Because the internet connection wastes a few seconds and is thus very annoying during repeated runs! - Manually remove the '-k' option that was passed to Make (when building the software). With the '-k', Make keeps going with the execution of other targets if something crashes and this usually causes confusions during the debugging. Doing the manual changes within the code was both very annoying and prone to errors (forgetting to correct it!). With this commit, the existing '--debug' option has been generalized to the software configuration phase of Maneage also. Until now, it was only available in the analysis phase (and would directly be passed to the 'make' command that would run the analysis). When this option is used, and the project is in the software configuration phase, the Zenodo check won't be done, it will use one single thread ('-j1'), and it will stop the execution as soon as an error occurs (Make is not run with '-k'). --- reproduce/software/shell/configure.sh | 48 ++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 12 deletions(-) (limited to 'reproduce/software/shell') diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 812f3d3..8ae74d2 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -3,6 +3,7 @@ # Necessary preparations/configurations for the reproducible project. # # Copyright (C) 2018-2021 Mohammad Akhlaghi +# Copyright (C) 2021 Raul Infante-Sainz # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -1458,18 +1459,19 @@ fi # which will download the DOI-resolved webpage, and extract the Zenodo-URL # of the most recent version from there (using the 'coreutils' tarball as # an example, the directory part of the URL for all the other software are -# the same). +# the same). This is not done if the option `--debug' is used. +zenodourl="" user_backup_urls="" zenodocheck=.build/software/zenodo-check.html -if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then - zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ - | sed -n -e'/http/p' \ - | tr ' ' '\n' \ - | grep http \ - | sed -e 's/href="//' -e 's|/coreutils| |' \ - | awk 'NR==1{print $1}') -else - zenodourl="" +if [ x$debug = x ]; then + if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then + zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ + | sed -n -e'/http/p' \ + | tr ' ' '\n' \ + | grep http \ + | sed -e 's/href="//' -e 's|/coreutils| |' \ + | awk 'NR==1{print $1}') + fi fi rm -f $zenodocheck @@ -1497,6 +1499,28 @@ user_backup_urls="$user_backup_urls $zenodourl" +# Corrections for debugging mode +# ------------------------------ +# +# If the user wants to debug the software configuration, they are usually +# focused on the building of the single problematic software. Therefore, +# the default multi-threaded execution of Make with the '--keep-going' +# option are very annoying and can even hide important warnings. Recall +# that with '--keep-going', Make will continue building other targets, even +# if one target fails. When the user runs './project configure --debug', +# the 'debug' variable will not be empty and this mode will be activated. +if [ x$debug = x ]; then + keepgoing="--keep-going" +else + jobs=1 + numthreads=1 + keepgoing="" +fi + + + + + # Build other basic tools our own GNU Make # ---------------------------------------- # @@ -1504,7 +1528,7 @@ user_backup_urls="$user_backup_urls $zenodourl" # Bash, Make, or AWK. In this step, we'll install such low-level basic # tools, but we have to be very portable (and use minimal features in all). echo; echo "Building necessary software (if necessary)..." -.local/bin/make -k -f reproduce/software/make/basic.mk \ +.local/bin/make $keepgoing -f reproduce/software/make/basic.mk \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ @@ -1532,7 +1556,7 @@ else numthreads=$jobs fi .local/bin/env -i HOME=$bdir \ - .local/bin/make -k -f reproduce/software/make/high-level.mk \ + .local/bin/make $keepgoing -f reproduce/software/make/high-level.mk \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ -- cgit v1.2.1 From a981196b2e2eea6fe0517040c49a32f757ad28cf Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 19 Mar 2021 18:11:12 +0000 Subject: Configuration: nullability-completeness warnings suppressed With a recent update of macOS systems (macOS Big Sur 11.2.3 and Xcode 12.4), there are many warnings when building C programs (for example the simple program we compile to check the compiler, or some of the software like `gzip'). It prints hundreds of warning lines for every source file that are irrelevant for our builds, but really clutters the output. With this commit, these warnings are disabled by adding `-Wno-nullability-completeness' to the 'CPPFLAGS' environment variable. This has also been added to the very first check of the C compiler in the configure step. --- reproduce/software/shell/configure.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'reproduce/software/shell') diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 8ae74d2..fe46cef 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -424,8 +424,17 @@ if ! [ -d $compilertestdir ]; then mkdir $compilertestdir; fi # Check C compiler # ---------------- +# +# Here we check if the C compiler works properly. About the "no warning" +# variable ('nowarnings'): +# +# -Wno-nullability-completeness: on macOS Big Sur 11.2.3 and Xcode 12.4, +# hundreds of 'nullability-completeness' warnings are printed which can +# be very annoying and even hide important errors or warnings. It is +# also harmless for our test here, so it is generally added. testprog=$compilertestdir/test testsource=$compilertestdir/test.c +noccwarnings="-Wno-nullability-completeness" echo; echo; echo "Checking host C compiler ('$CC')..."; cat > $testsource < @@ -433,7 +442,7 @@ cat > $testsource < Date: Sun, 28 Mar 2021 12:55:03 +0100 Subject: Configuration: corrected check of group name When built in 'group' mode, the write permissions of all created files will be activated for a certain group of users in the host operating system. The user specifies the name of the group with the '--group' option at configure time. At the very start, the './project' script checks to see if the given group name actually exists or not (to avoid hard-to-debug errors popping up later). Until now, the checking 'sg' command (that was used to build the project with group-writable permissions) would always fail due to the excessive number of redirections. Therefore, it would always print the error message and abort. With this commit, the output of 'sg' is no longer re-directed (which also helps users in debuggin). If the group does actually exist, it will just print a small statement saying so, and if it fails, the error message is printed. This fixed the problem, allowing maneage to be built in group-mode. I also noticed that the variable name keeping the group name ('reproducible_paper_group_name') used the old name for the project (which was "Reproducible paper template"! So it has been changed/corrected to 'maneage_group_name'. --- reproduce/software/shell/configure.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'reproduce/software/shell') diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index fe46cef..e15a8e9 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -771,7 +771,7 @@ fi # exists and we don't want to re-write it). if [ $rewritepconfig = no ]; then oldgroupname=$(awk '/GROUP-NAME/ {print $3; exit 0}' $pconf) - if [ "x$oldgroupname" = "x$reproducible_paper_group_name" ]; then + if [ "x$oldgroupname" = "x$maneage_group_name" ]; then just_a_place_holder_to_avoid_not_equal_test=1; else echo "-----------------------------" @@ -1089,7 +1089,7 @@ if [ $rewritepconfig = yes ]; then -e's|@ddir[@]|'"$ddir"'|' \ -e's|@sys_cpath[@]|'"$sys_cpath"'|' \ -e's|@downloader[@]|'"$downloader"'|' \ - -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \ + -e's|@groupname[@]|'"$maneage_group_name"'|' \ $pconf.in >> $pconf else # Read the values from existing configuration file. Note that the build @@ -1248,7 +1248,7 @@ if ! [ -d "$mtexdir" ]; then mkdir "$mtexdir"; fi # TeX build directory. If built in a group scenario, the TeX build # directory must be separate for each member (so they can work on their # relevant parts of the paper without conflicting with each other). -if [ "x$reproducible_paper_group_name" = x ]; then +if [ "x$maneage_group_name" = x ]; then texbdir="$texdir"/build else user=$(whoami) @@ -1758,10 +1758,10 @@ echo `.local/bin/date` > $finaltarget # # The configuration is now complete, we can inform the user on the next # step(s) to take. -if [ x$reproducible_paper_group_name = x ]; then +if [ x$maneage_group_name = x ]; then buildcommand="./project make -j8" else - buildcommand="./project make --group=$reproducible_paper_group_name -j8" + buildcommand="./project make --group=$maneage_group_name -j8" fi cat <