From 152421339a3a52485d9f6c4be28aad5c58cd089c Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 12 Feb 2021 23:49:16 +0000 Subject: Installation: minor correction in links to system libraries Until now when making a link to the system's 'dl' and 'pthread' libraries we were simply linking the installed location on the system (in '/usr/lib'). However, in some systems, these may themselves be links to other locations and this could cause linking problems. With this commit, we now use 'realpath' to extract the absolute address of the final file that the libraries may link to, and directly link to them. A minor cosmetic correction was also made in the build rule for CFITSIO: the long line was broken into two! --- reproduce/software/make/basic.mk | 5 ++++- reproduce/software/make/high-level.mk | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'reproduce/software') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 9217ee9..efb7377 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -210,7 +210,10 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ # POSIX Threads library for multi-threaded programs. for l in dl pthread; do if [ -f /usr/lib/lib$$l.a ]; then - ln -sf /usr/lib/lib$$l.* $(ildir)/ + for f in /usr/lib/lib$$l.*; do + ln -sf $$(realpath $$f) \ + $$(echo $$f | sed -e's|/usr/lib|$(ildir)|') + done fi done diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index d69722e..26a58f7 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -428,7 +428,8 @@ $(ibidir)/cfitsio-$(cfitsio-version): export gbuild_tar=$(ddir)/$$customtar $(call gbuild, cfitsio-$(cfitsio-version), , \ --enable-sse2 --enable-reentrant \ - --with-bzip2=$(idir), , make shared fpack funpack) + --with-bzip2=$(idir), , \ + make shared fpack funpack) rm $$customtar echo "CFITSIO $(cfitsio-version)" > $@ -- cgit v1.2.1 From c3e82b1c7b6c358719f3d1dd5cf4871cde3446a7 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 19 Mar 2021 17:52:08 +0000 Subject: Configuration: --debug option available in this phase also Until now, each time there was a problem in the configuration of Maneage'd projects and debugging was necessary, we had to take the following changes: - Run the configuration on a single thread ('-j1') to see the building of only the problematic software. - Disable the Zenodo check manually by commenting those parts of 'reproduce/software/shell/configure.sh'. Because the internet connection wastes a few seconds and is thus very annoying during repeated runs! - Manually remove the '-k' option that was passed to Make (when building the software). With the '-k', Make keeps going with the execution of other targets if something crashes and this usually causes confusions during the debugging. Doing the manual changes within the code was both very annoying and prone to errors (forgetting to correct it!). With this commit, the existing '--debug' option has been generalized to the software configuration phase of Maneage also. Until now, it was only available in the analysis phase (and would directly be passed to the 'make' command that would run the analysis). When this option is used, and the project is in the software configuration phase, the Zenodo check won't be done, it will use one single thread ('-j1'), and it will stop the execution as soon as an error occurs (Make is not run with '-k'). --- reproduce/software/shell/configure.sh | 48 ++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 12 deletions(-) (limited to 'reproduce/software') diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 812f3d3..8ae74d2 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -3,6 +3,7 @@ # Necessary preparations/configurations for the reproducible project. # # Copyright (C) 2018-2021 Mohammad Akhlaghi +# Copyright (C) 2021 Raul Infante-Sainz # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -1458,18 +1459,19 @@ fi # which will download the DOI-resolved webpage, and extract the Zenodo-URL # of the most recent version from there (using the 'coreutils' tarball as # an example, the directory part of the URL for all the other software are -# the same). +# the same). This is not done if the option `--debug' is used. +zenodourl="" user_backup_urls="" zenodocheck=.build/software/zenodo-check.html -if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then - zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ - | sed -n -e'/http/p' \ - | tr ' ' '\n' \ - | grep http \ - | sed -e 's/href="//' -e 's|/coreutils| |' \ - | awk 'NR==1{print $1}') -else - zenodourl="" +if [ x$debug = x ]; then + if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then + zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ + | sed -n -e'/http/p' \ + | tr ' ' '\n' \ + | grep http \ + | sed -e 's/href="//' -e 's|/coreutils| |' \ + | awk 'NR==1{print $1}') + fi fi rm -f $zenodocheck @@ -1497,6 +1499,28 @@ user_backup_urls="$user_backup_urls $zenodourl" +# Corrections for debugging mode +# ------------------------------ +# +# If the user wants to debug the software configuration, they are usually +# focused on the building of the single problematic software. Therefore, +# the default multi-threaded execution of Make with the '--keep-going' +# option are very annoying and can even hide important warnings. Recall +# that with '--keep-going', Make will continue building other targets, even +# if one target fails. When the user runs './project configure --debug', +# the 'debug' variable will not be empty and this mode will be activated. +if [ x$debug = x ]; then + keepgoing="--keep-going" +else + jobs=1 + numthreads=1 + keepgoing="" +fi + + + + + # Build other basic tools our own GNU Make # ---------------------------------------- # @@ -1504,7 +1528,7 @@ user_backup_urls="$user_backup_urls $zenodourl" # Bash, Make, or AWK. In this step, we'll install such low-level basic # tools, but we have to be very portable (and use minimal features in all). echo; echo "Building necessary software (if necessary)..." -.local/bin/make -k -f reproduce/software/make/basic.mk \ +.local/bin/make $keepgoing -f reproduce/software/make/basic.mk \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ @@ -1532,7 +1556,7 @@ else numthreads=$jobs fi .local/bin/env -i HOME=$bdir \ - .local/bin/make -k -f reproduce/software/make/high-level.mk \ + .local/bin/make $keepgoing -f reproduce/software/make/high-level.mk \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ -- cgit v1.2.1 From a981196b2e2eea6fe0517040c49a32f757ad28cf Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 19 Mar 2021 18:11:12 +0000 Subject: Configuration: nullability-completeness warnings suppressed With a recent update of macOS systems (macOS Big Sur 11.2.3 and Xcode 12.4), there are many warnings when building C programs (for example the simple program we compile to check the compiler, or some of the software like `gzip'). It prints hundreds of warning lines for every source file that are irrelevant for our builds, but really clutters the output. With this commit, these warnings are disabled by adding `-Wno-nullability-completeness' to the 'CPPFLAGS' environment variable. This has also been added to the very first check of the C compiler in the configure step. --- reproduce/software/make/basic.mk | 3 ++- reproduce/software/make/high-level.mk | 2 +- reproduce/software/shell/configure.sh | 11 ++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) (limited to 'reproduce/software') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index efb7377..fb378b8 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -86,7 +86,8 @@ export SHELL := $(ibdir)/dash export PATH := $(ibdir):$(PATH) export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) +export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) \ + -Wno-nullability-completeness export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) # This is the "basic" tools where we are relying on the host operating diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 26a58f7..9628df4 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -125,10 +125,10 @@ all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) \ export PATH := $(ibdir) export CCACHE_DISABLE := 1 export SHELL := $(ibdir)/bash -export CPPFLAGS := -I$(idir)/include .SHELLFLAGS := --noprofile --norc -ec export LDFLAGS := $(rpath_command) -L$(ildir) export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +export CPPFLAGS := -I$(idir)/include -Wno-nullability-completeness export PKG_CONFIG_PATH := $(ildir)/pkgconfig:$(idir)/share/pkgconfig # Settings specific to this Makefile. diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 8ae74d2..fe46cef 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -424,8 +424,17 @@ if ! [ -d $compilertestdir ]; then mkdir $compilertestdir; fi # Check C compiler # ---------------- +# +# Here we check if the C compiler works properly. About the "no warning" +# variable ('nowarnings'): +# +# -Wno-nullability-completeness: on macOS Big Sur 11.2.3 and Xcode 12.4, +# hundreds of 'nullability-completeness' warnings are printed which can +# be very annoying and even hide important errors or warnings. It is +# also harmless for our test here, so it is generally added. testprog=$compilertestdir/test testsource=$compilertestdir/test.c +noccwarnings="-Wno-nullability-completeness" echo; echo; echo "Checking host C compiler ('$CC')..."; cat > $testsource < @@ -433,7 +442,7 @@ cat > $testsource < Date: Wed, 24 Mar 2021 21:01:35 +0000 Subject: Maneage installation: removed TCL as a dependency of SWIG Until now the SWIG software would use the host operating system's packages to find the TCL configuraiton (which we don't install yet in Maneage). In particular, you can see the error during its configuration here: .... checking for pkg-config... pkg-config checking for Tcl configuration... found /usr/lib/tclConfig.sh /usr/lib/tclConfig.sh: line 2: dpkg-architecture: command not found /usr/lib//tcl8.6/tclConfig.sh: line 2: dpkg-architecture: com. not found With this commit, TCL has been disabled when building SWIG with the '--without-tcl' option. Later, when we add TCL in Maneage, we can remove this option. --- reproduce/software/make/high-level.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'reproduce/software') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 9628df4..6ea782c 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1317,7 +1317,8 @@ $(ibidir)/swig-$(swig-version): # pcr is a dependency of swig tarball=swig-$(swig-version).tar.gz $(call import-source, $(swig-url), $(swig-checksum)) - $(call gbuild, swig-$(swig-version), static, --without-pcre) + $(call gbuild, swig-$(swig-version), static, \ + --without-pcre --without-tcl) echo "Swig $(swig-version)" > $@ # The disables: -- cgit v1.2.1 From 20b6273a65ab4d99ad760fd715ca935494d4fc2e Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 28 Mar 2021 12:55:03 +0100 Subject: Configuration: corrected check of group name When built in 'group' mode, the write permissions of all created files will be activated for a certain group of users in the host operating system. The user specifies the name of the group with the '--group' option at configure time. At the very start, the './project' script checks to see if the given group name actually exists or not (to avoid hard-to-debug errors popping up later). Until now, the checking 'sg' command (that was used to build the project with group-writable permissions) would always fail due to the excessive number of redirections. Therefore, it would always print the error message and abort. With this commit, the output of 'sg' is no longer re-directed (which also helps users in debuggin). If the group does actually exist, it will just print a small statement saying so, and if it fails, the error message is printed. This fixed the problem, allowing maneage to be built in group-mode. I also noticed that the variable name keeping the group name ('reproducible_paper_group_name') used the old name for the project (which was "Reproducible paper template"! So it has been changed/corrected to 'maneage_group_name'. --- reproduce/software/shell/configure.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'reproduce/software') diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index fe46cef..e15a8e9 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -771,7 +771,7 @@ fi # exists and we don't want to re-write it). if [ $rewritepconfig = no ]; then oldgroupname=$(awk '/GROUP-NAME/ {print $3; exit 0}' $pconf) - if [ "x$oldgroupname" = "x$reproducible_paper_group_name" ]; then + if [ "x$oldgroupname" = "x$maneage_group_name" ]; then just_a_place_holder_to_avoid_not_equal_test=1; else echo "-----------------------------" @@ -1089,7 +1089,7 @@ if [ $rewritepconfig = yes ]; then -e's|@ddir[@]|'"$ddir"'|' \ -e's|@sys_cpath[@]|'"$sys_cpath"'|' \ -e's|@downloader[@]|'"$downloader"'|' \ - -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \ + -e's|@groupname[@]|'"$maneage_group_name"'|' \ $pconf.in >> $pconf else # Read the values from existing configuration file. Note that the build @@ -1248,7 +1248,7 @@ if ! [ -d "$mtexdir" ]; then mkdir "$mtexdir"; fi # TeX build directory. If built in a group scenario, the TeX build # directory must be separate for each member (so they can work on their # relevant parts of the paper without conflicting with each other). -if [ "x$reproducible_paper_group_name" = x ]; then +if [ "x$maneage_group_name" = x ]; then texbdir="$texdir"/build else user=$(whoami) @@ -1758,10 +1758,10 @@ echo `.local/bin/date` > $finaltarget # # The configuration is now complete, we can inform the user on the next # step(s) to take. -if [ x$reproducible_paper_group_name = x ]; then +if [ x$maneage_group_name = x ]; then buildcommand="./project make -j8" else - buildcommand="./project make --group=$reproducible_paper_group_name -j8" + buildcommand="./project make --group=$maneage_group_name -j8" fi cat <