IMPORTANT: better names for to-publish dirs and portability fixesHEADmaneage

Summary: this commit only affects the analysis of your project in case you
used the 'tex-publish-dir' or 'data-publish-dir' directories. They need to
be renamed to 'figdir' and 'drdir' respectively.

Until now, the directories containing the to-be-published data were
described deep in the middle of 'initialize.mk' (away from the other shared
directories), they also had hard-to-use (long) names. Furthermore,
'README-hacking.md' that is the main documentation of Maneage had grown
organically, such that the checklists were scattered across the file and
some things were repeated or not classified clearly. Finally, some
portability issues were reported by Stergios Amarantidis and Diana Korotun.

With this commit, the to-be-published directories have been renamed to the
more simple names of 'figdir' and 'drdir' respectively and their definition
in 'initialize.mk' has come right under the rest of the top-level
directories (to be easily found). Furthermore, the 'README-hacking.md' has
been fully restructured to be easier to navigate and follow for a new
reader. Finally, the portability issues mentioned above have been
addressed:

 - Within the '--help' output of './project', the '--keep-going' option was
   listed under "Configure and Make options", while it is only relevant as
   a Make option. It was therefore moved under Make options.

 - We had not explicitly set the locale of Maneage; causing a crash on one
   of the newly tested computers (a macOS with a non-C locale) during the
   execution of 'prep-source.sh'. With this commit, all the top-level
   Makefiles and scripts of Maneage set 'LC_ALL' to 'C' to ensure the same
   locale on any system.

 - The 'download-multi-try.sh' script would crash when building basic
   software in the following scenario: the downloader was still from the
   host, but one of its linked libraries had already been installed in
   Maneage and conflicted with the host. A step has been added to catch
   such cases and remove the Maneage'd library when this was the problem.

 - On macOS, we discovered that 'pkg-config' and 'wget' need a Python
   installation, so the necessary conditions have been added.

 - The 'apptainer.sh' script did not allow passing the '--debug' or
   '--host-cc' options to the execution of './project' within the
   container.

 - Within 'configure.sh', we wrote the absolute path of the downloader to
   use in 'LOCAL.conf'. However, we install both downloaders and it is
   safer to use Maneage's own downloader when it has been installed. So the
   downloader name in 'LOCAL.conf' now only has the name (without its
   absolute path).

6 files changed, 93 insertions, 10 deletions

diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk
index 000f62d..b045bb1 100644
--- a/reproduce/software/make/basic.mk
+++ b/reproduce/software/make/basic.mk
@@ -87,6 +87,7 @@ all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)); @echo > /dev/null
 # own build-directory comes first.
 .ONESHELL:
 .SHELLFLAGS := -e -c
+export LC_ALL := C
 export CCACHE_DISABLE := 1
 export SHELL := $(ibdir)/dash
 export PATH := $(ibdir):$(PATH)
@@ -123,6 +124,7 @@ export LD_LIBRARY_PATH=$(sys_library_sh_path)
 endif
 export LD_LIBRARY_PATH := $(shell echo $(ildir):$(LD_LIBRARY_PATH) \
                                   | sed -e's/::/:/g' -e's/^://' -e's/:$$//')
+ldpathorig := $(subst $(ildir):,,$(LD_LIBRARY_PATH))
 
 # RPATH is automatically written in macOS, so 'DYLD_LIBRARY_PATH' is
 # ultimately redundant. But on some systems, even having a single value
@@ -459,6 +461,21 @@ $(ibidir)/pkg-config-$(pkgconfig-version): $(ibidir)/tar-$(tar-version)
 	  printf                ' glib/configure\n' >> $$confsh
 	  printf 'exec ./configure "$$@"\n' >> $$confsh
 	  chmod +x $$confsh
+
+#	  For its 'glib' installation within macOS, pkg-config needs a
+#	  working Python for its build (not affecting its final code), but
+#	  only searches for '/usr/bin/python'. However, some users may have
+#	  Python available in another location, so we'll set it externally
+#	  here to be safe. Note that 'which' is not present on some
+#	  systems, 'command -v' is more portable).
+	  if type python > /dev/null 2> /dev/null; then
+	    pypath=$$(command -v python)
+	  elif type python3 > /dev/null 2> /dev/null; then
+	    pypath=$$(command -v python3)
+	  else
+	    printf "pkg-config: ERROR: a local Python is necessary"; exit 1
+	  fi
+	  export PYTHON=$$pypath
 	fi
 
 #	Configuration options:
@@ -479,6 +496,8 @@ $(ibidir)/pkg-config-$(pkgconfig-version): $(ibidir)/tar-$(tar-version)
 
 
 
+
+
 # Level 3 (THIRD MOST BASIC): Bash
 # --------------------------------
 #
@@ -687,7 +706,6 @@ $(ibidir)/bash-$(bash-version): \
 
 
 
-
 # Level 4: Most other programs
 # ----------------------------
 
@@ -940,6 +958,23 @@ $(ibidir)/wget-$(wget-version): \
 	tarball=wget-$(wget-version).tar.lz
 	$(call import-source, $(wget-url), $(wget-checksum))
 
+#	Wget needs a working Python for its tests (VERY BAD DESIGN!), this
+#	does not affect its final operation. But only searches for
+#	'/usr/bin/python'. However, some users may have Python available in
+#	another location, so we'll set it externally here to be safe. Note
+#	that 'which' is not present on some systems, 'command -v' is more
+#	portable).
+	if [ x$(on_mac_os) = xyes ]; then
+	  if type python > /dev/null 2> /dev/null; then
+	    pypath=$$(command -v python)
+	  elif type python3 > /dev/null 2> /dev/null; then
+	    pypath=$$(command -v python3)
+	  else
+	    printf "Wget: ERROR: a local Python is necessary"; exit 1
+	  fi
+	  export PYTHON=$$pypath
+	fi
+
 #	We need to explicitly disable 'libiconv', because of the
 #	'pkg-config' and 'libiconv' problem.
 	libs="-pthread"
diff --git a/reproduce/software/make/build-rules.mk b/reproduce/software/make/build-rules.mk
index 984d0bf..454a255 100644
--- a/reproduce/software/make/build-rules.mk
+++ b/reproduce/software/make/build-rules.mk
@@ -40,6 +40,14 @@
 # Necessary shell variables
 #   'tarball': This is the name of the actual tarball file without a
 #   directory.
+#
+# Notes:
+#  - The LD_LIBRARY_PATH correction is because we build the dependencies of
+#    the downloaders (Wget or cURL) before they are built them selves. In
+#    the time between the installation of the downloader dependencies and
+#    the downloader, the Gnuastro libraries may cause crashes in the host's
+#    downloader since normally, Gnuastro's library has higher precedence in
+#    'LD_LIBRARY_PATH'.
 import-source = final=$(tdir)/$$tarball; \
 	url=$(strip $(1)); \
 	exp_checksum="$(strip $(2))"; \
@@ -82,8 +90,14 @@ import-source = final=$(tdir)/$$tarball; \
 	  if [ x"$$exp_checksum" = x"NO-CHECK-SUM" ]; then \
 	    mv "$$unchecked" "$$final"; \
 	  else \
-	    if type sha512sum > /dev/null 2>/dev/null; then \
+	    if type sha512sum > /dev/null 2> /dev/null; then \
+	      if ! sha512sum --version > /dev/null 2> /dev/null; then \
+	        export LD_LIBRARY_PATH=$(ldpathorig); \
+	      fi; \
 	      checksum=$$(sha512sum "$$unchecked" | awk '{print $$1}'); \
+	      if [ LD_LIBRARY_PATH = $(ldpathorig) ]; then \
+	        export LD_LIBRARY_PATH=$(LD_LIBRARY_PATH); \
+	      fi; \
 	      if [ x"$$checksum" = x"$$exp_checksum" ]; then \
 	        mv "$$unchecked" "$$final"; \
 	      else \
diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk
index 281bd84..0f6f63d 100644
--- a/reproduce/software/make/high-level.mk
+++ b/reproduce/software/make/high-level.mk
@@ -166,6 +166,7 @@ export PKG_CONFIG_PATH := $(ildir)/pkgconfig:$(idir)/share/pkgconfig
 .SUFFIXES:
 
 # Settings specific to this Makefile.
+export LC_ALL := C
 export CC := $(ibdir)/gcc
 export CXX := $(ibdir)/g++
 export F77 := $(ibdir)/gfortran
diff --git a/reproduce/software/shell/apptainer.sh b/reproduce/software/shell/apptainer.sh
index eb24e9e..2aa3535 100755
--- a/reproduce/software/shell/apptainer.sh
+++ b/reproduce/software/shell/apptainer.sh
@@ -73,7 +73,9 @@ set -e
 # Default option values
 sif=""
 jobs=0
+debug=0
 quiet=0
+host_cc=0
 source_dir=
 build_only=
 base_sif=""
@@ -101,6 +103,10 @@ Top-level script to build and run a Maneage'd project within Apptainer.
       --base-os=STR        Base OS name (default: '$base_os').
       --base-sif=STR       Base OS apptainer image (a '.sif' file).
 
+ Configuration options
+      --host-cc            Use containers's C compiler, don't build GCC.
+  -d, --debug[=FLAGS]      Use -j1 and stop on first crash.
+
  Interactive shell
       --project-shell      Open the project's shell within the container.
       --container-shell    Open the container shell.
@@ -162,6 +168,12 @@ do
   --base-sif)            base_sif="$2";                               check_v "$1" "$base_sif";    shift;shift;;
   --base-sif=*)          base_sif="${1#*=}";                          check_v "$1" "$base_sif";    shift;;
 
+  # Configuration options.
+  --host-cc)             host_cc=1;                                                                 shift;;
+  --host-cc=*)           on_off_option_error --host-cc;;
+  -d|--debug)            debug=1;                                                                   shift;;
+  -d|--debug=*)          on_off_option_error --debug;;
+
   # Interactive shell.
   --project-shell)       project_shell=1;                                                           shift;;
   --project_shell=*)     on_off_option_error --project-shell;;
@@ -329,8 +341,11 @@ EOF
     #     software tarball directory, they will all be symbolic links that
     #     aren't valid when the user runs the container (since we only
     #     mount the software tarballs at build time).
+    confopts=""
     intbuild=/home/maneager/build
     maneage_def=$build_dir/maneage.def
+    if [ $host_cc = 1 ]; then confopts="--host-cc"; fi
+    if [ $debug = 1 ]; then confopts="$confopts --debug"; fi
     cat <<EOF > $maneage_def
 Bootstrap: localimage
 From: $base_sif
@@ -343,7 +358,7 @@ From: $base_sif
 
 %post
   cd /home/maneager/source
-  ./project configure --jobs=$jobs --no-pause \\
+  ./project configure --jobs=$jobs --no-pause $confopts \\
                       --input-dir=/home/maneager/input \\
                       --build-dir=$intbuild \\
                       --software-dir=/home/maneager/tarballs-software
@@ -351,7 +366,7 @@ From: $base_sif
 
 %runscript
   cd /home/maneager/source
-  if ./project configure --build-dir=$intbuild \\
+  if ./project configure --build-dir=$intbuild $confopts \\
                          --existing-conf --no-pause \\
                          --offline --quiet; then \\
      if   [ x"\$maneage_apptainer_stat" = xshell ]; then \\
diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh
index e17c9ad..19c14ac 100755
--- a/reproduce/software/shell/configure.sh
+++ b/reproduce/software/shell/configure.sh
@@ -1342,11 +1342,17 @@ elapsed_time_from_prev_step software-dir
 # complete the configuration, we may also need to download the source code
 # of some necessary software packages (including the downloaders). So we
 # need to check the host's available tool for downloading at this step.
+#
+# NOTE: it is important that the name of the programs are not recorded to
+# be absolute (for example 'wget' instead of '/usr/bin/wget'). This is
+# because we build both 'wget' and 'curl' within 'basic.mk'. After they are
+# built, Maneage's built software have higher precedence in 'PATH'. So
+# without an absolute address, our own built software will be used as soon
+# as they are installed/available.
 if [ $rewritelconfig = yes ]; then
-    if type wget > /dev/null 2>/dev/null; then
 
-        # 'which' isn't in POSIX, so we are using 'command -v' instead.
-        name=$(command -v wget)
+    # First look for wget on the host.
+    if type wget > /dev/null 2>/dev/null; then
 
         # See if the host wget has the '--no-use-server-timestamps' option
         # (for example wget 1.12 doesn't have it). If not, we'll have to
@@ -1363,13 +1369,16 @@ if [ $rewritelconfig = yes ]; then
 
         # By default Wget keeps the remote file's timestamp, so we'll have
         # to disable it manually.
-        downloader="$name $wgetts -O";
+        downloader="wget $wgetts -O";
+
+    # In case Wget doesn't exist, try cURL.
     elif type curl > /dev/null 2>/dev/null; then
-        name=$(command -v curl)
 
         # - cURL doesn't keep the remote file's timestamp by default.
         # - With the '-L' option, we tell cURL to follow redirects.
-        downloader="$name -L -o"
+        downloader="curl -L -o"
+
+    # Neither Wget or cURL could be found.
     else
         cat <<EOF
 
diff --git a/reproduce/software/shell/prep-source.sh b/reproduce/software/shell/prep-source.sh
index 2959255..7f49b5c 100755
--- a/reproduce/software/shell/prep-source.sh
+++ b/reproduce/software/shell/prep-source.sh
@@ -32,6 +32,15 @@ set -e
 
 
 
+
+# Set the locale to C to make sure local language settings do not
+# interefere (especially with the SED command).
+export LC_ALL=C
+
+
+
+
+
 # Read the first argument.
 bindir="$1"
 if [ x"$bindir" = x ]; then