diff options
Diffstat (limited to 'reproduce')
| -rwxr-xr-x | reproduce/analysis/bash/download-multi-try.sh | 24 | ||||
| -rw-r--r-- | reproduce/analysis/make/delete-me.mk | 14 | ||||
| -rw-r--r-- | reproduce/analysis/make/initialize.mk | 75 | ||||
| -rw-r--r-- | reproduce/software/make/basic.mk | 37 | ||||
| -rw-r--r-- | reproduce/software/make/build-rules.mk | 16 | ||||
| -rw-r--r-- | reproduce/software/make/high-level.mk | 1 | ||||
| -rwxr-xr-x | reproduce/software/shell/apptainer.sh | 19 | ||||
| -rwxr-xr-x | reproduce/software/shell/configure.sh | 21 | ||||
| -rwxr-xr-x | reproduce/software/shell/prep-source.sh | 9 |
9 files changed, 164 insertions, 52 deletions
diff --git a/reproduce/analysis/bash/download-multi-try.sh b/reproduce/analysis/bash/download-multi-try.sh index 5cc5407..2df8aab 100755 --- a/reproduce/analysis/bash/download-multi-try.sh +++ b/reproduce/analysis/bash/download-multi-try.sh @@ -96,6 +96,25 @@ download_func () { # Set the arguments. inurl="$1" + # During the installation of basic software, the dependencies of the + # downloaders that are installed in Maneage can conflict with the + # downloader that is not yet installed in Maneage. To avoid this, we + # check if the downloader works (with '--version') and remove the + # Maneage library temporarily in case it does not. + lpathorig="" + dprog=$(echo $downloader | awk '{print $1}') + if ! $dprog --version > /dev/null 2> /dev/null; then + echo "NOTE: the segmentation fault is expected, is not a problem" + basedir=$(echo $outname \ + | sed -e's|\/tarballs\/| |' \ + | awk '{print $1}') + libdir=$basedir/installed/lib + lpathorig=$LD_LIBRARY_PATH + lpath=$(echo $LD_LIBRARY_PATH \ + | sed -e's|'$libdir':||') + export LD_LIBRARY_PATH=$lpath + fi + # Attempt downloading the file. Note that the 'downloader' ends with # the respective option to specify the output name. For example "wget # -O" (so 'outname', that comes after it) will be the name of the @@ -107,6 +126,11 @@ download_func () { "if ! $downloader $outname \"$inurl\"; then rm -f $outname; fi" fi + # In case the LD_LIBRARY_PATH was changed, set it back to normal. + if ! [ x"$lpathorig" = x ]; then + export LD_LIBRARY_PATH=$lpathorig + fi + # Some servers return HTTP 4xx/5xx errors as an HTML page with a 200 # status, so the downloader exits 0 and saves the HTML body to disk. # Detect this by checking whether the response starts with an HTML tag diff --git a/reproduce/analysis/make/delete-me.mk b/reproduce/analysis/make/delete-me.mk index 1377395..6bba8ba 100644 --- a/reproduce/analysis/make/delete-me.mk +++ b/reproduce/analysis/make/delete-me.mk @@ -26,9 +26,9 @@ # and X^2 and draw its plot. # # Note that this dataset is directly read by LaTeX to generate a plot, so -# we need to put it in the $(tex-publish-dir) directory. -dm-squared = $(tex-publish-dir)/squared.txt -$(dm-squared): $(pconfdir)/delete-me-squared-num.conf | $(tex-publish-dir) +# we need to put it in the $(figdir) directory. +dm-squared = $(figdir)/squared.txt +$(dm-squared): $(pconfdir)/delete-me-squared-num.conf | $(figdir) # When the plotted values are re-made, it is necessary to also delete # the TiKZ externalized files so the plot is also re-made by @@ -87,10 +87,10 @@ $(dm-img-pdf): $(dm-histdir)/%.pdf: $(indir)/%.fits | $(dm-histdir) # # For an example plot, we'll show the pixel value histogram also. IMPORTANT # NOTE: because this histogram contains data that is included in a plot, we -# should publish it, so it will go into the $(tex-publish-dir). -dm-img-histogram = $(tex-publish-dir)/wfpc2-histogram.txt -$(dm-img-histogram): $(tex-publish-dir)/%-histogram.txt: $(indir)/%.fits \ - | $(tex-publish-dir) +# should publish it, so it will go into the $(figdir). +dm-img-histogram = $(figdir)/wfpc2-histogram.txt +$(dm-img-histogram): $(figdir)/%-histogram.txt: $(indir)/%.fits \ + | $(figdir) # When the plotted values are re-made, it is necessary to also delete # the TiKZ externalized files so the plot is also re-made. diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 2752f47..250cb25 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -53,6 +53,45 @@ pconfdir = reproduce/analysis/config +# Directories for to-be-published data +# ------------------------------------ +# +# Its good practice (so you don't forget in the last moment!) to have all +# the plot/figure/table data that you ultimately want to publish in a +# single directory. +# +# There are two types of to-publish data in the project. +# +# 1. Those data that also go into your figures/plots (for example to give +# to LateX's PGFPlots package to create the plot internally) should be +# under the '$(texdir)' directory (because other LaTeX producers may +# also need it for example when using './project make dist', or you may +# want to publish the raw data behind the plots, for example see +# Appendix E of [1]). The contents of this directory are also directly +# taken into the tarball. As you see in [1], it is best to keep these +# files in plain-text format. +# +# 2. The data that are not included directly in the paper's figures and +# can be seen as supplements, or a "data release" associated to the +# report. A good place to keep them is under your build-directory +# (since the data may be large). These files can be custom formats like +# FITS files. +# +# RECOMMENDATION: don't put the figure/plot/table number in the names of +# your to-be-published datasets! Given them a descriptive/short name that +# would be clear to anyone who has read the paper. Later, in the caption +# (or paper's tex/appendix), you will put links to the dataset on servers +# like Zenodo (see the "Publication checklist" in 'README-hacking.md'). +# +# [1] https://scixplorer.org/abs/2026A&A...709A.210E +drdir = $(badir)/data-release +figdir = $(texdir)/figures-data +$(figdir) $(drdir):; mkdir $@ + + + + + # Preparation phase # ----------------- # @@ -145,6 +184,7 @@ curdir := $(shell echo $$(pwd)) # avoid conflicts with existing TeX Live solutions. Later (in 'paper.mk'), # we are also going to overwrite 'TEXINPUTS' just before 'pdflatex'. .ONESHELL: +export LC_ALL := C export TEXINPUTS := export CCACHE_DISABLE := 1 export PATH := $(installdir)/bin @@ -594,41 +634,6 @@ $(inputdatasets): $(indir)/%: | $(indir) $(lockdir) -# Directory containing to-be-published datasets -# --------------------------------------------- -# -# Its good practice (so you don't forget in the last moment!) to have all -# the plot/figure/table data that you ultimately want to publish in a -# single directory. -# -# There are two types of to-publish data in the project. -# -# 1. Those data that also go into LaTeX (for example to give to LateX's -# PGFPlots package to create the plot internally) should be under the -# '$(texdir)' directory (because other LaTeX producers may also need it -# for example when using './project make dist', or you may want to -# publish the raw data behind the plots, like: -# https://zenodo.org/record/4291207/files/tools-per-year.txt). The -# contents of this directory are also directly taken into the tarball. -# -# 2. The data that aren't included directly in the LaTeX run of the paper, -# can be seen as supplements. A good place to keep them is under your -# build-directory. -# -# RECOMMENDATION: don't put the figure/plot/table number in the names of -# your to-be-published datasets! Given them a descriptive/short name that -# would be clear to anyone who has read the paper. Later, in the caption -# (or paper's tex/appendix), you will put links to the dataset on servers -# like Zenodo (see the "Publication checklist" in 'README-hacking.md'). -tex-publish-dir = $(texdir)/to-publish -data-publish-dir = $(badir)/data-to-publish -$(tex-publish-dir):; mkdir $@ -$(data-publish-dir):; mkdir $@ - - - - - # Print Copyright statement # ------------------------- # diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 000f62d..b045bb1 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -87,6 +87,7 @@ all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)); @echo > /dev/null # own build-directory comes first. .ONESHELL: .SHELLFLAGS := -e -c +export LC_ALL := C export CCACHE_DISABLE := 1 export SHELL := $(ibdir)/dash export PATH := $(ibdir):$(PATH) @@ -123,6 +124,7 @@ export LD_LIBRARY_PATH=$(sys_library_sh_path) endif export LD_LIBRARY_PATH := $(shell echo $(ildir):$(LD_LIBRARY_PATH) \ | sed -e's/::/:/g' -e's/^://' -e's/:$$//') +ldpathorig := $(subst $(ildir):,,$(LD_LIBRARY_PATH)) # RPATH is automatically written in macOS, so 'DYLD_LIBRARY_PATH' is # ultimately redundant. But on some systems, even having a single value @@ -459,6 +461,21 @@ $(ibidir)/pkg-config-$(pkgconfig-version): $(ibidir)/tar-$(tar-version) printf ' glib/configure\n' >> $$confsh printf 'exec ./configure "$$@"\n' >> $$confsh chmod +x $$confsh + +# For its 'glib' installation within macOS, pkg-config needs a +# working Python for its build (not affecting its final code), but +# only searches for '/usr/bin/python'. However, some users may have +# Python available in another location, so we'll set it externally +# here to be safe. Note that 'which' is not present on some +# systems, 'command -v' is more portable). + if type python > /dev/null 2> /dev/null; then + pypath=$$(command -v python) + elif type python3 > /dev/null 2> /dev/null; then + pypath=$$(command -v python3) + else + printf "pkg-config: ERROR: a local Python is necessary"; exit 1 + fi + export PYTHON=$$pypath fi # Configuration options: @@ -479,6 +496,8 @@ $(ibidir)/pkg-config-$(pkgconfig-version): $(ibidir)/tar-$(tar-version) + + # Level 3 (THIRD MOST BASIC): Bash # -------------------------------- # @@ -687,7 +706,6 @@ $(ibidir)/bash-$(bash-version): \ - # Level 4: Most other programs # ---------------------------- @@ -940,6 +958,23 @@ $(ibidir)/wget-$(wget-version): \ tarball=wget-$(wget-version).tar.lz $(call import-source, $(wget-url), $(wget-checksum)) +# Wget needs a working Python for its tests (VERY BAD DESIGN!), this +# does not affect its final operation. But only searches for +# '/usr/bin/python'. However, some users may have Python available in +# another location, so we'll set it externally here to be safe. Note +# that 'which' is not present on some systems, 'command -v' is more +# portable). + if [ x$(on_mac_os) = xyes ]; then + if type python > /dev/null 2> /dev/null; then + pypath=$$(command -v python) + elif type python3 > /dev/null 2> /dev/null; then + pypath=$$(command -v python3) + else + printf "Wget: ERROR: a local Python is necessary"; exit 1 + fi + export PYTHON=$$pypath + fi + # We need to explicitly disable 'libiconv', because of the # 'pkg-config' and 'libiconv' problem. libs="-pthread" diff --git a/reproduce/software/make/build-rules.mk b/reproduce/software/make/build-rules.mk index 984d0bf..454a255 100644 --- a/reproduce/software/make/build-rules.mk +++ b/reproduce/software/make/build-rules.mk @@ -40,6 +40,14 @@ # Necessary shell variables # 'tarball': This is the name of the actual tarball file without a # directory. +# +# Notes: +# - The LD_LIBRARY_PATH correction is because we build the dependencies of +# the downloaders (Wget or cURL) before they are built them selves. In +# the time between the installation of the downloader dependencies and +# the downloader, the Gnuastro libraries may cause crashes in the host's +# downloader since normally, Gnuastro's library has higher precedence in +# 'LD_LIBRARY_PATH'. import-source = final=$(tdir)/$$tarball; \ url=$(strip $(1)); \ exp_checksum="$(strip $(2))"; \ @@ -82,8 +90,14 @@ import-source = final=$(tdir)/$$tarball; \ if [ x"$$exp_checksum" = x"NO-CHECK-SUM" ]; then \ mv "$$unchecked" "$$final"; \ else \ - if type sha512sum > /dev/null 2>/dev/null; then \ + if type sha512sum > /dev/null 2> /dev/null; then \ + if ! sha512sum --version > /dev/null 2> /dev/null; then \ + export LD_LIBRARY_PATH=$(ldpathorig); \ + fi; \ checksum=$$(sha512sum "$$unchecked" | awk '{print $$1}'); \ + if [ LD_LIBRARY_PATH = $(ldpathorig) ]; then \ + export LD_LIBRARY_PATH=$(LD_LIBRARY_PATH); \ + fi; \ if [ x"$$checksum" = x"$$exp_checksum" ]; then \ mv "$$unchecked" "$$final"; \ else \ diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 281bd84..0f6f63d 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -166,6 +166,7 @@ export PKG_CONFIG_PATH := $(ildir)/pkgconfig:$(idir)/share/pkgconfig .SUFFIXES: # Settings specific to this Makefile. +export LC_ALL := C export CC := $(ibdir)/gcc export CXX := $(ibdir)/g++ export F77 := $(ibdir)/gfortran diff --git a/reproduce/software/shell/apptainer.sh b/reproduce/software/shell/apptainer.sh index eb24e9e..2aa3535 100755 --- a/reproduce/software/shell/apptainer.sh +++ b/reproduce/software/shell/apptainer.sh @@ -73,7 +73,9 @@ set -e # Default option values sif="" jobs=0 +debug=0 quiet=0 +host_cc=0 source_dir= build_only= base_sif="" @@ -101,6 +103,10 @@ Top-level script to build and run a Maneage'd project within Apptainer. --base-os=STR Base OS name (default: '$base_os'). --base-sif=STR Base OS apptainer image (a '.sif' file). + Configuration options + --host-cc Use containers's C compiler, don't build GCC. + -d, --debug[=FLAGS] Use -j1 and stop on first crash. + Interactive shell --project-shell Open the project's shell within the container. --container-shell Open the container shell. @@ -162,6 +168,12 @@ do --base-sif) base_sif="$2"; check_v "$1" "$base_sif"; shift;shift;; --base-sif=*) base_sif="${1#*=}"; check_v "$1" "$base_sif"; shift;; + # Configuration options. + --host-cc) host_cc=1; shift;; + --host-cc=*) on_off_option_error --host-cc;; + -d|--debug) debug=1; shift;; + -d|--debug=*) on_off_option_error --debug;; + # Interactive shell. --project-shell) project_shell=1; shift;; --project_shell=*) on_off_option_error --project-shell;; @@ -329,8 +341,11 @@ EOF # software tarball directory, they will all be symbolic links that # aren't valid when the user runs the container (since we only # mount the software tarballs at build time). + confopts="" intbuild=/home/maneager/build maneage_def=$build_dir/maneage.def + if [ $host_cc = 1 ]; then confopts="--host-cc"; fi + if [ $debug = 1 ]; then confopts="$confopts --debug"; fi cat <<EOF > $maneage_def Bootstrap: localimage From: $base_sif @@ -343,7 +358,7 @@ From: $base_sif %post cd /home/maneager/source - ./project configure --jobs=$jobs --no-pause \\ + ./project configure --jobs=$jobs --no-pause $confopts \\ --input-dir=/home/maneager/input \\ --build-dir=$intbuild \\ --software-dir=/home/maneager/tarballs-software @@ -351,7 +366,7 @@ From: $base_sif %runscript cd /home/maneager/source - if ./project configure --build-dir=$intbuild \\ + if ./project configure --build-dir=$intbuild $confopts \\ --existing-conf --no-pause \\ --offline --quiet; then \\ if [ x"\$maneage_apptainer_stat" = xshell ]; then \\ diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index e17c9ad..19c14ac 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -1342,11 +1342,17 @@ elapsed_time_from_prev_step software-dir # complete the configuration, we may also need to download the source code # of some necessary software packages (including the downloaders). So we # need to check the host's available tool for downloading at this step. +# +# NOTE: it is important that the name of the programs are not recorded to +# be absolute (for example 'wget' instead of '/usr/bin/wget'). This is +# because we build both 'wget' and 'curl' within 'basic.mk'. After they are +# built, Maneage's built software have higher precedence in 'PATH'. So +# without an absolute address, our own built software will be used as soon +# as they are installed/available. if [ $rewritelconfig = yes ]; then - if type wget > /dev/null 2>/dev/null; then - # 'which' isn't in POSIX, so we are using 'command -v' instead. - name=$(command -v wget) + # First look for wget on the host. + if type wget > /dev/null 2>/dev/null; then # See if the host wget has the '--no-use-server-timestamps' option # (for example wget 1.12 doesn't have it). If not, we'll have to @@ -1363,13 +1369,16 @@ if [ $rewritelconfig = yes ]; then # By default Wget keeps the remote file's timestamp, so we'll have # to disable it manually. - downloader="$name $wgetts -O"; + downloader="wget $wgetts -O"; + + # In case Wget doesn't exist, try cURL. elif type curl > /dev/null 2>/dev/null; then - name=$(command -v curl) # - cURL doesn't keep the remote file's timestamp by default. # - With the '-L' option, we tell cURL to follow redirects. - downloader="$name -L -o" + downloader="curl -L -o" + + # Neither Wget or cURL could be found. else cat <<EOF diff --git a/reproduce/software/shell/prep-source.sh b/reproduce/software/shell/prep-source.sh index 2959255..7f49b5c 100755 --- a/reproduce/software/shell/prep-source.sh +++ b/reproduce/software/shell/prep-source.sh @@ -32,6 +32,15 @@ set -e + +# Set the locale to C to make sure local language settings do not +# interefere (especially with the SED command). +export LC_ALL=C + + + + + # Read the first argument. bindir="$1" if [ x"$bindir" = x ]; then |