From ab5e0aa9fa8a6b94b1306fa120736b6fbb1a2f70 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 20 Jul 2026 20:20:56 +0200 Subject: IMPORTANT: better names for to-publish dirs and portability fixes Summary: this commit only affects the analysis of your project in case you used the 'tex-publish-dir' or 'data-publish-dir' directories. They need to be renamed to 'figdir' and 'drdir' respectively. Until now, the directories containing the to-be-published data were described deep in the middle of 'initialize.mk' (away from the other shared directories), they also had hard-to-use (long) names. Furthermore, 'README-hacking.md' that is the main documentation of Maneage had grown organically, such that the checklists were scattered across the file and some things were repeated or not classified clearly. Finally, some portability issues were reported by Stergios Amarantidis and Diana Korotun. With this commit, the to-be-published directories have been renamed to the more simple names of 'figdir' and 'drdir' respectively and their definition in 'initialize.mk' has come right under the rest of the top-level directories (to be easily found). Furthermore, the 'README-hacking.md' has been fully restructured to be easier to navigate and follow for a new reader. Finally, the portability issues mentioned above have been addressed: - Within the '--help' output of './project', the '--keep-going' option was listed under "Configure and Make options", while it is only relevant as a Make option. It was therefore moved under Make options. - We had not explicitly set the locale of Maneage; causing a crash on one of the newly tested computers (a macOS with a non-C locale) during the execution of 'prep-source.sh'. With this commit, all the top-level Makefiles and scripts of Maneage set 'LC_ALL' to 'C' to ensure the same locale on any system. - The 'download-multi-try.sh' script would crash when building basic software in the following scenario: the downloader was still from the host, but one of its linked libraries had already been installed in Maneage and conflicted with the host. A step has been added to catch such cases and remove the Maneage'd library when this was the problem. - On macOS, we discovered that 'pkg-config' and 'wget' need a Python installation, so the necessary conditions have been added. - The 'apptainer.sh' script did not allow passing the '--debug' or '--host-cc' options to the execution of './project' within the container. - Within 'configure.sh', we wrote the absolute path of the downloader to use in 'LOCAL.conf'. However, we install both downloaders and it is safer to use Maneage's own downloader when it has been installed. So the downloader name in 'LOCAL.conf' now only has the name (without its absolute path). --- reproduce/analysis/bash/download-multi-try.sh | 24 +++++++++ reproduce/analysis/make/delete-me.mk | 14 ++--- reproduce/analysis/make/initialize.mk | 75 ++++++++++++++------------- 3 files changed, 71 insertions(+), 42 deletions(-) (limited to 'reproduce/analysis') diff --git a/reproduce/analysis/bash/download-multi-try.sh b/reproduce/analysis/bash/download-multi-try.sh index 5cc5407..2df8aab 100755 --- a/reproduce/analysis/bash/download-multi-try.sh +++ b/reproduce/analysis/bash/download-multi-try.sh @@ -96,6 +96,25 @@ download_func () { # Set the arguments. inurl="$1" + # During the installation of basic software, the dependencies of the + # downloaders that are installed in Maneage can conflict with the + # downloader that is not yet installed in Maneage. To avoid this, we + # check if the downloader works (with '--version') and remove the + # Maneage library temporarily in case it does not. + lpathorig="" + dprog=$(echo $downloader | awk '{print $1}') + if ! $dprog --version > /dev/null 2> /dev/null; then + echo "NOTE: the segmentation fault is expected, is not a problem" + basedir=$(echo $outname \ + | sed -e's|\/tarballs\/| |' \ + | awk '{print $1}') + libdir=$basedir/installed/lib + lpathorig=$LD_LIBRARY_PATH + lpath=$(echo $LD_LIBRARY_PATH \ + | sed -e's|'$libdir':||') + export LD_LIBRARY_PATH=$lpath + fi + # Attempt downloading the file. Note that the 'downloader' ends with # the respective option to specify the output name. For example "wget # -O" (so 'outname', that comes after it) will be the name of the @@ -107,6 +126,11 @@ download_func () { "if ! $downloader $outname \"$inurl\"; then rm -f $outname; fi" fi + # In case the LD_LIBRARY_PATH was changed, set it back to normal. + if ! [ x"$lpathorig" = x ]; then + export LD_LIBRARY_PATH=$lpathorig + fi + # Some servers return HTTP 4xx/5xx errors as an HTML page with a 200 # status, so the downloader exits 0 and saves the HTML body to disk. # Detect this by checking whether the response starts with an HTML tag diff --git a/reproduce/analysis/make/delete-me.mk b/reproduce/analysis/make/delete-me.mk index 1377395..6bba8ba 100644 --- a/reproduce/analysis/make/delete-me.mk +++ b/reproduce/analysis/make/delete-me.mk @@ -26,9 +26,9 @@ # and X^2 and draw its plot. # # Note that this dataset is directly read by LaTeX to generate a plot, so -# we need to put it in the $(tex-publish-dir) directory. -dm-squared = $(tex-publish-dir)/squared.txt -$(dm-squared): $(pconfdir)/delete-me-squared-num.conf | $(tex-publish-dir) +# we need to put it in the $(figdir) directory. +dm-squared = $(figdir)/squared.txt +$(dm-squared): $(pconfdir)/delete-me-squared-num.conf | $(figdir) # When the plotted values are re-made, it is necessary to also delete # the TiKZ externalized files so the plot is also re-made by @@ -87,10 +87,10 @@ $(dm-img-pdf): $(dm-histdir)/%.pdf: $(indir)/%.fits | $(dm-histdir) # # For an example plot, we'll show the pixel value histogram also. IMPORTANT # NOTE: because this histogram contains data that is included in a plot, we -# should publish it, so it will go into the $(tex-publish-dir). -dm-img-histogram = $(tex-publish-dir)/wfpc2-histogram.txt -$(dm-img-histogram): $(tex-publish-dir)/%-histogram.txt: $(indir)/%.fits \ - | $(tex-publish-dir) +# should publish it, so it will go into the $(figdir). +dm-img-histogram = $(figdir)/wfpc2-histogram.txt +$(dm-img-histogram): $(figdir)/%-histogram.txt: $(indir)/%.fits \ + | $(figdir) # When the plotted values are re-made, it is necessary to also delete # the TiKZ externalized files so the plot is also re-made. diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 2752f47..250cb25 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -53,6 +53,45 @@ pconfdir = reproduce/analysis/config +# Directories for to-be-published data +# ------------------------------------ +# +# Its good practice (so you don't forget in the last moment!) to have all +# the plot/figure/table data that you ultimately want to publish in a +# single directory. +# +# There are two types of to-publish data in the project. +# +# 1. Those data that also go into your figures/plots (for example to give +# to LateX's PGFPlots package to create the plot internally) should be +# under the '$(texdir)' directory (because other LaTeX producers may +# also need it for example when using './project make dist', or you may +# want to publish the raw data behind the plots, for example see +# Appendix E of [1]). The contents of this directory are also directly +# taken into the tarball. As you see in [1], it is best to keep these +# files in plain-text format. +# +# 2. The data that are not included directly in the paper's figures and +# can be seen as supplements, or a "data release" associated to the +# report. A good place to keep them is under your build-directory +# (since the data may be large). These files can be custom formats like +# FITS files. +# +# RECOMMENDATION: don't put the figure/plot/table number in the names of +# your to-be-published datasets! Given them a descriptive/short name that +# would be clear to anyone who has read the paper. Later, in the caption +# (or paper's tex/appendix), you will put links to the dataset on servers +# like Zenodo (see the "Publication checklist" in 'README-hacking.md'). +# +# [1] https://scixplorer.org/abs/2026A&A...709A.210E +drdir = $(badir)/data-release +figdir = $(texdir)/figures-data +$(figdir) $(drdir):; mkdir $@ + + + + + # Preparation phase # ----------------- # @@ -145,6 +184,7 @@ curdir := $(shell echo $$(pwd)) # avoid conflicts with existing TeX Live solutions. Later (in 'paper.mk'), # we are also going to overwrite 'TEXINPUTS' just before 'pdflatex'. .ONESHELL: +export LC_ALL := C export TEXINPUTS := export CCACHE_DISABLE := 1 export PATH := $(installdir)/bin @@ -594,41 +634,6 @@ $(inputdatasets): $(indir)/%: | $(indir) $(lockdir) -# Directory containing to-be-published datasets -# --------------------------------------------- -# -# Its good practice (so you don't forget in the last moment!) to have all -# the plot/figure/table data that you ultimately want to publish in a -# single directory. -# -# There are two types of to-publish data in the project. -# -# 1. Those data that also go into LaTeX (for example to give to LateX's -# PGFPlots package to create the plot internally) should be under the -# '$(texdir)' directory (because other LaTeX producers may also need it -# for example when using './project make dist', or you may want to -# publish the raw data behind the plots, like: -# https://zenodo.org/record/4291207/files/tools-per-year.txt). The -# contents of this directory are also directly taken into the tarball. -# -# 2. The data that aren't included directly in the LaTeX run of the paper, -# can be seen as supplements. A good place to keep them is under your -# build-directory. -# -# RECOMMENDATION: don't put the figure/plot/table number in the names of -# your to-be-published datasets! Given them a descriptive/short name that -# would be clear to anyone who has read the paper. Later, in the caption -# (or paper's tex/appendix), you will put links to the dataset on servers -# like Zenodo (see the "Publication checklist" in 'README-hacking.md'). -tex-publish-dir = $(texdir)/to-publish -data-publish-dir = $(badir)/data-to-publish -$(tex-publish-dir):; mkdir $@ -$(data-publish-dir):; mkdir $@ - - - - - # Print Copyright statement # ------------------------- # -- cgit v1.2.1