diff options
| -rw-r--r-- | README-hacking.md | 23 | ||||
| -rw-r--r-- | reproduce/software/config/versions.conf | 9 | ||||
| -rw-r--r-- | reproduce/software/make/high-level.mk | 39 | ||||
| -rwxr-xr-x | reproduce/software/shell/apptainer.sh | 44 |
4 files changed, 84 insertions, 31 deletions
diff --git a/README-hacking.md b/README-hacking.md index fa14795..3c69f40 100644 --- a/README-hacking.md +++ b/README-hacking.md @@ -180,11 +180,22 @@ evolving rapidly, so some details will differ between the different versions. The more recent papers will tend to be the most useful as good working examples. + - Eskandarlou et + al. [2026](https://ui.adsabs.harvard.edu/abs/2025arXiv251012940E), + Astronomy and Astrophysics (A&A), accepted. The project's version + controlled source is on + [Gitlab](https://gitlab.com/Sepideh.Esk/psf-j-plus), necessary software, + outputs and backup of history are available at + [zenodo.17348653](http://doi.org/10.5281/zenodo.17348653); and the + archived git history is available at + [swh:1:dir:fc95ed9ad173de6fa64690e3d71ab041c630f32c](https://archive.softwareheritage.org/swh:1:dir:fc95ed9ad173de6fa64690e3d71ab041c630f32c;origin=https://gitlab.com/Sepideh.Esk/psf-j-plus;visit=swh:1:snp:31bb4550902f7902a0a65d0674106271de776d3e;anchor=swh:1:rev:4860c70d6285a60300e429889e8637e98568a915). + - Saremi et - al. [2025](https://ui.adsabs.harvard.edu/abs/2025arXiv250802780S), - Astronomy and Astrophysics (accepted): The project's version controlled - source is on [Gitlab](https://gitlab.com/nasim-projects/pipeline), - necessary software, outputs and backup of history are available at + al. [2025](https://ui.adsabs.harvard.edu/abs/2025A%26A...701A.116S), + Astronomy and Astrophysics (A&A), Volume 701, id.A116. The project's + version controlled source is on + [Gitlab](https://gitlab.com/nasim-projects/pipeline), necessary + software, outputs and backup of history are available at [zenodo.16152699](https://doi.org/10.5281/zenodo.16152699); and the archived git history is available at [swh:1:dir:b3657cfb6053fd976695bd63c15cb99e5095648a](https://archive.softwareheritage.org/swh:1:dir:b3657cfb6053fd976695bd63c15cb99e5095648a;origin=https://gitlab.com/nasim-projects/pipeline;visit=swh:1:snp:ab7c6f0b9999f42d77154103c1bc082fa23b325c;anchor=swh:1:rev:afeb282c01983cba2a11eb4b2f25d5a40d35c164). @@ -641,7 +652,7 @@ First custom commit uses a custom LaTeX style that resembles that of the Astrophysical Journal (because the precusor of Maneage was for [Akhlaghi & Ichikawa 2015](https://ui.adsabs.harvard.edu/abs/2015ApJS..220....1A)). The - journal you plan to submit your ppaer to will have its own separate + journal you plan to submit your paper to will have its own separate style. So it is best that you start your project by writing in the desired style. We have already customized Maneage for the official styles of some journals. To find them, run `git branch -r | grep @@ -1090,7 +1101,7 @@ $ git status # Just for a check above) with the classification `both modified:`. Open one of these files with your favorite text editor and correct the conflict (placed in between `<<<<<<<`, `=======` and `>>>>>>>`). Once all conflicts in a file are -remoted, the file will be automatically removed from the "Unmerged paths" +removed, the file will be automatically removed from the "Unmerged paths" of `git status`. So run `git status` after correcting the conflicts of each file just to make sure things are clean. TIP: If you want the changes in one file to be only from a special branch (`maneage` or `main`, completely diff --git a/reproduce/software/config/versions.conf b/reproduce/software/config/versions.conf index db5ac8e..f0c2a36 100644 --- a/reproduce/software/config/versions.conf +++ b/reproduce/software/config/versions.conf @@ -179,7 +179,6 @@ swig-version = 4.0.2 tides-version = 2.0 util-linux-version = 2.41.3 valgrind-version = 3.18.1 -vim-version = 9.0 wcslib-version = 8.5 xlsxio-version = 0.2.21 yaml-version = 0.2.5 @@ -200,8 +199,12 @@ xcb-proto-version = 1.17.0 xorgproto-version = 2024.1 xtrans-version = 1.5.2 -# Version-dependent build -# ----------------------- +# Not working with C23 (similar to same topic in basic): future versions +# may have fixed this problem, so when updating, first remove '-std=gnu17' +# and if it builds without crashing, put this back in the normal list. +vim-version = 9.0 + +# Version is in the build recipe lapack-version = 3.8.0 libgit2-version = 1.9.0 diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 685715f..cd628ec 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1236,22 +1236,58 @@ $(ibidir)/ghostscript-$(ghostscript-version): \ # Gnuastro can optionally depend on libgit2, but it is not included as a # dependency here for the two reasons below. If you would like to have it, # add it as a dependency (its build instruction and dependencies are here -# already) and remove the '--without-libgit2' option in the recipe). +# already), then remove the '--without-libgit2' configure option) and also +# comment the following two generic configuration lines: 'outfitsnocommit' +# and 'outfitsnoconfig'. # - Within Maneage, we have everything under Git already and users are # expected to include the version in all their products. # - libgit2 can only be built with CMake (which takes extremely long to # compile: possibly even longer than GCC!). +# - Maneage is often run on HPCs that can have network connections +# between the storage and processer and the necessary libgit2 +# operations on every created file can have a performance impact. $(ibidir)/gnuastro-$(gnuastro-version): \ $(ibidir)/gsl-$(gsl-version) \ $(ibidir)/wcslib-$(wcslib-version) \ $(ibidir)/libjpeg-$(libjpeg-version) \ $(ibidir)/libtiff-$(libtiff-version) \ $(ibidir)/ghostscript-$(ghostscript-version) + +# Generic installation. tarball=gnuastro-$(gnuastro-version).tar.lz $(call import-source, $(gnuastro-url), $(gnuastro-checksum)) $(call gbuild, gnuastro-$(gnuastro-version), static, \ --without-libgit2, -j$(numthreads)) cp $(dtexdir)/gnuastro.tex $(ictdir)/ + +# Generally, besides the Git commit, we are also disabling the +# default mode of printing any type of metadata and versions of +# dependencies in output headers (through the installation-wide +# configuration file). This is done because within a large pipeline, +# Gnuastro is used to create many intermediate files (that are +# deleted shortly after being created) and it is not worth the +# overhread to keep this information in those intermediate products: +# it is the pipeline's responsibility to put them in the final +# outputs. We recommend to use the Gnuastro Fits program's keyword +# writing options to add as much contextual metadata on your final +# products as possible. Some tips: +# - The creation date is not good (because it is not reproducible +# and will make simple validation hard). The project's Commit +# should be used instead. +# - Define a keyword to keep the public URL of the repository of +# the Maneage'd project. In this way, a person who gets your +# final product can easily check that for all the information +# (including software versions and configuration options). + gconf=$(idir)/etc/gnuastro/gnuastro.conf + echo "" >> $$gconf + echo "# Maneage specific (see Gnuastro build rule for details)." \ + >> $$gconf + echo " outfitsnocommit = 1" >> $$gconf + echo " outfitsnoconfig = 1" >> $$gconf + echo " outfitsnodate = 1" >> $$gconf + echo " outfitsnoversions = 1" >> $$gconf + +# Final target. echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ $(ibidir)/icu-$(icu-version): $(ibidir)/python-$(python-version) @@ -1830,6 +1866,7 @@ $(ibidir)/vim-$(vim-version): unpackdir=vim-$(vim-version) cd $(ddir)/$$unpackdir $(shsrcdir)/prep-source.sh $(ibdir) + export CFLAGS="-std=gnu17 $$CFLAGS" ./configure --prefix=$(idir) \ --disable-canberra \ --enable-multibyte \ diff --git a/reproduce/software/shell/apptainer.sh b/reproduce/software/shell/apptainer.sh index 51b36db..6ac9975 100755 --- a/reproduce/software/shell/apptainer.sh +++ b/reproduce/software/shell/apptainer.sh @@ -71,14 +71,14 @@ set -e # Default option values +sif="" jobs=0 quiet=0 source_dir= build_only= -base_name="" +base_sif="" shm_size=20gb scriptname="$0" -project_name="" project_shell=0 container_shell=0 base_os=debian:stable-slim @@ -97,9 +97,9 @@ Top-level script to build and run a Maneage'd project within Apptainer. --source-dir=STR Directory of source code (default: 'pwd -P'). Apptainer images + --sif=STR Project's apptainer image (a '.sif' file). --base-os=STR Base OS name (default: '$base_os'). - --base-name=STR Base OS apptainer image (a '.sif' file). - --project-name=STR Project's apptainer image (a '.sif' file). + --base-sif=STR Base OS apptainer image (a '.sif' file). Interactive shell --project-shell Open the project's shell within the container. @@ -155,10 +155,12 @@ do --source-dir=*) source_dir="${1#*=}"; check_v "$1" "$source_dir"; shift;; # Container options. - --base-name) base_name="$2"; check_v "$1" "$base_name"; shift;shift;; - --base-name=*) base_name="${1#*=}"; check_v "$1" "$base_name"; shift;; - --project-name) project_name="$2"; check_v "$1" "$project_name"; shift;shift;; - --project-name=*) project_name="${1#*=}"; check_v "$1" "$project_name"; shift;; + --sif) sif="$2"; check_v "$1" "$sif"; shift;shift;; + --sif=*) sif="${1#*=}"; check_v "$1" "$sif"; shift;; + --base-os) base_os="$2"; check_v "$1" "$base_os"; shift;shift;; + --base-os=*) base_os="${1#*=}"; check_v "$1" "$base_os"; shift;; + --base-sif) base_sif="$2"; check_v "$1" "$base_sif"; shift;shift;; + --base-sif=*) base_sif="${1#*=}"; check_v "$1" "$base_sif"; shift;; # Interactive shell. --project-shell) project_shell=1; shift;; @@ -206,8 +208,8 @@ fi # Set the default project and base-OS image names (inside the build # directory). -if [ x"$base_name" = x ]; then base_name=$build_dir/maneage-base.sif; fi -if [ x"$project_name" = x ]; then project_name=$build_dir/maneaged.sif; fi +if [ x"$base_sif" = x ]; then base_sif=$build_dir/maneage-base.sif; fi +if [ x"$sif" = x ]; then sif=$build_dir/maneaged.sif; fi @@ -278,9 +280,9 @@ fi # # Build the base operating system using Maneage's './project configure' # step. -if [ -f $project_name ]; then +if [ -f $sif ]; then if [ $quiet = 0 ]; then - printf "$scriptname: info: project's image ('$project_name') " + printf "$scriptname: info: project's image ('$sif') " printf "already exists and will be used. If you want to build a " printf "new project image, give a new name to '--project-name'. " printf "To remove this message run with '--quiet'\n" @@ -289,12 +291,12 @@ else # Build the basic definition, with just Debian-slim with minimal # necessary tools. - if [ -f $base_name ]; then + if [ -f $base_sif ]; then if [ $quiet = 0 ]; then printf "$scriptname: info: base OS apptainer image " - printf "('$base_name') already exists and will be used. " + printf "('$base_sif') already exists and will be used. " printf "If you want to build a new base OS image, give " - printf "a new name to '--base-name'. To remove this " + printf "a new name to '--base-sif'. To remove this " printf "message run with '--quiet'\n" fi else @@ -309,7 +311,7 @@ From: $base_os EOF # Build the base operating system container and delete the # temporary definition file. - apptainer build $base_name $base_def + apptainer build $base_sif $base_def rm $base_def fi @@ -330,7 +332,7 @@ EOF maneage_def=$build_dir/maneage.def cat <<EOF > $maneage_def Bootstrap: localimage -From: $base_name +From: $base_sif %setup mkdir -p \${APPTAINER_ROOTFS}/home/maneager/input @@ -385,7 +387,7 @@ EOF $software_dir_mnt \ --ignore-fakeroot-command \ \ - $project_name \ + $sif \ $maneage_def # Clean up. @@ -397,7 +399,7 @@ fi if ! [ x"$build_only" = x ]; then if [ $quiet = 0 ]; then printf "$scriptname: info: Maneaged project has been configured " - printf "successfully in the '$project_name' image" + printf "successfully in the '$sif' image" fi exit 0 fi @@ -419,7 +421,7 @@ else fi # Build the hostname from the name of the SIF file of the project name. -hstname=$(echo "$project_name" \ +hstname=$(echo "$sif" \ | awk 'BEGIN{FS="/"}{print $NF}' \ | sed -e's|.sif$||') @@ -454,4 +456,4 @@ apptainer $aopt \ --hostname $hstname \ --cwd /home/maneager/source \ \ - $project_name + $sif |