aboutsummaryrefslogtreecommitdiff
path: root/reproduce
diff options
context:
space:
mode:
authorGiacomo Lorenzetti <glorenzetti@cefca.es>2025-02-27 17:49:28 +0100
committerMohammad Akhlaghi <mohammad@akhlaghi.org>2025-04-23 15:38:19 +0200
commita1f8947ab7784af4b7e66c617ce19a8bdd9c99ed (patch)
tree5a0d781e28ba467e590f4e4fac6f23a6bfefc1b2 /reproduce
parentcb936287ff70f278eb3040d38007c47ae6b05360 (diff)
IMPORTANT: Apptainer and Docker containers, minor restructuringHEADmaneage
Summary: it is necessary to re-configure your project (just running './project configure -e', not deleting 'build/software' to re-build software) after this commit, see "Affected files" item below). Until now, we only had a relatively long set of manual instructions for building Maneage within Docker in the top-level README. This was hard to automate, focing Maneage users to write custom commands based on the instructions and maintain those scripts outside of Maneage. As a result, experience could not be shared between projects (or at most in the README file!). With this commit, a new 'reproduce/software/containers' directory has been created within Maneage that contains two scripts (with a unified interface) greatly simplifying the building of the project's software environment within a container (one script for Apptainer and one for Docker). Two READMEs have been added for each container to help in their first time usage. Also, the old checklist within the main README has been replaced with a short introduction on containers and points the interested readers to the custom README of each container technology. Since we wanted the containers to be read-only after build, we needed to fully decouple the 'build/software' and 'build/analysis', such that './project configure' only writes to the former and './project make' only writes the latter. The file and directories mentioned in the affected files are cases that both project phases was writing to the 'build/software' and 'build/analysis' directories. Affected files: 'preparation-done.mk' and 'lockdir' which were previously in the 'build/software' directory are now made during the 'make' phase and the 'configure' phase no longer builds the 'build/analysis' or anything within it. Also, the software version LaTeX macros (which were previously written during the 'configure' phase in the 'analysis' directory) are now written in the software directory and copied into the analysis for usage in LaTeX while building the paper. Other minor additions in this commit: - The './project' script has a new '--timing' option to write the starting and ending times of the project in a file. It also builds the high-level analysis directories when './project make' is called (but before calling 'top-make.mk'. - The 'tar' calls in the custom build commands of the software building Makefiles now have the '--no-same-owner --no-same-permissions' options like the 'tar' call within the 'uncompress' function of 'build-rules.mk'. This commit was originally written by Giacomo Lorenzetti only for Apptainer on the registered commit date. It was later re-implemented from scratch by Mohammad Akhlaghi to have a unified interface for both Apptainer and Docker and merged into Maneage on 2025-04-23.
Diffstat (limited to 'reproduce')
-rw-r--r--reproduce/analysis/make/initialize.mk6
-rw-r--r--reproduce/analysis/make/paper.mk41
-rw-r--r--reproduce/analysis/make/prepare.mk2
-rw-r--r--reproduce/analysis/make/top-prepare.mk2
-rw-r--r--reproduce/software/containers/README-apptainer.md69
-rw-r--r--reproduce/software/containers/README-docker.md180
-rwxr-xr-xreproduce/software/containers/apptainer.sh441
-rwxr-xr-xreproduce/software/containers/docker.sh486
-rw-r--r--reproduce/software/make/basic.mk46
-rw-r--r--reproduce/software/make/build-rules.mk2
-rw-r--r--reproduce/software/make/high-level.mk54
-rw-r--r--reproduce/software/make/python.mk4
-rwxr-xr-xreproduce/software/shell/configure.sh33
-rwxr-xr-xreproduce/software/shell/pre-make-build.sh2
14 files changed, 1283 insertions, 85 deletions
diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk
index 92e5eff..c51f910 100644
--- a/reproduce/analysis/make/initialize.mk
+++ b/reproduce/analysis/make/initialize.mk
@@ -41,7 +41,7 @@ bsdir=$(BDIR)/software
# Derived directories (the locks directory can be shared with software
# which already has this directory.).
texdir = $(badir)/tex
-lockdir = $(bsdir)/locks
+lockdir = $(badir)/.locks
indir = $(badir)/inputs
prepdir = $(badir)/prepare
mtexdir = $(texdir)/macros
@@ -73,7 +73,7 @@ pconfdir = reproduce/analysis/config
ifeq (x$(project-phase),xprepare)
$(prepdir):; mkdir $@
else
--include $(bsdir)/preparation-done.mk
+-include $(badir)/preparation-done.mk
ifeq (x$(include-prepare-results),xyes)
-include $(prepdir)/*.mk $(prepdir)/*.conf
endif
@@ -274,7 +274,7 @@ clean:
rm -rf $(texdir)/macros/!(dependencies.tex|dependencies-bib.tex|hardware-parameters.tex)
rm -rf $(badir)/!(tex) $(texdir)/!(macros|$(texbtopdir))
rm -rf $(texdir)/build/!(tikz) $(texdir)/build/tikz/*
- rm -rf $(bsdir)/preparation-done.mk
+ rm -rf $(badir)/preparation-done.mk
distclean: clean
# Without cleaning the Git hooks, we won't be able to easily commit
diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk
index 66c6859..3c06ce3 100644
--- a/reproduce/analysis/make/paper.mk
+++ b/reproduce/analysis/make/paper.mk
@@ -18,6 +18,7 @@
+
# LaTeX macros for paper
# ----------------------
#
@@ -92,6 +93,40 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex
+# TeX build directory
+# -------------------
+#
+# If built in a group scenario, the TeX build directory must be separate
+# for each member (so they can work on their relevant parts of the paper
+# without conflicting with each other).
+ifeq ($(strip $(maneage_group_name)),)
+texbdir:=$(texdir)/build
+else
+texbdir:=$(texdir)/build-$(shell whoami)
+endif
+tikzdir:=$(texbdir)/tikz
+$(texbdir):; mkdir $@
+$(tikzdir): | $(texbdir); mkdir $@
+
+
+
+
+
+# Software info in TeX
+# --------------------
+#
+# The information of the installed software is placed in the
+# '.build/software' directory (which the TeX build should not depend
+# on). Therefore, we should copy those macros here in the LaTeX build
+# directory, so the TeX directory is completely independent from each
+# other.
+$(mtexdir)/dependencies.tex: $(bsdir)/tex/dependencies.tex
+ cp $(bsdir)/tex/*.tex $(mtexdir)/
+
+
+
+
+
# The bibliography
# ----------------
#
@@ -104,8 +139,9 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex
# recipe and the 'paper.pdf' recipe. But if 'tex/src/references.tex' hasn't
# been modified, we don't want to re-build the bibliography, only the final
# PDF.
-$(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \
- | $(mtexdir)/project.tex
+$(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies.tex \
+ | $(mtexdir)/project.tex $(tikzdir)
+
# If '$(mtexdir)/project.tex' is empty, don't build PDF.
@macros=$$(cat $(mtexdir)/project.tex)
if [ x"$$macros" != x ]; then
@@ -135,7 +171,6 @@ $(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \
export LD_LIBRARY_PATH="$(sys_library_sh_path):$$LD_LIBRARY_PATH"
pdflatex -shell-escape -halt-on-error "$$p"/paper.tex
biber paper
-
fi
diff --git a/reproduce/analysis/make/prepare.mk b/reproduce/analysis/make/prepare.mk
index 2cc1187..ffb2a3c 100644
--- a/reproduce/analysis/make/prepare.mk
+++ b/reproduce/analysis/make/prepare.mk
@@ -25,7 +25,7 @@
#
# We need to remove the 'prepare' word from the list of 'makesrc'.
prepare-dep = $(filter-out prepare, $(makesrc))
-$(bsdir)/preparation-done.mk: \
+$(badir)/preparation-done.mk: \
$(foreach s, $(prepare-dep), $(mtexdir)/$(s).tex)
# If you need to add preparations (mainly automatically generated
diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk
index 7d92d72..ea40f39 100644
--- a/reproduce/analysis/make/top-prepare.mk
+++ b/reproduce/analysis/make/top-prepare.mk
@@ -36,7 +36,7 @@ include reproduce/software/config/LOCAL.conf
#
# See 'top-make.mk' for complete explanation.
ifeq (x$(maneage_group_name),x$(GROUP-NAME))
-all: $(BDIR)/software/preparation-done.mk
+all: $(BDIR)/analysis/preparation-done.mk
@echo "Project preparation is complete.";
else
all:
diff --git a/reproduce/software/containers/README-apptainer.md b/reproduce/software/containers/README-apptainer.md
new file mode 100644
index 0000000..9608dc8
--- /dev/null
+++ b/reproduce/software/containers/README-apptainer.md
@@ -0,0 +1,69 @@
+# Maneage'd projects in Apptainer
+
+Copyright (C) 2025-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org>\
+Copyright (C) 2025-2025 Giacomo Lorenzetti <glorenzetti@cefca.es>\
+See the end of the file for license conditions.
+
+For an introduction on containers, see the "Building in containers" section
+of the `README.md` file within the top-level directory of this
+project. Here, we focus on Apptainer with a simple checklist on how to use
+the `apptainer-run.sh` script that we have already prepared in this
+directory for easy usage in a Maneage'd project.
+
+
+
+
+
+## Building your Maneage'd project in Apptainer
+
+Through the steps below, you will create an Apptainer image that will only
+contain the software environment and keep the project source and built
+analysis files (data and PDF) on your host operating system. This enables
+you to keep the size of the image to a minimum (only containing the built
+software environment) to easily move it from one computer to another.
+
+ 1. Using your favorite text editor, create a `apptainer-local.sh` in your
+ project's top directory that contains the usage command shown at the
+ top of the 'apptainer.sh' script and take the following steps:
+ * Set the respective directories based on your own preferences.
+ * The `--software-dir` is optional (if you don't have the source
+ tarballs, Maneage will download them automatically. But that requires
+ internet (which may not always be available). If you regularly build
+ Maneage'd projects, you can clone the repository containing all the
+ tarballs at https://gitlab.cefca.es/maneage/tarballs-software
+ * Add an extra `--build-only` for the first run so it doesn't go onto
+ doing the analysis and just builds the image. After it has completed,
+ remove the `--build-only` and it will only run the analysis of your
+ project.
+
+ 2. Once step one finishes, the build directory will contain two
+ Singularity Image Format (SIF) files listed below. You can move them to
+ any other (more permanent) positions in your filesystem or to other
+ computers as needed.
+ * `maneage-base.sif`: image containing the base operating system that
+ was used to build your project. You can safely delete this unless you
+ need to keep it for future builds without internet (you can give it
+ to the `--base-name` option of this script). If you want a different
+ name for this, put the same option in your
+ * `maneaged.sif`: image with the full software environment of your
+ project. This file is necessary for future runs of your project
+ within the container.
+
+
+
+
+
+## Copyright information
+
+This file is free software: you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
+
+This file is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+more details.
+
+You should have received a copy of the GNU General Public License along
+with this file. If not, see <https://www.gnu.org/licenses/>.
diff --git a/reproduce/software/containers/README-docker.md b/reproduce/software/containers/README-docker.md
new file mode 100644
index 0000000..f86dceb
--- /dev/null
+++ b/reproduce/software/containers/README-docker.md
@@ -0,0 +1,180 @@
+# Maneage'd projects in Docker
+
+Copyright (C) 2021-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org>\
+See the end of the file for license conditions.
+
+For an introduction on containers, see the "Building in containers" section
+of the `README.md` file within the top-level directory of this
+project. Here, we focus on Docker with a simple checklist on how to use the
+`docker.sh` script that we have already prepared in this directory for easy
+usage in a Maneage'd project.
+
+
+
+
+
+## Building your Maneage'd project in Docker
+
+Through the steps below, you will create a Docker image that will only
+contain the software environment and keep the project source and built
+analysis files (data and PDF) on your host operating system. This enables
+you to keep the size of the image to a minimum (only containing the built
+software environment) to easily move it from one computer to another.
+
+ 0. Add your user to the `docker` group: `usermod -aG docker
+ USERNAME`. This is only necessary once on an operating system.
+
+ 1. Start the Docker daemon (root permissions required). If the operating
+ system uses systemd you can use the command below. If you want the
+ Docker daemon to be available after a reboot also (so you don't have to
+ restart it after turning off your computer), run this command again but
+ replacing `start` with `enable` (this is not recommended if you don't
+ regularly use Docker: it will slow the boot time of your OS).
+
+ ```shell
+ systemctl start docker
+ ```
+
+ 2. Using your favorite text editor, create a `docker-local.sh` in your top
+ Maneage directory (as described in the comments at the start of the
+ `docker.sh` script in this directory). Just activate `--build-only` on
+ the first run so it doesn't go onto doing the analysis and just sets up
+ the software environment.
+
+ 3. After the setup is complete, run the following command to confirm that
+ the `maneage-base` (the OS of the container) and `maneaged` (your
+ project's full Maneage'd environment) images are available. If you want
+ different names for these images, add the `--project-name` and
+ `--base-name` options to the `docker.sh` call.
+
+ ```shell
+ docker image list
+ ```
+
+ 4. You are now ready to do your analysis by removing the `--build-only`
+ option.
+
+
+
+
+
+## Script usage tips
+
+The `docker.sh` script introduced above has many options allowing certain
+customizations that you can see when running it with the `--help`
+option. The tips below are some of the more useful scenarios that we have
+encountered so far.
+
+### Docker image in a single file
+
+In case you want to store the image as a single file as backup or to move
+to another computer. For such cases, run the `docker.sh` script with the
+`--image-file` option (for example `--image-file=myproj.tar.gz`). After
+moving the file to the other system, run `docker.sh` with the same option.
+
+When the given file to `docker.sh` already exists, it will only be used for
+loading the environment. When it doesn't exist, the script will save the
+image into it.
+
+
+
+
+
+## Docker usage tips
+
+Below are some useful Docker usage scenarios that have proved to be
+relevant for us in Maneage'd projects.
+
+### Cleaning up
+
+Docker has stored many large files in your operating system that can drain
+valuable storage space. The storage of the cached files are usually orders
+of magnitudes larger than what you see in `docker image list`! So after
+doing your work, it is best to clean up all those files. If you feel you
+may need the image later, you can save it in a single file as mentioned
+above and delete all the un-necessary cached files. Afterwards, when you
+load the image, only that image will be present with nothing extra.
+
+The easiest and most powerful way to clean up everything in Docker is the
+two commands below. The first will close all open containers. The second
+will remove all stopped containers, all networks not used by at least one
+container, all images without at least one container associated to them,
+and all build cache.
+
+```shell
+docker ps -a -q | xargs docker rm
+docker system prune -a
+```
+
+If you only want to delete the existing used images, run the command
+below. But be careful that the cache is the largest storage consumer! So
+the command above is the solution if your OS's root partition is close to
+getting filled.
+
+```shell
+docker images -a -q | xargs docker rmi -f
+```
+
+
+### Preserving the state of an open container
+
+All interactive changes in a container will be deleted as soon as you exit
+it. This is a very good feature of Docker in general! If you want to make
+persistent changes, you should do it in the project's plain-text source and
+commit them into your project's online Git repository. But in certain
+situations, it is necessary to preserve the state of an interactive
+container. To do this, you need to `commit` the container (and thus save it
+as a Docker "image"). To do this, while the container is still running,
+open another terminal and run these commands:
+
+```shell
+# These two commands should be done in another terminal
+docker container list
+
+# Get the 'XXXXXXX' of your desired container from the first column above.
+# Give the new image a name by replacing 'NEW-IMAGE-NAME'.
+docker commit XXXXXXX NEW-IMAGE-NAME
+```
+
+
+### Interactive tests on built container
+
+If you later want to start a container with the built image and enter it in
+interactive mode (for example for temporary tests), run the following
+command. Just replace `NAME` with the same name you specified when building
+the project. You can always exit the container with the `exit` command
+(note that all your changes will be discarded once you exit, see below if
+you want to preserve your changes after you exit).
+
+```shell
+docker run -it NAME
+```
+
+
+### Copying files from the Docker image to host operating system
+
+Except for the mounted directories, the Docker environment's file system is
+indepenent of your host operating system. One easy way to copy files to and
+from an open container is to use the `docker cp` command (very similar to
+the shell's `cp` command).
+
+```shell
+docker cp CONTAINER:/file/path/within/container /host/path/target
+```
+
+
+
+## Copyright information
+
+This file is free software: you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
+
+This file is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+more details.
+
+You should have received a copy of the GNU General Public License along
+with this file. If not, see <https://www.gnu.org/licenses/>.
diff --git a/reproduce/software/containers/apptainer.sh b/reproduce/software/containers/apptainer.sh
new file mode 100755
index 0000000..52315f6
--- /dev/null
+++ b/reproduce/software/containers/apptainer.sh
@@ -0,0 +1,441 @@
+#!/bin/sh
+#
+# Create a Apptainer container from an existing image of the built software
+# environment, but with the source, data and build (analysis) directories
+# directly within the host file system. This script is assumed to be run in
+# the top project source directory (that has 'README.md' and
+# 'paper.tex'). If not, use the '--source-dir' option to specify where the
+# Maneage'd project source is located.
+#
+# Usage:
+#
+# - When you are at the top Maneage'd project directory, you can run this
+# script like the example below. Just set all the '/PATH/TO/...'
+# directories. See the items below for optional values.
+#
+# ./reproduce/software/containers/apptainer.sh \
+# --build-dir=/PATH/TO/BUILD/DIRECTORY \
+# --software-dir=/PATH/TO/SOFTWARE/TARBALLS
+#
+# - Non-mandatory options:
+#
+# - If you already have the input data that is necessary for your
+# project's, use the '--input-dir' option to specify its location
+# on your host file system. Otherwise the necessary analysis
+# files will be downloaded directly into the build
+# directory. Note that this is only necessary when '--build-only'
+# is not given.
+#
+# - The '--software-dir' is only useful if you want to build a
+# container. Even in that case, it is not mandatory: if not
+# given, the software tarballs will be downloaded (thus requiring
+# internet).
+#
+# - To avoid having to set them every time you want to start the
+# apptainer environment, you can put this command (with the proper
+# directories) into a 'run.sh' script in the top Maneage'd project
+# source directory and simply execute that. The special name 'run.sh'
+# is in Maneage's '.gitignore', so it will not be included in your
+# git history by mistake.
+#
+# Known problems:
+#
+# Copyright (C) 2025-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org>
+# Copyright (C) 2025-2025 Giacomo Lorenzetti <glorenzetti@cefca.es>
+#
+# This script is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This script is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this script. If not, see <http://www.gnu.org/licenses/>.
+
+
+
+
+
+# Script settings
+# ---------------
+# Stop the script if there are any errors.
+set -e
+
+
+
+
+
+# Default option values
+jobs=
+quiet=0
+source_dir=
+build_only=
+base_name=""
+shm_size=20gb
+scriptname="$0"
+project_name=""
+project_shell=0
+container_shell=0
+base_os=debian:stable-slim
+
+print_help() {
+ # Print the output.
+ cat <<EOF
+Usage: $scriptname [OPTIONS]
+
+Top-level script to build and run a Maneage'd project within Apptainer.
+
+ Host OS directories (to be mounted in the container):
+ -b, --build-dir=STR Dir. to build in (only analysis in host).
+ -i, --input-dir=STR Dir. of input datasets (optional).
+ -s, --software-dir=STR Directory of necessary software tarballs.
+ --source-dir=STR Directory of source code (default: 'pwd -P').
+
+ Apptainer images
+ --base-os=STR Base OS name (default: '$base_os').
+ --base-name=STR Base OS apptainer image (a '.sif' file).
+ --project-name=STR Project's apptainer image (a '.sif' file).
+
+ Interactive shell
+ --project-shell Open the project's shell within the container.
+ --container-shell Open the container shell.
+
+ Operating mode:
+ --quiet Do not print informative statements.
+ -?, --help Give this help list.
+ -j, --jobs=INT Number of threads to use in each phase.
+ --build-only Just build the container, don't run it.
+
+Mandatory or optional arguments to long options are also mandatory or
+optional for any corresponding short options.
+
+Maneage URL: https://maneage.org
+
+Report bugs to mohammad@akhlaghi.org
+EOF
+}
+
+on_off_option_error() {
+ if [ "x$2" = x ]; then
+ echo "$scriptname: '$1' doesn't take any values"
+ else
+ echo "$scriptname: '$1' (or '$2') doesn't take any values"
+ fi
+ exit 1
+}
+
+check_v() {
+ if [ x"$2" = x ]; then
+ printf "$scriptname: option '$1' requires an argument. "
+ printf "Try '$scriptname --help' for more information\n"
+ exit 1;
+ fi
+}
+
+while [ $# -gt 0 ]
+do
+ case $1 in
+
+ # OS directories
+ -b|--build-dir) build_dir="$2"; check_v "$1" "$build_dir"; shift;shift;;
+ -b=*|--build-dir=*) build_dir="${1#*=}"; check_v "$1" "$build_dir"; shift;;
+ -b*) build_dir=$(echo "$1" | sed -e's/-b//'); check_v "$1" "$build_dir"; shift;;
+ -i|--input-dir) input_dir="$2"; check_v "$1" "$input_dir"; shift;shift;;
+ -i=*|--input-dir=*) input_dir="${1#*=}"; check_v "$1" "$input_dir"; shift;;
+ -i*) input_dir=$(echo "$1" | sed -e's/-i//'); check_v "$1" "$input_dir"; shift;;
+ -s|--software-dir) software_dir="$2"; check_v "$1" "$software_dir"; shift;shift;;
+ -s=*|--software-dir=*) software_dir="${1#*=}"; check_v "$1" "$software_dir"; shift;;
+ -s*) software_dir=$(echo "$1" | sed -e's/-s//'); check_v "$1" "$software_dir"; shift;;
+ --source-dir) source_dir="$2"; check_v "$1" "$source_dir"; shift;shift;;
+ --source-dir=*) source_dir="${1#*=}"; check_v "$1" "$source_dir"; shift;;
+
+ # Container options.
+ --base-name) base_name="$2"; check_v "$1" "$base_name"; shift;shift;;
+ --base-name=*) base_name="${1#*=}"; check_v "$1" "$base_name"; shift;;
+ --project-name) project_name="$2"; check_v "$1" "$project_name"; shift;shift;;
+ --project-name=*) project_name="${1#*=}"; check_v "$1" "$project_name"; shift;;
+
+ # Interactive shell.
+ --project-shell) project_shell=1; shift;;
+ --project_shell=*) on_off_option_error --project-shell;;
+ --container-shell) container_shell=1; shift;;
+ --container_shell=*) on_off_option_error --container-shell;;
+
+ # Operating mode
+ --quiet) quiet=1; shift;;
+ --quiet=*) on_off_option_error --quiet;;
+ -j|--jobs) jobs="$2"; check_v "$1" "$jobs"; shift;shift;;
+ -j=*|--jobs=*) jobs="${1#*=}"; check_v "$1" "$jobs"; shift;;
+ -j*) jobs=$(echo "$1" | sed -e's/-j//'); check_v "$1" "$jobs"; shift;;
+ --build-only) build_only=1; shift;;
+ --build-only=*) on_off_option_error --build-only;;
+ --shm-size) shm_size="$2"; check_v "$1" "$shm_size"; shift;shift;;
+ --shm-size=*) shm_size="${1#*=}"; check_v "$1" "$shm_size"; shift;;
+ -'?'|--help) print_help; exit 0;;
+ -'?'*|--help=*) on_off_option_error --help -?;;
+
+ # Unrecognized option:
+ -*) echo "$scriptname: unknown option '$1'"; exit 1;;
+ esac
+done
+
+
+
+
+
+# Sanity checks
+# -------------
+#
+# Make sure that the build directory is given and that it exists.
+if [ x$build_dir = x ]; then
+ printf "$scriptname: '--build-dir' not provided, this is the location "
+ printf "that all built analysis files will be kept on the host OS\n"
+ exit 1;
+else
+ if ! [ -d $build_dir ]; then
+ printf "$scriptname: '$build_dir' (value to '--build-dir') doesn't "
+ printf "exist\n"
+ exit 1;
+ fi
+fi
+
+# Set the default project and base-OS image names (inside the build
+# directory).
+if [ x"$base_name" = x ]; then base_name=$build_dir/maneage-base.sif; fi
+if [ x"$project_name" = x ]; then project_name=$build_dir/maneaged.sif; fi
+
+
+
+
+
+# Directory preparations
+# ----------------------
+#
+# If the host operating system has '/dev/shm', then give Apptainer access
+# to it also for improved speed in some scenarios (like configuration).
+if [ -d /dev/shm ]; then
+ shm_mnt="--mount type=bind,src=/dev/shm,dst=/dev/shm";
+else shm_mnt="";
+fi
+
+# If the following directories do not exist within the build directory,
+# create them to make sure the '--mount' commands always work and
+# that any file. Ideally, the 'input' directory should not be under the 'build'
+# directory, but if the user hasn't given it then they don't care about
+# potentially deleting it later (Maneage will download the inputs), so put
+# it in the build directory.
+analysis_dir="$build_dir"/analysis
+if ! [ -d $analysis_dir ]; then mkdir $analysis_dir; fi
+analysis_dir_mnt="--mount type=bind,src=$analysis_dir,dst=/home/maneager/build/analysis"
+
+# If no '--source-dir' was given, set it to the output of 'pwd -P' (to get
+# the direct path without potential symbolic links) in the running directory.
+if [ x"$source_dir" = x ]; then source_dir=$(pwd -P); fi
+source_dir_mnt="--mount type=bind,src=$source_dir,dst=/home/maneager/source"
+
+# Only when an an input directory is given, we need the respective 'mount'
+# option for the 'apptainer run' command.
+input_dir_mnt=""
+if ! [ x"$input_dir" = x ]; then
+ input_dir_mnt="--mount type=bind,src=$input_dir,dst=/home/maneager/input"
+fi
+
+# If no '--jobs' has been specified, use the maximum available jobs to the
+# operating system.
+if [ x$jobs = x ]; then jobs=$(nproc); fi
+
+# [APPTAINER-ONLY] Optional mounting option for the software directory.
+software_dir_mnt=""
+if ! [ x"$software_dir" = x ]; then
+ software_dir_mnt="--mount type=bind,src=$software_dir,dst=/home/maneager/tarballs-software"
+fi
+
+# [APPTAINER-ONLY] Since the container is read-only and is run with the
+# '--contain' option (which makes an empty '/tmp'), we need to make a
+# dedicated directory for the container to be able to write to. This is
+# necessary because some software (Biber in particular on the default
+# branch) need to write there! See https://github.com/plk/biber/issues/494.
+# We'll keep the directory on the host OS within the build directory, but
+# as a hidden file (since it is not necessary in other types of build and
+# ultimately only contains temporary files of programs that need it).
+toptmp=$build_dir/.apptainer-tmp-$(whoami)
+if ! [ -d $toptmp ]; then mkdir $toptmp; fi
+rm -rf $toptmp/* # So previous runs don't affect this run.
+
+
+
+
+
+# Maneage'd Apptainer SIF container
+# ---------------------------------
+#
+# Build the base operating system using Maneage's './project configure'
+# step.
+if [ -f $project_name ]; then
+ if [ $quiet = 0 ]; then
+ printf "$scriptname: info: project's image ('$project_name') "
+ printf "already exists and will be used. If you want to build a "
+ printf "new project image, give a new name to '--project-name'. "
+ printf "To remove this message run with '--quiet'\n"
+ fi
+else
+
+ # Build the basic definition, with just Debian and gcc/g++
+ if [ -f $base_name ]; then
+ if [ $quiet = 0 ]; then
+ printf "$scriptname: info: base OS docker image ('$base_name') "
+ printf "already exists and will be used. If you want to build a "
+ printf "new base OS image, give a new name to '--base-name'. "
+ printf "To remove this message run with '--quiet'\n"
+ fi
+ else
+
+ base_def=$build_dir/base.def
+ cat <<EOF > $base_def
+Bootstrap: docker
+From: $base_os
+
+%post
+ apt-get update && apt-get install -y gcc g++
+EOF
+ # Build the base operating system container and delete the
+ # temporary definition file.
+ apptainer build $base_name $base_def
+ rm $base_def
+ fi
+
+ # Build the Maneage definition file.
+ # - About the '$jobs' variable: this definition file is temporarily
+ # built and deleted immediately after the SIF file is created. So
+ # instead of using Apptainer's more complex '{{ jobs }}' format to
+ # pass an argument, we simply write the value of the configure
+ # script's '--jobs' option as a shell variable here when we are
+ # building that file.
+ # - About the removal of Maneage'd tarballs: we are doing this so if
+ # Maneage has downloaded tarballs during the build they do not
+ # unecessarily bloat the container. Even when the user has given a
+ # software tarball directory, they will all be symbolic links that
+ # aren't valid when the user runs the container (since we only
+ # mount the software tarballs at build time).
+ maneage_def=$build_dir/maneage.def
+ cat <<EOF > $maneage_def
+Bootstrap: localimage
+From: $base_name
+
+%setup
+ mkdir -p \${APPTAINER_ROOTFS}/home/maneager/input
+ mkdir -p \${APPTAINER_ROOTFS}/home/maneager/source
+ mkdir -p \${APPTAINER_ROOTFS}/home/maneager/build/analysis
+ mkdir -p \${APPTAINER_ROOTFS}/home/maneager/tarballs-software
+
+%post
+ cd /home/maneager/source
+ ./project configure --jobs=$jobs \\
+ --input-dir=/home/maneager/input \\
+ --build-dir=/home/maneager/build \\
+ --software-dir=/home/maneager/tarballs-software
+ rm /home/maneager/build/software/tarballs/*
+
+%runscript
+ cd /home/maneager/source
+ if [ x"\$maneage_apptainer_stat" = xshell ]; then \\
+ ./project shell; \\
+ elif [ x"\$maneage_apptainer_stat" = xrun ]; then \\
+ if [ x"\$maneage_jobs" = x ]; then \\
+ ./project make; \\
+ else \\
+ ./project make --jobs=\$maneage_jobs; \\
+ fi; \\
+ else \\
+ printf "$scriptname: '\$maneage_apptainer_stat' (value "; \\
+ printf "to 'maneage_apptainer_stat' environment variable) "; \\
+ printf "is not recognized: should be either 'shell' or 'run'"; \\
+ exit 1; \\
+ fi
+EOF
+
+ # Build the maneage container. The last two are arguments (where order
+ # matters). The first few are options where order does not matter (so
+ # we have sorted them by line length).
+ apptainer build \
+ $shm_mnt \
+ $input_dir_mnt \
+ $source_dir_mnt \
+ $analysis_dir_mnt \
+ $software_dir_mnt \
+ --ignore-fakeroot-command \
+ \
+ $project_name \
+ $maneage_def
+
+ # Clean up.
+ rm $maneage_def
+fi
+
+# If the user just wanted to build the base operating system, abort the
+# script here.
+if ! [ x"$build_only" = x ]; then
+ if [ $quiet = 0 ]; then
+ printf "$scriptname: info: Maneaged project has been configured "
+ printf "successfully in the '$project_name' image"
+ fi
+ exit 0
+fi
+
+
+
+
+
+# Run the Maneage'd container
+# ---------------------------
+#
+# Set the high-level Apptainer operational mode.
+if [ $container_shell = 1 ]; then
+ aopt="shell"
+elif [ $project_shell = 1 ]; then
+ aopt="run --env maneage_apptainer_stat=shell"
+else
+ aopt="run --env maneage_apptainer_stat=run --env maneage_jobs=$jobs"
+fi
+
+# Build the hostname from the name of the SIF file of the project name.
+hstname=$(echo "$project_name" \
+ | awk 'BEGIN{FS="/"}{print $NF}' \
+ | sed -e's|.sif$||')
+
+# Execute Apptainer:
+#
+# - We are not using '--unsquash' (to run within a sandbox) because it
+# loads the full multi-gigabyte container into RAM (which we usually
+# need for data processing). The container is read-only and we are
+# using the following two options instead to ensure that we have no
+# influence from outside the container. (description of each is from
+# the Apptainer manual)
+# --contain: use minimal /dev and empty other directories (e.g. /tmp
+# and $HOME) instead of sharing filesystems from your host.
+# --cleanenv: clean environment before running container".
+#
+# - We are not mounting '/dev/shm' since Apptainer prints a warning that
+# it is already mounted (apparently does not need it at run time).
+#
+# --no-home and --home: the first ensures that the 'HOME' variable is
+# different from the user's home on the host operating system, the
+# second sets it to a directory we specify (to keep things like
+# '.bash_history').
+apptainer $aopt \
+ --no-home \
+ --contain \
+ --cleanenv \
+ --home $toptmp \
+ $input_dir_mnt \
+ $source_dir_mnt \
+ $analysis_dir_mnt \
+ --workdir $toptmp \
+ --hostname $hstname \
+ --cwd /home/maneager/source \
+ \
+ $project_name
diff --git a/reproduce/software/containers/docker.sh b/reproduce/software/containers/docker.sh
new file mode 100755
index 0000000..d5b5682
--- /dev/null
+++ b/reproduce/software/containers/docker.sh
@@ -0,0 +1,486 @@
+#!/bin/sh
+#
+# Create a Docker container from an existing image of the built software
+# environment, but with the source, data and build (analysis) directories
+# directly within the host file system. This script is assumed to be run in
+# the top project source directory (that has 'README.md' and
+# 'paper.tex'). If not, use the '--source-dir' option to specify where the
+# Maneage'd project source is located.
+#
+# Usage:
+#
+# - When you are at the top Maneage'd project directory, you can run this
+# script like the example below. Just set all the '/PATH/TO/...'
+# directories (see below for '--tmp-dir'). See the items below for
+# optional values.
+#
+# ./reproduce/software/containers/docker.sh --shm-size=15gb \
+# --software-dir=/PATH/TO/SOFTWARE/TARBALLS \
+# --build-dir=/PATH/TO/BUILD/DIRECTORY
+#
+# - Non-mandatory options:
+#
+# - If you already have the input data that is necessary for your
+# project's, use the '--input-dir' option to specify its location
+# on your host file system. Otherwise the necessary analysis
+# files will be downloaded directly into the build
+# directory. Note that this is only necessary when '--build-only'
+# is not given.
+#
+# - The '--software-dir' is only useful if you want to build a
+# container. Even in that case, it is not mandatory: if not
+# given, the software tarballs will be downloaded (thus requiring
+# internet).
+#
+# - To avoid having to set the directory(s) every time you want to
+# start the docker environment, you can put this command (with the
+# proper directories) into a 'run.sh' script in the top Maneage'd
+# project source directory and simply execute that. The special name
+# 'run.sh' is in Maneage's '.gitignore', so it will not be included
+# in your git history by mistake.
+#
+# Known problems:
+#
+# - As of 2025-04-06 the log file containing the output of the 'docker
+# build' command that configures the Maneage'd project does not keep
+# all the output (which gets clipped by Docker). with a "[output
+# clipped, log limit 2MiB reached]" message. We need to find a way to
+# fix this (so nothing gets clipped: useful for debugging).
+#
+# Copyright (C) 2021-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org>
+#
+# This script is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This script is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this script. If not, see <http://www.gnu.org/licenses/>.
+
+
+
+
+
+# Script settings
+# ---------------
+# Stop the script if there are any errors.
+set -e
+
+
+
+
+
+# Default option values
+jobs=
+quiet=0
+source_dir=
+build_only=
+image_file=""
+shm_size=20gb
+scriptname="$0"
+project_shell=0
+container_shell=0
+project_name=maneaged
+base_name=maneage-base
+base_os=debian:stable-slim
+
+print_help() {
+ # Print the output.
+ cat <<EOF
+Usage: $scriptname [OPTIONS]
+
+Top-level script to build and run a Maneage'd project within Docker.
+
+ Host OS directories (to be mounted in the container):
+ -b, --build-dir=STR Dir. to build in (only analysis in host).
+ -i, --input-dir=STR Dir. of input datasets (optional).
+ -s, --software-dir=STR Directory of necessary software tarballs.
+ --source-dir=STR Directory of source code (default: 'pwd -P').
+
+ Docker images
+ --base-os=STR Base OS name (default: '$base_os').
+ --base-name=STR Base OS docker image (default: $base_name).
+ --project-name=STR Project's docker image (default: $project_name).
+ --image-file=STR [Docker only] Load (if given file exists), or
+ save (if given file does not exist), the image.
+ For saving, the given name has to have an
+ '.tar.gz' suffix.
+
+ Interactive shell
+ --project-shell Open the project's shell within the container.
+ --container-shell Open the container shell.
+
+ Operating mode:
+ --quiet Do not print informative statements.
+ -?, --help Give this help list.
+ --shm-size=STR Passed to 'docker build' (default: $shm_size).
+ -j, --jobs=INT Number of threads to use in each phase.
+ --build-only Just build the container, don't run it.
+
+Mandatory or optional arguments to long options are also mandatory or
+optional for any corresponding short options.
+
+Maneage URL: https://maneage.org
+
+Report bugs to mohammad@akhlaghi.org
+EOF
+}
+
+on_off_option_error() {
+ if [ "x$2" = x ]; then
+ echo "$scriptname: '$1' doesn't take any values"
+ else
+ echo "$scriptname: '$1' (or '$2') doesn't take any values"
+ fi
+ exit 1
+}
+
+check_v() {
+ if [ x"$2" = x ]; then
+ printf "$scriptname: option '$1' requires an argument. "
+ printf "Try '$scriptname --help' for more information\n"
+ exit 1;
+ fi
+}
+
+while [ $# -gt 0 ]
+do
+ case $1 in
+
+ # OS directories
+ -b|--build-dir) build_dir="$2"; check_v "$1" "$build_dir"; shift;shift;;
+ -b=*|--build-dir=*) build_dir="${1#*=}"; check_v "$1" "$build_dir"; shift;;
+ -b*) build_dir=$(echo "$1" | sed -e's/-b//'); check_v "$1" "$build_dir"; shift;;
+ -i|--input-dir) input_dir="$2"; check_v "$1" "$input_dir"; shift;shift;;
+ -i=*|--input-dir=*) input_dir="${1#*=}"; check_v "$1" "$input_dir"; shift;;
+ -i*) input_dir=$(echo "$1" | sed -e's/-i//'); check_v "$1" "$input_dir"; shift;;
+ -s|--software-dir) software_dir="$2"; check_v "$1" "$software_dir"; shift;shift;;
+ -s=*|--software-dir=*) software_dir="${1#*=}"; check_v "$1" "$software_dir"; shift;;
+ -s*) software_dir=$(echo "$1" | sed -e's/-s//'); check_v "$1" "$software_dir"; shift;;
+ --source-dir) source_dir="$2"; check_v "$1" "$source_dir"; shift;shift;;
+ --source-dir=*) source_dir="${1#*=}"; check_v "$1" "$source_dir"; shift;;
+
+ # Container options.
+ --base-name) base_name="$2"; check_v "$1" "$base_name"; shift;shift;;
+ --base-name=*) base_name="${1#*=}"; check_v "$1" "$base_name"; shift;;
+
+ # Interactive shell.
+ --project-shell) project_shell=1; shift;;
+ --project_shell=*) on_off_option_error --project-shell;;
+ --container-shell) container_shell=1; shift;;
+ --container_shell=*) on_off_option_error --container-shell;;
+
+ # Operating mode
+ --quiet) quiet=1; shift;;
+ --quiet=*) on_off_option_error --quiet;;
+ -j|--jobs) jobs="$2"; check_v "$1" "$jobs"; shift;shift;;
+ -j=*|--jobs=*) jobs="${1#*=}"; check_v "$1" "$jobs"; shift;;
+ -j*) jobs=$(echo "$1" | sed -e's/-j//'); check_v "$1" "$jobs"; shift;;
+ --build-only) build_only=1; shift;;
+ --build-only=*) on_off_option_error --build-only;;
+ --shm-size) shm_size="$2"; check_v "$1" "$shm_size"; shift;shift;;
+ --shm-size=*) shm_size="${1#*=}"; check_v "$1" "$shm_size"; shift;;
+ -'?'|--help) print_help; exit 0;;
+ -'?'*|--help=*) on_off_option_error --help -?;;
+
+ # Output file
+ --image-file) image_file="$2"; check_v "$1" "$image_file"; shift;shift;;
+ --image-file=*) image_file="${1#*=}"; check_v "$1" "$image_file"; shift;;
+
+ # Unrecognized option:
+ -*) echo "$scriptname: unknown option '$1'"; exit 1;;
+ esac
+done
+
+
+
+
+
+# Sanity checks
+# -------------
+#
+# Make sure that the build directory is given and that it exists.
+if [ x$build_dir = x ]; then
+ printf "$scriptname: '--build-dir' not provided, this is the location "
+ printf "that all built analysis files will be kept on the host OS\n"
+ exit 1;
+else
+ if ! [ -d $build_dir ]; then
+ printf "$scriptname: '$build_dir' (value to '--build-dir') doesn't "
+ printf "exist\n"; exit 1;
+ fi
+fi
+
+# The temporary directory to place the Dockerfile.
+tmp_dir="$build_dir"/temporary-docker-container-dir
+
+
+
+
+# Directory preparations
+# ----------------------
+#
+# If the host operating system has '/dev/shm', then give Docker access
+# to it also for improved speed in some scenarios (like configuration).
+if [ -d /dev/shm ]; then shm_mnt="-v /dev/shm:/dev/shm";
+else shm_mnt=""; fi
+
+# If the following directories do not exist within the build directory,
+# create them to make sure the '--mount' commands always work and
+# that any file. Ideally, the 'input' directory should not be under the 'build'
+# directory, but if the user hasn't given it then they don't care about
+# potentially deleting it later (Maneage will download the inputs), so put
+# it in the build directory.
+analysis_dir="$build_dir"/analysis
+if ! [ -d $analysis_dir ]; then mkdir $analysis_dir; fi
+
+# If no '--source-dir' was given, set it to the output of 'pwd -P' (to get
+# the path without potential symbolic links) in the running directory.
+if [ x"$source_dir" = x ]; then source_dir=$(pwd -P); fi
+
+# Only when an an input directory is given, we need the respective 'mount'
+# option for the 'docker run' command.
+input_dir_mnt=""
+if ! [ x"$input_dir" = x ]; then
+ input_dir_mnt="-v $input_dir:/home/maneager/input"
+fi
+
+# If no '--jobs' has been specified, use the maximum available jobs to the
+# operating system.
+if [ x$jobs = x ]; then jobs=$(nproc); fi
+
+# [DOCKER-ONLY] Make sure the user is a member of the 'docker' group:
+glist=$(groups $(whoami) | awk '/docker/')
+if [ x"$glist" = x ]; then
+ printf "$scriptname: you are not a member of the 'docker' group "
+ printf "You can run the following command as root to fix this: "
+ printf "'usermod -aG docker $(whoami)'\n"
+ exit 1
+fi
+
+# [DOCKER-ONLY] Function to check the temporary directory for building the
+# base operating system docker image. It is necessary that this directory
+# be empty because Docker will inherit the sub-directories of the directory
+# that the Dockerfile is located in.
+tmp_dir_check () {
+ if [ -d $tmp_dir ]; then
+ printf "$scriptname: '$tmp_dir' already exists, please "
+ printf "delete it and re-run this script. This is a temporary "
+ printf "directory only necessary when building a Docker image "
+ printf "and gets deleted automatically after a successful "
+ printf "build. The fact that it remains hints at a problem "
+ printf "in a previous attempt to build a Docker image\n"
+ exit 1
+ else
+ mkdir $tmp_dir
+ fi
+}
+
+
+
+
+
+# Base operating system
+# ---------------------
+#
+# If the base image does not exist, then create it. If it does, inform the
+# user that it will be used.
+if docker image list | grep $base_name &> /dev/null; then
+ if [ $quiet = 0 ]; then
+ printf "$scriptname: info: base OS docker image ('$base_name') "
+ printf "already exists and will be used. If you want to build a "
+ printf "new base OS image, give a new name to '--base-name'. "
+ printf "To remove this message run with '--quiet'\n"
+ fi
+else
+
+ # In case an image file is given, load the environment from that (no
+ # need to build the environment from scratch).
+ if ! [ x"$image_file" = x ] && [ -f "$image_file" ]; then
+ docker load --input $image_file
+ else
+
+ # Build the temporary directory.
+ tmp_dir_check
+
+ # Build the Dockerfile.
+ uid=$(id -u)
+ cat <<EOF > $tmp_dir/Dockerfile
+FROM $base_os
+RUN useradd -ms /bin/sh --uid $uid maneager; \\
+ printf '123\n123' | passwd maneager; \\
+ printf '456\n456' | passwd root
+RUN apt update; apt install -y gcc g++ wget; echo 'export PS1="[\[\033[01;31m\]\u@\h \W\[\033[32m\]\[\033[00m\]]# "' >> ~/.bashrc
+USER maneager
+WORKDIR /home/maneager
+RUN mkdir build; mkdir build/analysis; echo 'export PS1="[\[\033[01;35m\]\u@\h \W\[\033[32m\]\[\033[00m\]]$ "' >> ~/.bashrc
+EOF
+
+ # Build the base-OS container and delete the temporary directory.
+ curdir="$(pwd)"
+ cd $tmp_dir
+ docker build ./ \
+ -t $base_name \
+ --shm-size=$shm_size
+ cd "$curdir"
+ rm -rf $tmp_dir
+ fi
+fi
+
+
+
+
+
+# Maneage software configuration
+# ------------------------------
+#
+# Having the base operating system in place, we can now construct the
+# project's docker file.
+if docker image list | grep $project_name &> /dev/null; then
+ if [ $quiet = 0 ]; then
+ printf "$scriptname: info: project's image ('$project_name') "
+ printf "already exists and will be used. If you want to build a "
+ printf "new project image, give a new name to '--project-name'. "
+ printf "To remove this message run with '--quiet'\n"
+ fi
+else
+
+ # Build the temporary directory.
+ tmp_dir_check
+ df=$tmp_dir/Dockerfile
+
+ # The only way to mount a directory inside the Docker build environment
+ # is the 'RUN --mount' command. But Docker doesn't recognize things
+ # like symbolic links. So we need to copy the project's source under
+ # this temporary directory.
+ sdir=source
+ mkdir $tmp_dir/$sdir
+ dsr=/home/maneager/source-raw
+ cp -r $source_dir/* $source_dir/.git $tmp_dir/$sdir
+
+ # Start constructing the Dockerfile.
+ #
+ # Note on the printf's '\x5C\n' part: this will print out as a
+ # backslash at the end of the line to allow easy human readability of
+ # the Dockerfile (necessary for debugging!).
+ echo "FROM $base_name" > $df
+ printf "RUN --mount=type=bind,source=$sdir,target=$dsr \x5C\n" >> $df
+
+ # If a software directory was given, copy it and add its line.
+ tsdir=tarballs-software
+ dts=/home/maneager/tarballs-software
+ if ! [ x"$software_dir" = x ]; then
+
+ # Make the directory to host the software and copy the contents
+ # that the user gave there.
+ mkdir $tmp_dir/$tsdir
+ cp -r "$software_dir"/* $tmp_dir/$tsdir/
+ printf " --mount=type=bind,source=$tsdir,target=$dts \x5C\n" >> $df
+ fi
+
+ # Construct the rest of the 'RUN' command.
+ printf " cp -r $dsr /home/maneager/source; \x5C\n" >> $df
+ printf " cd /home/maneager/source; \x5C\n" >> $df
+ printf " ./project configure --jobs=$jobs \x5C\n" >> $df
+ printf " --build-dir=/home/maneager/build \x5C\n" >> $df
+ printf " --input-dir=/home/maneager/input \x5C\n" >> $df
+ printf " --software-dir=$dts; \x5C\n" >> $df
+
+ # We are deleting the '.build/software/tarballs' directory because this
+ # directory is not relevant for the analysis of the project. But in
+ # case any tarball was downloaded, it will consume space within the
+ # container.
+ printf " rm -rf .build/software/tarballs; \x5C\n" >> $df
+
+ # We are deleting the source directory becaues later (at 'docker run'
+ # time), the 'source' will be mounted directly from the host operating
+ # system.
+ printf " cd /home/maneager; \x5C\n" >> $df
+ printf " rm -rf source\n" >> $df
+
+ # Build the Maneage container and delete the temporary directory. The
+ # '--progress plain' option is for Docker to print all the outputs
+ # (otherwise, it will only print a very small part!).
+ cd $tmp_dir
+ docker build ./ -t $project_name \
+ --progress=plain \
+ --shm-size=$shm_size \
+ --no-cache \
+ 2>&1 | tee build.log
+ cd ..
+ rm -rf $tmp_dir
+fi
+
+# If the user wants to save the container (into a file that does not
+# exist), do it here. If the file exists, it will only be used for creating
+# the container in the previous stages.
+if ! [ x"$image_file" = x ] && ! [ -f "$image_file" ]; then
+
+ # Save the image into a tarball
+ tarname=$(echo $image_file | sed -e's|.gz$||')
+ if [ $quiet = 0 ]; then
+ printf "$scriptname: info: saving docker image to '$tarname'"
+ fi
+ docker save -o $tarname $project_name
+
+ # Compress the saved image
+ if [ $quiet = 0 ]; then
+ printf "$scriptname: info: compressing to '$image_file' (can "
+ printf "take +10 minutes, but volume decreases by more than half!)"
+ fi
+ gzip --best $tarname
+fi
+
+# If the user just wanted to build the base operating system, abort the
+# script here.
+if ! [ x"$build_only" = x ]; then
+ if [ $quiet = 0 ]; then
+ printf "$scriptname: info: Maneaged project has been configured "
+ printf "successfully in the '$project_name' image"
+ fi
+ exit 0
+fi
+
+
+
+
+
+# Run the analysis within the Maneage'd container
+# -----------------------------------------------
+#
+# The startup command of the container is managed though the 'shellopt'
+# variable that starts here.
+shellopt=""
+if [ $container_shell = 1 ] || [ $project_shell = 1 ]; then
+
+ # If the user wants to start the project shell within the container,
+ # add the necessary command.
+ if [ $project_shell = 1 ]; then
+ shellopt="/bin/bash -c 'cd source; ./project shell;'"
+ fi
+
+ # Finish the 'shellop' string with a single quote (necessary in any
+ # case) and run Docker.
+ interactiveopt="-it"
+
+# No interactive shell requested, just run the project.
+else
+ interactiveopt=""
+ shellopt="/bin/bash -c 'cd source; ./project make --jobs=$jobs;'"
+fi
+
+# Execute Docker. The 'eval' is because the 'shellopt' variable contains a
+# single-quote that the shell should "evaluate".
+eval docker run \
+ -v "$analysis_dir":/home/maneager/build/analysis \
+ -v "$source_dir":/home/maneager/source \
+ $input_dir_mnt \
+ $shm_mnt \
+ $interactiveopt \
+ $project_name \
+ $shellopt
diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk
index 745aeca..40c5a4e 100644
--- a/reproduce/software/make/basic.mk
+++ b/reproduce/software/make/basic.mk
@@ -144,10 +144,17 @@ backupservers_all = $(user_backup_urls) $(maneage_backup_urls)
topbackupserver = $(word 1, $(backupservers_all))
backupservers = $(filter-out $(topbackupserver),$(backupservers_all))
-
-
-
-
+# When building in Apptainer containers, as of 2025-04-18, we need to
+# configure Maneage as root (within the container). In such cases, we need
+# to activate the 'FORCE_UNSAFE_CONFIGURE' environment variable to build
+# some of the software. The 'if' statement is here to make sure we are in
+# Apptainer: in other situations, the "unsafe" configure script shouldn't
+# be activated. Note that this doesn't happen in Docker (where the Maneage
+# source is in the same directory) because we build a non-root ('maneager'
+# user there who executes the configure command.
+unsafe-config = if [ $$(pwd) = "/home/maneager/source" ] \
+ && [ $$(whoami) = root ]; then \
+ export FORCE_UNSAFE_CONFIGURE=1; fi
@@ -261,11 +268,8 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \
#
# The first set of programs to be built are those that we need to unpack
# the source code tarballs of each program. We have already installed Lzip
-# before calling 'basic.mk', so it is present and working. Hence we first
-# build the Lzipped tarball of Gzip, then use our own Gzip to unpack the
-# tarballs of the other compression programs. Once all the compression
-# programs/libraries are complete, we build our own GNU Tar and continue
-# with other software.
+# before calling 'basic.mk', so it is present and working. So the only
+# prerequisites of these (until reaching Tar) is the necessary directories.
$(lockdir): | $(BDIR); mkdir $@
$(ibidir)/gzip-$(gzip-version): | $(ibdir) $(ildir) $(lockdir)
tarball=gzip-$(gzip-version).tar.lz
@@ -273,13 +277,13 @@ $(ibidir)/gzip-$(gzip-version): | $(ibdir) $(ildir) $(lockdir)
$(call gbuild, gzip-$(gzip-version), static, , V=1)
echo "GNU Gzip $(gzip-version)" > $@
-$(ibidir)/xz-$(xz-version): $(ibidir)/gzip-$(gzip-version)
+$(ibidir)/xz-$(xz-version): | $(ibdir) $(ildir) $(lockdir)
tarball=xz-$(xz-version).tar.lz
$(call import-source, $(xz-url), $(xz-checksum))
$(call gbuild, xz-$(xz-version), static)
echo "XZ Utils $(xz-version)" > $@
-$(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version)
+$(ibidir)/bzip2-$(bzip2-version): | $(ibdir) $(ildir) $(lockdir)
# Download the tarball.
tarball=bzip2-$(bzip2-version).tar.lz
@@ -308,7 +312,7 @@ $(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version)
fi
cd $(ddir)
rm -rf $$tdir
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd $$tdir
$(shsrcdir)/prep-source.sh $(ibdir)
sed -e 's@\(ln -s -f \)$$(PREFIX)/bin/@\1@' Makefile \
@@ -330,7 +334,7 @@ $(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version)
#
# Note for a static-only build: Zlib's './configure' doesn't use Autoconf's
# configure script, it just accepts a direct '--static' option.
-$(ibidir)/zlib-$(zlib-version): $(ibidir)/gzip-$(gzip-version)
+$(ibidir)/zlib-$(zlib-version): | $(ibdir) $(ildir) $(lockdir)
tarball=zlib-$(zlib-version).tar.lz
$(call import-source, $(zlib-url), $(zlib-checksum))
$(call gbuild, zlib-$(zlib-version))
@@ -351,6 +355,7 @@ $(ibidir)/tar-$(tar-version): \
# a bottleneck here: only making Tar. So its more efficient to built
# it on multiple threads (even when the user's Make doesn't pass down
# the number of threads).
+ $(call unsafe-config)
tarball=tar-$(tar-version).tar.lz
$(call import-source, $(tar-url), $(tar-checksum))
$(call gbuild, tar-$(tar-version), , , -j$(numthreads) V=1)
@@ -615,7 +620,7 @@ $(ibidir)/perl-$(perl-version): $(ibidir)/patchelf-$(patchelf-version)
| awk '{printf("%d.%d", $$1, $$2)}')
cd $(ddir)
rm -rf perl-$(perl-version)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd perl-$(perl-version)
$(shsrcdir)/prep-source.sh $(ibdir)
./Configure -des \
@@ -675,14 +680,15 @@ $(ibidir)/coreutils-$(coreutils-version): \
$(ibidir)/perl-$(perl-version) \
$(ibidir)/openssl-$(openssl-version)
-# Import the source tarball.
+# Import, unpack and enter the source directory.
+ $(call unsafe-config)
tarball=coreutils-$(coreutils-version).tar.lz
$(call import-source, $(coreutils-url), $(coreutils-checksum))
# Unpack and enter the source.
cd $(ddir)
rm -rf coreutils-$(coreutils-version)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd coreutils-$(coreutils-version)
$(shsrcdir)/prep-source.sh $(ibdir)
@@ -696,7 +702,7 @@ $(ibidir)/coreutils-$(coreutils-version): \
# Fix RPATH if necessary.
if [ -f $(ibdir)/patchelf ]; then
make SHELL=$(ibdir)/bash install DESTDIR=junkinst
- unalias ls || true # avoid decorated 'ls' commands with extra characters
+ unalias ls || true # Not decorated 'ls' (with extra characters).
instprogs=$$(ls junkinst/$(ibdir))
for f in $$instprogs; do
$(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/$$f
@@ -721,7 +727,7 @@ $(ibidir)/podlators-$(podlators-version): $(ibidir)/perl-$(perl-version)
$(call import-source, $(podlators-url), $(podlators-checksum))
cd $(ddir)
rm -rf podlators-$(podlators-version)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd podlators-$(podlators-version)
$(shsrcdir)/prep-source.sh $(ibdir)
perl Makefile.PL
@@ -1400,7 +1406,7 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version)
# Unpack GCC and prepare the 'build' directory inside it for all
# the built files.
rm -rf gcc-$(gcc-version)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
if [ $$odir != $(ddir) ]; then
ln -s $$odir/gcc-$(gcc-version) $(ddir)/gcc-$(gcc-version)
fi
@@ -1530,7 +1536,7 @@ $(ibidir)/lzip-$(lzip-version): $(ibidir)/gcc-$(gcc-version)
unpackdir=lzip-$(lzip-version)
cd $(ddir)
rm -rf $$unpackdir
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd $$unpackdir
$(shsrcdir)/prep-source.sh $(ibdir)
./configure --build --check --installdir="$(ibdir)"
diff --git a/reproduce/software/make/build-rules.mk b/reproduce/software/make/build-rules.mk
index 62cb6d5..463fbbf 100644
--- a/reproduce/software/make/build-rules.mk
+++ b/reproduce/software/make/build-rules.mk
@@ -160,7 +160,7 @@ uncompress = csuffix=$$(echo $$utarball \
intarrm=0; \
intar=$$utarball; \
fi; \
- if tar -xf $$intar; then \
+ if tar -xf $$intar --no-same-owner --no-same-permissions; then \
if [ x$$intarrm = x1 ]; then rm $$intar; fi; \
else \
echo; echo "Tar error"; exit 1; \
diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk
index c81f0b9..4ed5d62 100644
--- a/reproduce/software/make/high-level.mk
+++ b/reproduce/software/make/high-level.mk
@@ -348,7 +348,8 @@ $(ibidir)/atlas-$(atlas-version):
# 'rpath_command'.
export LDFLAGS=-L$(ildir)
cd $(ddir)
- tar -xf $(tdir)/atlas-$(atlas-version).tar.lz
+ tar -xf $(tdir)/atlas-$(atlas-version).tar.lz \
+ --no-same-owner --no-same-permissions
cd ATLAS
$(shsrcdir)/prep-source.sh $(ibdir)
rm -rf build
@@ -405,7 +406,7 @@ $(ibidir)/boost-$(boost-version): \
rm -rf $(ddir)/$$unpackdir
topdir=$(pwd)
cd $(ddir)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd $$unpackdir
$(shsrcdir)/prep-source.sh $(ibdir)
./bootstrap.sh --prefix=$(idir) --with-libraries=all \
@@ -427,7 +428,8 @@ $(ibidir)/cfitsio-$(cfitsio-version):
# systems. So we need to change it to our library installation
# path. It doesn't affect GNU/Linux, so we'll just do it in any case
# to keep things clean.
- topdir=$(pwd); cd $(ddir); tar -xf $(tdir)/$$tarball
+ topdir=$(pwd); cd $(ddir)
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
customtar=cfitsio-$(cfitsio-version)-custom.tar.gz
cd cfitsio-$(cfitsio-version)
sed -i -e's|@rpath|$(ildir)|g' configure
@@ -467,7 +469,8 @@ $(ibidir)/eigen-$(eigen-version):
tarball=eigen-$(eigen-version).tar.lz
$(call import-source, $(eigen-url), $(eigen-checksum))
rm -rf $(ddir)/eigen-eigen-*
- topdir=$(pwd); cd $(ddir); tar -xf $(tdir)/$$tarball
+ topdir=$(pwd); cd $(ddir)
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd eigen-$(eigen-version)
if ! [ -d $(iidir)/eigen3 ]; then mkdir $(iidir)/eigen3; fi
cp -r Eigen/* $(iidir)/eigen3/ # Some expect 'eigen3'.
@@ -597,7 +600,7 @@ $(ibidir)/healpix-$(healpix-version): $(healpix-python-dep) \
fi
rm -rf $(ddir)/Healpix_$(healpix-version)
topdir=$(pwd); cd $(ddir);
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd Healpix_$(healpix-version)
$(shsrcdir)/prep-source.sh $(ibdir)
cd src/C/autotools
@@ -689,7 +692,7 @@ $(ibidir)/libpaper-$(libpaper-version): \
# Unpack, build the configure system, build and install.
cd $(ddir)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
unpackdir=libpaper-$(libpaper-version)
cd $$unpackdir
$(shsrcdir)/prep-source.sh $(ibdir)
@@ -824,7 +827,7 @@ $(ibidir)/ninjabuild-$(ninjabuild-version): $(ibidir)/cmake-$(cmake-version)
tarball=ninjabuild-$(ninjabuild-version).tar.lz
$(call import-source, $(ninjabuild-url), $(ninjabuild-checksum))
cd $(ddir)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd ninjabuild-$(ninjabuild-version)
cmake -Bbuild-cmake
cmake --build build-cmake -j$(numthreads)
@@ -839,7 +842,7 @@ $(ibidir)/openblas-$(openblas-version):
$(call import-source, $(openblas-url), $(openblas-checksum))
if [ x$(on_mac_os) = xyes ]; then export CC=clang; fi
cd $(ddir)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd openblas-$(openblas-version)
$(shsrcdir)/prep-source.sh $(ibdir)
make -j$(numthreads)
@@ -1048,7 +1051,7 @@ $(ibidir)/astrometrynet-$(astrometrynet-version): \
# 'astrometrynet'
cd $(ddir)
rm -rf astrometry.net-$(astrometrynet-version)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd astrometry.net-$(astrometrynet-version)
$(shsrcdir)/prep-source.sh $(ibdir)
sed -e 's|cat /proc/cpuinfo|echo "Ignoring CPU info"|' \
@@ -1087,7 +1090,7 @@ $(ibidir)/cdsclient-$(cdsclient-version):
tarball=cdsclient-$(cdsclient-version).tar.lz
$(call import-source, $(cdsclient-url), $(cdsclient-checksum))
cd $(ddir)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd cdsclient-$(cdsclient-version)
$(shsrcdir)/prep-source.sh $(ibdir)
touch *
@@ -1120,7 +1123,7 @@ $(ibidir)/cmake-$(cmake-version):
# Go into the unpacked directory and prepare CMake.
cd $(ddir)
rm -rf cmake-$(cmake-version)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd cmake-$(cmake-version)
$(shsrcdir)/prep-source.sh $(ibdir)
@@ -1186,7 +1189,7 @@ $(ibidir)/ghostscript-$(ghostscript-version): \
# '-DPNG_ARM_NEON_OPT=0' prevents an arm64 'neon' library from being
# required at compile time.
cd $(ddir)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd ghostscript-$(ghostscript-version)
$(shsrcdir)/prep-source.sh $(ibdir)
./configure --prefix=$(idir) \
@@ -1209,9 +1212,9 @@ $(ibidir)/ghostscript-$(ghostscript-version): \
# Install the fonts.
tar -xvf $(tdir)/ghostscript-fonts-std-$(ghostscript-fonts-std-version).tar.lz \
- -C $(idir)/share/ghostscript
+ -C $(idir)/share/ghostscript --no-same-owner --no-same-permissions
tar -xvf $(tdir)/ghostscript-fonts-gnu-$(ghostscript-fonts-gnu-version).tar.lz \
- -C $(idir)/share/ghostscript
+ -C $(idir)/share/ghostscript --no-same-owner --no-same-permissions
fc-cache -v $(idir)/share/ghostscript/fonts/
echo; echo "Ghostscript fonts added to Fontconfig."; echo;
@@ -1248,7 +1251,7 @@ $(ibidir)/icu-$(icu-version): $(ibidir)/python-$(python-version)
tarball=icu-$(icu-version).tar.lz
$(call import-source, $(icu-url), $(icu-checksum))
cd $(ddir)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
unpackdir=icu-$(icu-version)
cd $$unpackdir
$(shsrcdir)/prep-source.sh $(ibdir)
@@ -1317,7 +1320,7 @@ $(ibidir)/imfit-$(imfit-version): \
cd $(ddir)
unpackdir=imfit-$(imfit-version)
rm -rf $$unpackdir
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd $$unpackdir
$(shsrcdir)/prep-source.sh $(ibdir)
sed -i 's|/usr/local|$(idir)|g' SConstruct
@@ -1365,7 +1368,8 @@ $(ibidir)/minizip-$(minizip-version): $(ibidir)/automake-$(automake-version)
unpackdir=minizip-$(minizip-version)
rm -rf $$unpackdir
mkdir $$unpackdir
- tar -xf $(tdir)/$$tarball -C$$unpackdir --strip-components=1
+ tar -xf $(tdir)/$$tarball -C$$unpackdir --strip-components=1 \
+ --no-same-owner --no-same-permissions
cd $$unpackdir
$(shsrcdir)/prep-source.sh $(ibdir)
./configure --prefix=$(idir)
@@ -1427,7 +1431,7 @@ $(ibidir)/netpbm-$(netpbm-version): \
cd $(ddir)
unpackdir=netpbm-$(netpbm-version)
rm -rf $$unpackdir
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd $$unpackdir
$(shsrcdir)/prep-source.sh $(ibdir)
@@ -1547,7 +1551,7 @@ $(ibidir)/scons-$(scons-version): $(ibidir)/python-$(python-version)
cd $(ddir)
unpackdir=scons-$(scons-version)
rm -rf $$unpackdir
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd $$unpackdir
$(shsrcdir)/prep-source.sh $(ibdir)
@@ -1593,7 +1597,7 @@ $(ibidir)/sextractor-$(sextractor-version): \
unpackdir=sextractor-$(sextractor-version)
cd $(ddir)
rm -rf $$unpackdir
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd $$unpackdir
# See comment above 'missfits' for '-fcommon'.
@@ -1694,7 +1698,7 @@ $(ibidir)/util-linux-$(util-linux-version): \
# explained above). As shown below, later, we'll put a symbolic link
# of all the necessary binaries in the main '$(idir)/bin'.
cd $(ddir)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd util-linux-$(util-linux-version)
$(shsrcdir)/prep-source.sh $(ibdir)
@@ -1799,7 +1803,7 @@ $(ibidir)/vim-$(vim-version):
tarball=vim-$(vim-version).tar.lz
$(call import-source, $(vim-url), $(vim-checksum))
cd $(ddir)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
unpackdir=vim-$(vim-version)
cd $(ddir)/$$unpackdir
$(shsrcdir)/prep-source.sh $(ibdir)
@@ -1869,7 +1873,8 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf
@topdir=$$(pwd)
cd $(ddir)
rm -rf install-tl-*
- tar -xf $(tdir)/install-tl-unx.tar.gz
+ tar -xf $(tdir)/install-tl-unx.tar.gz \
+ --no-same-owner --no-same-permissions
cd install-tl-*
$(shsrcdir)/prep-source.sh $(ibdir)
sed -e's|@installdir[@]|$(idir)|g' \
@@ -1956,7 +1961,8 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf
"$(backupservers)"; then
cd $(ddir)
rm -rf install-tl-*
- tar -xf $(tdir)/install-tl-unx.tar.gz
+ tar -xf $(tdir)/install-tl-unx.tar.gz \
+ --no-same-owner --no-same-permissions
cd install-tl-*
$(shsrcdir)/prep-source.sh $(ibdir)
sed -e's|@installdir[@]|$(idir)|g' \
diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk
index c017fd2..c994e3f 100644
--- a/reproduce/software/make/python.mk
+++ b/reproduce/software/make/python.mk
@@ -99,7 +99,7 @@ $(ibidir)/python-$(python-version): $(ibidir)/libffi-$(libffi-version)
# Unpack the tarball (see below for the necessary modification).
cd $(ddir)
unpackdir=python-$(python-version)
- tar -xf $(tdir)/$$tarball
+ tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions
cd $$unpackdir
$(shsrcdir)/prep-source.sh $(ibdir)
@@ -215,7 +215,7 @@ pybuild = cd $(ddir); \
packagedir=$(strip $(2)); \
if (printf "$$packagedir" | grep "[a-z][a-z]"); then rm -rf $$packagedir; fi; \
printf "\nStarting to install python package with maneage pybuild rule: $(4)\n ..."; \
- if ! $(1) $(tdir)/$$tarball; then \
+ if ! $(1) $(tdir)/$$tarball --no-same-owner --no-same-permissions; then \
echo; echo "Tar error"; exit 1; \
fi; \
cd $$packagedir; \
diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh
index 517e1ed..e291f7b 100755
--- a/reproduce/software/shell/configure.sh
+++ b/reproduce/software/shell/configure.sh
@@ -1273,33 +1273,10 @@ chmod +x $makewshell
# Project's top-level built analysis directories
# ----------------------------------------------
-# Top-level built analysis directories.
-badir="$bdir"/analysis
-if ! [ -d "$badir" ]; then mkdir "$badir"; fi
-
# Top-level LaTeX.
-texdir="$badir"/tex
+texdir="$sdir"/tex
if ! [ -d "$texdir" ]; then mkdir "$texdir"; fi
-# LaTeX macros.
-mtexdir="$texdir"/macros
-if ! [ -d "$mtexdir" ]; then mkdir "$mtexdir"; fi
-
-# TeX build directory. If built in a group scenario, the TeX build
-# directory must be separate for each member (so they can work on their
-# relevant parts of the paper without conflicting with each other).
-if [ "x$maneage_group_name" = x ]; then
- texbdir="$texdir"/build
-else
- user=$(whoami)
- texbdir="$texdir"/build-$user
-fi
-if ! [ -d "$texbdir" ]; then mkdir "$texbdir"; fi
-
-# TiKZ (for building figures within LaTeX).
-tikzdir="$texbdir"/tikz
-if ! [ -d "$tikzdir" ]; then mkdir "$tikzdir"; fi
-
# If 'tex/build' and 'tex/tikz' are symbolic links then 'rm -f' will delete
# them and we can continue. However, when the project is being built from
# the tarball, these two are not symbolic links but actual directories with
@@ -1328,8 +1305,6 @@ rm -f .build .local
ln -s "$bdir" .build
ln -s "$instdir" .local
-ln -s "$texdir" tex/build
-ln -s "$tikzdir" tex/tikz
# --------- Delete for no Gnuastro ---------
rm -f .gnuastro
@@ -1875,7 +1850,7 @@ pymodules=$(prepare_name_version $verdir/python/*)
texpkg=$(prepare_name_version $verdir/tex/texlive)
# Acknowledge these software packages in a LaTeX paragraph.
-pkgver=$mtexdir/dependencies.tex
+pkgver=$texdir/dependencies.tex
# Add the text to the ${pkgver} file.
.local/bin/echo "$thank_software_introduce " > $pkgver
@@ -1892,7 +1867,7 @@ bibfiles="$ictdir/*"
for f in $bibfiles; do if [ -f $f ]; then hasentry=1; break; fi; done;
# Make sure we start with an empty output file.
-pkgbib=$mtexdir/dependencies-bib.tex
+pkgbib=$texdir/dependencies-bib.tex
echo "" > $pkgbib
# Fill it in with all the BibTeX entries in this directory. We'll just
@@ -1912,7 +1887,7 @@ fi
# ---------------------------
#
# Report hardware
-hwparam="$mtexdir/hardware-parameters.tex"
+hwparam="$texdir/hardware-parameters.tex"
# Add the text to the ${hwparam} file. Since harware class might include
# underscore, it must be replaced with '\_', otherwise pdftex would
diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh
index 93d3266..28b7385 100755
--- a/reproduce/software/shell/pre-make-build.sh
+++ b/reproduce/software/shell/pre-make-build.sh
@@ -177,7 +177,7 @@ build_program() {
fi
# Unpack the tarball and go into it.
- tar xf "$intar"
+ tar xf "$intar" --no-same-owner --no-same-permissions
if [ x$intarrm = x1 ]; then rm "$intar"; fi
cd "$unpackdir"