From a1f8947ab7784af4b7e66c617ce19a8bdd9c99ed Mon Sep 17 00:00:00 2001 From: Giacomo Lorenzetti Date: Thu, 27 Feb 2025 17:49:28 +0100 Subject: IMPORTANT: Apptainer and Docker containers, minor restructuring Summary: it is necessary to re-configure your project (just running './project configure -e', not deleting 'build/software' to re-build software) after this commit, see "Affected files" item below). Until now, we only had a relatively long set of manual instructions for building Maneage within Docker in the top-level README. This was hard to automate, focing Maneage users to write custom commands based on the instructions and maintain those scripts outside of Maneage. As a result, experience could not be shared between projects (or at most in the README file!). With this commit, a new 'reproduce/software/containers' directory has been created within Maneage that contains two scripts (with a unified interface) greatly simplifying the building of the project's software environment within a container (one script for Apptainer and one for Docker). Two READMEs have been added for each container to help in their first time usage. Also, the old checklist within the main README has been replaced with a short introduction on containers and points the interested readers to the custom README of each container technology. Since we wanted the containers to be read-only after build, we needed to fully decouple the 'build/software' and 'build/analysis', such that './project configure' only writes to the former and './project make' only writes the latter. The file and directories mentioned in the affected files are cases that both project phases was writing to the 'build/software' and 'build/analysis' directories. Affected files: 'preparation-done.mk' and 'lockdir' which were previously in the 'build/software' directory are now made during the 'make' phase and the 'configure' phase no longer builds the 'build/analysis' or anything within it. Also, the software version LaTeX macros (which were previously written during the 'configure' phase in the 'analysis' directory) are now written in the software directory and copied into the analysis for usage in LaTeX while building the paper. Other minor additions in this commit: - The './project' script has a new '--timing' option to write the starting and ending times of the project in a file. It also builds the high-level analysis directories when './project make' is called (but before calling 'top-make.mk'. - The 'tar' calls in the custom build commands of the software building Makefiles now have the '--no-same-owner --no-same-permissions' options like the 'tar' call within the 'uncompress' function of 'build-rules.mk'. This commit was originally written by Giacomo Lorenzetti only for Apptainer on the registered commit date. It was later re-implemented from scratch by Mohammad Akhlaghi to have a unified interface for both Apptainer and Docker and merged into Maneage on 2025-04-23. --- reproduce/analysis/make/initialize.mk | 6 ++--- reproduce/analysis/make/paper.mk | 41 +++++++++++++++++++++++++++++++--- reproduce/analysis/make/prepare.mk | 2 +- reproduce/analysis/make/top-prepare.mk | 2 +- 4 files changed, 43 insertions(+), 8 deletions(-) (limited to 'reproduce/analysis') diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 92e5eff..c51f910 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -41,7 +41,7 @@ bsdir=$(BDIR)/software # Derived directories (the locks directory can be shared with software # which already has this directory.). texdir = $(badir)/tex -lockdir = $(bsdir)/locks +lockdir = $(badir)/.locks indir = $(badir)/inputs prepdir = $(badir)/prepare mtexdir = $(texdir)/macros @@ -73,7 +73,7 @@ pconfdir = reproduce/analysis/config ifeq (x$(project-phase),xprepare) $(prepdir):; mkdir $@ else --include $(bsdir)/preparation-done.mk +-include $(badir)/preparation-done.mk ifeq (x$(include-prepare-results),xyes) -include $(prepdir)/*.mk $(prepdir)/*.conf endif @@ -274,7 +274,7 @@ clean: rm -rf $(texdir)/macros/!(dependencies.tex|dependencies-bib.tex|hardware-parameters.tex) rm -rf $(badir)/!(tex) $(texdir)/!(macros|$(texbtopdir)) rm -rf $(texdir)/build/!(tikz) $(texdir)/build/tikz/* - rm -rf $(bsdir)/preparation-done.mk + rm -rf $(badir)/preparation-done.mk distclean: clean # Without cleaning the Git hooks, we won't be able to easily commit diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk index 66c6859..3c06ce3 100644 --- a/reproduce/analysis/make/paper.mk +++ b/reproduce/analysis/make/paper.mk @@ -18,6 +18,7 @@ + # LaTeX macros for paper # ---------------------- # @@ -92,6 +93,40 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex +# TeX build directory +# ------------------- +# +# If built in a group scenario, the TeX build directory must be separate +# for each member (so they can work on their relevant parts of the paper +# without conflicting with each other). +ifeq ($(strip $(maneage_group_name)),) +texbdir:=$(texdir)/build +else +texbdir:=$(texdir)/build-$(shell whoami) +endif +tikzdir:=$(texbdir)/tikz +$(texbdir):; mkdir $@ +$(tikzdir): | $(texbdir); mkdir $@ + + + + + +# Software info in TeX +# -------------------- +# +# The information of the installed software is placed in the +# '.build/software' directory (which the TeX build should not depend +# on). Therefore, we should copy those macros here in the LaTeX build +# directory, so the TeX directory is completely independent from each +# other. +$(mtexdir)/dependencies.tex: $(bsdir)/tex/dependencies.tex + cp $(bsdir)/tex/*.tex $(mtexdir)/ + + + + + # The bibliography # ---------------- # @@ -104,8 +139,9 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex # recipe and the 'paper.pdf' recipe. But if 'tex/src/references.tex' hasn't # been modified, we don't want to re-build the bibliography, only the final # PDF. -$(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ - | $(mtexdir)/project.tex +$(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies.tex \ + | $(mtexdir)/project.tex $(tikzdir) + # If '$(mtexdir)/project.tex' is empty, don't build PDF. @macros=$$(cat $(mtexdir)/project.tex) if [ x"$$macros" != x ]; then @@ -135,7 +171,6 @@ $(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ export LD_LIBRARY_PATH="$(sys_library_sh_path):$$LD_LIBRARY_PATH" pdflatex -shell-escape -halt-on-error "$$p"/paper.tex biber paper - fi diff --git a/reproduce/analysis/make/prepare.mk b/reproduce/analysis/make/prepare.mk index 2cc1187..ffb2a3c 100644 --- a/reproduce/analysis/make/prepare.mk +++ b/reproduce/analysis/make/prepare.mk @@ -25,7 +25,7 @@ # # We need to remove the 'prepare' word from the list of 'makesrc'. prepare-dep = $(filter-out prepare, $(makesrc)) -$(bsdir)/preparation-done.mk: \ +$(badir)/preparation-done.mk: \ $(foreach s, $(prepare-dep), $(mtexdir)/$(s).tex) # If you need to add preparations (mainly automatically generated diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk index 7d92d72..ea40f39 100644 --- a/reproduce/analysis/make/top-prepare.mk +++ b/reproduce/analysis/make/top-prepare.mk @@ -36,7 +36,7 @@ include reproduce/software/config/LOCAL.conf # # See 'top-make.mk' for complete explanation. ifeq (x$(maneage_group_name),x$(GROUP-NAME)) -all: $(BDIR)/software/preparation-done.mk +all: $(BDIR)/analysis/preparation-done.mk @echo "Project preparation is complete."; else all: -- cgit v1.2.1