From a1f8947ab7784af4b7e66c617ce19a8bdd9c99ed Mon Sep 17 00:00:00 2001 From: Giacomo Lorenzetti Date: Thu, 27 Feb 2025 17:49:28 +0100 Subject: IMPORTANT: Apptainer and Docker containers, minor restructuring Summary: it is necessary to re-configure your project (just running './project configure -e', not deleting 'build/software' to re-build software) after this commit, see "Affected files" item below). Until now, we only had a relatively long set of manual instructions for building Maneage within Docker in the top-level README. This was hard to automate, focing Maneage users to write custom commands based on the instructions and maintain those scripts outside of Maneage. As a result, experience could not be shared between projects (or at most in the README file!). With this commit, a new 'reproduce/software/containers' directory has been created within Maneage that contains two scripts (with a unified interface) greatly simplifying the building of the project's software environment within a container (one script for Apptainer and one for Docker). Two READMEs have been added for each container to help in their first time usage. Also, the old checklist within the main README has been replaced with a short introduction on containers and points the interested readers to the custom README of each container technology. Since we wanted the containers to be read-only after build, we needed to fully decouple the 'build/software' and 'build/analysis', such that './project configure' only writes to the former and './project make' only writes the latter. The file and directories mentioned in the affected files are cases that both project phases was writing to the 'build/software' and 'build/analysis' directories. Affected files: 'preparation-done.mk' and 'lockdir' which were previously in the 'build/software' directory are now made during the 'make' phase and the 'configure' phase no longer builds the 'build/analysis' or anything within it. Also, the software version LaTeX macros (which were previously written during the 'configure' phase in the 'analysis' directory) are now written in the software directory and copied into the analysis for usage in LaTeX while building the paper. Other minor additions in this commit: - The './project' script has a new '--timing' option to write the starting and ending times of the project in a file. It also builds the high-level analysis directories when './project make' is called (but before calling 'top-make.mk'. - The 'tar' calls in the custom build commands of the software building Makefiles now have the '--no-same-owner --no-same-permissions' options like the 'tar' call within the 'uncompress' function of 'build-rules.mk'. This commit was originally written by Giacomo Lorenzetti only for Apptainer on the registered commit date. It was later re-implemented from scratch by Mohammad Akhlaghi to have a unified interface for both Apptainer and Docker and merged into Maneage on 2025-04-23. --- reproduce/software/make/basic.mk | 46 ++++++++++++++++------------- reproduce/software/make/build-rules.mk | 2 +- reproduce/software/make/high-level.mk | 54 +++++++++++++++++++--------------- reproduce/software/make/python.mk | 4 +-- 4 files changed, 59 insertions(+), 47 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 745aeca..40c5a4e 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -144,10 +144,17 @@ backupservers_all = $(user_backup_urls) $(maneage_backup_urls) topbackupserver = $(word 1, $(backupservers_all)) backupservers = $(filter-out $(topbackupserver),$(backupservers_all)) - - - - +# When building in Apptainer containers, as of 2025-04-18, we need to +# configure Maneage as root (within the container). In such cases, we need +# to activate the 'FORCE_UNSAFE_CONFIGURE' environment variable to build +# some of the software. The 'if' statement is here to make sure we are in +# Apptainer: in other situations, the "unsafe" configure script shouldn't +# be activated. Note that this doesn't happen in Docker (where the Maneage +# source is in the same directory) because we build a non-root ('maneager' +# user there who executes the configure command. +unsafe-config = if [ $$(pwd) = "/home/maneager/source" ] \ + && [ $$(whoami) = root ]; then \ + export FORCE_UNSAFE_CONFIGURE=1; fi @@ -261,11 +268,8 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ # # The first set of programs to be built are those that we need to unpack # the source code tarballs of each program. We have already installed Lzip -# before calling 'basic.mk', so it is present and working. Hence we first -# build the Lzipped tarball of Gzip, then use our own Gzip to unpack the -# tarballs of the other compression programs. Once all the compression -# programs/libraries are complete, we build our own GNU Tar and continue -# with other software. +# before calling 'basic.mk', so it is present and working. So the only +# prerequisites of these (until reaching Tar) is the necessary directories. $(lockdir): | $(BDIR); mkdir $@ $(ibidir)/gzip-$(gzip-version): | $(ibdir) $(ildir) $(lockdir) tarball=gzip-$(gzip-version).tar.lz @@ -273,13 +277,13 @@ $(ibidir)/gzip-$(gzip-version): | $(ibdir) $(ildir) $(lockdir) $(call gbuild, gzip-$(gzip-version), static, , V=1) echo "GNU Gzip $(gzip-version)" > $@ -$(ibidir)/xz-$(xz-version): $(ibidir)/gzip-$(gzip-version) +$(ibidir)/xz-$(xz-version): | $(ibdir) $(ildir) $(lockdir) tarball=xz-$(xz-version).tar.lz $(call import-source, $(xz-url), $(xz-checksum)) $(call gbuild, xz-$(xz-version), static) echo "XZ Utils $(xz-version)" > $@ -$(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) +$(ibidir)/bzip2-$(bzip2-version): | $(ibdir) $(ildir) $(lockdir) # Download the tarball. tarball=bzip2-$(bzip2-version).tar.lz @@ -308,7 +312,7 @@ $(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) fi cd $(ddir) rm -rf $$tdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$tdir $(shsrcdir)/prep-source.sh $(ibdir) sed -e 's@\(ln -s -f \)$$(PREFIX)/bin/@\1@' Makefile \ @@ -330,7 +334,7 @@ $(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) # # Note for a static-only build: Zlib's './configure' doesn't use Autoconf's # configure script, it just accepts a direct '--static' option. -$(ibidir)/zlib-$(zlib-version): $(ibidir)/gzip-$(gzip-version) +$(ibidir)/zlib-$(zlib-version): | $(ibdir) $(ildir) $(lockdir) tarball=zlib-$(zlib-version).tar.lz $(call import-source, $(zlib-url), $(zlib-checksum)) $(call gbuild, zlib-$(zlib-version)) @@ -351,6 +355,7 @@ $(ibidir)/tar-$(tar-version): \ # a bottleneck here: only making Tar. So its more efficient to built # it on multiple threads (even when the user's Make doesn't pass down # the number of threads). + $(call unsafe-config) tarball=tar-$(tar-version).tar.lz $(call import-source, $(tar-url), $(tar-checksum)) $(call gbuild, tar-$(tar-version), , , -j$(numthreads) V=1) @@ -615,7 +620,7 @@ $(ibidir)/perl-$(perl-version): $(ibidir)/patchelf-$(patchelf-version) | awk '{printf("%d.%d", $$1, $$2)}') cd $(ddir) rm -rf perl-$(perl-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd perl-$(perl-version) $(shsrcdir)/prep-source.sh $(ibdir) ./Configure -des \ @@ -675,14 +680,15 @@ $(ibidir)/coreutils-$(coreutils-version): \ $(ibidir)/perl-$(perl-version) \ $(ibidir)/openssl-$(openssl-version) -# Import the source tarball. +# Import, unpack and enter the source directory. + $(call unsafe-config) tarball=coreutils-$(coreutils-version).tar.lz $(call import-source, $(coreutils-url), $(coreutils-checksum)) # Unpack and enter the source. cd $(ddir) rm -rf coreutils-$(coreutils-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd coreutils-$(coreutils-version) $(shsrcdir)/prep-source.sh $(ibdir) @@ -696,7 +702,7 @@ $(ibidir)/coreutils-$(coreutils-version): \ # Fix RPATH if necessary. if [ -f $(ibdir)/patchelf ]; then make SHELL=$(ibdir)/bash install DESTDIR=junkinst - unalias ls || true # avoid decorated 'ls' commands with extra characters + unalias ls || true # Not decorated 'ls' (with extra characters). instprogs=$$(ls junkinst/$(ibdir)) for f in $$instprogs; do $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/$$f @@ -721,7 +727,7 @@ $(ibidir)/podlators-$(podlators-version): $(ibidir)/perl-$(perl-version) $(call import-source, $(podlators-url), $(podlators-checksum)) cd $(ddir) rm -rf podlators-$(podlators-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd podlators-$(podlators-version) $(shsrcdir)/prep-source.sh $(ibdir) perl Makefile.PL @@ -1400,7 +1406,7 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) # Unpack GCC and prepare the 'build' directory inside it for all # the built files. rm -rf gcc-$(gcc-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions if [ $$odir != $(ddir) ]; then ln -s $$odir/gcc-$(gcc-version) $(ddir)/gcc-$(gcc-version) fi @@ -1530,7 +1536,7 @@ $(ibidir)/lzip-$(lzip-version): $(ibidir)/gcc-$(gcc-version) unpackdir=lzip-$(lzip-version) cd $(ddir) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir $(shsrcdir)/prep-source.sh $(ibdir) ./configure --build --check --installdir="$(ibdir)" diff --git a/reproduce/software/make/build-rules.mk b/reproduce/software/make/build-rules.mk index 62cb6d5..463fbbf 100644 --- a/reproduce/software/make/build-rules.mk +++ b/reproduce/software/make/build-rules.mk @@ -160,7 +160,7 @@ uncompress = csuffix=$$(echo $$utarball \ intarrm=0; \ intar=$$utarball; \ fi; \ - if tar -xf $$intar; then \ + if tar -xf $$intar --no-same-owner --no-same-permissions; then \ if [ x$$intarrm = x1 ]; then rm $$intar; fi; \ else \ echo; echo "Tar error"; exit 1; \ diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index c81f0b9..4ed5d62 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -348,7 +348,8 @@ $(ibidir)/atlas-$(atlas-version): # 'rpath_command'. export LDFLAGS=-L$(ildir) cd $(ddir) - tar -xf $(tdir)/atlas-$(atlas-version).tar.lz + tar -xf $(tdir)/atlas-$(atlas-version).tar.lz \ + --no-same-owner --no-same-permissions cd ATLAS $(shsrcdir)/prep-source.sh $(ibdir) rm -rf build @@ -405,7 +406,7 @@ $(ibidir)/boost-$(boost-version): \ rm -rf $(ddir)/$$unpackdir topdir=$(pwd) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir $(shsrcdir)/prep-source.sh $(ibdir) ./bootstrap.sh --prefix=$(idir) --with-libraries=all \ @@ -427,7 +428,8 @@ $(ibidir)/cfitsio-$(cfitsio-version): # systems. So we need to change it to our library installation # path. It doesn't affect GNU/Linux, so we'll just do it in any case # to keep things clean. - topdir=$(pwd); cd $(ddir); tar -xf $(tdir)/$$tarball + topdir=$(pwd); cd $(ddir) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions customtar=cfitsio-$(cfitsio-version)-custom.tar.gz cd cfitsio-$(cfitsio-version) sed -i -e's|@rpath|$(ildir)|g' configure @@ -467,7 +469,8 @@ $(ibidir)/eigen-$(eigen-version): tarball=eigen-$(eigen-version).tar.lz $(call import-source, $(eigen-url), $(eigen-checksum)) rm -rf $(ddir)/eigen-eigen-* - topdir=$(pwd); cd $(ddir); tar -xf $(tdir)/$$tarball + topdir=$(pwd); cd $(ddir) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd eigen-$(eigen-version) if ! [ -d $(iidir)/eigen3 ]; then mkdir $(iidir)/eigen3; fi cp -r Eigen/* $(iidir)/eigen3/ # Some expect 'eigen3'. @@ -597,7 +600,7 @@ $(ibidir)/healpix-$(healpix-version): $(healpix-python-dep) \ fi rm -rf $(ddir)/Healpix_$(healpix-version) topdir=$(pwd); cd $(ddir); - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd Healpix_$(healpix-version) $(shsrcdir)/prep-source.sh $(ibdir) cd src/C/autotools @@ -689,7 +692,7 @@ $(ibidir)/libpaper-$(libpaper-version): \ # Unpack, build the configure system, build and install. cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions unpackdir=libpaper-$(libpaper-version) cd $$unpackdir $(shsrcdir)/prep-source.sh $(ibdir) @@ -824,7 +827,7 @@ $(ibidir)/ninjabuild-$(ninjabuild-version): $(ibidir)/cmake-$(cmake-version) tarball=ninjabuild-$(ninjabuild-version).tar.lz $(call import-source, $(ninjabuild-url), $(ninjabuild-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd ninjabuild-$(ninjabuild-version) cmake -Bbuild-cmake cmake --build build-cmake -j$(numthreads) @@ -839,7 +842,7 @@ $(ibidir)/openblas-$(openblas-version): $(call import-source, $(openblas-url), $(openblas-checksum)) if [ x$(on_mac_os) = xyes ]; then export CC=clang; fi cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd openblas-$(openblas-version) $(shsrcdir)/prep-source.sh $(ibdir) make -j$(numthreads) @@ -1048,7 +1051,7 @@ $(ibidir)/astrometrynet-$(astrometrynet-version): \ # 'astrometrynet' cd $(ddir) rm -rf astrometry.net-$(astrometrynet-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd astrometry.net-$(astrometrynet-version) $(shsrcdir)/prep-source.sh $(ibdir) sed -e 's|cat /proc/cpuinfo|echo "Ignoring CPU info"|' \ @@ -1087,7 +1090,7 @@ $(ibidir)/cdsclient-$(cdsclient-version): tarball=cdsclient-$(cdsclient-version).tar.lz $(call import-source, $(cdsclient-url), $(cdsclient-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd cdsclient-$(cdsclient-version) $(shsrcdir)/prep-source.sh $(ibdir) touch * @@ -1120,7 +1123,7 @@ $(ibidir)/cmake-$(cmake-version): # Go into the unpacked directory and prepare CMake. cd $(ddir) rm -rf cmake-$(cmake-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd cmake-$(cmake-version) $(shsrcdir)/prep-source.sh $(ibdir) @@ -1186,7 +1189,7 @@ $(ibidir)/ghostscript-$(ghostscript-version): \ # '-DPNG_ARM_NEON_OPT=0' prevents an arm64 'neon' library from being # required at compile time. cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd ghostscript-$(ghostscript-version) $(shsrcdir)/prep-source.sh $(ibdir) ./configure --prefix=$(idir) \ @@ -1209,9 +1212,9 @@ $(ibidir)/ghostscript-$(ghostscript-version): \ # Install the fonts. tar -xvf $(tdir)/ghostscript-fonts-std-$(ghostscript-fonts-std-version).tar.lz \ - -C $(idir)/share/ghostscript + -C $(idir)/share/ghostscript --no-same-owner --no-same-permissions tar -xvf $(tdir)/ghostscript-fonts-gnu-$(ghostscript-fonts-gnu-version).tar.lz \ - -C $(idir)/share/ghostscript + -C $(idir)/share/ghostscript --no-same-owner --no-same-permissions fc-cache -v $(idir)/share/ghostscript/fonts/ echo; echo "Ghostscript fonts added to Fontconfig."; echo; @@ -1248,7 +1251,7 @@ $(ibidir)/icu-$(icu-version): $(ibidir)/python-$(python-version) tarball=icu-$(icu-version).tar.lz $(call import-source, $(icu-url), $(icu-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions unpackdir=icu-$(icu-version) cd $$unpackdir $(shsrcdir)/prep-source.sh $(ibdir) @@ -1317,7 +1320,7 @@ $(ibidir)/imfit-$(imfit-version): \ cd $(ddir) unpackdir=imfit-$(imfit-version) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir $(shsrcdir)/prep-source.sh $(ibdir) sed -i 's|/usr/local|$(idir)|g' SConstruct @@ -1365,7 +1368,8 @@ $(ibidir)/minizip-$(minizip-version): $(ibidir)/automake-$(automake-version) unpackdir=minizip-$(minizip-version) rm -rf $$unpackdir mkdir $$unpackdir - tar -xf $(tdir)/$$tarball -C$$unpackdir --strip-components=1 + tar -xf $(tdir)/$$tarball -C$$unpackdir --strip-components=1 \ + --no-same-owner --no-same-permissions cd $$unpackdir $(shsrcdir)/prep-source.sh $(ibdir) ./configure --prefix=$(idir) @@ -1427,7 +1431,7 @@ $(ibidir)/netpbm-$(netpbm-version): \ cd $(ddir) unpackdir=netpbm-$(netpbm-version) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir $(shsrcdir)/prep-source.sh $(ibdir) @@ -1547,7 +1551,7 @@ $(ibidir)/scons-$(scons-version): $(ibidir)/python-$(python-version) cd $(ddir) unpackdir=scons-$(scons-version) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir $(shsrcdir)/prep-source.sh $(ibdir) @@ -1593,7 +1597,7 @@ $(ibidir)/sextractor-$(sextractor-version): \ unpackdir=sextractor-$(sextractor-version) cd $(ddir) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir # See comment above 'missfits' for '-fcommon'. @@ -1694,7 +1698,7 @@ $(ibidir)/util-linux-$(util-linux-version): \ # explained above). As shown below, later, we'll put a symbolic link # of all the necessary binaries in the main '$(idir)/bin'. cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd util-linux-$(util-linux-version) $(shsrcdir)/prep-source.sh $(ibdir) @@ -1799,7 +1803,7 @@ $(ibidir)/vim-$(vim-version): tarball=vim-$(vim-version).tar.lz $(call import-source, $(vim-url), $(vim-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions unpackdir=vim-$(vim-version) cd $(ddir)/$$unpackdir $(shsrcdir)/prep-source.sh $(ibdir) @@ -1869,7 +1873,8 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf @topdir=$$(pwd) cd $(ddir) rm -rf install-tl-* - tar -xf $(tdir)/install-tl-unx.tar.gz + tar -xf $(tdir)/install-tl-unx.tar.gz \ + --no-same-owner --no-same-permissions cd install-tl-* $(shsrcdir)/prep-source.sh $(ibdir) sed -e's|@installdir[@]|$(idir)|g' \ @@ -1956,7 +1961,8 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf "$(backupservers)"; then cd $(ddir) rm -rf install-tl-* - tar -xf $(tdir)/install-tl-unx.tar.gz + tar -xf $(tdir)/install-tl-unx.tar.gz \ + --no-same-owner --no-same-permissions cd install-tl-* $(shsrcdir)/prep-source.sh $(ibdir) sed -e's|@installdir[@]|$(idir)|g' \ diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index c017fd2..c994e3f 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -99,7 +99,7 @@ $(ibidir)/python-$(python-version): $(ibidir)/libffi-$(libffi-version) # Unpack the tarball (see below for the necessary modification). cd $(ddir) unpackdir=python-$(python-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir $(shsrcdir)/prep-source.sh $(ibdir) @@ -215,7 +215,7 @@ pybuild = cd $(ddir); \ packagedir=$(strip $(2)); \ if (printf "$$packagedir" | grep "[a-z][a-z]"); then rm -rf $$packagedir; fi; \ printf "\nStarting to install python package with maneage pybuild rule: $(4)\n ..."; \ - if ! $(1) $(tdir)/$$tarball; then \ + if ! $(1) $(tdir)/$$tarball --no-same-owner --no-same-permissions; then \ echo; echo "Tar error"; exit 1; \ fi; \ cd $$packagedir; \ -- cgit v1.2.1