From 3c9bf3aff30f02c7d31bd86f36c4db2520f8ffa4 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 3 May 2024 13:07:41 +0200 Subject: Configuration: no dependency on /bin/sh and useful run-time options SUMMARY: no change necessary in your project, this commit only involves changes in how already-existing software are built. Some handy options have also been added to the top-level project script and the copyright years have been updated. Until now, if the host's '/bin/sh' had conflicts with the Maneage environment, the configuration of Maneage would crash as soon as we entered the building of high-level software. The full scenario is described in the comments of the newly added 'reproduce/software/shell/prep-source.sh'. This is most relevant when building older Maneage'd project in newer environments. With this commit, the following changes were made to avoid the problem above: - Maneage edits the source code of all installed software to replace '/bin/sh' with Maneage's own shell before the programs are built. Through this, we were able to solve the problem described above. - The portable '#!/usr/bin/env sh' shebangs are now used at the start of the scripts that run during configure time so it uses the first available shell that it finds in its PATH (the system's before Dash is built), then Dash, and after Dash is built, Bash. - For TeXLive, since we don't install it from source, it was necessary to add the libraries necessary for the local '/bin/sh' in LD_LIBRARY_PATH. Some high-level options have been added to the './project' script to simplify certain operations: --keep-going: do not stop upon the first crash, but keep going on to build targets until all build-able targets have been built. This is very useful for debugging large pipelines and allows you to isolate the problematic part of your project. --highlight-all: equivalent to calling both '--highlight-new' & '--highlight-notes'. --- reproduce/software/make/high-level.mk | 90 +++++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 19 deletions(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 3f9c40c..2274c45 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -12,8 +12,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2018-2023 Mohammad Akhlaghi -# Copyright (C) 2019-2023 Raul Infante-Sainz +# Copyright (C) 2018-2025 Mohammad Akhlaghi +# Copyright (C) 2019-2025 Raul Infante-Sainz # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -64,6 +64,12 @@ ipydir = $(BDIR)/software/installed/version-info/python ircrandir = $(BDIR)/software/installed/version-info/r-cran ilibrcrandir = $(BDIR)/software/installed/lib/R/library +# Special files. +makewshell = $(ibdir)/make-with-shell + + + + # Targets to build. ifeq ($(strip $(all_highlevel)),1) @@ -232,7 +238,7 @@ $(idircustom):; mkdir $@ # Afer putting everything together, we use the first server as the # reference for all software if their '-url' variable isn't defined (in # 'reproduce/software/config/urls.conf'). -downloadwrapper = ./reproduce/analysis/bash/download-multi-try +downloadwrapper = ./reproduce/analysis/bash/download-multi-try.sh maneage_backup_urls := $(shell awk '!/^#/{printf "%s ", $$1}' \ reproduce/software/config/servers-backup.conf) backupservers_all = $(user_backup_urls) $(maneage_backup_urls) @@ -339,6 +345,7 @@ $(ibidir)/atlas-$(atlas-version): cd $(ddir) tar -xf $(tdir)/atlas-$(atlas-version).tar.lz cd ATLAS + $(shsrcdir)/prep-source.sh $(ibdir) rm -rf build mkdir build cd build @@ -395,6 +402,7 @@ $(ibidir)/boost-$(boost-version): \ cd $(ddir) tar -xf $(tdir)/$$tarball cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) ./bootstrap.sh --prefix=$(idir) --with-libraries=all \ --with-python=python3 echo "using mpi ;" > project-config.jam @@ -417,9 +425,7 @@ $(ibidir)/cfitsio-$(cfitsio-version): topdir=$(pwd); cd $(ddir); tar -xf $(tdir)/$$tarball customtar=cfitsio-$(cfitsio-version)-custom.tar.gz cd cfitsio-$(cfitsio-version) - sed configure -e's|@rpath|$(ildir)|g' > configure_tmp - mv configure_tmp configure - chmod +x configure + sed -i -e's|@rpath|$(ildir)|g' configure cd .. tar cf $$customtar cfitsio-$(cfitsio-version) cd $$topdir @@ -518,6 +524,12 @@ $(ibidir)/fftw-$(fftw-version): echo "FFTW $(fftw-version) \citep{fftw}" > $@ $(ibidir)/freetype-$(freetype-version): $(ibidir)/libpng-$(libpng-version) +# As of version 2.13.2, FreeType doesn't account for the 'SHELL' +# environment variable. The issue has been reported to the +# developers. But until future versions, the work-around was +# discoverd to be setting the 'GNUMAKE' environment variable so it +# includes 'SHELL'. + export GNUMAKE="$(makewshell)" tarball=freetype-$(freetype-version).tar.lz $(call import-source, $(freetype-url), $(freetype-checksum)) $(call gbuild, freetype-$(freetype-version), static) @@ -578,7 +590,9 @@ $(ibidir)/healpix-$(healpix-version): $(healpix-python-dep) \ rm -rf $(ddir)/Healpix_$(healpix-version) topdir=$(pwd); cd $(ddir); tar -xf $(tdir)/$$tarball - cd Healpix_$(healpix-version)/src/C/autotools/ + cd Healpix_$(healpix-version) + $(shsrcdir)/prep-source.sh $(ibdir) + cd src/C/autotools autoreconf --install ./configure --prefix=$(idir) make V=1 -j$(numthreads) SHELL=$(ibdir)/bash @@ -670,6 +684,7 @@ $(ibidir)/libpaper-$(libpaper-version): \ tar -xf $(tdir)/$$tarball unpackdir=libpaper-$(libpaper-version) cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) autoreconf -fi ./configure --prefix=$(idir) --sysconfdir=$(idir)/etc \ --disable-static @@ -799,6 +814,7 @@ $(ibidir)/openblas-$(openblas-version): cd $(ddir) tar -xf $(tdir)/$$tarball cd OpenBLAS-$(openblas-version) + $(shsrcdir)/prep-source.sh $(ibdir) make -j$(numthreads) make PREFIX=$(idir) install cd .. @@ -1007,6 +1023,7 @@ $(ibidir)/astrometrynet-$(astrometrynet-version): \ rm -rf astrometry.net-$(astrometrynet-version) tar -xf $(tdir)/$$tarball cd astrometry.net-$(astrometrynet-version) + $(shsrcdir)/prep-source.sh $(ibdir) sed -e 's|cat /proc/cpuinfo|echo "Ignoring CPU info"|' \ -e 's|-free|echo "Ignoring RAM info"|' Makefile > Makefile.tmp mv Makefile.tmp Makefile @@ -1045,6 +1062,7 @@ $(ibidir)/cdsclient-$(cdsclient-version): cd $(ddir) tar -xf $(tdir)/$$tarball cd cdsclient-$(cdsclient-version) + $(shsrcdir)/prep-source.sh $(ibdir) touch * ./configure --prefix=$(idir) make @@ -1061,25 +1079,29 @@ $(ibidir)/cmake-$(cmake-version): tarball=cmake-$(cmake-version).tar.lz $(call import-source, $(cmake-url), $(cmake-checksum)) -# After searching in 'bootstrap', I couldn't find 'LIBS', only -# 'LDFLAGS'. So the extra libraries are being added to 'LDFLAGS', not -# 'LIBS'. -# # On Mac systems, the build complains about 'clang' specific # features, so we can't use our own GCC build here. if [ x$(on_mac_os) = xyes ]; then export CC=clang export CXX=clang++ fi + +# CMake wants a single executable for 'MAKE', so we can't use 'make +# SHELL=$(SHELL) and we have defined this script. + export MAKE="$(makewshell)" + +# Go into the unpacked directory and build CMake. cd $(ddir) rm -rf cmake-$(cmake-version) tar -xf $(tdir)/$$tarball cd cmake-$(cmake-version) + $(shsrcdir)/prep-source.sh $(ibdir) ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ --system-bzip2 --system-liblzma --no-qt-gui \ --parallel=$(numthreads) - make -j$(numthreads) LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 - make install + $(makewshell) VERBOSE=1 LIBS="$$LIBS -lssl -lcrypto -lz" \ + -j$(numthreads) + $(makewshell) install cd .. rm -rf cmake-$(cmake-version) echo "CMake $(cmake-version)" > $@ @@ -1129,6 +1151,7 @@ $(ibidir)/ghostscript-$(ghostscript-version): \ cd $(ddir) tar -xf $(tdir)/$$tarball cd ghostscript-$(ghostscript-version) + $(shsrcdir)/prep-source.sh $(ibdir) ./configure --prefix=$(idir) \ --disable-cups \ --enable-dynamic \ @@ -1170,7 +1193,7 @@ $(ibidir)/gnuastro-$(gnuastro-version): \ $(call gbuild, gnuastro-$(gnuastro-version), static, , \ -j$(numthreads)) cp $(dtexdir)/gnuastro.tex $(ictdir)/ - echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro,akhlaghi19}" > $@ + echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ $(ibidir)/icu-$(icu-version): $(ibidir)/python-$(python-version) @@ -1188,7 +1211,9 @@ $(ibidir)/icu-$(icu-version): $(ibidir)/python-$(python-version) cd $(ddir) tar -xf $(tdir)/$$tarball unpackdir=icu-$(icu-version) - cd $$unpackdir/icu4c/source + cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) + cd icu4c/source ./configure --enable-static --prefix=$(idir) make -j$(numthreads) V=1 make install @@ -1255,6 +1280,7 @@ $(ibidir)/imfit-$(imfit-version): \ rm -rf $$unpackdir tar -xf $(tdir)/$$tarball cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) sed -i 's|/usr/local|$(idir)|g' SConstruct sed -i 's|/usr/include|$(idir)/include|g' SConstruct sed -i 's|.append(|.insert(0,|g' SConstruct @@ -1302,6 +1328,7 @@ $(ibidir)/minizip-$(minizip-version): $(ibidir)/automake-$(automake-version) mkdir $$unpackdir tar -xf $(tdir)/$$tarball -C$$unpackdir --strip-components=1 cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) ./configure --prefix=$(idir) make cd contrib/minizip @@ -1363,6 +1390,7 @@ $(ibidir)/netpbm-$(netpbm-version): \ rm -rf $$unpackdir tar -xf $(tdir)/$$tarball cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) # As of NetPBM 10.73.39 and Flex 2.6.4-410-74a89fd (commit 74a89fd in # Flex's Git that is 410 commits after version 2.6.4), there is the @@ -1482,6 +1510,7 @@ $(ibidir)/scons-$(scons-version): $(ibidir)/python-$(python-version) rm -rf $$unpackdir tar -xf $(tdir)/$$tarball cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) # Unfortuantely SCons hard-codes its search PATH in its source (to # use POSIX operating system defaults)! So the only way to modify it @@ -1625,8 +1654,9 @@ $(ibidir)/util-linux-$(util-linux-version): | $(idircustom) cd $(ddir) tar -xf $(tdir)/$$tarball cd util-linux-$(util-linux-version) + $(shsrcdir)/prep-source.sh $(ibdir) -# If a patch exists for the current version, apply it. +# If a patch is necessary, apply it. if [ -f $(patchdir)/util-linux-$(util-linux-version)-macos.patch ]; then cp $(patchdir)/util-linux-$(util-linux-version)-macos.patch \ util-linux-$(util-linux-version)-macos.patch @@ -1727,6 +1757,7 @@ $(ibidir)/vim-$(vim-version): tar -xf $(tdir)/$$tarball unpackdir=vim-$(vim-version) cd $(ddir)/$$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) ./configure --prefix=$(idir) \ --disable-canberra \ --enable-multibyte \ @@ -1795,10 +1826,16 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf rm -rf install-tl-* tar -xf $(tdir)/install-tl-unx.tar.gz cd install-tl-* + $(shsrcdir)/prep-source.sh $(ibdir) sed -e's|@installdir[@]|$(idir)|g' \ "$$topdir"/reproduce/software/config/texlive.conf \ > texlive.conf +# We do not build TeXLive from source and for its installation it +# downloads components from the web internally; and those components +# can use '/bin/sh' (which will need '$(sys_library_sh_path)'). + export LD_LIBRARY_PATH="$(sys_library_sh_path):$$LD_LIBRARY_PATH" + # TeX Live's installation may fail due to any reason. But TeX Live is # optional (only necessary for building the final PDF). So we don't # want the configure script to fail if it can't run. Possible error @@ -1867,13 +1904,16 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf tarballurl=$$url/install-tl-unx.tar.gz touch $(lockdir)/download downloader="wget --no-use-server-timestamps -O" - if $(downloadwrapper) "$$downloader" $(lockdir)/download \ - $$tarballurl "$(tdir)/install-tl-unx.tar.gz" \ - "$(backupservers)"; then + if $(downloadwrapper) "$$downloader" \ + $(lockdir)/download \ + $$tarballurl \ + "$(tdir)/install-tl-unx.tar.gz" \ + "$(backupservers)"; then cd $(ddir) rm -rf install-tl-* tar -xf $(tdir)/install-tl-unx.tar.gz cd install-tl-* + $(shsrcdir)/prep-source.sh $(ibdir) sed -e's|@installdir[@]|$(idir)|g' \ $$topdir/reproduce/software/config/texlive.conf \ > texlive.conf @@ -1917,6 +1957,11 @@ $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ echo "" > $@ else +# We do not build TeXLive from source and for its installation it +# downloads components from the web internally; and those +# components can use '/bin/sh' (which needs 'sys_library_sh_path'). + export LD_LIBRARY_PATH="$(sys_library_sh_path):$$LD_LIBRARY_PATH" + # To update itself, tlmgr needs a backup directory. backupdir=$(idir)/texlive/backups mkdir -p $$backupdir @@ -1940,6 +1985,13 @@ $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ # directory so we don't have to modify 'PATH'. ln -fs $(idir)/texlive/maneage/bin/*/* $(ibdir)/ +# Correct any reference to '/bin/sh' within the installed LaTeX +# files (this is because we do no yet install LaTeX from source): + cdir=$$(pwd) + cd $(idir)/texlive + $(shsrcdir)/prep-source.sh $(ibdir) + cd $$cdir + # Get all the necessary versions. texlive=$$(pdflatex --version \ | awk 'NR==1' \ -- cgit v1.2.1