From ab811d0952ac93ce608c81ab2cc44d67c7b02dbe Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 10 Jun 2022 01:43:30 +0200 Subject: Housekeeping: some portability issues fixed; four software updates Until now, there were several portability issues in Maneage: 1. Maneage would crash on older operating systems (checked on Debian 6), where Wget didn't have the '--no-use-server-timestamps'. 2. On a Linux kernel 2.6.32 (of the same Debian 6 above) some features in 'util-linux' (like 'swapon' or 'libmount') wouldn't build and wouldn't let 'util-linux' complete. These features need root permissions to be useful, so the wouldn't be used in Maneage any way! But they wouldn't let Maneage get built 3. The './project shell' command would still read the host's '~/.bashrc', letting the host environment leak-in to Maneage's interactive shell. 4. The building of Flex 2.64 wouldn't complete due to a segmentation fault an Ubuntu, but NetPBM (which depends on Flex) would crash with a wrong usage of 'yyunput'. This had actually caused a non-update to Flex in a previous Maneage software update. 5. The update Astrometry.net would assume SExtractor's executable name is 'source-extractor'; causing a crash in usage. This forced the users to manually create a 'source-extractor' symbolic link in the '.local/bin' directory. 6. The 'reproduce/software/shell/tarball-prepare.sh' script (that is used for making Maneage-standard tarballs) wouldn't accept option values with an '=' between the option name and value! It also didnt' print sufficiently informative messages and errors (for example it would say "skipping ..." (making the user think there is a problem!), but it was actually that the file already existed! 7. The 'reproduce/analysis/make/prepare.mk' and 'reproduce/analysis/make/verify.mk' Makefiles that needed to reject some of the 'makesrc' sub-Makefiles would simply substitute their names with nothing. But this would cause problems when the name is part of the name of another sub-Makefile. 8. On the Debian 6 system mentioned above the raw 'df' command's output wasn't in the expected format; so Maneage would fail to properly detect the free space in the disk. With these commit, all the issues above have been solved: for 1, A check has been added to avoid using that option. For 2, those 'util-linux' features have been disabled. For 3, the '--norc' and '--noprofile' options have beed added to the call to Bash. For 4, see below. For 5, the symbolic link is now automatically made with SExtractor. For 6, the option reading components of that script have been fully re-written and more robust sanity checks are also added, with more informative warnings. For 7, the 'subst' function of Make was replaced with 'filter-out' and this fixed the problem. For 8, 'df' is called with the '-P' option so it has a unified format in all versions. For 4, the versions of 'flex' and 'netpbm' have been updated. Since they were the dependency of 'astrometrynet', that has also been updated. In the process, we discovered that 'lzip' has a new version which claims to be faster, so that is also updated. lzip 1.22 --> 1.23 astrometrynet 0.85 --> 0.89 flex 2.6.4 --> 2.6.4-410-74a89fd netpbm 10.73.39 --> 10.73.39 NetPBM needed some manual manipulation in its source (to remove the extra line), so the necessary steps have been added to its build recipe in 'reproduce/software/make/high-level.mk'. --- reproduce/analysis/config/INPUTS.conf | 2 +- reproduce/analysis/make/initialize.mk | 2 +- reproduce/analysis/make/prepare.mk | 6 +- reproduce/analysis/make/verify.mk | 2 +- reproduce/software/config/checksums.conf | 8 +- reproduce/software/config/versions.conf | 8 +- reproduce/software/make/high-level.mk | 32 ++++++++ reproduce/software/shell/configure.sh | 17 +++- reproduce/software/shell/tarball-prepare.sh | 116 ++++++++++++++++++---------- 9 files changed, 139 insertions(+), 54 deletions(-) (limited to 'reproduce') diff --git a/reproduce/analysis/config/INPUTS.conf b/reproduce/analysis/config/INPUTS.conf index 5a58758..3958153 100644 --- a/reproduce/analysis/config/INPUTS.conf +++ b/reproduce/analysis/config/INPUTS.conf @@ -30,7 +30,7 @@ # # The naming convension is critical for the input files to be properly # imported into the project. In the patterns below, the '%' is the full -# file name (including its prefix): for example in the demo input of this +# file name (including its suffix): for example in the demo input of this # file in the 'maneage' branch, we have 'INPUT-wfpc2.fits-sha256': # therefore, the input file (within the project's '$(indir)') is called # 'wfpc2.fits'. This allows you to simply set '$(indir)/wfpc2.fits' as the diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 4e8ee68..6893962 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -138,10 +138,10 @@ curdir := $(shell echo $$(pwd)) # avoid conflicts with existing TeX Live solutions. Later (in 'paper.mk'), # we are also going to overwrite 'TEXINPUTS' just before 'pdflatex'. .ONESHELL: -.SHELLFLAGS = -ec export TEXINPUTS := export CCACHE_DISABLE := 1 export PATH := $(installdir)/bin +.SHELLFLAGS = --noprofile --norc -ec export LDFLAGS := -L$(installdir)/lib export SHELL := $(installdir)/bin/bash export CPPFLAGS := -I$(installdir)/include diff --git a/reproduce/analysis/make/prepare.mk b/reproduce/analysis/make/prepare.mk index ecb6842..aed2b5f 100644 --- a/reproduce/analysis/make/prepare.mk +++ b/reproduce/analysis/make/prepare.mk @@ -1,4 +1,4 @@ -# Basic preparations, called by './project prepare'. +# Basic preparations, called by './project make'. # # Copyright (C) 2019-2022 Mohammad Akhlaghi # @@ -22,7 +22,9 @@ # Final-target # # Without this file, './project make' won't work. -prepare-dep = $(subst prepare, ,$(makesrc)) +# +# We need to remove the 'prepare' word from the list of 'makesrc'. +prepare-dep = $(filter-out prepare, ,$(makesrc)) $(bsdir)/preparation-done.mk: \ $(foreach s, $(prepare-dep), $(mtexdir)/$(s).tex) diff --git a/reproduce/analysis/make/verify.mk b/reproduce/analysis/make/verify.mk index d3f3282..a645713 100644 --- a/reproduce/analysis/make/verify.mk +++ b/reproduce/analysis/make/verify.mk @@ -110,7 +110,7 @@ verify-txt-no-comments-no-space = \ # essentially the same as 'verify-dep', but it has removed # the 'initialize' step (which is information about the # pipeline, not the results). -verify-dep = $(subst verify,,$(subst paper,,$(makesrc))) +verify-dep = $(filter-out verify paper, $(makesrc)) verify-check = $(subst initialize,,$(verify-dep)) $(mtexdir)/verify.tex: $(foreach s, $(verify-dep), $(mtexdir)/$(s).tex) diff --git a/reproduce/software/config/checksums.conf b/reproduce/software/config/checksums.conf index 7450e6d..736a5e5 100644 --- a/reproduce/software/config/checksums.conf +++ b/reproduce/software/config/checksums.conf @@ -39,7 +39,7 @@ libiconv-checksum = 396be5305f899619596b72ccc8a192c5fcc8e12163fd23376833c957ee44 libtool-checksum = 379a16baf2123d3cdd28cf8e35e4bb1e0326ae7f431c42ab2a3bd42ccac99a622e5ff0a68703b81614c147bc02178612a5cecb105b8943937a05c0b3ec91fea0 libunistring-checksum = f1e92717fc043c2d7c454e44700559b0bd75457fdf3da5e803d9d58d8ebdff261bc73f380f6350bb04ce04071687c0317d09b4e8615003187948be1cd4d0bab3 libxml2-checksum = 0fdc16aa7536adfe1deaadd19cdc20101dd3b081bd0a2e59d8ecc2965b2ef4c30d2e2447ab5b0cdb861c57ddb89ace1bd4ce120d9436b751b37724a90981ba8c -lzip-checksum = 8d063a5f0609caa4d568f7bad4c67fcdee70a16c37e67b9d3cccbae53f09c14fa8590816aeb37a3fe0bc71a642c55e9fb4b371c0f68585be9564af22584d5c99 +lzip-checksum = 9b18ab9b7ef9a7d1c2da06b48fd018399e94405a5ada457885b778c59caa3faed0e73e1da398516621dc9e721587785b57ef1f8a12c1b0e116ebce5129f0f24d m4-checksum = 7f8845f99e64d6a45859b9d80b03352a5526b3de0311ca4d6dd6850e504d26dfc90cd21d1640b10382f786213f8fdf20183bff424b3c41ea11432315993ab829 make-checksum = ddf0fdcb9ee1b182ef294c5da70c1275288c99bef60e63a25c0abed2ddd44aba1770be4aab1db8cac81e5f624576f2127c5d825a1824e1c7a49df4f16445526b mpc-checksum = 85e044749c2a001a70af2c11fc4bac58709ca7df11bf6da3b03cca852ef336eb37c4fb1dafdffde2d13df35caa7c296a7cba19a4625b82dc3cccc5066ac95774 @@ -77,7 +77,7 @@ zlib-checksum = 97493906504451830655b300b4144deeeb9f41bdb9b3f3a3d5635c2b16bf90d5 apachelog4cxx-checksum = 2c4b907a47ae00c38ae9bcd5d215be5115bfb677646cac50538eb1c75824f594aa997dafc9a7828dc6bfd5f22f4c486f13e4210b9a1d8a9c98098d96384ef9c3 apr-checksum = d0ddf8b10dd8d3a831b94e541d387414cc4d507ad48ff752d3274fb808afe6628d7951364aa3e0f71f373c78d0081411f6a0595d01c62bfb0a54bbc86002c82a apr-util-checksum = 6589948e3f9daf4ecd700bdbd2053fcb83005cec9d339278dda067996ba696e4a947116f066cee03214458aa15e0cb6f0df0103e7bc6f8f32327722eb7265f35 -astrometrynet-checksum = 066d8f6820d6bd9cdeef9334d150aef15a934c43944a96ac91b7184eff6c59876bcfed4ca221650b26b12b5e839228a8a7cb4ac3c72e599717d8cedfe6c77510 +astrometrynet-checksum = 9d07cf03f6408a19164914fa7b389aaedec8ae27ec9dcdef75b5a0db6a71000a15cdc1d9a8d9b2256c5299a890f499792407cf362ca0bed27f4f182f4a93b789 atlas-checksum = 13634a63c686800bef8affcf4e8f5f89d08f485fc557eb1b9ba51a405926014e659e439268f655984bb7f59ffe497d594cd7a607103826f1046325d89796e1b0 autoconf-checksum = daf3d17178d8c2d0f872a50217ba4d8a1481bad7e70ee5a58c5599717a01209cbd58e6149c72f6c4829b96dcf602bcfb828d47e32a9cfe3e5710c45f4f1736e7 automake-checksum = 728284b89ab4607b5de5db6b8bf5ad8730cbd8ac094f583b93bb92c59dee35e021e823cddd412b0507b18a12cbbbfbec0fb0447c23d0afd167769641fe0755e5 @@ -91,7 +91,7 @@ eigen-checksum = 39c1944e0daca50bb01e929edc98e2bfc234accb30ca019dcccfc7b02db5cc8 emacs-checksum = 91313dced8cd74f9e4a996eef42806aad00fb2576fc77b3694bb7b670624b6a1551ff49c2a3326dbeeba916e5732ca5975b5dc40b3b063f6e73a22839009abe9 expat-checksum = 526f2ec03978f853ff76a911b5a168f58a7eac83b8405536afe2a6acf4611ebcb96acbc5df809fd18e4edeb7ca5059a3ce56dcbd12cb33eecf742c6361d83d19 fftw-checksum = 28bd2f620399a415181027d30d7ea193aa487c7a277c9943d0051488908fd87e2731de0dfc3bebd22a6121d1deaa46037be8296a8a9cdb711f9fde4510c3d368 -flex-checksum = f15c5a1efc78d3de36572ece8ec1694f368ecf13a01ac3851be142bd63e16e9e5c62d10ce9ac586aabcbed533a47d0d97f7c9f467b729d5efa3316c7788230e2 +flex-checksum = a18e0dcdd21bd51779fd70e5efc2d77fa7b2a56d4d0e929d8abb437f86d7fcfa2cd09b8268185698200866dac8bec4444c5a7cff3140c515eaa49fc77c5aea66 freetype-checksum = b3c7289cda3f89fa6e51143f2d1f1c82fa34808d0caa9262e7be5498c8f0fdb14e88d31ec9928e3add71f03208e40adb2fa94c2a4b4c6b7bfb81daf64b1578cb gdb-checksum = 4afd6660d2bbc4c48ce726062fb42217bdaa5974774407358b2624036a2ace48ed853746e4d6b83b9a409cd8fefc622b0727c79da8826e28e3d1444fa3a9d148 ghostscript-checksum = 6dd1713bf40648ba6ed234a6c68582f87fa4075a191cf9af4f4ffa1854b94439c5ee4748e27ade3e8e0b7168fee1df3e9a9f3a32f70ac1ab9dd68e3878358ff2 @@ -120,7 +120,7 @@ libtiff-checksum = 628e9e6b3ff5cdf661ac4c9cbee7fc10d5555554d39e447d9a0c4e99b9a81 libtirpc-checksum = bcb6b5c062c1301aa1246ec93ae0a5c1d221b8421126d020863517cb814b43ed038fb6c0c2faf4e68ff133b69abefe4f4d42bfc870671da6c27ca941a30b155a metastore-checksum = ac1230686535a652e95024abaf6c5585bdab36f4e092bee5fa4deff2a913cd60f3a6bd8020c6887ccab97f0da3a284a0d4619fad5464a269e7b2040d6f7e6aaf missfits-checksum = 32727f5eb30573a1cedacb8900e2536867e4815059eee32e64e3db65be9291b8a91b9f45b2c9f3cf6fc2a8cc448012ea3d502bdd9dee516008e17d5086aee795 -netpbm-checksum = 051c05aa080e6563483053c16bc416b0e1368d6d7b130dec12a01017245a99252f80ddf7ae7322a4f8e9c90d833fc90d94324585659605d8dbb766d5fa2238c5 +netpbm-checksum = aef81d2e46850fae1fb34a38fa9f634be3c47c7c4e80f300c61aadd5532bcbccdec636e6c701e54caca67ac2d6ce3a7b6a9f090deb00cd4db718439986f89d8e openblas-checksum = 363588495451f872d4332a1c5e5ff4422202935641f490515e6f8144d3e1838ef737f39e77aaca396d23f9011b981961aa503e5d5935260231e31bb2280f890d openmpi-checksum = 88c73fd708dd5fe3a893d5517425a8a4c3e252fae4715df9a9b0f2311abe94cf5f71be92f153820fbaeb32c7d0ac0175d5165bdadcadc731427309102182c1bd openssh-checksum = e280fa2d56f550efd37c5d2477670326261aa8b94d991f9eb17aad90e0c6c9c939efa90fe87d33260d0f709485cb05c379f0fd1bd44fc0d5190298b6398c9982 diff --git a/reproduce/software/config/versions.conf b/reproduce/software/config/versions.conf index 34d074c..04de56f 100644 --- a/reproduce/software/config/versions.conf +++ b/reproduce/software/config/versions.conf @@ -39,7 +39,7 @@ libiconv-version = 1.16 libtool-version = 2.4.6 libunistring-version = 1.0 libxml2-version = 2.9.12 -lzip-version = 1.22 +lzip-version = 1.23 m4-version = 1.4.19 make-version = 4.3 mpc-version = 1.2.1 @@ -95,7 +95,7 @@ certpem-version = 2021-10-15 apachelog4cxx-version = 0.12.1 apr-version = 1.7.0 apr-util-version = 1.6.1 -astrometrynet-version = 0.85 +astrometrynet-version = 0.89 atlas-version = 3.10.3 autoconf-version = 2.71 automake-version = 1.16.5 @@ -109,7 +109,7 @@ eigen-version = 3.4.0 emacs-version = 28.1 expat-version = 2.4.1 fftw-version = 3.3.10 -flex-version = 2.6.4 +flex-version = 2.6.4-410-74a89fd freetype-version = 2.11.0 gdb-version = 11.1 ghostscript-version = 9.55.0 @@ -179,7 +179,7 @@ wcslib-version = 7.7 # Netpbm's questions in the configure steps maybe change with different or # new versions. -netpbm-version = 10.73.38 +netpbm-version = 10.73.39 # Minizip is installed with the same 'zlib' tarball, and they have the same # version. diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 34c829b..62004a8 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1293,16 +1293,34 @@ $(ibidir)/netpbm-$(netpbm-version): \ $(ibidir)/libxml2-$(libxml2-version) tarball=netpbm-$(netpbm-version).tar.lz $(call import-source, $(netpbm-url), $(netpbm-checksum)) + +# Answers to the configuration questions. if [ x$(on_mac_os) = xyes ]; then answers='\n\n$(ildir)\n\n\n\n\n\n$(ildir)/include\n\n$(ildir)/include\n\n$(ildir)/include\nnone\n\n' else answers='\n\n\n\n\n\n\n\n\n\n\n\n\nnone\n\n\n' fi + +# Go into the temporary directory and unpack the tarball. cd $(ddir) unpackdir=netpbm-$(netpbm-version) rm -rf $$unpackdir tar -xf $(tdir)/$$tarball cd $$unpackdir + +# As of NetPBM 10.73.39 and Flex 2.6.4-410-74a89fd (commit 74a89fd in +# Flex's Git that is 410 commits after version 2.6.4), there is the +# following line: 'if (0) yyunput(0, NULL);'. It will cause a crash +# and is just to avoid compiler warnings! So we are setting the +# 'yyunput(0, NULL);' to the redundant 'yyunput(0, NULL);' to let the +# compilation finish! + awk '$$1=="if" && $$2=="(0)"{inif=1} \ + {if(inif==1 && $$1=="yyunput(0,") \ + print "{int a=1;}"; else print $$0}' \ + converter/pbm/thinkjettopbm.l > thinkjettopbm.tmp + mv thinkjettopbm.tmp converter/pbm/thinkjettopbm.l + +# Pass the answers to the configure script then build and install it. printf "$$answers" | ./configure make rm -rf $(ddir)/$$unpackdir/install @@ -1396,6 +1414,7 @@ $(ibidir)/sextractor-$(sextractor-version): \ --with-openblas-libdir=$(ildir) \ --with-openblas-incdir=$(idir)/include) ln -fs $(ibdir)/sex $(ibdir)/sextractor + ln -fs $(ibdir)/sex $(ibdir)/source-extractor cp $(dtexdir)/sextractor.tex $(ictdir)/ echo "SExtractor $(sextractor-version) \citep{sextractor}" > $@ @@ -1431,6 +1450,14 @@ $(ibidir)/swig-$(swig-version): # --disable-mount # --disable-wall # --disable-su +# Because they fail on older kernels (tested on Linux 2.6.32) +# and they need root (to actually use; so are not relevant to +# Maneage): +# --disable-swapon +# --disable-unshare +# --disable-libmount +# --disable-mountpoint +# --enable-libmount-support-mtab # # NOTE ON INSTALLATION DIRECTORY: Util-linux libraries are relatively # low-level and may cause conflicts with system libraries (especilly when @@ -1459,9 +1486,14 @@ $(ibidir)/util-linux-$(util-linux-version): | $(idircustom) cd util-linux-$(util-linux-version) ./configure --prefix=$(idircustom)/util-linux \ --disable-dependency-tracking \ + --enable-libmount-support-mtab \ --disable-silent-rules \ + --disable-mountpoint \ + --disable-libmount \ + --disable-unshare \ --without-systemd \ --enable-libuuid \ + --disable-swapon \ --disable-mount \ --disable-ipcrm \ --disable-ipcs \ diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 07381e6..7f5f9c4 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -191,7 +191,7 @@ free_space_warning() { fs_threshold=$1 fs_destpath="$2" - return $(df "$fs_destpath" \ + return $(df -P "$fs_destpath" \ | awk 'FNR==2 {if($4>'$fs_threshold') print 1; \ else print 0; }') } @@ -814,9 +814,22 @@ if [ $rewritepconfig = yes ]; then # 'which' isn't in POSIX, so we are using 'command -v' instead. name=$(command -v wget) + # See if the host wget has the '--no-use-server-timestamps' option + # (for example wget 1.12 doesn't have it). If not, we'll have to + # remove it. This won't affect the analysis of Maneage in anyway, + # its just to avoid re-downloading if the server timestamps are + # bad; at the worst case, it will just cause a re-download of an + # input software source code (for data inputs, we will use our own + # wget that has this option). + tsname="no-use-server-timestamps" + tscheck=$(wget --help | grep $tsname || true) + if [ x"$tscheck" = x ]; then wgetts="" + else wgetts="--$tsname"; + fi + # By default Wget keeps the remote file's timestamp, so we'll have # to disable it manually. - downloader="$name --no-use-server-timestamps -O"; + downloader="$name $wgetts -O"; elif type curl > /dev/null 2>/dev/null; then name=$(command -v curl) diff --git a/reproduce/software/shell/tarball-prepare.sh b/reproduce/software/shell/tarball-prepare.sh index ccc9318..69e6afa 100755 --- a/reproduce/software/shell/tarball-prepare.sh +++ b/reproduce/software/shell/tarball-prepare.sh @@ -31,25 +31,21 @@ odir= idir= quiet= basedir=$PWD +scriptname=$0 # The --help output print_help() { cat <