diff options
Diffstat (limited to 'reproduce')
| -rw-r--r-- | reproduce/analysis/config/INPUTS.conf | 2 | ||||
| -rw-r--r-- | reproduce/analysis/make/initialize.mk | 2 | ||||
| -rw-r--r-- | reproduce/analysis/make/prepare.mk | 6 | ||||
| -rw-r--r-- | reproduce/analysis/make/verify.mk | 2 | ||||
| -rw-r--r-- | reproduce/software/config/checksums.conf | 8 | ||||
| -rw-r--r-- | reproduce/software/config/versions.conf | 8 | ||||
| -rw-r--r-- | reproduce/software/make/high-level.mk | 32 | ||||
| -rwxr-xr-x | reproduce/software/shell/configure.sh | 17 | ||||
| -rwxr-xr-x | reproduce/software/shell/tarball-prepare.sh | 116 |
9 files changed, 139 insertions, 54 deletions
diff --git a/reproduce/analysis/config/INPUTS.conf b/reproduce/analysis/config/INPUTS.conf index 5a58758..3958153 100644 --- a/reproduce/analysis/config/INPUTS.conf +++ b/reproduce/analysis/config/INPUTS.conf @@ -30,7 +30,7 @@ # # The naming convension is critical for the input files to be properly # imported into the project. In the patterns below, the '%' is the full -# file name (including its prefix): for example in the demo input of this +# file name (including its suffix): for example in the demo input of this # file in the 'maneage' branch, we have 'INPUT-wfpc2.fits-sha256': # therefore, the input file (within the project's '$(indir)') is called # 'wfpc2.fits'. This allows you to simply set '$(indir)/wfpc2.fits' as the diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 4e8ee68..6893962 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -138,10 +138,10 @@ curdir := $(shell echo $$(pwd)) # avoid conflicts with existing TeX Live solutions. Later (in 'paper.mk'), # we are also going to overwrite 'TEXINPUTS' just before 'pdflatex'. .ONESHELL: -.SHELLFLAGS = -ec export TEXINPUTS := export CCACHE_DISABLE := 1 export PATH := $(installdir)/bin +.SHELLFLAGS = --noprofile --norc -ec export LDFLAGS := -L$(installdir)/lib export SHELL := $(installdir)/bin/bash export CPPFLAGS := -I$(installdir)/include diff --git a/reproduce/analysis/make/prepare.mk b/reproduce/analysis/make/prepare.mk index ecb6842..aed2b5f 100644 --- a/reproduce/analysis/make/prepare.mk +++ b/reproduce/analysis/make/prepare.mk @@ -1,4 +1,4 @@ -# Basic preparations, called by './project prepare'. +# Basic preparations, called by './project make'. # # Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> # @@ -22,7 +22,9 @@ # Final-target # # Without this file, './project make' won't work. -prepare-dep = $(subst prepare, ,$(makesrc)) +# +# We need to remove the 'prepare' word from the list of 'makesrc'. +prepare-dep = $(filter-out prepare, ,$(makesrc)) $(bsdir)/preparation-done.mk: \ $(foreach s, $(prepare-dep), $(mtexdir)/$(s).tex) diff --git a/reproduce/analysis/make/verify.mk b/reproduce/analysis/make/verify.mk index d3f3282..a645713 100644 --- a/reproduce/analysis/make/verify.mk +++ b/reproduce/analysis/make/verify.mk @@ -110,7 +110,7 @@ verify-txt-no-comments-no-space = \ # essentially the same as 'verify-dep', but it has removed # the 'initialize' step (which is information about the # pipeline, not the results). -verify-dep = $(subst verify,,$(subst paper,,$(makesrc))) +verify-dep = $(filter-out verify paper, $(makesrc)) verify-check = $(subst initialize,,$(verify-dep)) $(mtexdir)/verify.tex: $(foreach s, $(verify-dep), $(mtexdir)/$(s).tex) diff --git a/reproduce/software/config/checksums.conf b/reproduce/software/config/checksums.conf index 7450e6d..736a5e5 100644 --- a/reproduce/software/config/checksums.conf +++ b/reproduce/software/config/checksums.conf @@ -39,7 +39,7 @@ libiconv-checksum = 396be5305f899619596b72ccc8a192c5fcc8e12163fd23376833c957ee44 libtool-checksum = 379a16baf2123d3cdd28cf8e35e4bb1e0326ae7f431c42ab2a3bd42ccac99a622e5ff0a68703b81614c147bc02178612a5cecb105b8943937a05c0b3ec91fea0 libunistring-checksum = f1e92717fc043c2d7c454e44700559b0bd75457fdf3da5e803d9d58d8ebdff261bc73f380f6350bb04ce04071687c0317d09b4e8615003187948be1cd4d0bab3 libxml2-checksum = 0fdc16aa7536adfe1deaadd19cdc20101dd3b081bd0a2e59d8ecc2965b2ef4c30d2e2447ab5b0cdb861c57ddb89ace1bd4ce120d9436b751b37724a90981ba8c -lzip-checksum = 8d063a5f0609caa4d568f7bad4c67fcdee70a16c37e67b9d3cccbae53f09c14fa8590816aeb37a3fe0bc71a642c55e9fb4b371c0f68585be9564af22584d5c99 +lzip-checksum = 9b18ab9b7ef9a7d1c2da06b48fd018399e94405a5ada457885b778c59caa3faed0e73e1da398516621dc9e721587785b57ef1f8a12c1b0e116ebce5129f0f24d m4-checksum = 7f8845f99e64d6a45859b9d80b03352a5526b3de0311ca4d6dd6850e504d26dfc90cd21d1640b10382f786213f8fdf20183bff424b3c41ea11432315993ab829 make-checksum = ddf0fdcb9ee1b182ef294c5da70c1275288c99bef60e63a25c0abed2ddd44aba1770be4aab1db8cac81e5f624576f2127c5d825a1824e1c7a49df4f16445526b mpc-checksum = 85e044749c2a001a70af2c11fc4bac58709ca7df11bf6da3b03cca852ef336eb37c4fb1dafdffde2d13df35caa7c296a7cba19a4625b82dc3cccc5066ac95774 @@ -77,7 +77,7 @@ zlib-checksum = 97493906504451830655b300b4144deeeb9f41bdb9b3f3a3d5635c2b16bf90d5 apachelog4cxx-checksum = 2c4b907a47ae00c38ae9bcd5d215be5115bfb677646cac50538eb1c75824f594aa997dafc9a7828dc6bfd5f22f4c486f13e4210b9a1d8a9c98098d96384ef9c3 apr-checksum = d0ddf8b10dd8d3a831b94e541d387414cc4d507ad48ff752d3274fb808afe6628d7951364aa3e0f71f373c78d0081411f6a0595d01c62bfb0a54bbc86002c82a apr-util-checksum = 6589948e3f9daf4ecd700bdbd2053fcb83005cec9d339278dda067996ba696e4a947116f066cee03214458aa15e0cb6f0df0103e7bc6f8f32327722eb7265f35 -astrometrynet-checksum = 066d8f6820d6bd9cdeef9334d150aef15a934c43944a96ac91b7184eff6c59876bcfed4ca221650b26b12b5e839228a8a7cb4ac3c72e599717d8cedfe6c77510 +astrometrynet-checksum = 9d07cf03f6408a19164914fa7b389aaedec8ae27ec9dcdef75b5a0db6a71000a15cdc1d9a8d9b2256c5299a890f499792407cf362ca0bed27f4f182f4a93b789 atlas-checksum = 13634a63c686800bef8affcf4e8f5f89d08f485fc557eb1b9ba51a405926014e659e439268f655984bb7f59ffe497d594cd7a607103826f1046325d89796e1b0 autoconf-checksum = daf3d17178d8c2d0f872a50217ba4d8a1481bad7e70ee5a58c5599717a01209cbd58e6149c72f6c4829b96dcf602bcfb828d47e32a9cfe3e5710c45f4f1736e7 automake-checksum = 728284b89ab4607b5de5db6b8bf5ad8730cbd8ac094f583b93bb92c59dee35e021e823cddd412b0507b18a12cbbbfbec0fb0447c23d0afd167769641fe0755e5 @@ -91,7 +91,7 @@ eigen-checksum = 39c1944e0daca50bb01e929edc98e2bfc234accb30ca019dcccfc7b02db5cc8 emacs-checksum = 91313dced8cd74f9e4a996eef42806aad00fb2576fc77b3694bb7b670624b6a1551ff49c2a3326dbeeba916e5732ca5975b5dc40b3b063f6e73a22839009abe9 expat-checksum = 526f2ec03978f853ff76a911b5a168f58a7eac83b8405536afe2a6acf4611ebcb96acbc5df809fd18e4edeb7ca5059a3ce56dcbd12cb33eecf742c6361d83d19 fftw-checksum = 28bd2f620399a415181027d30d7ea193aa487c7a277c9943d0051488908fd87e2731de0dfc3bebd22a6121d1deaa46037be8296a8a9cdb711f9fde4510c3d368 -flex-checksum = f15c5a1efc78d3de36572ece8ec1694f368ecf13a01ac3851be142bd63e16e9e5c62d10ce9ac586aabcbed533a47d0d97f7c9f467b729d5efa3316c7788230e2 +flex-checksum = a18e0dcdd21bd51779fd70e5efc2d77fa7b2a56d4d0e929d8abb437f86d7fcfa2cd09b8268185698200866dac8bec4444c5a7cff3140c515eaa49fc77c5aea66 freetype-checksum = b3c7289cda3f89fa6e51143f2d1f1c82fa34808d0caa9262e7be5498c8f0fdb14e88d31ec9928e3add71f03208e40adb2fa94c2a4b4c6b7bfb81daf64b1578cb gdb-checksum = 4afd6660d2bbc4c48ce726062fb42217bdaa5974774407358b2624036a2ace48ed853746e4d6b83b9a409cd8fefc622b0727c79da8826e28e3d1444fa3a9d148 ghostscript-checksum = 6dd1713bf40648ba6ed234a6c68582f87fa4075a191cf9af4f4ffa1854b94439c5ee4748e27ade3e8e0b7168fee1df3e9a9f3a32f70ac1ab9dd68e3878358ff2 @@ -120,7 +120,7 @@ libtiff-checksum = 628e9e6b3ff5cdf661ac4c9cbee7fc10d5555554d39e447d9a0c4e99b9a81 libtirpc-checksum = bcb6b5c062c1301aa1246ec93ae0a5c1d221b8421126d020863517cb814b43ed038fb6c0c2faf4e68ff133b69abefe4f4d42bfc870671da6c27ca941a30b155a metastore-checksum = ac1230686535a652e95024abaf6c5585bdab36f4e092bee5fa4deff2a913cd60f3a6bd8020c6887ccab97f0da3a284a0d4619fad5464a269e7b2040d6f7e6aaf missfits-checksum = 32727f5eb30573a1cedacb8900e2536867e4815059eee32e64e3db65be9291b8a91b9f45b2c9f3cf6fc2a8cc448012ea3d502bdd9dee516008e17d5086aee795 -netpbm-checksum = 051c05aa080e6563483053c16bc416b0e1368d6d7b130dec12a01017245a99252f80ddf7ae7322a4f8e9c90d833fc90d94324585659605d8dbb766d5fa2238c5 +netpbm-checksum = aef81d2e46850fae1fb34a38fa9f634be3c47c7c4e80f300c61aadd5532bcbccdec636e6c701e54caca67ac2d6ce3a7b6a9f090deb00cd4db718439986f89d8e openblas-checksum = 363588495451f872d4332a1c5e5ff4422202935641f490515e6f8144d3e1838ef737f39e77aaca396d23f9011b981961aa503e5d5935260231e31bb2280f890d openmpi-checksum = 88c73fd708dd5fe3a893d5517425a8a4c3e252fae4715df9a9b0f2311abe94cf5f71be92f153820fbaeb32c7d0ac0175d5165bdadcadc731427309102182c1bd openssh-checksum = e280fa2d56f550efd37c5d2477670326261aa8b94d991f9eb17aad90e0c6c9c939efa90fe87d33260d0f709485cb05c379f0fd1bd44fc0d5190298b6398c9982 diff --git a/reproduce/software/config/versions.conf b/reproduce/software/config/versions.conf index 34d074c..04de56f 100644 --- a/reproduce/software/config/versions.conf +++ b/reproduce/software/config/versions.conf @@ -39,7 +39,7 @@ libiconv-version = 1.16 libtool-version = 2.4.6 libunistring-version = 1.0 libxml2-version = 2.9.12 -lzip-version = 1.22 +lzip-version = 1.23 m4-version = 1.4.19 make-version = 4.3 mpc-version = 1.2.1 @@ -95,7 +95,7 @@ certpem-version = 2021-10-15 apachelog4cxx-version = 0.12.1 apr-version = 1.7.0 apr-util-version = 1.6.1 -astrometrynet-version = 0.85 +astrometrynet-version = 0.89 atlas-version = 3.10.3 autoconf-version = 2.71 automake-version = 1.16.5 @@ -109,7 +109,7 @@ eigen-version = 3.4.0 emacs-version = 28.1 expat-version = 2.4.1 fftw-version = 3.3.10 -flex-version = 2.6.4 +flex-version = 2.6.4-410-74a89fd freetype-version = 2.11.0 gdb-version = 11.1 ghostscript-version = 9.55.0 @@ -179,7 +179,7 @@ wcslib-version = 7.7 # Netpbm's questions in the configure steps maybe change with different or # new versions. -netpbm-version = 10.73.38 +netpbm-version = 10.73.39 # Minizip is installed with the same 'zlib' tarball, and they have the same # version. diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 34c829b..62004a8 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1293,16 +1293,34 @@ $(ibidir)/netpbm-$(netpbm-version): \ $(ibidir)/libxml2-$(libxml2-version) tarball=netpbm-$(netpbm-version).tar.lz $(call import-source, $(netpbm-url), $(netpbm-checksum)) + +# Answers to the configuration questions. if [ x$(on_mac_os) = xyes ]; then answers='\n\n$(ildir)\n\n\n\n\n\n$(ildir)/include\n\n$(ildir)/include\n\n$(ildir)/include\nnone\n\n' else answers='\n\n\n\n\n\n\n\n\n\n\n\n\nnone\n\n\n' fi + +# Go into the temporary directory and unpack the tarball. cd $(ddir) unpackdir=netpbm-$(netpbm-version) rm -rf $$unpackdir tar -xf $(tdir)/$$tarball cd $$unpackdir + +# As of NetPBM 10.73.39 and Flex 2.6.4-410-74a89fd (commit 74a89fd in +# Flex's Git that is 410 commits after version 2.6.4), there is the +# following line: 'if (0) yyunput(0, NULL);'. It will cause a crash +# and is just to avoid compiler warnings! So we are setting the +# 'yyunput(0, NULL);' to the redundant 'yyunput(0, NULL);' to let the +# compilation finish! + awk '$$1=="if" && $$2=="(0)"{inif=1} \ + {if(inif==1 && $$1=="yyunput(0,") \ + print "{int a=1;}"; else print $$0}' \ + converter/pbm/thinkjettopbm.l > thinkjettopbm.tmp + mv thinkjettopbm.tmp converter/pbm/thinkjettopbm.l + +# Pass the answers to the configure script then build and install it. printf "$$answers" | ./configure make rm -rf $(ddir)/$$unpackdir/install @@ -1396,6 +1414,7 @@ $(ibidir)/sextractor-$(sextractor-version): \ --with-openblas-libdir=$(ildir) \ --with-openblas-incdir=$(idir)/include) ln -fs $(ibdir)/sex $(ibdir)/sextractor + ln -fs $(ibdir)/sex $(ibdir)/source-extractor cp $(dtexdir)/sextractor.tex $(ictdir)/ echo "SExtractor $(sextractor-version) \citep{sextractor}" > $@ @@ -1431,6 +1450,14 @@ $(ibidir)/swig-$(swig-version): # --disable-mount # --disable-wall # --disable-su +# Because they fail on older kernels (tested on Linux 2.6.32) +# and they need root (to actually use; so are not relevant to +# Maneage): +# --disable-swapon +# --disable-unshare +# --disable-libmount +# --disable-mountpoint +# --enable-libmount-support-mtab # # NOTE ON INSTALLATION DIRECTORY: Util-linux libraries are relatively # low-level and may cause conflicts with system libraries (especilly when @@ -1459,9 +1486,14 @@ $(ibidir)/util-linux-$(util-linux-version): | $(idircustom) cd util-linux-$(util-linux-version) ./configure --prefix=$(idircustom)/util-linux \ --disable-dependency-tracking \ + --enable-libmount-support-mtab \ --disable-silent-rules \ + --disable-mountpoint \ + --disable-libmount \ + --disable-unshare \ --without-systemd \ --enable-libuuid \ + --disable-swapon \ --disable-mount \ --disable-ipcrm \ --disable-ipcs \ diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 07381e6..7f5f9c4 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -191,7 +191,7 @@ free_space_warning() { fs_threshold=$1 fs_destpath="$2" - return $(df "$fs_destpath" \ + return $(df -P "$fs_destpath" \ | awk 'FNR==2 {if($4>'$fs_threshold') print 1; \ else print 0; }') } @@ -814,9 +814,22 @@ if [ $rewritepconfig = yes ]; then # 'which' isn't in POSIX, so we are using 'command -v' instead. name=$(command -v wget) + # See if the host wget has the '--no-use-server-timestamps' option + # (for example wget 1.12 doesn't have it). If not, we'll have to + # remove it. This won't affect the analysis of Maneage in anyway, + # its just to avoid re-downloading if the server timestamps are + # bad; at the worst case, it will just cause a re-download of an + # input software source code (for data inputs, we will use our own + # wget that has this option). + tsname="no-use-server-timestamps" + tscheck=$(wget --help | grep $tsname || true) + if [ x"$tscheck" = x ]; then wgetts="" + else wgetts="--$tsname"; + fi + # By default Wget keeps the remote file's timestamp, so we'll have # to disable it manually. - downloader="$name --no-use-server-timestamps -O"; + downloader="$name $wgetts -O"; elif type curl > /dev/null 2>/dev/null; then name=$(command -v curl) diff --git a/reproduce/software/shell/tarball-prepare.sh b/reproduce/software/shell/tarball-prepare.sh index ccc9318..69e6afa 100755 --- a/reproduce/software/shell/tarball-prepare.sh +++ b/reproduce/software/shell/tarball-prepare.sh @@ -31,25 +31,21 @@ odir= idir= quiet= basedir=$PWD +scriptname=$0 # The --help output print_help() { cat <<EOF -Usage: $0 [OPTIONS] +Usage: $scriptname [OPTIONS] Low-level script to create maneage-standard tarballs. - -o, --output-dir Target directory to write the packed tarballs. Current: $odir - - -i, --input-dir Directory containing original tarballs. Current: $idir - -q, --quiet Suppress logging information. Only print the final packed file and its sha512sum. - Maneage URL: https://maneage.org Report bugs: https://savannah.nongnu.org/bugs/?group=reproduce @@ -59,45 +55,87 @@ EOF + +# Functions to check option values and complain if necessary. +on_off_option_error() { + if [ x"$2" = x ]; then + echo "$scriptname: '$1' doesn't take any values" + else + echo "$scriptname: '$1' (or '$2') doesn't take any values" + fi + exit 1 +} + +check_v() { + if [ x"$2" = x ]; then + cat <<EOF +$scriptname: option '$1' requires an argument. Try '$scriptname --help' for more information +EOF + exit 1; + fi +} + +option_given_and_valid() { + dirname="$1" + optionlong="$2" + optionshort="$3" + if [ x"$dirname" = x ]; then + cat <<EOF +$scriptname: no '--$optionlong' (or '-$optionshort') given: use this for identifying the directory containing the input tarballs +EOF + exit 1 + else + dirname=$(echo "$dirname" | sed 's|/$||'); # Remove possible trailing slash + if [ ! -d "$dirname" ]; then + cat <<EOF +$scriptname: '$dirname' that is given to '--$optionlong' (or '-$optionshort') couldn't be opened +EOF + exit 1 + else + outdir=$(realpath $dirname) + fi + fi + ogvout=$outdir +} + + + + + # Parse the arguments while [ $# -gt 0 ] do - case $1 in - -q|--quiet) quiet=1; shift;; - -h|--help|-'?') print_help; exit 0;; - -i|--input-dir) - # Remove the trailing '/' introduced by autocomplete - idir=$(echo "$2" | sed 's|/$||'); - shift; # past argument - shift;; # past value - -o|--output-dir) - # Remove the trailing '/' introduced by autocomplete - odir=$(echo "$2" | sed 's|/$||'); - shift; # past argument - shift;; # past value - *) echo "$0: unknown option '$1'"; exit 1;; + case $1 in + # Input and Output directories + -i|--input-dir) idir="$2"; check_v "$1" "$idir"; shift;shift;; + -i=*|--input-dir=*) idir="${1#*=}"; check_v "$1" "$idir"; shift;; + -i*) idir=$(echo "$1" | sed -e's/-i//'); check_v "$1" "$idir"; shift;; + -o|--output-dir) odir="$2"; check_v "$1" "$odir"; shift;shift;; + -o=*|--output-dir=*) odir="${1#*=}"; check_v "$1" "$odir"; shift;; + -o*) odir=$(echo "$1" | sed -e's/-o//'); check_v "$1" "$odir"; shift;; + + # Operating mode options + -?|--help) print_help; exit 0;; + -'?'*|--help=*) on_off_option_error --help -?;; + -q|--quiet) quiet=1; shift;; + -q*|--quiet=*) on_off_option_error --quiet -q;; + *) echo "$scriptname: unknown option '$1'"; exit 1;; esac done -# Extract the 'absolute path' to input and output directories. Working with -# relative path is a great source of confusion and unwanted side-effects -# like moving/removing files by accident. -if [ ! -d "$idir" ]; then - echo "$0: please pass the input directory (option --input-dir or -i)." - exit 1 -else - idir=$(realpath $idir) -fi -if [ ! -d "$odir" ]; then - echo "$0: please pass the output directory (option --output-dir or -o)." - exit 1 -else - odir=$(realpath $odir) -fi +# Basic sanity checks +# +# Make sure the input and output directories are given. Also extract +# the absolute path to input and output directories and remove any +# possible trailing '/'. Working with a relative path is a great +# source of confusion and unwanted side-effects like moving/removing +# files by accident. +option_given_and_valid "$idir" "input-dir" "i" && idir=$ogvout +option_given_and_valid "$odir" "output-dir" "o" && odir=$ogvout @@ -117,15 +155,15 @@ fi for f in $allfiles; do # Seperate name and version number - name=$(echo $f | sed -e 's/.tar.*//' | \ - awk 'BEGIN { FS = "[-_ ]" } {print $1 "-" $2}') + name=$(echo $f | sed -e 's/.tar.*//' \ + | awk 'BEGIN { FS = "[-_ ]" } {print $1 "-" $2}') # Skip previously packed files if [ -f $odir/$name.tar.lz ]; then # Print the info message if not in quiet mode if [ -z $quiet ]; then - echo "$0: skipping '$odir/$name.tar.lz'" + echo "$scriptname: $odir/$name.tar.lz: already present in output directory" fi # skip this file @@ -134,7 +172,7 @@ for f in $allfiles; do # Print the info message if not in quiet mode if [ -z $quiet ]; then - echo "$0: processing '$idir/$f'" + echo "$scriptname: processing '$idir/$f'" fi fi |