aboutsummaryrefslogtreecommitdiff
path: root/reproduce
diff options
context:
space:
mode:
Diffstat (limited to 'reproduce')
-rw-r--r--reproduce/analysis/config/INPUTS.conf2
-rw-r--r--reproduce/analysis/make/initialize.mk2
-rw-r--r--reproduce/analysis/make/prepare.mk6
-rw-r--r--reproduce/analysis/make/verify.mk2
-rw-r--r--reproduce/software/config/checksums.conf8
-rw-r--r--reproduce/software/config/versions.conf8
-rw-r--r--reproduce/software/make/high-level.mk32
-rwxr-xr-xreproduce/software/shell/configure.sh17
-rwxr-xr-xreproduce/software/shell/tarball-prepare.sh116
9 files changed, 139 insertions, 54 deletions
diff --git a/reproduce/analysis/config/INPUTS.conf b/reproduce/analysis/config/INPUTS.conf
index 5a58758..3958153 100644
--- a/reproduce/analysis/config/INPUTS.conf
+++ b/reproduce/analysis/config/INPUTS.conf
@@ -30,7 +30,7 @@
#
# The naming convension is critical for the input files to be properly
# imported into the project. In the patterns below, the '%' is the full
-# file name (including its prefix): for example in the demo input of this
+# file name (including its suffix): for example in the demo input of this
# file in the 'maneage' branch, we have 'INPUT-wfpc2.fits-sha256':
# therefore, the input file (within the project's '$(indir)') is called
# 'wfpc2.fits'. This allows you to simply set '$(indir)/wfpc2.fits' as the
diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk
index 4e8ee68..6893962 100644
--- a/reproduce/analysis/make/initialize.mk
+++ b/reproduce/analysis/make/initialize.mk
@@ -138,10 +138,10 @@ curdir := $(shell echo $$(pwd))
# avoid conflicts with existing TeX Live solutions. Later (in 'paper.mk'),
# we are also going to overwrite 'TEXINPUTS' just before 'pdflatex'.
.ONESHELL:
-.SHELLFLAGS = -ec
export TEXINPUTS :=
export CCACHE_DISABLE := 1
export PATH := $(installdir)/bin
+.SHELLFLAGS = --noprofile --norc -ec
export LDFLAGS := -L$(installdir)/lib
export SHELL := $(installdir)/bin/bash
export CPPFLAGS := -I$(installdir)/include
diff --git a/reproduce/analysis/make/prepare.mk b/reproduce/analysis/make/prepare.mk
index ecb6842..aed2b5f 100644
--- a/reproduce/analysis/make/prepare.mk
+++ b/reproduce/analysis/make/prepare.mk
@@ -1,4 +1,4 @@
-# Basic preparations, called by './project prepare'.
+# Basic preparations, called by './project make'.
#
# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org>
#
@@ -22,7 +22,9 @@
# Final-target
#
# Without this file, './project make' won't work.
-prepare-dep = $(subst prepare, ,$(makesrc))
+#
+# We need to remove the 'prepare' word from the list of 'makesrc'.
+prepare-dep = $(filter-out prepare, ,$(makesrc))
$(bsdir)/preparation-done.mk: \
$(foreach s, $(prepare-dep), $(mtexdir)/$(s).tex)
diff --git a/reproduce/analysis/make/verify.mk b/reproduce/analysis/make/verify.mk
index d3f3282..a645713 100644
--- a/reproduce/analysis/make/verify.mk
+++ b/reproduce/analysis/make/verify.mk
@@ -110,7 +110,7 @@ verify-txt-no-comments-no-space = \
# essentially the same as 'verify-dep', but it has removed
# the 'initialize' step (which is information about the
# pipeline, not the results).
-verify-dep = $(subst verify,,$(subst paper,,$(makesrc)))
+verify-dep = $(filter-out verify paper, $(makesrc))
verify-check = $(subst initialize,,$(verify-dep))
$(mtexdir)/verify.tex: $(foreach s, $(verify-dep), $(mtexdir)/$(s).tex)
diff --git a/reproduce/software/config/checksums.conf b/reproduce/software/config/checksums.conf
index 7450e6d..736a5e5 100644
--- a/reproduce/software/config/checksums.conf
+++ b/reproduce/software/config/checksums.conf
@@ -39,7 +39,7 @@ libiconv-checksum = 396be5305f899619596b72ccc8a192c5fcc8e12163fd23376833c957ee44
libtool-checksum = 379a16baf2123d3cdd28cf8e35e4bb1e0326ae7f431c42ab2a3bd42ccac99a622e5ff0a68703b81614c147bc02178612a5cecb105b8943937a05c0b3ec91fea0
libunistring-checksum = f1e92717fc043c2d7c454e44700559b0bd75457fdf3da5e803d9d58d8ebdff261bc73f380f6350bb04ce04071687c0317d09b4e8615003187948be1cd4d0bab3
libxml2-checksum = 0fdc16aa7536adfe1deaadd19cdc20101dd3b081bd0a2e59d8ecc2965b2ef4c30d2e2447ab5b0cdb861c57ddb89ace1bd4ce120d9436b751b37724a90981ba8c
-lzip-checksum = 8d063a5f0609caa4d568f7bad4c67fcdee70a16c37e67b9d3cccbae53f09c14fa8590816aeb37a3fe0bc71a642c55e9fb4b371c0f68585be9564af22584d5c99
+lzip-checksum = 9b18ab9b7ef9a7d1c2da06b48fd018399e94405a5ada457885b778c59caa3faed0e73e1da398516621dc9e721587785b57ef1f8a12c1b0e116ebce5129f0f24d
m4-checksum = 7f8845f99e64d6a45859b9d80b03352a5526b3de0311ca4d6dd6850e504d26dfc90cd21d1640b10382f786213f8fdf20183bff424b3c41ea11432315993ab829
make-checksum = ddf0fdcb9ee1b182ef294c5da70c1275288c99bef60e63a25c0abed2ddd44aba1770be4aab1db8cac81e5f624576f2127c5d825a1824e1c7a49df4f16445526b
mpc-checksum = 85e044749c2a001a70af2c11fc4bac58709ca7df11bf6da3b03cca852ef336eb37c4fb1dafdffde2d13df35caa7c296a7cba19a4625b82dc3cccc5066ac95774
@@ -77,7 +77,7 @@ zlib-checksum = 97493906504451830655b300b4144deeeb9f41bdb9b3f3a3d5635c2b16bf90d5
apachelog4cxx-checksum = 2c4b907a47ae00c38ae9bcd5d215be5115bfb677646cac50538eb1c75824f594aa997dafc9a7828dc6bfd5f22f4c486f13e4210b9a1d8a9c98098d96384ef9c3
apr-checksum = d0ddf8b10dd8d3a831b94e541d387414cc4d507ad48ff752d3274fb808afe6628d7951364aa3e0f71f373c78d0081411f6a0595d01c62bfb0a54bbc86002c82a
apr-util-checksum = 6589948e3f9daf4ecd700bdbd2053fcb83005cec9d339278dda067996ba696e4a947116f066cee03214458aa15e0cb6f0df0103e7bc6f8f32327722eb7265f35
-astrometrynet-checksum = 066d8f6820d6bd9cdeef9334d150aef15a934c43944a96ac91b7184eff6c59876bcfed4ca221650b26b12b5e839228a8a7cb4ac3c72e599717d8cedfe6c77510
+astrometrynet-checksum = 9d07cf03f6408a19164914fa7b389aaedec8ae27ec9dcdef75b5a0db6a71000a15cdc1d9a8d9b2256c5299a890f499792407cf362ca0bed27f4f182f4a93b789
atlas-checksum = 13634a63c686800bef8affcf4e8f5f89d08f485fc557eb1b9ba51a405926014e659e439268f655984bb7f59ffe497d594cd7a607103826f1046325d89796e1b0
autoconf-checksum = daf3d17178d8c2d0f872a50217ba4d8a1481bad7e70ee5a58c5599717a01209cbd58e6149c72f6c4829b96dcf602bcfb828d47e32a9cfe3e5710c45f4f1736e7
automake-checksum = 728284b89ab4607b5de5db6b8bf5ad8730cbd8ac094f583b93bb92c59dee35e021e823cddd412b0507b18a12cbbbfbec0fb0447c23d0afd167769641fe0755e5
@@ -91,7 +91,7 @@ eigen-checksum = 39c1944e0daca50bb01e929edc98e2bfc234accb30ca019dcccfc7b02db5cc8
emacs-checksum = 91313dced8cd74f9e4a996eef42806aad00fb2576fc77b3694bb7b670624b6a1551ff49c2a3326dbeeba916e5732ca5975b5dc40b3b063f6e73a22839009abe9
expat-checksum = 526f2ec03978f853ff76a911b5a168f58a7eac83b8405536afe2a6acf4611ebcb96acbc5df809fd18e4edeb7ca5059a3ce56dcbd12cb33eecf742c6361d83d19
fftw-checksum = 28bd2f620399a415181027d30d7ea193aa487c7a277c9943d0051488908fd87e2731de0dfc3bebd22a6121d1deaa46037be8296a8a9cdb711f9fde4510c3d368
-flex-checksum = f15c5a1efc78d3de36572ece8ec1694f368ecf13a01ac3851be142bd63e16e9e5c62d10ce9ac586aabcbed533a47d0d97f7c9f467b729d5efa3316c7788230e2
+flex-checksum = a18e0dcdd21bd51779fd70e5efc2d77fa7b2a56d4d0e929d8abb437f86d7fcfa2cd09b8268185698200866dac8bec4444c5a7cff3140c515eaa49fc77c5aea66
freetype-checksum = b3c7289cda3f89fa6e51143f2d1f1c82fa34808d0caa9262e7be5498c8f0fdb14e88d31ec9928e3add71f03208e40adb2fa94c2a4b4c6b7bfb81daf64b1578cb
gdb-checksum = 4afd6660d2bbc4c48ce726062fb42217bdaa5974774407358b2624036a2ace48ed853746e4d6b83b9a409cd8fefc622b0727c79da8826e28e3d1444fa3a9d148
ghostscript-checksum = 6dd1713bf40648ba6ed234a6c68582f87fa4075a191cf9af4f4ffa1854b94439c5ee4748e27ade3e8e0b7168fee1df3e9a9f3a32f70ac1ab9dd68e3878358ff2
@@ -120,7 +120,7 @@ libtiff-checksum = 628e9e6b3ff5cdf661ac4c9cbee7fc10d5555554d39e447d9a0c4e99b9a81
libtirpc-checksum = bcb6b5c062c1301aa1246ec93ae0a5c1d221b8421126d020863517cb814b43ed038fb6c0c2faf4e68ff133b69abefe4f4d42bfc870671da6c27ca941a30b155a
metastore-checksum = ac1230686535a652e95024abaf6c5585bdab36f4e092bee5fa4deff2a913cd60f3a6bd8020c6887ccab97f0da3a284a0d4619fad5464a269e7b2040d6f7e6aaf
missfits-checksum = 32727f5eb30573a1cedacb8900e2536867e4815059eee32e64e3db65be9291b8a91b9f45b2c9f3cf6fc2a8cc448012ea3d502bdd9dee516008e17d5086aee795
-netpbm-checksum = 051c05aa080e6563483053c16bc416b0e1368d6d7b130dec12a01017245a99252f80ddf7ae7322a4f8e9c90d833fc90d94324585659605d8dbb766d5fa2238c5
+netpbm-checksum = aef81d2e46850fae1fb34a38fa9f634be3c47c7c4e80f300c61aadd5532bcbccdec636e6c701e54caca67ac2d6ce3a7b6a9f090deb00cd4db718439986f89d8e
openblas-checksum = 363588495451f872d4332a1c5e5ff4422202935641f490515e6f8144d3e1838ef737f39e77aaca396d23f9011b981961aa503e5d5935260231e31bb2280f890d
openmpi-checksum = 88c73fd708dd5fe3a893d5517425a8a4c3e252fae4715df9a9b0f2311abe94cf5f71be92f153820fbaeb32c7d0ac0175d5165bdadcadc731427309102182c1bd
openssh-checksum = e280fa2d56f550efd37c5d2477670326261aa8b94d991f9eb17aad90e0c6c9c939efa90fe87d33260d0f709485cb05c379f0fd1bd44fc0d5190298b6398c9982
diff --git a/reproduce/software/config/versions.conf b/reproduce/software/config/versions.conf
index 34d074c..04de56f 100644
--- a/reproduce/software/config/versions.conf
+++ b/reproduce/software/config/versions.conf
@@ -39,7 +39,7 @@ libiconv-version = 1.16
libtool-version = 2.4.6
libunistring-version = 1.0
libxml2-version = 2.9.12
-lzip-version = 1.22
+lzip-version = 1.23
m4-version = 1.4.19
make-version = 4.3
mpc-version = 1.2.1
@@ -95,7 +95,7 @@ certpem-version = 2021-10-15
apachelog4cxx-version = 0.12.1
apr-version = 1.7.0
apr-util-version = 1.6.1
-astrometrynet-version = 0.85
+astrometrynet-version = 0.89
atlas-version = 3.10.3
autoconf-version = 2.71
automake-version = 1.16.5
@@ -109,7 +109,7 @@ eigen-version = 3.4.0
emacs-version = 28.1
expat-version = 2.4.1
fftw-version = 3.3.10
-flex-version = 2.6.4
+flex-version = 2.6.4-410-74a89fd
freetype-version = 2.11.0
gdb-version = 11.1
ghostscript-version = 9.55.0
@@ -179,7 +179,7 @@ wcslib-version = 7.7
# Netpbm's questions in the configure steps maybe change with different or
# new versions.
-netpbm-version = 10.73.38
+netpbm-version = 10.73.39
# Minizip is installed with the same 'zlib' tarball, and they have the same
# version.
diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk
index 34c829b..62004a8 100644
--- a/reproduce/software/make/high-level.mk
+++ b/reproduce/software/make/high-level.mk
@@ -1293,16 +1293,34 @@ $(ibidir)/netpbm-$(netpbm-version): \
$(ibidir)/libxml2-$(libxml2-version)
tarball=netpbm-$(netpbm-version).tar.lz
$(call import-source, $(netpbm-url), $(netpbm-checksum))
+
+# Answers to the configuration questions.
if [ x$(on_mac_os) = xyes ]; then
answers='\n\n$(ildir)\n\n\n\n\n\n$(ildir)/include\n\n$(ildir)/include\n\n$(ildir)/include\nnone\n\n'
else
answers='\n\n\n\n\n\n\n\n\n\n\n\n\nnone\n\n\n'
fi
+
+# Go into the temporary directory and unpack the tarball.
cd $(ddir)
unpackdir=netpbm-$(netpbm-version)
rm -rf $$unpackdir
tar -xf $(tdir)/$$tarball
cd $$unpackdir
+
+# As of NetPBM 10.73.39 and Flex 2.6.4-410-74a89fd (commit 74a89fd in
+# Flex's Git that is 410 commits after version 2.6.4), there is the
+# following line: 'if (0) yyunput(0, NULL);'. It will cause a crash
+# and is just to avoid compiler warnings! So we are setting the
+# 'yyunput(0, NULL);' to the redundant 'yyunput(0, NULL);' to let the
+# compilation finish!
+ awk '$$1=="if" && $$2=="(0)"{inif=1} \
+ {if(inif==1 && $$1=="yyunput(0,") \
+ print "{int a=1;}"; else print $$0}' \
+ converter/pbm/thinkjettopbm.l > thinkjettopbm.tmp
+ mv thinkjettopbm.tmp converter/pbm/thinkjettopbm.l
+
+# Pass the answers to the configure script then build and install it.
printf "$$answers" | ./configure
make
rm -rf $(ddir)/$$unpackdir/install
@@ -1396,6 +1414,7 @@ $(ibidir)/sextractor-$(sextractor-version): \
--with-openblas-libdir=$(ildir) \
--with-openblas-incdir=$(idir)/include)
ln -fs $(ibdir)/sex $(ibdir)/sextractor
+ ln -fs $(ibdir)/sex $(ibdir)/source-extractor
cp $(dtexdir)/sextractor.tex $(ictdir)/
echo "SExtractor $(sextractor-version) \citep{sextractor}" > $@
@@ -1431,6 +1450,14 @@ $(ibidir)/swig-$(swig-version):
# --disable-mount
# --disable-wall
# --disable-su
+# Because they fail on older kernels (tested on Linux 2.6.32)
+# and they need root (to actually use; so are not relevant to
+# Maneage):
+# --disable-swapon
+# --disable-unshare
+# --disable-libmount
+# --disable-mountpoint
+# --enable-libmount-support-mtab
#
# NOTE ON INSTALLATION DIRECTORY: Util-linux libraries are relatively
# low-level and may cause conflicts with system libraries (especilly when
@@ -1459,9 +1486,14 @@ $(ibidir)/util-linux-$(util-linux-version): | $(idircustom)
cd util-linux-$(util-linux-version)
./configure --prefix=$(idircustom)/util-linux \
--disable-dependency-tracking \
+ --enable-libmount-support-mtab \
--disable-silent-rules \
+ --disable-mountpoint \
+ --disable-libmount \
+ --disable-unshare \
--without-systemd \
--enable-libuuid \
+ --disable-swapon \
--disable-mount \
--disable-ipcrm \
--disable-ipcs \
diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh
index 07381e6..7f5f9c4 100755
--- a/reproduce/software/shell/configure.sh
+++ b/reproduce/software/shell/configure.sh
@@ -191,7 +191,7 @@ free_space_warning()
{
fs_threshold=$1
fs_destpath="$2"
- return $(df "$fs_destpath" \
+ return $(df -P "$fs_destpath" \
| awk 'FNR==2 {if($4>'$fs_threshold') print 1; \
else print 0; }')
}
@@ -814,9 +814,22 @@ if [ $rewritepconfig = yes ]; then
# 'which' isn't in POSIX, so we are using 'command -v' instead.
name=$(command -v wget)
+ # See if the host wget has the '--no-use-server-timestamps' option
+ # (for example wget 1.12 doesn't have it). If not, we'll have to
+ # remove it. This won't affect the analysis of Maneage in anyway,
+ # its just to avoid re-downloading if the server timestamps are
+ # bad; at the worst case, it will just cause a re-download of an
+ # input software source code (for data inputs, we will use our own
+ # wget that has this option).
+ tsname="no-use-server-timestamps"
+ tscheck=$(wget --help | grep $tsname || true)
+ if [ x"$tscheck" = x ]; then wgetts=""
+ else wgetts="--$tsname";
+ fi
+
# By default Wget keeps the remote file's timestamp, so we'll have
# to disable it manually.
- downloader="$name --no-use-server-timestamps -O";
+ downloader="$name $wgetts -O";
elif type curl > /dev/null 2>/dev/null; then
name=$(command -v curl)
diff --git a/reproduce/software/shell/tarball-prepare.sh b/reproduce/software/shell/tarball-prepare.sh
index ccc9318..69e6afa 100755
--- a/reproduce/software/shell/tarball-prepare.sh
+++ b/reproduce/software/shell/tarball-prepare.sh
@@ -31,25 +31,21 @@ odir=
idir=
quiet=
basedir=$PWD
+scriptname=$0
# The --help output
print_help() {
cat <<EOF
-Usage: $0 [OPTIONS]
+Usage: $scriptname [OPTIONS]
Low-level script to create maneage-standard tarballs.
-
-o, --output-dir Target directory to write the packed tarballs.
Current: $odir
-
-
-i, --input-dir Directory containing original tarballs.
Current: $idir
-
-q, --quiet Suppress logging information. Only print the
final packed file and its sha512sum.
-
Maneage URL: https://maneage.org
Report bugs: https://savannah.nongnu.org/bugs/?group=reproduce
@@ -59,45 +55,87 @@ EOF
+
+# Functions to check option values and complain if necessary.
+on_off_option_error() {
+ if [ x"$2" = x ]; then
+ echo "$scriptname: '$1' doesn't take any values"
+ else
+ echo "$scriptname: '$1' (or '$2') doesn't take any values"
+ fi
+ exit 1
+}
+
+check_v() {
+ if [ x"$2" = x ]; then
+ cat <<EOF
+$scriptname: option '$1' requires an argument. Try '$scriptname --help' for more information
+EOF
+ exit 1;
+ fi
+}
+
+option_given_and_valid() {
+ dirname="$1"
+ optionlong="$2"
+ optionshort="$3"
+ if [ x"$dirname" = x ]; then
+ cat <<EOF
+$scriptname: no '--$optionlong' (or '-$optionshort') given: use this for identifying the directory containing the input tarballs
+EOF
+ exit 1
+ else
+ dirname=$(echo "$dirname" | sed 's|/$||'); # Remove possible trailing slash
+ if [ ! -d "$dirname" ]; then
+ cat <<EOF
+$scriptname: '$dirname' that is given to '--$optionlong' (or '-$optionshort') couldn't be opened
+EOF
+ exit 1
+ else
+ outdir=$(realpath $dirname)
+ fi
+ fi
+ ogvout=$outdir
+}
+
+
+
+
+
# Parse the arguments
while [ $# -gt 0 ]
do
- case $1 in
- -q|--quiet) quiet=1; shift;;
- -h|--help|-'?') print_help; exit 0;;
- -i|--input-dir)
- # Remove the trailing '/' introduced by autocomplete
- idir=$(echo "$2" | sed 's|/$||');
- shift; # past argument
- shift;; # past value
- -o|--output-dir)
- # Remove the trailing '/' introduced by autocomplete
- odir=$(echo "$2" | sed 's|/$||');
- shift; # past argument
- shift;; # past value
- *) echo "$0: unknown option '$1'"; exit 1;;
+ case $1 in
+ # Input and Output directories
+ -i|--input-dir) idir="$2"; check_v "$1" "$idir"; shift;shift;;
+ -i=*|--input-dir=*) idir="${1#*=}"; check_v "$1" "$idir"; shift;;
+ -i*) idir=$(echo "$1" | sed -e's/-i//'); check_v "$1" "$idir"; shift;;
+ -o|--output-dir) odir="$2"; check_v "$1" "$odir"; shift;shift;;
+ -o=*|--output-dir=*) odir="${1#*=}"; check_v "$1" "$odir"; shift;;
+ -o*) odir=$(echo "$1" | sed -e's/-o//'); check_v "$1" "$odir"; shift;;
+
+ # Operating mode options
+ -?|--help) print_help; exit 0;;
+ -'?'*|--help=*) on_off_option_error --help -?;;
+ -q|--quiet) quiet=1; shift;;
+ -q*|--quiet=*) on_off_option_error --quiet -q;;
+ *) echo "$scriptname: unknown option '$1'"; exit 1;;
esac
done
-# Extract the 'absolute path' to input and output directories. Working with
-# relative path is a great source of confusion and unwanted side-effects
-# like moving/removing files by accident.
-if [ ! -d "$idir" ]; then
- echo "$0: please pass the input directory (option --input-dir or -i)."
- exit 1
-else
- idir=$(realpath $idir)
-fi
-if [ ! -d "$odir" ]; then
- echo "$0: please pass the output directory (option --output-dir or -o)."
- exit 1
-else
- odir=$(realpath $odir)
-fi
+# Basic sanity checks
+#
+# Make sure the input and output directories are given. Also extract
+# the absolute path to input and output directories and remove any
+# possible trailing '/'. Working with a relative path is a great
+# source of confusion and unwanted side-effects like moving/removing
+# files by accident.
+option_given_and_valid "$idir" "input-dir" "i" && idir=$ogvout
+option_given_and_valid "$odir" "output-dir" "o" && odir=$ogvout
@@ -117,15 +155,15 @@ fi
for f in $allfiles; do
# Seperate name and version number
- name=$(echo $f | sed -e 's/.tar.*//' | \
- awk 'BEGIN { FS = "[-_ ]" } {print $1 "-" $2}')
+ name=$(echo $f | sed -e 's/.tar.*//' \
+ | awk 'BEGIN { FS = "[-_ ]" } {print $1 "-" $2}')
# Skip previously packed files
if [ -f $odir/$name.tar.lz ]; then
# Print the info message if not in quiet mode
if [ -z $quiet ]; then
- echo "$0: skipping '$odir/$name.tar.lz'"
+ echo "$scriptname: $odir/$name.tar.lz: already present in output directory"
fi
# skip this file
@@ -134,7 +172,7 @@ for f in $allfiles; do
# Print the info message if not in quiet mode
if [ -z $quiet ]; then
- echo "$0: processing '$idir/$f'"
+ echo "$scriptname: processing '$idir/$f'"
fi
fi