From 4cfa34b1e101adcd050a476307b7680db8afcd8e Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 3 Jul 2026 16:23:00 +0200 Subject: Configuration: fixed portability issues, add jq and updated GCC Summary: this commit will not affect your analysis: only the version of GCC has been updated for portability and jq (a JSON parser) has been added. Until now, several portability issues existed in Maneage, causing problems mostl on macOS, but some also some on GNU/Linux. They are listed below along with the fix. With this commit, the problems mentioned above have been fixed: - 'pkg-config' on macOS needed some manual steps for a successful installation. The cause was its improper checks on the macOS "carbon" library. So we now manually fix this check within the code to fix the problem. This was found and fixed by Martin Wiesmann. - Apple chip variants like M1 Pro, M1 Max, M1 Ultra, M2 Pro, etc could not be properly recognized as a macOS system since we expected a single word. This was solved by using pattern matching instead. Found and fixed by Martin Wiesmann. - macOS's sha512sum wraps the 128-character hex hash across two output lines (80 + 48 chars) and the command that we used to read the output would only captured the first 80 characters, so checksums never matched. The solution was to merge the lines before the check. Found and fixed by Martin Wiesmann. - When a checksum failed, the script exited with an error but left the bad '.unchecked' file on disk. On the next run, the download script saw the file already existed and skipped downloading entirely. As a result, the corrupt file was passed to the checksum check again, failing forever. The fix was to remove the file when we confirmed a bad download. Found and fixed by Martin Wiesmann. - When the server of a file returns a HTML output instead of the file we want, the download script could not detect and would assume that the download was successful. Therefore, it would always crash with a checksum error. The fix was to check the first few bytes of the output and consider it failed if we detect '/dev/null | grep -qi ' /dev/null; then c=$$(git merge-base HEAD maneage) v=$$(git describe --always --long $$c) d=$$(git show -s --format=%aD $$v | awk '{print $$2, $$3, $$4}') + +# No 'maneage' branch found else - echo - echo "WARNING: no 'maneage' branch found! Without it, the latest merge of " - echo "this project with Maneage can't be reported in the paper (which is bad " - echo "for your readers; that includes yourself in a few years). Please run " - echo "the commands below to fetch the 'maneage' branch from its own server " - echo "and remove this warning (these commands will not affect your project):" - echo " $ git remote add origin-maneage http://git.maneage.org/project.git" - echo " $ git fetch origin-maneage" - echo " $ git branch maneage --track origin-maneage/maneage" - echo + +# Only print a warning if a PDF is to be created. + if [ x$(pdf-build-final) = xyes ]; then + echo + printf "WARNING: no 'maneage' branch found! Without it, " + printf "the latest merge of this project with Maneage can't " + printf "be reported in the paper (which is bad for your " + printf "readers; that includes yourself in a few years). " + printf "Please run the commands below to fetch the " + printf "'maneage' branch from its own server and remove " + printf "this warning (these commands will not affect your " + printf "project):\n" + printf " $ git remote add origin-maneage " + printf "http://git.maneage.org/project.git\n" + printf " $ git fetch origin-maneage\n" + printf " $ git branch maneage --track " + printf "origin-maneage/maneage\n" + echo + fi v="\textcolor{red}{NO-MANEAGE-BRANCH (see printed warning to fix this)}" d="\textcolor{red}{NO-MANEAGE-DATE}" fi diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk index 8d1c2e0..913a062 100644 --- a/reproduce/analysis/make/top-prepare.mk +++ b/reproduce/analysis/make/top-prepare.mk @@ -42,10 +42,10 @@ else all: @if [ "x$(GROUP-NAME)" = x ]; then \ echo "Project is NOT configured for groups, please run"; \ - echo " $$ ./project prepare"; \ + echo " $$ ./project configure"; \ else \ echo "Project is configured for groups, please run"; \ - echo " $$ ./project prepare --group=$(GROUP-NAME) -j8"; \ + echo " $$ ./project configure --group=$(GROUP-NAME) -j8"; \ fi exit 1 endif @@ -61,7 +61,7 @@ endif # # To ensure that 'prepare' and 'make' have the same basic definitions and # environment and that all 'downloads' are managed in one place, both -# './project prepare' and './project make' will first read 'initialize.mk' +# './project configure' and './project make' will first read 'initialize.mk' # and 'downloads.mk'. makesrc = initialize \ prepare diff --git a/reproduce/software/config/checksums.conf b/reproduce/software/config/checksums.conf index 9e1c437..d3717cb 100644 --- a/reproduce/software/config/checksums.conf +++ b/reproduce/software/config/checksums.conf @@ -27,7 +27,7 @@ file-checksum = b843b3c25656e8dec52e64eed6f581b29faf36540e0604a803d61c0f0eca830a findutils-checksum = 826c643e7f5c5d6976a47eabcd9807e51350d09ee8fc7dc931f2d9276f938f65aa0bd97e6213aa979742234784c120e1a6850a52207c327e1c1a465feb374053 flock-checksum = f711815035e21b46572bf80e730a55822e5abf4cb29749e476ee6cf4d5027e9a7deeacf5f6b8c37f18f17a0cc7a6d98fb0be3936e97b122707f1cb2306d1e1d9 gawk-checksum = 9013f5cb4c08aa13029ecf0c6b5e02ac93a3b500683b07ae8d34d698f0451dd5146a0a5aef96a249112834e27b379869d8e541f6a1b99accf701ba6a48f58f33 -gcc-checksum = f7386218e0993e19da1092bf625791cb9ec269667f7bdf378fef359a304aa08e4d79cb7b9f56d5d7028b53b29071005707a1cad0fd1b770884ab02310ed7b028 +gcc-checksum = b2191bf7005c21dee7675cc1f9ed615de418eac6afaf8dd6b3fc6022660d2ae63aec66a3fd072f227fa05da1654e04c2d06fb331fd50798c768de7ef1a7d9590 gettext-checksum = dbdc5016a776f12feb7134c05e2568ba9134bfa66b824967b4cc81d2093328c0f3bbcf269ad5a60ff2257625c763ac96c071685ee2b1d397730b06a058f565aa git-checksum = e2a3d2cf272554c0e9fa2abc4cf7b1eff17884911859f17567b76f170996802caa01675a42dbf29af7fc283a07e8dc4a11382ffbd2a03d7d62238e8c598f0b56 gmp-checksum = ad65de00ecb46cf454ed6c40d2a57ce2528f5fa64df1284dfa15036f1e8cf27760a09a4ecdfcc39048faffb71339bba30d99dd365c54173dbc2ba629bee2fad9 @@ -105,6 +105,7 @@ help2man-checksum = 83dca38c2020c85a66da882cd994b4e291eb6a0584149b7b3a74fec14443 icu-checksum = ab14b52a3fdf2dcde6b5160ab7218eac381b850d3c278324379741c49d71fa6040fbacca94c6937e6c9fc15843761121deff302ca6854da5ca1cd5b26a34e839 imagemagick-checksum = 2132614540b7422c9772fcebe7e8e358994efcfb53d8e48fa52992313b09b191847e395bad305322c377a4697014353bb8c15adc4edfd712e038504fc7f17c5e imfit-checksum = 15edd2349232c1c8e611b31d3a46b0700112d274515f54d0a0085bb4bfa6d3d5f8a15cd926516e043a29ce841accf3534ae58dbfb952d858dc9445199c957096 +jq-checksum = f83613b26ed6f4f963865bbc28fdc5246644e0e21287d0707ce62c5d10e84d70e622c7ca7f83147682e6052d96892697a2fe32a47d47bd3f32e3d74548877743 lalsuite-checksum = 330f2576698bd3107621d49596da66c7629d314e5744b3a267b3fa346e7926bb255979506627abaeeba62ff9a776c6091e9150ab5c471c3b4a43bec40a1a5de1 lapack-checksum = ff670e194a1d8c998f05e6143e01a09e6b43176c511217ea3c77742afd9f2566251c50fc23aeb916442401f7118c1d1fe21f0172382a7f4f2c516c1d7d873e24 libbsd-checksum = 5c7d98474000af1271a36ab769e54aba41578e0b0f06e47af2986d6821b6586ac430ec04cc51b7836823834dd9d0aec9f4ab3af088b94f963b89729fa2cc95d8 diff --git a/reproduce/software/config/servers-backup.conf b/reproduce/software/config/servers-backup.conf index 26c992f..04ad406 100644 --- a/reproduce/software/config/servers-backup.conf +++ b/reproduce/software/config/servers-backup.conf @@ -9,6 +9,6 @@ # permitted in any medium without royalty provided the copyright notice and # this notice are preserved. This file is offered as-is, without any # warranty. -http://gitlab.cefca.es/maneage/tarballs-software/-/raw/master/ +http://gitlab.cefca.es/maneage/tarballs-software/-/raw/master http://gitlab.com/maneage/tarballs-software/-/raw/master http://git.maneage.org/tarballs-software.git/plain diff --git a/reproduce/software/config/urls.conf b/reproduce/software/config/urls.conf index f976930..8010c9d 100644 --- a/reproduce/software/config/urls.conf +++ b/reproduce/software/config/urls.conf @@ -111,6 +111,7 @@ #icu-url = https://github.com/unicode-org/icu/releases #imagemagick-url = https://download.imagemagick.org/ImageMagick/download #imfit-url = http://www.mpe.mpg.de/~erwin/resources/imfit +#jq-url = https://jqlang.org/download #lapack-url = http://www.netlib.org/lapack #libbsd-url = http://libbsd.freedesktop.org/releases #libffi-url = https://github.com/libffi/libffi diff --git a/reproduce/software/config/versions.conf b/reproduce/software/config/versions.conf index 3e4ec9a..552043f 100644 --- a/reproduce/software/config/versions.conf +++ b/reproduce/software/config/versions.conf @@ -26,6 +26,7 @@ diffutils-version = 3.12 findutils-version = 4.10.0 flock-version = 0.4.0 gawk-version = 5.3.2 +gcc-version = 16.1.0 gettext-version = 1.0 git-version = 2.52.0 grep-version = 3.12 @@ -107,12 +108,6 @@ file-version = 5.46 ncurses-version = 6.5 pkgconfig-version = 0.29.2 -# GCC -# --- -# -# There is a version-specific 'sed -e' fix for a bug in this version of GCC -# that should no longer be necessary in future versions. -gcc-version = 15.2.0 @@ -156,6 +151,7 @@ healpix-version = 3.83-2024Nov13 icu-version = 70.1 imagemagick-version = 7.1.0-13 imfit-version = 1.6.1 +jq-version = 1.8.2 libbsd-version = 0.11.3 libffi-version = 3.4.7 libidn-version = 1.42 diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 92fa8f0..000f62d 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -369,14 +369,17 @@ $(ibidir)/tar-$(tar-version): \ $(ibidir)/zlib-$(zlib-version) \ $(ibidir)/bzip2-$(bzip2-version) -# About the onfigurations: nls and iconv were creating problems with -# the dependencies on MacOs and are not relevant in the context of -# Maneage, hence, they are disabled. +# About the configurations: disabled because they are not relevant in +# the context of Maneage usage. +# - nls: conflicts with iconv on macOS. +# - acl: causing crashes when Tar was compiled with GCC 16.1.1. $(call unsafe-config) tarball=tar-$(tar-version).tar.lz $(call import-source, $(tar-url), $(tar-checksum)) $(call gbuild, tar-$(tar-version), , \ - --disable-nls am_cv_func_iconv=no, \ + --disable-nls \ + --disable-acl \ + am_cv_func_iconv=no, \ -j$(numthreads) V=1) echo "GNU Tar $(tar-version)" > $@ @@ -417,35 +420,56 @@ $(ibidir)/pkg-config-$(pkgconfig-version): $(ibidir)/tar-$(tar-version) # 'pkg-config' is built first. But when we don't have a clean build # (and 'libiconv' exists) there will be a problem. So before # re-building 'pkg-config', we'll remove any installation of -# 'libiconv'. +# 'libiconv'. This can create a known bug: +# https://savannah.nongnu.org/bugs/?67001 rm -f $(ildir)/libiconv* $(idir)/include/iconv.h -# Some Mac OS systems may have a version of the GNU C Compiler (GCC) -# installed that doesn't support some necessary features of building -# Glib (as part of pkg-config), so we will disable pkg-config's -# internal Glib for Mac systems, and to be further safe, we'll make -# sure it will use LLVM's Clang. -# -# On macOS systems, to ensure that Clang can build pkg-config, take -# the following steps: -# 1. Install the latest Glib via Homebrew: -# brew install glib -# 2. Set these environment variables before configuring Maneage: -# export GLIB_CFLAGS=$(pkg-config --cflags glib-2.0) -# export GLIB_LIBS=$(pkg-config --libs glib-2.0) -# 3. Ensure PKG_CONFIG_PATH includes Homebrew's pkgconfig: -# export PKG_CONFIG_PATH=/opt/homebrew/lib/pkgconfig:$PKG_CONFIG_PATH +# On GNU/Linux no changes necessary, so we'll start with those. If +# the runner is on a macOS, some changes are necessary. + export compiler="" + confsh=./configure if [ x$(on_mac_os) = xyes ]; then - extra_pkgconf="" + +# On macOS we force Clang, because some GCC versions shipped with +# Xcode lack features needed to build Glib. export compiler="CC=clang" - else - export compiler="" - extra_pkgconf="--with-internal-glib" + +# The bundled glib's gatomic.c uses integer-to-pointer conversions +# that newer Clang (Xcode 26+) treats as hard errors on all macOS +# architectures. + CFLAGS="-Wno-int-conversion $$CFLAGS" + +# On macOS ARM64 the bundled glib sub-configure checks for Carbon +# support with the C preprocessor only. Since 'Carbon/Carbon.h' +# exists in the Xcode SDK on Apple Silicon, the check passes and +# '-framework Carbon' is appended to LDFLAGS. But Carbon cannot +# actually be linked on ARM64, so every subsequent linker test in +# glib's configure fails silently and libglib-2.0.la is never +# created. We work around this by passing gbuild a tiny wrapper +# script (instead of ./configure) that patches glib/configure to +# force glib_have_carbon=no before the real configure runs. +# +# NOTE: the initial two SPACES in the SED command are very +# important (see 'glib/configure'): without them, the wrong +# variable will be set. + confsh=$(ddir)/pkg-config-configure + printf '#!/bin/sh\n' > $$confsh + printf 'sed -i.bak "s/^ glib_have_carbon=yes$$/' >> $$confsh + printf ' glib_have_carbon=no/"' >> $$confsh + printf ' glib/configure\n' >> $$confsh + printf 'exec ./configure "$$@"\n' >> $$confsh + chmod +x $$confsh fi + +# Configuration options: +# --with-internal-glib: the internal glib is necessary because +# without it, we have many problems in macOS. export CFLAGS="-std=$(std_c_old) $$CFLAGS" $(call gbuild, pkg-config-$(pkgconfig-version), static, \ - $$compiler $$extra_pkgconf \ - --with-pc-path=$(ildir)/pkgconfig, V=1) + $$compiler \ + --with-internal-glib \ + --with-pc-path=$(ildir)/pkgconfig, V=1, , $$confsh) + if [ -f $$confsh ]; then rm $$confsh; fi # Only on macOS. echo "pkg-config $(pkgconfig-version)" > $@ @@ -1458,32 +1482,6 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) # first look into its own 'include' directory before anything else. export CPPFLAGS="-I$$(pwd)/include $(CPPFLAGS)" -# In the GNU C Library 2.36 (which is more recent than GCC 12.1.0), -# the 'linux/mount.h' (loaded by 'linux/fs.h', which is loaded by -# 'libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cpp' -# in GCC) conflicts with 'sys/mount.h' which is directly loaded by -# the same file! This is a known conflict in glibc 2.36 (see -# [1]). As described in [1], one solution is the final job done in -# [2]. We therefore do this process here: 1) Not loading -# 'linux/fs.h', and adding the necessary macros directly. -# -# [1] https://sourceware.org/glibc/wiki/Release/2.36#Usage_of_.3Clinux.2Fmount.h.3E_and_.3Csys.2Fmount.h.3E -# [2] https://reviews.llvm.org/D129471 - sed -e's|\#include ||' \ - -e"s|FS_IOC_GETFLAGS;|_IOR('f', 1, long);|" \ - -e"s|FS_IOC_GETVERSION;|_IOR('v', 1, long);|" \ - -e"s|FS_IOC_SETFLAGS;|_IOW('f', 2, long);|" \ - -e"s|FS_IOC_SETVERSION;|_IOW('v', 2, long);|" \ - -i libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cpp - -# Bug in GCC 15.2.0 using glibc 2.43 (see [1]) as fixed in [2]. -# [1] https://patchwork.ozlabs.org/project/gcc/patch/e1679277-d7c9-49aa-8365-a8dca082d9bd@web.de -# [2] https://github.com/johnny-mnemonic/toolchain-autobuilds/commit/9585fdfc - sed -e's|char \*q = strchr (p + 1,|const char \*q = strchr (p + 1,|' \ - libgomp/affinity-fmt.c > affinity-fmt-tmp.c - mv affinity-fmt-tmp.c libgomp/affinity-fmt.c - - # Set the build directory for the processing. mkdir build cd build diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index d059544..281bd84 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -465,7 +465,7 @@ $(ibidir)/cfitsio-$(cfitsio-version): $(call gbuild, cfitsio-$(cfitsio-version), , \ $$confopts \ --disable-curl \ - --disable-fortran \ + --without-fortran \ --enable-reentrant \ --with-bzip2=$(idir), , \ make fpack funpack) @@ -1764,6 +1764,20 @@ $(ibidir)/imfit-$(imfit-version): \ rm -rf $$unpackdir echo "Imfit $(imfit-version) \citep{imfit2015}" > $@ +# jq (to parse JSON files on the command-line) uses the external Oniguruma +# regular expression library that has ended development in 2025-04-24, see +# https://github.com/kkos/oniguruma. If this library ('libonig') is present +# on the host, jq will build with that: causing a leaking of os environment +# into the Maneage'd environment. Fortunately, jq maintains a copy of this +# library in its own source (within its tarball) and will build it with the +# '--with-oniguruma=builtin' configuration option. +$(ibidir)/jq-$(jq-version): + tarball=jq-$(jq-version).tar.lz + $(call import-source, $(jq-url), $(jq-checksum)) + $(call gbuild, jq-$(jq-version), static, \ + --with-oniguruma=builtin, V=1) + echo "jq $(jq-version)" > $@ + # This is the LIGO lscsoft package 'metaio' [1] (to be distinguished from # some other packages with the same name). # @@ -1771,7 +1785,7 @@ $(ibidir)/imfit-$(imfit-version): \ $(ibidir)/metaio-$(metaio-version): tarball=metaio-$(metaio-version).tar.lz $(call import-source, $(metaio-url), $(metaio-checksum)) - $(call gbuild, metaio-$(metaio-version), static) + $(call gbuild, metaio-$(metaio-version), static,, V=1) echo "Lscsoft Metaio $(metaio-version)" > $@ # Minizip 1.x is actually distributed within zlib. It doesn't have its own @@ -2279,14 +2293,45 @@ $(ibidir)/vim-$(vim-version): rm -rf $$unpackdir echo "VIM $(vim-version)" > $@ -$(ibidir)/unzip-$(unzip-version): $(ibidir)/gzip-$(gzip-version) +$(ibidir)/unzip-$(unzip-version): + +# Prepare the tarball and build directory. tarball=unzip-$(unzip-version).tar.lz $(call import-source, $(unzip-url), $(unzip-checksum)) - $(call gbuild, unzip-$(unzip-version), static,, \ - -f unix/Makefile generic \ - CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ - -f unix/Makefile generic \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) + +# Unpack and go into the directory. + export LDFLAGS="$$LDFLAGS -static" + cd $(ddir) + rm -fr unzip-$(unzip-version)/ + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd unzip-$(unzip-version)/ + +# Remove reference to the old 'gmtime' function and include the +# system's 'time.h instead. + sed -i -e's|struct tm *gmtime(), *localtime();|#include |' \ + -e'/gmtime/d; /localtime/d' unix/unxcfg.h + +# The 'NO_DIR' variable causes conflicts: the unzip code has its own +# internal fallback implementation called 'NO_DIR'. The modern Linux +# system already provides a standard implementation using +# 'dirent.h'. These two systems conflict with each other. This causes +# compilation errors such as conflicting types for DIR and invalid +# use of DIR. + sed -i -e's/-DNO_DIR//g' unix/configure + sed -i -e's/#ifdef NO_DIR/#if 0/g' unix/unix.c + sed -i -e's/-DNO_DIR//g' -e's/-U_NO_DIR//g' unix/Makefile + +# Since unzip is no longer maintained, it uses an old C standard. + export CFLAGS="-std=gnu89 -fcommon" + +# Build and install + make SHELL=$(ibdir)/bash -f unix/Makefile generic + make SHELL=$(ibdir)/bash install -f unix/Makefile \ + generic BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 + +# Go back to the top build directory, clean up and finalize. + cd $(ddir) + rm -rf unzip-$(unzip-version)/ echo "Unzip $(unzip-version)" > $@ $(ibidir)/zip-$(zip-version): diff --git a/reproduce/software/shell/apptainer.sh b/reproduce/software/shell/apptainer.sh index 8760db6..eb24e9e 100755 --- a/reproduce/software/shell/apptainer.sh +++ b/reproduce/software/shell/apptainer.sh @@ -343,7 +343,7 @@ From: $base_sif %post cd /home/maneager/source - ./project configure --jobs=$jobs \\ + ./project configure --jobs=$jobs --no-pause \\ --input-dir=/home/maneager/input \\ --build-dir=$intbuild \\ --software-dir=/home/maneager/tarballs-software diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 3e0163e..e17c9ad 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -375,20 +375,23 @@ if [ $built_container = 0 ]; then fi # On macOS, the way of obtaining the number of cores is different - # between Intel or Apple M1 CPUs. Here we disinguish between Apple - # M1 or others. - maccputype=$(sysctl -n machdep.cpu.brand_string) - if [ x"$maccputype" = x"Apple M1" ] \ - || [ x"$maccputype" = x"Apple M2" ] \ - || [ x"$maccputype" = x"Apple M3" ] \ - || [ x"$maccputype" = x"Apple M4" ] \ - || [ x"$maccputype" = x"Apple M5" ] ; then - address_size_physical=$(sysctl -n machdep.cpu.thread_count) - address_size_virtual=$(sysctl -n machdep.cpu.logical_per_package) - else - address_size_physical=$(sysctl -n machdep.cpu.address_bits.physical) - address_size_virtual=$(sysctl -n machdep.cpu.address_bits.virtual) - fi + # between Intel or Apple Silicon CPUs. Here we distinguish between + # Apple Silicon (any "Apple M*" brand string, e.g. M1, M1 Pro, M1 + # Max, M2, M2 Pro, ...) and Intel-based Macs. Note that the Apple + # Silicon group can be in the formats of "Apple M1" or "Apple M1 + # Pro", so we use a prefix match via a 'case' pattern to include + # both in one check. + maccputype=$(sysctl -n machdep.cpu.brand_string 2>/dev/null) + case "$maccputype" in + "Apple M"*) + address_size_physical=$(sysctl -n machdep.cpu.thread_count) + address_size_virtual=$(sysctl -n machdep.cpu.logical_per_package) + ;; + *) + address_size_physical=$(sysctl -n machdep.cpu.address_bits.physical) + address_size_virtual=$(sysctl -n machdep.cpu.address_bits.virtual) + ;; + esac address_sizes="$address_size_physical bits physical, " address_sizes+="$address_size_virtual bits virtual" else @@ -1781,30 +1784,37 @@ fi # an example, the directory part of the URL for all the other software are # the same). This is not done if the options '--debug' or `--offline` are # used. -zenodourl="" -user_backup_urls="" -zenodocheck="$bdir"/software/zenodo-check.html -if [ $built_container = 0 ]; then - if [ x$debug = x ] && [ x$offline = x ]; then - if $downloader $zenodocheck \ - https://doi.org/10.5281/zenodo.3883409; then - zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ - | sed -n -e'/http/p' \ - | tr ' ' '\n' \ - | grep http \ - | sed -e 's/href="//' -e 's|/coreutils| |' \ - | awk 'NR==1{print $1}') - fi - fi - rm -f $zenodocheck - - # Add the Zenodo URL to the user's given back software URLs. Since the - # user can specify 'user_backup_urls' (not yet implemented as an option - # in './project'), we'll give preference to their specified servers, - # then add the Zenodo URL afterwards. - user_backup_urls="$user_backup_urls $zenodourl" - elapsed_time_from_prev_step zenodo-url -fi +# +# NOTE on commented region below (starting with '##'): As described [1], +# Zenodo's upload strategy does not allow a repository with more than one +# hundred files. So until the solution there has been implemented, this +# step has been commented. +# +# [1] https://savannah.nongnu.org/task/?16621 +## zenodourl="" +## user_backup_urls="" +## zenodocheck="$bdir"/software/zenodo-check.html +## if [ $built_container = 0 ]; then +## if [ x$debug = x ] && [ x$offline = x ]; then +## if $downloader $zenodocheck \ +## https://doi.org/10.5281/zenodo.3883409; then +## zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ +## | sed -n -e'/http/p' \ +## | tr ' ' '\n' \ +## | grep http \ +## | sed -e 's/href="//' -e 's|/coreutils| |' \ +## | awk 'NR==1{print $1}') +## fi +## fi +## rm -f $zenodocheck +## +## # Add the Zenodo URL to the user's given back software URLs. Since the +## # user can specify 'user_backup_urls' (not yet implemented as an option +## # in './project'), we'll give preference to their specified servers, +## # then add the Zenodo URL afterwards. +## user_backup_urls="$user_backup_urls $zenodourl" +## elapsed_time_from_prev_step zenodo-url +## fi diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh index 8236247..3163f74 100755 --- a/reproduce/software/shell/pre-make-build.sh +++ b/reproduce/software/shell/pre-make-build.sh @@ -120,16 +120,24 @@ download_tarball() { "$bservers" fi + # Make sure this is the correct tarball. if type sha512sum > /dev/null 2> /dev/null; then - checksum=$(sha512sum "$ucname" | awk '{print $1}') + + # On macOS, sha512sum wraps the hash in two lines. The 'tr -d '\n'' + # part joins them into one before awk extracts the first field. On + # Linux (GNU coreutils) the output is already a single line, so the + # 'tr' command has no effect. + checksum=$(sha512sum "$ucname" | tr -d '\n' | awk '{print $1}') expectedchecksum=$(awk '/^'$progname'-checksum/{print $3}' \ "$checksumsfile") - if [ x$checksum = x$expectedchecksum ]; then mv "$ucname" "$maneagetar" + if [ x$checksum = x$expectedchecksum ]; then + mv "$ucname" "$maneagetar" else - echo "ERROR: Non-matching checksum: $tarball" - echo "Checksum should be: $expectedchecksum" - echo "Checksum is: $checksum" + echo "ERROR: Non-matching checksum in $tarball" + echo " Checksum should be: $expectedchecksum" + echo " Checksum is: $checksum" + rm -f "$ucname" exit 1 fi; else mv "$ucname" "$maneagetar" -- cgit v1.2.1