aboutsummaryrefslogtreecommitdiff
path: root/reproduce
diff options
context:
space:
mode:
authorMohammad Akhlaghi <mohammad@akhlaghi.org>2021-04-17 04:55:58 +0100
committerMohammad Akhlaghi <mohammad@akhlaghi.org>2021-04-17 04:55:58 +0100
commitd11725a09133a422dbad19f46caa088bd0bca3a9 (patch)
tree94ec18b1ff1324837888e3b16025c835f7958f5b /reproduce
parent30bf4624adf40e9611ad8f6a0214e725b2ea88af (diff)
parent6e4ec9a305f7021643fe22e08fe0ad17dd363a93 (diff)
Imported recent work in Maneage, minor conflicts fixed
Some minor conflicts (all expected from the commit messages in the Maneage branch) occurred but were easily fixed.
Diffstat (limited to 'reproduce')
-rw-r--r--reproduce/analysis/config/metadata.conf21
-rw-r--r--reproduce/analysis/make/initialize.mk35
-rw-r--r--reproduce/analysis/make/top-make.mk4
-rw-r--r--reproduce/analysis/make/top-prepare.mk2
-rw-r--r--reproduce/software/make/basic.mk8
-rw-r--r--reproduce/software/make/high-level.mk8
-rwxr-xr-xreproduce/software/shell/configure.sh69
7 files changed, 104 insertions, 43 deletions
diff --git a/reproduce/analysis/config/metadata.conf b/reproduce/analysis/config/metadata.conf
index 470e429..b1f657b 100644
--- a/reproduce/analysis/config/metadata.conf
+++ b/reproduce/analysis/config/metadata.conf
@@ -1,6 +1,19 @@
-# Project meta-data that can be used in a project's output datasets and
+# Project meta-data that will be used in a project's output datasets and
# final paper. Please set the values here and use them in your analysis or
-# paper, don't repeat them
+# paper, don't repeat them.
+#
+# These variables are used in 'reproduce/analysis/make/initialize.mk': 1)
+# to create a Make variable called 'print-general-metadata'. You can simply
+# print this variable's value in any plain-text output.
+#
+# Why add a Copyright for the data? people need to know if they can "use"
+# the dataset (i.e., modify it), or possibly re-distribute it and their
+# derived products. They also need to know how they can contact the creator
+# of the datset (who is usually also the copyright owner). So take this
+# seriously and add your name and email-address (or the name of the person
+# and email of the person who was in charge of that part of the project),
+# and the copyright license name and standard link to the fully copyright
+# license.
#
# Copyright (C) 2020-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org>
#
@@ -12,7 +25,9 @@
# Project information
metadata-title = Towards Long-term and Archivable Reproducibility
-# DOIs and identifiers.
+# DOIs and identifiers (don't include fixed URL prefixes like
+# 'https://doi.org/' or 'https://arxiv.org/abs'), they will be added
+# automatically where necessary.
metadata-arxiv = 2006.03018
metadata-doi-zenodo = 10.5281/zenodo.4291207
metadata-doi-journal = 10.1109/MCSE.2021.3072860
diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk
index 8af3199..bc73df8 100644
--- a/reproduce/analysis/make/initialize.mk
+++ b/reproduce/analysis/make/initialize.mk
@@ -224,7 +224,7 @@ project-package-contents = $(texdir)/$(project-package-name)
$(project-package-contents) $(mtexdir)/initialize.tex
texclean:
- rm *.pdf
+ rm -f *.pdf
rm -rf $(texdir)/build/*
mkdir $(texdir)/build/tikz # 'tikz' is assumed to already exist.
@@ -253,8 +253,7 @@ distclean: clean
# `rm' program. So for this recipe, we'll use the host system's
# `rm', not our own.
$$sys_rm -rf $(BDIR)
- $$sys_rm -f $(pconfdir)/LOCAL.conf
- $$sys_rm -f Makefile .gnuastro .local .build
+ $$sys_rm -f .local .build $(pconfdir)/LOCAL.conf
@@ -367,18 +366,18 @@ $(project-package-contents): paper.pdf | $(texdir)
sed -e's|{tikz/}|{tex/tikz/}|' $$pgfsettings > $$pgfsettings.new
mv $$pgfsettings.new $$pgfsettings
- # Clean temporary (currently those ending in `~') files.
- cd $(texdir)
- find $(project-package-name) -name \*~ -delete
- find $(project-package-name) -name \*.swp -delete
-
# PROJECT SPECIFIC
# ----------------
- # Put any project specific distribution steps here.
+ # Put any project-specific distribution steps here.
cd $$curdir
cp tex/build/build/appendix.bbl $$dir/
# ----------------
+ # Clean temporary files that may have been created by text editors.
+ cd $(texdir)
+ find $(project-package-name) -name \*~ -delete
+ find $(project-package-name) -name \*.swp -delete
+
# Package into `.tar.gz' or '.tar.lz'.
dist dist-lzip: $(project-package-contents)
curdir=$$(pwd)
@@ -460,15 +459,21 @@ $(data-publish-dir):; mkdir $@
# Print Copyright statement
# -------------------------
#
-# This statement can be used in published datasets that are in plain-text
-# format. It assumes you have already put the data-specific statements in
-# its first argument, it will supplement them with general project links.
-doi-prefix-url = https://doi.org
+# The 'print-general-metadata' can be used to print the general metadata in
+# published datasets that are in plain-text format. It should be called
+# with make's 'call' function like this (where 'FILENAME' is the name of
+# the file it will append this content to):
+#
+# $(call print-general-metadata, FILENAME)
+#
+# See 'reproduce/analysis/make/delete-me.mk' (in the Maneage branch) for a
+# real-world usage of this variable.
+doi-prefix-url = https://doi.org
arxiv-prefix-url = https://arxiv.org/abs
print-general-metadata = \
echo "\# Project title: $(metadata-title)" >> $(1); \
echo "\# Git commit (that produced this dataset): $(project-commit-hash)" >> $(1); \
- echo "\# Project's Git repository: $(metadata-git-repository)" >> $(1); \
+ echo "\# Git repository: $(metadata-git-repository)" >> $(1); \
if [ x$(metadata-arxiv) != x ]; then \
echo "\# Pre-print: $(arxiv-prefix-url)/abs/$(metadata-arxiv)" >> $(1); fi; \
if [ x$(metadata-doi-journal) != x ]; then \
@@ -501,6 +506,8 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
echo "\newcommand{\projectdate}{$$d}" > $@
echo "\newcommand{\projecttitle}{$(metadata-title)}" >> $@
echo "\newcommand{\projectversion}{$(project-commit-hash)}" >> $@
+ echo "\newcommand{\projectgitrepo}{$(metadata-git-repository)}" >> $@
+ echo "\newcommand{\projectcopyrightowner}{$(metadata-copyright-owner)}" >> $@
# arXiv/Zenodo identifier (necessary for download link):
echo "\newcommand{\projectarxivid}{$(metadata-arxiv)}" >> $@
diff --git a/reproduce/analysis/make/top-make.mk b/reproduce/analysis/make/top-make.mk
index 6b055f7..27c1b5b 100644
--- a/reproduce/analysis/make/top-make.mk
+++ b/reproduce/analysis/make/top-make.mk
@@ -51,7 +51,7 @@ include reproduce/software/config/LOCAL.conf
# Controlling this requires two variables that are available at this stage:
#
# - `GROUP-NAME': from `LOCAL.conf' (which was built by `./project configure').
-# - `reproducible_paper_group_name': value to the `--group' option.
+# - `maneage_group_name': value to the `--group' option.
#
# The analysis is only done when both have the same group name. Note that
# when the project isn't being built for a group, both variables will be an
@@ -69,7 +69,7 @@ top-pdfs = paper.pdf
else
top-pdfs = paper.pdf supplement.pdf
endif
-ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME))
+ifeq (x$(maneage_group_name),x$(GROUP-NAME))
all: $(top-pdfs)
else
all:
diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk
index f81ac07..fb5700e 100644
--- a/reproduce/analysis/make/top-prepare.mk
+++ b/reproduce/analysis/make/top-prepare.mk
@@ -35,7 +35,7 @@ include reproduce/software/config/LOCAL.conf
# -------------------------------
#
# See `top-make.mk' for complete explanation.
-ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME))
+ifeq (x$(maneage_group_name),x$(GROUP-NAME))
all: $(BDIR)/software/preparation-done.mk
@echo "Project preparation is complete.";
else
diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk
index 9217ee9..fb378b8 100644
--- a/reproduce/software/make/basic.mk
+++ b/reproduce/software/make/basic.mk
@@ -86,7 +86,8 @@ export SHELL := $(ibdir)/dash
export PATH := $(ibdir):$(PATH)
export PKG_CONFIG_PATH := $(ildir)/pkgconfig
export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig
-export CPPFLAGS := -I$(idir)/include $(CPPFLAGS)
+export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) \
+ -Wno-nullability-completeness
export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS)
# This is the "basic" tools where we are relying on the host operating
@@ -210,7 +211,10 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \
# POSIX Threads library for multi-threaded programs.
for l in dl pthread; do
if [ -f /usr/lib/lib$$l.a ]; then
- ln -sf /usr/lib/lib$$l.* $(ildir)/
+ for f in /usr/lib/lib$$l.*; do
+ ln -sf $$(realpath $$f) \
+ $$(echo $$f | sed -e's|/usr/lib|$(ildir)|')
+ done
fi
done
diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk
index d69722e..6ea782c 100644
--- a/reproduce/software/make/high-level.mk
+++ b/reproduce/software/make/high-level.mk
@@ -125,10 +125,10 @@ all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) \
export PATH := $(ibdir)
export CCACHE_DISABLE := 1
export SHELL := $(ibdir)/bash
-export CPPFLAGS := -I$(idir)/include
.SHELLFLAGS := --noprofile --norc -ec
export LDFLAGS := $(rpath_command) -L$(ildir)
export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig
+export CPPFLAGS := -I$(idir)/include -Wno-nullability-completeness
export PKG_CONFIG_PATH := $(ildir)/pkgconfig:$(idir)/share/pkgconfig
# Settings specific to this Makefile.
@@ -428,7 +428,8 @@ $(ibidir)/cfitsio-$(cfitsio-version):
export gbuild_tar=$(ddir)/$$customtar
$(call gbuild, cfitsio-$(cfitsio-version), , \
--enable-sse2 --enable-reentrant \
- --with-bzip2=$(idir), , make shared fpack funpack)
+ --with-bzip2=$(idir), , \
+ make shared fpack funpack)
rm $$customtar
echo "CFITSIO $(cfitsio-version)" > $@
@@ -1316,7 +1317,8 @@ $(ibidir)/swig-$(swig-version):
# pcr is a dependency of swig
tarball=swig-$(swig-version).tar.gz
$(call import-source, $(swig-url), $(swig-checksum))
- $(call gbuild, swig-$(swig-version), static, --without-pcre)
+ $(call gbuild, swig-$(swig-version), static, \
+ --without-pcre --without-tcl)
echo "Swig $(swig-version)" > $@
# The disables:
diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh
index 812f3d3..e15a8e9 100755
--- a/reproduce/software/shell/configure.sh
+++ b/reproduce/software/shell/configure.sh
@@ -3,6 +3,7 @@
# Necessary preparations/configurations for the reproducible project.
#
# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org>
+# Copyright (C) 2021 Raul Infante-Sainz <infantesainz@gmail.com>
#
# This script is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -423,8 +424,17 @@ if ! [ -d $compilertestdir ]; then mkdir $compilertestdir; fi
# Check C compiler
# ----------------
+#
+# Here we check if the C compiler works properly. About the "no warning"
+# variable ('nowarnings'):
+#
+# -Wno-nullability-completeness: on macOS Big Sur 11.2.3 and Xcode 12.4,
+# hundreds of 'nullability-completeness' warnings are printed which can
+# be very annoying and even hide important errors or warnings. It is
+# also harmless for our test here, so it is generally added.
testprog=$compilertestdir/test
testsource=$compilertestdir/test.c
+noccwarnings="-Wno-nullability-completeness"
echo; echo; echo "Checking host C compiler ('$CC')...";
cat > $testsource <<EOF
#include <stdio.h>
@@ -432,7 +442,7 @@ cat > $testsource <<EOF
int main(void){printf("...C compiler works.\n");
return EXIT_SUCCESS;}
EOF
-if $CC $testsource -o$testprog && $testprog; then
+if $CC $noccwarnings $testsource -o$testprog && $testprog; then
rm $testsource $testprog
else
rm $testsource
@@ -761,7 +771,7 @@ fi
# exists and we don't want to re-write it).
if [ $rewritepconfig = no ]; then
oldgroupname=$(awk '/GROUP-NAME/ {print $3; exit 0}' $pconf)
- if [ "x$oldgroupname" = "x$reproducible_paper_group_name" ]; then
+ if [ "x$oldgroupname" = "x$maneage_group_name" ]; then
just_a_place_holder_to_avoid_not_equal_test=1;
else
echo "-----------------------------"
@@ -1079,7 +1089,7 @@ if [ $rewritepconfig = yes ]; then
-e's|@ddir[@]|'"$ddir"'|' \
-e's|@sys_cpath[@]|'"$sys_cpath"'|' \
-e's|@downloader[@]|'"$downloader"'|' \
- -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \
+ -e's|@groupname[@]|'"$maneage_group_name"'|' \
$pconf.in >> $pconf
else
# Read the values from existing configuration file. Note that the build
@@ -1238,7 +1248,7 @@ if ! [ -d "$mtexdir" ]; then mkdir "$mtexdir"; fi
# TeX build directory. If built in a group scenario, the TeX build
# directory must be separate for each member (so they can work on their
# relevant parts of the paper without conflicting with each other).
-if [ "x$reproducible_paper_group_name" = x ]; then
+if [ "x$maneage_group_name" = x ]; then
texbdir="$texdir"/build
else
user=$(whoami)
@@ -1458,18 +1468,19 @@ fi
# which will download the DOI-resolved webpage, and extract the Zenodo-URL
# of the most recent version from there (using the 'coreutils' tarball as
# an example, the directory part of the URL for all the other software are
-# the same).
+# the same). This is not done if the option `--debug' is used.
+zenodourl=""
user_backup_urls=""
zenodocheck=.build/software/zenodo-check.html
-if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then
- zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \
- | sed -n -e'/http/p' \
- | tr ' ' '\n' \
- | grep http \
- | sed -e 's/href="//' -e 's|/coreutils| |' \
- | awk 'NR==1{print $1}')
-else
- zenodourl=""
+if [ x$debug = x ]; then
+ if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then
+ zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \
+ | sed -n -e'/http/p' \
+ | tr ' ' '\n' \
+ | grep http \
+ | sed -e 's/href="//' -e 's|/coreutils| |' \
+ | awk 'NR==1{print $1}')
+ fi
fi
rm -f $zenodocheck
@@ -1497,6 +1508,28 @@ user_backup_urls="$user_backup_urls $zenodourl"
+# Corrections for debugging mode
+# ------------------------------
+#
+# If the user wants to debug the software configuration, they are usually
+# focused on the building of the single problematic software. Therefore,
+# the default multi-threaded execution of Make with the '--keep-going'
+# option are very annoying and can even hide important warnings. Recall
+# that with '--keep-going', Make will continue building other targets, even
+# if one target fails. When the user runs './project configure --debug',
+# the 'debug' variable will not be empty and this mode will be activated.
+if [ x$debug = x ]; then
+ keepgoing="--keep-going"
+else
+ jobs=1
+ numthreads=1
+ keepgoing=""
+fi
+
+
+
+
+
# Build other basic tools our own GNU Make
# ----------------------------------------
#
@@ -1504,7 +1537,7 @@ user_backup_urls="$user_backup_urls $zenodourl"
# Bash, Make, or AWK. In this step, we'll install such low-level basic
# tools, but we have to be very portable (and use minimal features in all).
echo; echo "Building necessary software (if necessary)..."
-.local/bin/make -k -f reproduce/software/make/basic.mk \
+.local/bin/make $keepgoing -f reproduce/software/make/basic.mk \
user_backup_urls="$user_backup_urls" \
sys_library_path=$sys_library_path \
rpath_command=$rpath_command \
@@ -1532,7 +1565,7 @@ else
numthreads=$jobs
fi
.local/bin/env -i HOME=$bdir \
- .local/bin/make -k -f reproduce/software/make/high-level.mk \
+ .local/bin/make $keepgoing -f reproduce/software/make/high-level.mk \
user_backup_urls="$user_backup_urls" \
sys_library_path=$sys_library_path \
rpath_command=$rpath_command \
@@ -1725,10 +1758,10 @@ echo `.local/bin/date` > $finaltarget
#
# The configuration is now complete, we can inform the user on the next
# step(s) to take.
-if [ x$reproducible_paper_group_name = x ]; then
+if [ x$maneage_group_name = x ]; then
buildcommand="./project make -j8"
else
- buildcommand="./project make --group=$reproducible_paper_group_name -j8"
+ buildcommand="./project make --group=$maneage_group_name -j8"
fi
cat <<EOF