diff options
-rw-r--r-- | README-hacking.md | 34 | ||||
-rwxr-xr-x | project | 17 | ||||
-rw-r--r-- | reproduce/analysis/config/metadata.conf | 21 | ||||
-rw-r--r-- | reproduce/analysis/make/initialize.mk | 35 | ||||
-rw-r--r-- | reproduce/analysis/make/top-make.mk | 4 | ||||
-rw-r--r-- | reproduce/analysis/make/top-prepare.mk | 2 | ||||
-rw-r--r-- | reproduce/software/make/basic.mk | 8 | ||||
-rw-r--r-- | reproduce/software/make/high-level.mk | 8 | ||||
-rwxr-xr-x | reproduce/software/shell/configure.sh | 69 | ||||
-rw-r--r-- | tex/src/preamble-project.tex | 8 |
10 files changed, 144 insertions, 62 deletions
diff --git a/README-hacking.md b/README-hacking.md index 92c878e..e42bf42 100644 --- a/README-hacking.md +++ b/README-hacking.md @@ -1050,11 +1050,13 @@ future. information for plain-text data and will put it in the `$(print-general-metadata)` variable. It is thus recommended to print this variable into your plain-text file before printing the actual - data (so it shows on top of the file). If you are publishing your data - in binary formats, please add all the metadata you see in - `$(print-general-metadata)` into each dataset file (for example - keywords in the FITS format). If there are many files, its easy to - define a tiny shell-script to do the job on each dataset. + data (so it shows on top of the file). For a real-world example, see + its usage in `reproduce/analysis/make/delete-me.mk` (in the `maneage` + branch). If you are publishing your data in binary formats, please add + all the metadata you see in `$(print-general-metadata)` into each + dataset file (for example keywords in the FITS format). If there are + many files, its easy to define a tiny shell-script to do the job on + each dataset. - **Link to figure datasets in caption**: all the datasets that go into the plots should be uploaded directly to Zenodo so they can be @@ -1209,15 +1211,31 @@ future. - **After acceptance (before publication)**: Congratulations on the acceptance! The main science content of your paper can't be changed any - more, but the paper will not go to the publication editor (for language + more, but the paper will now go to the publication editor (for language and style). Your approval of the final proof is necessary before the - paper is finally published. Some journals associate your paper's DOI - during this process. So before approving the final proof do these steps: + paper is finally published. Use this period to finalize the final + metadata of your project: the journal's DOI. Some journals associate + your paper's DOI during this process. So before approving the final + proof do these steps: * Add the Journal DOI in `reproduce/analysis/config/metadata.conf`, and re-build your final data products, so this important metadata is added. + * Once you get the final proof, and if everything is OK for you, + implement all the good language corrections/edits they have made + inside your own copy here and commit it into your project. This will + be the final commit of your project before publication. + + * Submit your final project as a new version to Zenodo (and + arXiv). The Zenodo one is most important because your plots will + link to it and you want the commit hash in the data files that + readers will get from Zenodo to be the same hash as the paper. + + * Tell the journal's publication editor to correct the hash and Zenodo + ID in your final proof confirmation (so the links point to the + correct place). Recall that on every new version upload in Zenodo, + you get a new DOI (or Zenodo ID). @@ -4,6 +4,7 @@ # Run `./project --help' for a description of how to use it. # # Copyright (C) 2019-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2021 Raul Infante-Sainz <infantesainz@gmail.com> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -47,7 +48,6 @@ existing_conf=0 highlightnotes=0 separatesupplement=0 scriptname="./project" -minmapsize=10000000000 @@ -109,7 +109,6 @@ Configure options: -e, --existing-conf Use (possibly existing) local configuration. --host-cc Use host system's C compiler, don't build GCC. -i, --input-dir=STR Directory containing input datasets (optional). - -m, --minmapsize=INT [Gnuastro] Minimum number of bytes to use RAM. -s, --software-dir=STR Directory containing necessary software tarballs. --check-config During configuration, show what is being built. --clean-texdir Remove possibly existing build-time subdirectories @@ -118,12 +117,13 @@ Configure options: --all-highlevel Build all high-level software (for development). Configure and Make options: + -d, --debug[=FLAGS] In configure: use -j1, no -k, and no Zenodo check. + In make: 'FLAGS' will be directly passed to 'make'. -g, --group=STR Build and run with write permissions for a group. -j, --jobs=INT Number of threads to build/run the software. -?, --help Print this help list. Make (analysis) options: - -d, --debug=FLAGS Print various types of debugging information. -p, --prepare-redo Re-do preparation (only done automatically once). Make (final PDF) options: @@ -186,9 +186,6 @@ do -i|--input-dir) input_dir="$2"; check_v "$1" "$input_dir"; shift;shift;; -i=*|--input-dir=*) input_dir="${1#*=}"; check_v "$1" "$input_dir"; shift;; -i*) input_dir=$(echo "$1" | sed -e's/-i//'); check_v "$1" "$input_dir"; shift;; - -m|--minmapsize) minmapsize="$2"; check_v "$1" "$minmapsize"; shift;shift;; - -m=*|--minmapsize=*) minmapsize="${1#*=}"; check_v "$1" "$minmapsize"; shift;; - -m*) minmapsize=$(echo "$1" | sed -e's/-m//'); check_v "$1" "$minmapsize"; shift;; -s|--software-dir) software_dir="$2"; check_v "$1" "$software_dir"; shift;shift;; -s=*|--software-dir=*) software_dir="${1#*=}"; check_v "$1" "$software_dir"; shift;; -s*) software_dir=$(echo "$1" | sed -e's/-s//'); check_v "$1" "$software_dir"; shift;; @@ -333,14 +330,14 @@ fi if ! [ x$group = x ]; then # Check if group is usable. - if ! sg "$group" "echo test &> /dev/null" &> /dev/null; then + if ! sg "$group" "echo Group \'$group\' exists"; then echo "$scriptname: '$group' is not a usable group name on this system."; echo "(TIP: you can use the 'groups' command to see your groups)" exit 1 fi # Set the group option for running Make. - gopt="reproducible_paper_group_name=$group" + gopt="maneage_group_name=$group" fi @@ -444,15 +441,15 @@ case $operation in # Variables to pass to the configuration script. export jobs=$jobs + export debug=$debug export host_cc=$host_cc export build_dir=$build_dir export input_dir=$input_dir export scriptname=$scriptname - export minmapsize=$minmapsize + export maneage_group_name=$group export software_dir=$software_dir export existing_conf=$existing_conf export all_highlevel=$all_highlevel - export reproducible_paper_group_name=$group # Run the configuration script if [ x"$group" = x ]; then diff --git a/reproduce/analysis/config/metadata.conf b/reproduce/analysis/config/metadata.conf index 470e429..b1f657b 100644 --- a/reproduce/analysis/config/metadata.conf +++ b/reproduce/analysis/config/metadata.conf @@ -1,6 +1,19 @@ -# Project meta-data that can be used in a project's output datasets and +# Project meta-data that will be used in a project's output datasets and # final paper. Please set the values here and use them in your analysis or -# paper, don't repeat them +# paper, don't repeat them. +# +# These variables are used in 'reproduce/analysis/make/initialize.mk': 1) +# to create a Make variable called 'print-general-metadata'. You can simply +# print this variable's value in any plain-text output. +# +# Why add a Copyright for the data? people need to know if they can "use" +# the dataset (i.e., modify it), or possibly re-distribute it and their +# derived products. They also need to know how they can contact the creator +# of the datset (who is usually also the copyright owner). So take this +# seriously and add your name and email-address (or the name of the person +# and email of the person who was in charge of that part of the project), +# and the copyright license name and standard link to the fully copyright +# license. # # Copyright (C) 2020-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> # @@ -12,7 +25,9 @@ # Project information metadata-title = Towards Long-term and Archivable Reproducibility -# DOIs and identifiers. +# DOIs and identifiers (don't include fixed URL prefixes like +# 'https://doi.org/' or 'https://arxiv.org/abs'), they will be added +# automatically where necessary. metadata-arxiv = 2006.03018 metadata-doi-zenodo = 10.5281/zenodo.4291207 metadata-doi-journal = 10.1109/MCSE.2021.3072860 diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 8af3199..bc73df8 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -224,7 +224,7 @@ project-package-contents = $(texdir)/$(project-package-name) $(project-package-contents) $(mtexdir)/initialize.tex texclean: - rm *.pdf + rm -f *.pdf rm -rf $(texdir)/build/* mkdir $(texdir)/build/tikz # 'tikz' is assumed to already exist. @@ -253,8 +253,7 @@ distclean: clean # `rm' program. So for this recipe, we'll use the host system's # `rm', not our own. $$sys_rm -rf $(BDIR) - $$sys_rm -f $(pconfdir)/LOCAL.conf - $$sys_rm -f Makefile .gnuastro .local .build + $$sys_rm -f .local .build $(pconfdir)/LOCAL.conf @@ -367,18 +366,18 @@ $(project-package-contents): paper.pdf | $(texdir) sed -e's|{tikz/}|{tex/tikz/}|' $$pgfsettings > $$pgfsettings.new mv $$pgfsettings.new $$pgfsettings - # Clean temporary (currently those ending in `~') files. - cd $(texdir) - find $(project-package-name) -name \*~ -delete - find $(project-package-name) -name \*.swp -delete - # PROJECT SPECIFIC # ---------------- - # Put any project specific distribution steps here. + # Put any project-specific distribution steps here. cd $$curdir cp tex/build/build/appendix.bbl $$dir/ # ---------------- + # Clean temporary files that may have been created by text editors. + cd $(texdir) + find $(project-package-name) -name \*~ -delete + find $(project-package-name) -name \*.swp -delete + # Package into `.tar.gz' or '.tar.lz'. dist dist-lzip: $(project-package-contents) curdir=$$(pwd) @@ -460,15 +459,21 @@ $(data-publish-dir):; mkdir $@ # Print Copyright statement # ------------------------- # -# This statement can be used in published datasets that are in plain-text -# format. It assumes you have already put the data-specific statements in -# its first argument, it will supplement them with general project links. -doi-prefix-url = https://doi.org +# The 'print-general-metadata' can be used to print the general metadata in +# published datasets that are in plain-text format. It should be called +# with make's 'call' function like this (where 'FILENAME' is the name of +# the file it will append this content to): +# +# $(call print-general-metadata, FILENAME) +# +# See 'reproduce/analysis/make/delete-me.mk' (in the Maneage branch) for a +# real-world usage of this variable. +doi-prefix-url = https://doi.org arxiv-prefix-url = https://arxiv.org/abs print-general-metadata = \ echo "\# Project title: $(metadata-title)" >> $(1); \ echo "\# Git commit (that produced this dataset): $(project-commit-hash)" >> $(1); \ - echo "\# Project's Git repository: $(metadata-git-repository)" >> $(1); \ + echo "\# Git repository: $(metadata-git-repository)" >> $(1); \ if [ x$(metadata-arxiv) != x ]; then \ echo "\# Pre-print: $(arxiv-prefix-url)/abs/$(metadata-arxiv)" >> $(1); fi; \ if [ x$(metadata-doi-journal) != x ]; then \ @@ -501,6 +506,8 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\projectdate}{$$d}" > $@ echo "\newcommand{\projecttitle}{$(metadata-title)}" >> $@ echo "\newcommand{\projectversion}{$(project-commit-hash)}" >> $@ + echo "\newcommand{\projectgitrepo}{$(metadata-git-repository)}" >> $@ + echo "\newcommand{\projectcopyrightowner}{$(metadata-copyright-owner)}" >> $@ # arXiv/Zenodo identifier (necessary for download link): echo "\newcommand{\projectarxivid}{$(metadata-arxiv)}" >> $@ diff --git a/reproduce/analysis/make/top-make.mk b/reproduce/analysis/make/top-make.mk index 6b055f7..27c1b5b 100644 --- a/reproduce/analysis/make/top-make.mk +++ b/reproduce/analysis/make/top-make.mk @@ -51,7 +51,7 @@ include reproduce/software/config/LOCAL.conf # Controlling this requires two variables that are available at this stage: # # - `GROUP-NAME': from `LOCAL.conf' (which was built by `./project configure'). -# - `reproducible_paper_group_name': value to the `--group' option. +# - `maneage_group_name': value to the `--group' option. # # The analysis is only done when both have the same group name. Note that # when the project isn't being built for a group, both variables will be an @@ -69,7 +69,7 @@ top-pdfs = paper.pdf else top-pdfs = paper.pdf supplement.pdf endif -ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) +ifeq (x$(maneage_group_name),x$(GROUP-NAME)) all: $(top-pdfs) else all: diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk index f81ac07..fb5700e 100644 --- a/reproduce/analysis/make/top-prepare.mk +++ b/reproduce/analysis/make/top-prepare.mk @@ -35,7 +35,7 @@ include reproduce/software/config/LOCAL.conf # ------------------------------- # # See `top-make.mk' for complete explanation. -ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) +ifeq (x$(maneage_group_name),x$(GROUP-NAME)) all: $(BDIR)/software/preparation-done.mk @echo "Project preparation is complete."; else diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 9217ee9..fb378b8 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -86,7 +86,8 @@ export SHELL := $(ibdir)/dash export PATH := $(ibdir):$(PATH) export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) +export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) \ + -Wno-nullability-completeness export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) # This is the "basic" tools where we are relying on the host operating @@ -210,7 +211,10 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ # POSIX Threads library for multi-threaded programs. for l in dl pthread; do if [ -f /usr/lib/lib$$l.a ]; then - ln -sf /usr/lib/lib$$l.* $(ildir)/ + for f in /usr/lib/lib$$l.*; do + ln -sf $$(realpath $$f) \ + $$(echo $$f | sed -e's|/usr/lib|$(ildir)|') + done fi done diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index d69722e..6ea782c 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -125,10 +125,10 @@ all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) \ export PATH := $(ibdir) export CCACHE_DISABLE := 1 export SHELL := $(ibdir)/bash -export CPPFLAGS := -I$(idir)/include .SHELLFLAGS := --noprofile --norc -ec export LDFLAGS := $(rpath_command) -L$(ildir) export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +export CPPFLAGS := -I$(idir)/include -Wno-nullability-completeness export PKG_CONFIG_PATH := $(ildir)/pkgconfig:$(idir)/share/pkgconfig # Settings specific to this Makefile. @@ -428,7 +428,8 @@ $(ibidir)/cfitsio-$(cfitsio-version): export gbuild_tar=$(ddir)/$$customtar $(call gbuild, cfitsio-$(cfitsio-version), , \ --enable-sse2 --enable-reentrant \ - --with-bzip2=$(idir), , make shared fpack funpack) + --with-bzip2=$(idir), , \ + make shared fpack funpack) rm $$customtar echo "CFITSIO $(cfitsio-version)" > $@ @@ -1316,7 +1317,8 @@ $(ibidir)/swig-$(swig-version): # pcr is a dependency of swig tarball=swig-$(swig-version).tar.gz $(call import-source, $(swig-url), $(swig-checksum)) - $(call gbuild, swig-$(swig-version), static, --without-pcre) + $(call gbuild, swig-$(swig-version), static, \ + --without-pcre --without-tcl) echo "Swig $(swig-version)" > $@ # The disables: diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 812f3d3..e15a8e9 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -3,6 +3,7 @@ # Necessary preparations/configurations for the reproducible project. # # Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2021 Raul Infante-Sainz <infantesainz@gmail.com> # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -423,8 +424,17 @@ if ! [ -d $compilertestdir ]; then mkdir $compilertestdir; fi # Check C compiler # ---------------- +# +# Here we check if the C compiler works properly. About the "no warning" +# variable ('nowarnings'): +# +# -Wno-nullability-completeness: on macOS Big Sur 11.2.3 and Xcode 12.4, +# hundreds of 'nullability-completeness' warnings are printed which can +# be very annoying and even hide important errors or warnings. It is +# also harmless for our test here, so it is generally added. testprog=$compilertestdir/test testsource=$compilertestdir/test.c +noccwarnings="-Wno-nullability-completeness" echo; echo; echo "Checking host C compiler ('$CC')..."; cat > $testsource <<EOF #include <stdio.h> @@ -432,7 +442,7 @@ cat > $testsource <<EOF int main(void){printf("...C compiler works.\n"); return EXIT_SUCCESS;} EOF -if $CC $testsource -o$testprog && $testprog; then +if $CC $noccwarnings $testsource -o$testprog && $testprog; then rm $testsource $testprog else rm $testsource @@ -761,7 +771,7 @@ fi # exists and we don't want to re-write it). if [ $rewritepconfig = no ]; then oldgroupname=$(awk '/GROUP-NAME/ {print $3; exit 0}' $pconf) - if [ "x$oldgroupname" = "x$reproducible_paper_group_name" ]; then + if [ "x$oldgroupname" = "x$maneage_group_name" ]; then just_a_place_holder_to_avoid_not_equal_test=1; else echo "-----------------------------" @@ -1079,7 +1089,7 @@ if [ $rewritepconfig = yes ]; then -e's|@ddir[@]|'"$ddir"'|' \ -e's|@sys_cpath[@]|'"$sys_cpath"'|' \ -e's|@downloader[@]|'"$downloader"'|' \ - -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \ + -e's|@groupname[@]|'"$maneage_group_name"'|' \ $pconf.in >> $pconf else # Read the values from existing configuration file. Note that the build @@ -1238,7 +1248,7 @@ if ! [ -d "$mtexdir" ]; then mkdir "$mtexdir"; fi # TeX build directory. If built in a group scenario, the TeX build # directory must be separate for each member (so they can work on their # relevant parts of the paper without conflicting with each other). -if [ "x$reproducible_paper_group_name" = x ]; then +if [ "x$maneage_group_name" = x ]; then texbdir="$texdir"/build else user=$(whoami) @@ -1458,18 +1468,19 @@ fi # which will download the DOI-resolved webpage, and extract the Zenodo-URL # of the most recent version from there (using the 'coreutils' tarball as # an example, the directory part of the URL for all the other software are -# the same). +# the same). This is not done if the option `--debug' is used. +zenodourl="" user_backup_urls="" zenodocheck=.build/software/zenodo-check.html -if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then - zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ - | sed -n -e'/http/p' \ - | tr ' ' '\n' \ - | grep http \ - | sed -e 's/href="//' -e 's|/coreutils| |' \ - | awk 'NR==1{print $1}') -else - zenodourl="" +if [ x$debug = x ]; then + if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then + zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ + | sed -n -e'/http/p' \ + | tr ' ' '\n' \ + | grep http \ + | sed -e 's/href="//' -e 's|/coreutils| |' \ + | awk 'NR==1{print $1}') + fi fi rm -f $zenodocheck @@ -1497,6 +1508,28 @@ user_backup_urls="$user_backup_urls $zenodourl" +# Corrections for debugging mode +# ------------------------------ +# +# If the user wants to debug the software configuration, they are usually +# focused on the building of the single problematic software. Therefore, +# the default multi-threaded execution of Make with the '--keep-going' +# option are very annoying and can even hide important warnings. Recall +# that with '--keep-going', Make will continue building other targets, even +# if one target fails. When the user runs './project configure --debug', +# the 'debug' variable will not be empty and this mode will be activated. +if [ x$debug = x ]; then + keepgoing="--keep-going" +else + jobs=1 + numthreads=1 + keepgoing="" +fi + + + + + # Build other basic tools our own GNU Make # ---------------------------------------- # @@ -1504,7 +1537,7 @@ user_backup_urls="$user_backup_urls $zenodourl" # Bash, Make, or AWK. In this step, we'll install such low-level basic # tools, but we have to be very portable (and use minimal features in all). echo; echo "Building necessary software (if necessary)..." -.local/bin/make -k -f reproduce/software/make/basic.mk \ +.local/bin/make $keepgoing -f reproduce/software/make/basic.mk \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ @@ -1532,7 +1565,7 @@ else numthreads=$jobs fi .local/bin/env -i HOME=$bdir \ - .local/bin/make -k -f reproduce/software/make/high-level.mk \ + .local/bin/make $keepgoing -f reproduce/software/make/high-level.mk \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ @@ -1725,10 +1758,10 @@ echo `.local/bin/date` > $finaltarget # # The configuration is now complete, we can inform the user on the next # step(s) to take. -if [ x$reproducible_paper_group_name = x ]; then +if [ x$maneage_group_name = x ]; then buildcommand="./project make -j8" else - buildcommand="./project make --group=$reproducible_paper_group_name -j8" + buildcommand="./project make --group=$maneage_group_name -j8" fi cat <<EOF diff --git a/tex/src/preamble-project.tex b/tex/src/preamble-project.tex index 7351d83..7fabca1 100644 --- a/tex/src/preamble-project.tex +++ b/tex/src/preamble-project.tex @@ -24,7 +24,13 @@ -%% Import graphics +%% Packages you may need in your project +%% ------------------------------------- +% +%% Here you can add/remove any custom LaTeX package that you need for this +%% project that aren't provided by the journal's style. + +%% For loading images into the output (with '\includegraphics'). \usepackage{graphicx} %% IEEEtran V1.6 and later pre-defines the format of the cite.sty package |