From cbe30304c50c659b088a698c148b86a24da6b7d6 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 10 Jan 2021 03:32:54 +0000 Subject: make dist: removing temp files moved after project-specific files Until now, when you ran './project make dist', first it would delete the temporary files (like files ending in '~' or '.swp' created by some editors), then it had a place to add project-specific operations for the distribution. However, in the process of cleaning the temporary files, it would 'cd' into the directory that would later be packaged. So project-specific operations would first have to 'cd' back into the top source directory. This was prone to hard-to-find bugs. With this commit, to avoid the problem the project-specific operations are now placed before the cleaning phase. This is also technically good because in the project-specific operations there may also be temporary files that shouldn't go into the distribution tarball. --- reproduce/analysis/make/initialize.mk | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 3b1ffe5..81943db 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -374,16 +374,17 @@ $(project-package-contents): paper.pdf | $(texdir) sed -e's|{tikz/}|{tex/tikz/}|' $$pgfsettings > $$pgfsettings.new mv $$pgfsettings.new $$pgfsettings - # Clean temporary (currently those ending in `~') files. - cd $(texdir) - find $(project-package-name) -name \*~ -delete - find $(project-package-name) -name \*.swp -delete - # PROJECT SPECIFIC # ---------------- - # Put any project specific distribution steps here. + # Put any project-specific distribution steps here. + # ---------------- + # Clean temporary files that may have been created by text editors. + cd $(texdir) + find $(project-package-name) -name \*~ -delete + find $(project-package-name) -name \*.swp -delete + # Package into `.tar.gz' or '.tar.lz'. dist dist-lzip: $(project-package-contents) curdir=$$(pwd) -- cgit v1.2.1 From ecbaadcc88ae38adb34e08755459318372ec13b8 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Tue, 12 Jan 2021 16:15:47 +0000 Subject: Default LaTeX preamble: some packages moved to preamble-project.tex Until now, important LaTeX packages like 'caption' (for managing figure captions), 'hyperref' (for managing links) and 'xcolor' (for managing colors) were being loaded inside the optional 'tex/src/preamble-maneagge-defualt-style.tex' file. We recommend to remove this file from loading when you use custom journal sytels. However, these packages will often be necessary after loading special journal styles also. With this commit, these packages are now loaded into LaTeX as part of the 'tex/src/preamble-project.tex' file. This file is in charge of LaTeX settings that are custom to the project and independent of its style. Several other small corrections are made with this commit: - I noticed that './project make texclean' crashes if no PDF exists in the working directory! So a '-f' was added to the 'rm' command of the 'texclean' rule. - As part of the LaTeX Hyperref, we can set general metadata or properties for the PDF (that aren't written into the printable PDF, but into the file metadata). They can be viewed in many PDF viewers as PDF properties. Until now, we were only using the '\projecttitle' macro here to write the paper's title. However, thanks to the recently added 'reproduce/analysis/config/metadata.conf', we now have a lot of useful information that can also go here. So the 'metadata-copyright-owner' is now used to define the PDF author, and the project's 'metadata-git-repository' and commit hash are written into the PDF subject. But to import these, it was necessary to define them as LaTeX macros, hence the addition of these macros in 'initialize.mk'. - Some extra packages that aren't necessary to build the default PDF were removed in 'preamble-project.tex'. --- reproduce/analysis/make/initialize.mk | 4 ++- tex/src/preamble-maneage-default-style.tex | 54 ------------------------------ tex/src/preamble-project.tex | 54 +++++++++++++++++++++--------- 3 files changed, 42 insertions(+), 70 deletions(-) diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 81943db..886c0a7 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -237,7 +237,7 @@ clean-mmap:; rm -f reproduce/config/gnuastro/mmap* # ------------------------------------------ texclean: - rm *.pdf + rm -f *.pdf rm -rf $(texdir)/build/* mkdir $(texdir)/build/tikz # 'tikz' is assumed to already exist. @@ -505,6 +505,8 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\projectdate}{$$d}" > $@ echo "\newcommand{\projecttitle}{$(metadata-title)}" >> $@ echo "\newcommand{\projectversion}{$(project-commit-hash)}" >> $@ + echo "\newcommand{\projectgitrepo}{$(metadata-git-repository)}" >> $@ + echo "\newcommand{\projectcopyrightowner}{$(metadata-copyright-owner)}" >> $@ # Calculate the latest Maneage commit used to build this # project: diff --git a/tex/src/preamble-maneage-default-style.tex b/tex/src/preamble-maneage-default-style.tex index 4e294a5..0f1c84a 100644 --- a/tex/src/preamble-maneage-default-style.tex +++ b/tex/src/preamble-maneage-default-style.tex @@ -64,37 +64,6 @@ -%% Color related settings: -\usepackage{xcolor} -\color{black} % Text color -\definecolor{DarkBlue}{RGB}{0,0,90} - - - - - - -% figure and figure* ordering correction: -\usepackage{fixltx2e} - - - - - -%% For editing the caption appearence. The `setspace' package defines -%% the `stretch' variable. `abovecaptionskip' is the distance between -%% the figure and the caption. -\usepackage{setspace, caption} -\captionsetup{font=footnotesize, labelfont={color=DarkBlue,bf}, skip=1pt} -\captionsetup[figure]{font={stretch=1, small}} -\setlength{\abovecaptionskip}{3pt plus 1pt minus 1pt} -\setlength{\belowcaptionskip}{-1.25em} - - - - - - %% To make the footnotes align: \usepackage[hang]{footmisc} \setlength\footnotemargin{10pt} @@ -110,20 +79,6 @@ -%To make links to webpages and include document information in the -%properties of the PDF -\usepackage[ - colorlinks, - urlcolor=blue, - citecolor=blue, - linkcolor=blue, - linktocpage]{hyperref} -\renewcommand\UrlFont{\rmfamily} - - - - - %% Define the abstract environment \renewenvironment{abstract} {\vspace{-0.5cm}\small% @@ -199,15 +154,6 @@ {0em} { } -% Basic Document information that goes into the PDF meta-data. -\hypersetup -{ - pdfauthor={YOUR NAME}, - pdfsubject={\projecttitle}, - pdftitle={\projecttitle}, - pdfkeywords={SOME, KEYWORDS, FOR, THE, PDF} -} - %% Title and author information \usepackage{authblk} \renewcommand\Authfont{\small\scshape} diff --git a/tex/src/preamble-project.tex b/tex/src/preamble-project.tex index 2a3dd0e..98e797d 100644 --- a/tex/src/preamble-project.tex +++ b/tex/src/preamble-project.tex @@ -24,27 +24,51 @@ -%% Packages you need in your project -%% --------------------------------- +%% Packages you may need in your project +%% ------------------------------------- % %% Here you can add/remove any custom LaTeX package that you need for this %% project that aren't provided by the journal's style. -% Better than verbatim for displaying typed text. -\usepackage{alltt} - -% For arithmetic opertions within LaTeX -\usepackage[nomessages]{fp} - -%To add a code font to the text: -\usepackage{courier} - -%To add some enumerating styles -\usepackage{enumerate} - -%Including images if necessary +%% For loading images into the output (with '\includegraphics'). \usepackage{graphicx} +%% Ordering correction between 'figure' and 'figure*' ('figure*' is +%% commonly used in two-column documents, where the figure should span both +%% columns). +\usepackage{fixltx2e} + +%% Color management. +\usepackage{xcolor} +\color{black} % Color of main text. +\definecolor{DarkBlue}{RGB}{0,0,90} + +%% Caption management: The `setspace' package defines the `stretch' +%% variable. `abovecaptionskip' is the distance between the figure and the +%% caption. You can use 'captionof{figure}{...}' to use these custom +%% 'figure' caption that is defined here. +\usepackage{setspace, caption} +\captionsetup{font=footnotesize, labelfont={color=DarkBlue,bf}, skip=1pt} +\captionsetup[figure]{font={stretch=1, small}} +\setlength{\abovecaptionskip}{3pt plus 1pt minus 1pt} +\setlength{\belowcaptionskip}{-1.25em} + +%% Manage links in the produced paper (for example their colors), and +%% include document information in the "Properties" of the PDF. +\usepackage[ + colorlinks, + urlcolor=blue, + citecolor=blue, + linkcolor=blue, + linktocpage]{hyperref} +\renewcommand\UrlFont{\rmfamily} +\hypersetup{ + pdftitle={\projecttitle}, + pdfauthor={\projectcopyrightowner}, + pdfsubject={\projectgitrepo{} (commit \projectversion)}, + pdfkeywords={Reproducible research, Maneage, ADD YOUR OWN} +} + -- cgit v1.2.1 From 152421339a3a52485d9f6c4be28aad5c58cd089c Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 12 Feb 2021 23:49:16 +0000 Subject: Installation: minor correction in links to system libraries Until now when making a link to the system's 'dl' and 'pthread' libraries we were simply linking the installed location on the system (in '/usr/lib'). However, in some systems, these may themselves be links to other locations and this could cause linking problems. With this commit, we now use 'realpath' to extract the absolute address of the final file that the libraries may link to, and directly link to them. A minor cosmetic correction was also made in the build rule for CFITSIO: the long line was broken into two! --- reproduce/software/make/basic.mk | 5 ++++- reproduce/software/make/high-level.mk | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 9217ee9..efb7377 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -210,7 +210,10 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ # POSIX Threads library for multi-threaded programs. for l in dl pthread; do if [ -f /usr/lib/lib$$l.a ]; then - ln -sf /usr/lib/lib$$l.* $(ildir)/ + for f in /usr/lib/lib$$l.*; do + ln -sf $$(realpath $$f) \ + $$(echo $$f | sed -e's|/usr/lib|$(ildir)|') + done fi done diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index d69722e..26a58f7 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -428,7 +428,8 @@ $(ibidir)/cfitsio-$(cfitsio-version): export gbuild_tar=$(ddir)/$$customtar $(call gbuild, cfitsio-$(cfitsio-version), , \ --enable-sse2 --enable-reentrant \ - --with-bzip2=$(idir), , make shared fpack funpack) + --with-bzip2=$(idir), , \ + make shared fpack funpack) rm $$customtar echo "CFITSIO $(cfitsio-version)" > $@ -- cgit v1.2.1 From c3e82b1c7b6c358719f3d1dd5cf4871cde3446a7 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 19 Mar 2021 17:52:08 +0000 Subject: Configuration: --debug option available in this phase also Until now, each time there was a problem in the configuration of Maneage'd projects and debugging was necessary, we had to take the following changes: - Run the configuration on a single thread ('-j1') to see the building of only the problematic software. - Disable the Zenodo check manually by commenting those parts of 'reproduce/software/shell/configure.sh'. Because the internet connection wastes a few seconds and is thus very annoying during repeated runs! - Manually remove the '-k' option that was passed to Make (when building the software). With the '-k', Make keeps going with the execution of other targets if something crashes and this usually causes confusions during the debugging. Doing the manual changes within the code was both very annoying and prone to errors (forgetting to correct it!). With this commit, the existing '--debug' option has been generalized to the software configuration phase of Maneage also. Until now, it was only available in the analysis phase (and would directly be passed to the 'make' command that would run the analysis). When this option is used, and the project is in the software configuration phase, the Zenodo check won't be done, it will use one single thread ('-j1'), and it will stop the execution as soon as an error occurs (Make is not run with '-k'). --- project | 5 +++- reproduce/software/shell/configure.sh | 48 ++++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/project b/project index 900e02c..7fef1fe 100755 --- a/project +++ b/project @@ -4,6 +4,7 @@ # Run `./project --help' for a description of how to use it. # # Copyright (C) 2019-2021 Mohammad Akhlaghi +# Copyright (C) 2021 Raul Infante-Sainz # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -117,12 +118,13 @@ Configure options: --all-highlevel Build all high-level software (for development). Configure and Make options: + -d, --debug[=FLAGS] In configure: use -j1, no -k, and no Zenodo check. + In make: 'FLAGS' will be directly passed to 'make'. -g, --group=STR Build and run with write permissions for a group. -j, --jobs=INT Number of threads to build/run the software. -?, --help Print this help list. Make (analysis) options: - -d, --debug=FLAGS Print various types of debugging information. -p, --prepare-redo Re-do preparation (only done automatically once). Make (final PDF) options: @@ -439,6 +441,7 @@ case $operation in # Variables to pass to the configuration script. export jobs=$jobs + export debug=$debug export host_cc=$host_cc export build_dir=$build_dir export input_dir=$input_dir diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 812f3d3..8ae74d2 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -3,6 +3,7 @@ # Necessary preparations/configurations for the reproducible project. # # Copyright (C) 2018-2021 Mohammad Akhlaghi +# Copyright (C) 2021 Raul Infante-Sainz # # This script is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -1458,18 +1459,19 @@ fi # which will download the DOI-resolved webpage, and extract the Zenodo-URL # of the most recent version from there (using the 'coreutils' tarball as # an example, the directory part of the URL for all the other software are -# the same). +# the same). This is not done if the option `--debug' is used. +zenodourl="" user_backup_urls="" zenodocheck=.build/software/zenodo-check.html -if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then - zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ - | sed -n -e'/http/p' \ - | tr ' ' '\n' \ - | grep http \ - | sed -e 's/href="//' -e 's|/coreutils| |' \ - | awk 'NR==1{print $1}') -else - zenodourl="" +if [ x$debug = x ]; then + if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then + zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ + | sed -n -e'/http/p' \ + | tr ' ' '\n' \ + | grep http \ + | sed -e 's/href="//' -e 's|/coreutils| |' \ + | awk 'NR==1{print $1}') + fi fi rm -f $zenodocheck @@ -1497,6 +1499,28 @@ user_backup_urls="$user_backup_urls $zenodourl" +# Corrections for debugging mode +# ------------------------------ +# +# If the user wants to debug the software configuration, they are usually +# focused on the building of the single problematic software. Therefore, +# the default multi-threaded execution of Make with the '--keep-going' +# option are very annoying and can even hide important warnings. Recall +# that with '--keep-going', Make will continue building other targets, even +# if one target fails. When the user runs './project configure --debug', +# the 'debug' variable will not be empty and this mode will be activated. +if [ x$debug = x ]; then + keepgoing="--keep-going" +else + jobs=1 + numthreads=1 + keepgoing="" +fi + + + + + # Build other basic tools our own GNU Make # ---------------------------------------- # @@ -1504,7 +1528,7 @@ user_backup_urls="$user_backup_urls $zenodourl" # Bash, Make, or AWK. In this step, we'll install such low-level basic # tools, but we have to be very portable (and use minimal features in all). echo; echo "Building necessary software (if necessary)..." -.local/bin/make -k -f reproduce/software/make/basic.mk \ +.local/bin/make $keepgoing -f reproduce/software/make/basic.mk \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ @@ -1532,7 +1556,7 @@ else numthreads=$jobs fi .local/bin/env -i HOME=$bdir \ - .local/bin/make -k -f reproduce/software/make/high-level.mk \ + .local/bin/make $keepgoing -f reproduce/software/make/high-level.mk \ user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ -- cgit v1.2.1 From a981196b2e2eea6fe0517040c49a32f757ad28cf Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 19 Mar 2021 18:11:12 +0000 Subject: Configuration: nullability-completeness warnings suppressed With a recent update of macOS systems (macOS Big Sur 11.2.3 and Xcode 12.4), there are many warnings when building C programs (for example the simple program we compile to check the compiler, or some of the software like `gzip'). It prints hundreds of warning lines for every source file that are irrelevant for our builds, but really clutters the output. With this commit, these warnings are disabled by adding `-Wno-nullability-completeness' to the 'CPPFLAGS' environment variable. This has also been added to the very first check of the C compiler in the configure step. --- reproduce/software/make/basic.mk | 3 ++- reproduce/software/make/high-level.mk | 2 +- reproduce/software/shell/configure.sh | 11 ++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index efb7377..fb378b8 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -86,7 +86,8 @@ export SHELL := $(ibdir)/dash export PATH := $(ibdir):$(PATH) export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) +export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) \ + -Wno-nullability-completeness export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) # This is the "basic" tools where we are relying on the host operating diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 26a58f7..9628df4 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -125,10 +125,10 @@ all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) \ export PATH := $(ibdir) export CCACHE_DISABLE := 1 export SHELL := $(ibdir)/bash -export CPPFLAGS := -I$(idir)/include .SHELLFLAGS := --noprofile --norc -ec export LDFLAGS := $(rpath_command) -L$(ildir) export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +export CPPFLAGS := -I$(idir)/include -Wno-nullability-completeness export PKG_CONFIG_PATH := $(ildir)/pkgconfig:$(idir)/share/pkgconfig # Settings specific to this Makefile. diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 8ae74d2..fe46cef 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -424,8 +424,17 @@ if ! [ -d $compilertestdir ]; then mkdir $compilertestdir; fi # Check C compiler # ---------------- +# +# Here we check if the C compiler works properly. About the "no warning" +# variable ('nowarnings'): +# +# -Wno-nullability-completeness: on macOS Big Sur 11.2.3 and Xcode 12.4, +# hundreds of 'nullability-completeness' warnings are printed which can +# be very annoying and even hide important errors or warnings. It is +# also harmless for our test here, so it is generally added. testprog=$compilertestdir/test testsource=$compilertestdir/test.c +noccwarnings="-Wno-nullability-completeness" echo; echo; echo "Checking host C compiler ('$CC')..."; cat > $testsource < @@ -433,7 +442,7 @@ cat > $testsource < Date: Wed, 24 Mar 2021 21:01:35 +0000 Subject: Maneage installation: removed TCL as a dependency of SWIG Until now the SWIG software would use the host operating system's packages to find the TCL configuraiton (which we don't install yet in Maneage). In particular, you can see the error during its configuration here: .... checking for pkg-config... pkg-config checking for Tcl configuration... found /usr/lib/tclConfig.sh /usr/lib/tclConfig.sh: line 2: dpkg-architecture: command not found /usr/lib//tcl8.6/tclConfig.sh: line 2: dpkg-architecture: com. not found With this commit, TCL has been disabled when building SWIG with the '--without-tcl' option. Later, when we add TCL in Maneage, we can remove this option. --- reproduce/software/make/high-level.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 9628df4..6ea782c 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1317,7 +1317,8 @@ $(ibidir)/swig-$(swig-version): # pcr is a dependency of swig tarball=swig-$(swig-version).tar.gz $(call import-source, $(swig-url), $(swig-checksum)) - $(call gbuild, swig-$(swig-version), static, --without-pcre) + $(call gbuild, swig-$(swig-version), static, \ + --without-pcre --without-tcl) echo "Swig $(swig-version)" > $@ # The disables: -- cgit v1.2.1 From ac8890dfbf06c8b98a3163f798ad3fe65b193a33 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 26 Mar 2021 18:19:22 +0000 Subject: ./project: unused --minmapsize option is removed Until now, the './project' script included an '--minmapsize' option which is an option to one of the original programs that was used in Maneage (Gnuastro). Such an option doesn't exist in many other programs, so it is not a suitable option for the generic Maneage project (and can just cause confusion). It was also not used in any part of Maneage any more! With this commit, this option is removed from the core Maneage './project' script and if any project uses it, they can implement it in their own branch. --- project | 6 ------ 1 file changed, 6 deletions(-) diff --git a/project b/project index 7fef1fe..1e3aeb4 100755 --- a/project +++ b/project @@ -47,7 +47,6 @@ all_highlevel=0 existing_conf=0 highlightnotes=0 scriptname="./project" -minmapsize=10000000000 @@ -109,7 +108,6 @@ Configure options: -e, --existing-conf Use (possibly existing) local configuration. --host-cc Use host system's C compiler, don't build GCC. -i, --input-dir=STR Directory containing input datasets (optional). - -m, --minmapsize=INT [Gnuastro] Minimum number of bytes to use RAM. -s, --software-dir=STR Directory containing necessary software tarballs. --check-config During configuration, show what is being built. --clean-texdir Remove possibly existing build-time subdirectories @@ -186,9 +184,6 @@ do -i|--input-dir) input_dir="$2"; check_v "$1" "$input_dir"; shift;shift;; -i=*|--input-dir=*) input_dir="${1#*=}"; check_v "$1" "$input_dir"; shift;; -i*) input_dir=$(echo "$1" | sed -e's/-i//'); check_v "$1" "$input_dir"; shift;; - -m|--minmapsize) minmapsize="$2"; check_v "$1" "$minmapsize"; shift;shift;; - -m=*|--minmapsize=*) minmapsize="${1#*=}"; check_v "$1" "$minmapsize"; shift;; - -m*) minmapsize=$(echo "$1" | sed -e's/-m//'); check_v "$1" "$minmapsize"; shift;; -s|--software-dir) software_dir="$2"; check_v "$1" "$software_dir"; shift;shift;; -s=*|--software-dir=*) software_dir="${1#*=}"; check_v "$1" "$software_dir"; shift;; -s*) software_dir=$(echo "$1" | sed -e's/-s//'); check_v "$1" "$software_dir"; shift;; @@ -446,7 +441,6 @@ case $operation in export build_dir=$build_dir export input_dir=$input_dir export scriptname=$scriptname - export minmapsize=$minmapsize export software_dir=$software_dir export existing_conf=$existing_conf export all_highlevel=$all_highlevel -- cgit v1.2.1 From 611c2f1e6ebbd4e4f83b66325707afb7038d1026 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 26 Mar 2021 20:37:35 +0000 Subject: Initialization: removed other Gnuastro-specific features In the previous commit, some Gnuastro-specific initializations were removed but a few more cases remained that are removed with this commit. --- reproduce/analysis/make/initialize.mk | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 886c0a7..15a4dbf 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -229,19 +229,15 @@ project-package-contents = $(texdir)/$(project-package-name) # we want to ensure that the file is always built in every run: it contains # the project version which may change between two separate runs, even when # no file actually differs. -.PHONY: all clean dist dist-zip dist-lzip texclean distclean clean-mmap \ +.PHONY: all clean dist dist-zip dist-lzip texclean distclean \ $(project-package-contents) $(mtexdir)/initialize.tex -# --------- Delete for no Gnuastro --------- -clean-mmap:; rm -f reproduce/config/gnuastro/mmap* -# ------------------------------------------ - texclean: rm -f *.pdf rm -rf $(texdir)/build/* mkdir $(texdir)/build/tikz # 'tikz' is assumed to already exist. -clean: clean-mmap +clean: # Delete the top-level PDF file. rm -f *.pdf @@ -266,8 +262,7 @@ distclean: clean # `rm' program. So for this recipe, we'll use the host system's # `rm', not our own. $$sys_rm -rf $(BDIR) - $$sys_rm -f $(pconfdir)/LOCAL.conf - $$sys_rm -f Makefile .gnuastro .local .build + $$sys_rm -f .local .build $(pconfdir)/LOCAL.conf -- cgit v1.2.1 From 20b6273a65ab4d99ad760fd715ca935494d4fc2e Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 28 Mar 2021 12:55:03 +0100 Subject: Configuration: corrected check of group name When built in 'group' mode, the write permissions of all created files will be activated for a certain group of users in the host operating system. The user specifies the name of the group with the '--group' option at configure time. At the very start, the './project' script checks to see if the given group name actually exists or not (to avoid hard-to-debug errors popping up later). Until now, the checking 'sg' command (that was used to build the project with group-writable permissions) would always fail due to the excessive number of redirections. Therefore, it would always print the error message and abort. With this commit, the output of 'sg' is no longer re-directed (which also helps users in debuggin). If the group does actually exist, it will just print a small statement saying so, and if it fails, the error message is printed. This fixed the problem, allowing maneage to be built in group-mode. I also noticed that the variable name keeping the group name ('reproducible_paper_group_name') used the old name for the project (which was "Reproducible paper template"! So it has been changed/corrected to 'maneage_group_name'. --- project | 6 +++--- reproduce/analysis/make/top-make.mk | 4 ++-- reproduce/analysis/make/top-prepare.mk | 2 +- reproduce/software/shell/configure.sh | 10 +++++----- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/project b/project index 1e3aeb4..53452ff 100755 --- a/project +++ b/project @@ -326,14 +326,14 @@ fi if ! [ x$group = x ]; then # Check if group is usable. - if ! sg "$group" "echo test &> /dev/null" &> /dev/null; then + if ! sg "$group" "echo Group \'$group\' exists"; then echo "$scriptname: '$group' is not a usable group name on this system."; echo "(TIP: you can use the 'groups' command to see your groups)" exit 1 fi # Set the group option for running Make. - gopt="reproducible_paper_group_name=$group" + gopt="maneage_group_name=$group" fi @@ -441,10 +441,10 @@ case $operation in export build_dir=$build_dir export input_dir=$input_dir export scriptname=$scriptname + export maneage_group_name=$group export software_dir=$software_dir export existing_conf=$existing_conf export all_highlevel=$all_highlevel - export reproducible_paper_group_name=$group # Run the configuration script if [ x"$group" = x ]; then diff --git a/reproduce/analysis/make/top-make.mk b/reproduce/analysis/make/top-make.mk index 026220e..596cc0d 100644 --- a/reproduce/analysis/make/top-make.mk +++ b/reproduce/analysis/make/top-make.mk @@ -51,7 +51,7 @@ include reproduce/software/config/LOCAL.conf # Controlling this requires two variables that are available at this stage: # # - `GROUP-NAME': from `LOCAL.conf' (which was built by `./project configure'). -# - `reproducible_paper_group_name': value to the `--group' option. +# - `maneage_group_name': value to the `--group' option. # # The analysis is only done when both have the same group name. Note that # when the project isn't being built for a group, both variables will be an @@ -64,7 +64,7 @@ include reproduce/software/config/LOCAL.conf # If you are just interested in the processing and don't want to build the # PDF, you can skip the creation of the final PDF by giving a value of # `yes' to `pdf-build-final' in `reproduce/analysis/config/pdf-build.conf'. -ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) +ifeq (x$(maneage_group_name),x$(GROUP-NAME)) all: paper.pdf else all: diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk index f81ac07..fb5700e 100644 --- a/reproduce/analysis/make/top-prepare.mk +++ b/reproduce/analysis/make/top-prepare.mk @@ -35,7 +35,7 @@ include reproduce/software/config/LOCAL.conf # ------------------------------- # # See `top-make.mk' for complete explanation. -ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) +ifeq (x$(maneage_group_name),x$(GROUP-NAME)) all: $(BDIR)/software/preparation-done.mk @echo "Project preparation is complete."; else diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index fe46cef..e15a8e9 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -771,7 +771,7 @@ fi # exists and we don't want to re-write it). if [ $rewritepconfig = no ]; then oldgroupname=$(awk '/GROUP-NAME/ {print $3; exit 0}' $pconf) - if [ "x$oldgroupname" = "x$reproducible_paper_group_name" ]; then + if [ "x$oldgroupname" = "x$maneage_group_name" ]; then just_a_place_holder_to_avoid_not_equal_test=1; else echo "-----------------------------" @@ -1089,7 +1089,7 @@ if [ $rewritepconfig = yes ]; then -e's|@ddir[@]|'"$ddir"'|' \ -e's|@sys_cpath[@]|'"$sys_cpath"'|' \ -e's|@downloader[@]|'"$downloader"'|' \ - -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \ + -e's|@groupname[@]|'"$maneage_group_name"'|' \ $pconf.in >> $pconf else # Read the values from existing configuration file. Note that the build @@ -1248,7 +1248,7 @@ if ! [ -d "$mtexdir" ]; then mkdir "$mtexdir"; fi # TeX build directory. If built in a group scenario, the TeX build # directory must be separate for each member (so they can work on their # relevant parts of the paper without conflicting with each other). -if [ "x$reproducible_paper_group_name" = x ]; then +if [ "x$maneage_group_name" = x ]; then texbdir="$texdir"/build else user=$(whoami) @@ -1758,10 +1758,10 @@ echo `.local/bin/date` > $finaltarget # # The configuration is now complete, we can inform the user on the next # step(s) to take. -if [ x$reproducible_paper_group_name = x ]; then +if [ x$maneage_group_name = x ]; then buildcommand="./project make -j8" else - buildcommand="./project make --group=$reproducible_paper_group_name -j8" + buildcommand="./project make --group=$maneage_group_name -j8" fi cat < Date: Sat, 17 Apr 2021 04:31:31 +0100 Subject: IMPORTANT: print-general-metadata new name for print-copyright Summary: - Use the new name of this variable in your Makefiles. - In 'metadata.conf', remove fixed URL prefixes for DOIs ('https://doi.org/') or arXiv ('https://arxiv.org/abs'). Until now, the Make variable that would print the general metadata (of whole project) into each to-be-published dataset was called 'print-copyright'! But it now does much more than simply printing the copyright, it will also print a lot of metadata like arXiv ID, Zenodo DOI and etc into plain-text outputs. The out-dated name could thus be misleading and cause confusions. With this commit, the variable is therefore called 'print-general-metadata'. After merging your project with the Maneage branch, please replace any usage of 'print-copyright' to 'print-general-metadata'. Also with this commit, 'README-hacking.md' mentions 'metadata.conf' and 'print-general-metadata' in the "Publication checklist" section and reminds you to keep the first up to date, and use the second in your to-be-published datasets. --- README-hacking.md | 143 ++++++++++++++++++-------------- reproduce/analysis/config/metadata.conf | 21 ++++- reproduce/analysis/make/delete-me.mk | 4 +- reproduce/analysis/make/initialize.mk | 24 ++++-- 4 files changed, 115 insertions(+), 77 deletions(-) diff --git a/README-hacking.md b/README-hacking.md index 475f2ca..e42bf42 100644 --- a/README-hacking.md +++ b/README-hacking.md @@ -945,14 +945,14 @@ effectively no cost in keeping multiple redundancies on different servers, just in case one (or more) of them are discontinued in the (near/far) future. - - **Reserve a DOI for your dataset**: There are multiple data servers that - give this functionality, one of the most well known and (currently!) - well-funded is [Zenodo](https://zenodo.org) so we'll focus on it - here. Of course, you can use any other service that provides a similar - functionality. Once you complete these steps, you can start using/citing - your dataset's DOI in the source of your project to finalize the rest of - the points. With Zenodo, you can even use the given identifier - for things like downloading. + - **Reserve a DOI for your datasets**: There are multiple data servers + that give this functionality, one of the most well known and + (currently!) well-funded is [Zenodo](https://zenodo.org) so we'll focus + on it here. Of course, you can use any other service that provides a + similar functionality. Once you complete these steps, you can start + using/citing your dataset's DOI in the source of your project to + finalize the rest of the points. With Zenodo, you can even use the given + identifier for things like downloading. * *Start new upload*: After you log in to Zenodo, you can start a new upload by clicking on the "New Upload button". @@ -961,16 +961,23 @@ future. Identifier", click on the "Reserve DOI" button. * *Fill basic info*: You need to at least fill in the "required fields" - (marked with a red star). You will always be able to change any - metadata (even after you "Publish"), so don't worry too much about - values in the fields, at this phase, its just important that they - are not empty. + (marked with a red star). You will always be able to change any + metadata (even after you "Publish"), so don't worry too much about + values in the fields, at this phase, its just important that they are + not empty. * *Save your project but do not yet publish*: Press the "Save" button - (at the top or bottom of the page). Do not yet press "Publish" - though, since that would make the project public, and freeze the DOI - with any possible file you may have uploaded already. We will get to - the publication phase in the next steps. + (at the top or bottom of the page). Do not yet press "Publish" though, + since that would make the project public, and freeze the DOI with any + possible file you may have uploaded already. We will get to the + publication phase in the next steps. + + - **Record the metadata**: Maneage comes with a file to store all the + project's metadata: `reproduce/analysis/config/metadata.conf`. Open this + file and store all the information that you currently have: for example + the Zenodo DOI, project's Git repository, Copyright owner and license of + the data after it becomes public. Keep the empty fields in mind and + after obtaining them, don't forget to fill them up. - **Request archival on SoftwareHeritage**: [Software Heritage](https://archive.softwareheritage.org/save/) is an online @@ -989,7 +996,7 @@ future. - **Zenodo/SoftwareHeritage links in paper**: put links to the Zenodo-DOI (and SoftwareHeritage source when you make it public) in your - paper. Somewhere close the start, maybe under the keywords/abstract, + paper. Somewhere close to the start, maybe under the keywords/abstract, highlighting that they are supplements for reproducibility. These help readers easily access these resources for supplementary material directly from your PDF paper (sources on SoftwareHeritage and @@ -1013,14 +1020,14 @@ future. (for example with columns separated by white-space characters) or in the more formal [Comma-separated values](https://en.wikipedia.org/wiki/Comma-separated_values) or CSV, - format). In the former case, its best to set the suffixes to `.txt` - (because most browsers/OSs will automatically know they are plain-text - and open them without needing any other software. If you have other - types of data (for example images, or very large tables with millions - of rows/columns that can be inconvenient in plain-text), feel free to - use custom binary formats, but later, in the description of your - project on the server, add a note, explaining what software they - should use to open them. + format). Generally, its best to set the suffixes to `.txt` (because + most browsers/OSs will automatically know they are plain-text and open + them without needing any other software). If you have other types of + data (for example images, or very large tables with millions of + rows/columns that can be inconvenient in plain-text), feel free to use + custom binary formats, but later, in the description of your project + on the server, add a note, explaining what software they should use to + open them. * *Descriptive names*: In some papers there are many files and having cryptic names will only confuse your readers (actually, yourself in @@ -1033,45 +1040,23 @@ future. to rename everything related to each figure (which is very frustrating and prone to errors). - * *Good metadata*: Raw data are not too useful merely as a series of + * *Good metadata*: Raw data are not too useful merely as a series of raw numbers! So don't forget to have **good metadata in every file**. If its a plain-text file, usually lines starting with a `#` are ignored. So in the command that generates each dataset, add some extra - information about the dataset as lines starting with `#`. A minimal - set of recommended metadata are listed below. Feel free to add - more. You can use a configuration file to keep this information in one - place and automatically include them in all your output files. - - * *Project Title and authors*: This is very important to give a - general perspective of the figure. - - * *Links to project*: For example Zenodo-DOI, Journal-DOI (after it is - accepted), SoftwareHeritage page, arXiv-ID (or any other pre-print - server) and ofcourse, your Git repository. - - * *Commit hash* of the project that produced the dataset. This - directly links the dataset to a particular point in your project's - history. It is stored in the `$(project-commit-hash)` variable that - is defined in `initialize.mk`. So you can use it anywhere in your - project. - - * *Same commit hashes*: each dataset may have been created at - different phases of your project's history. If you simply upload the - produced datasets, they may therefore have different commits on - them. To avoid confusing your readers (and your self in the future), - it is best that they all have the same commit hash (which will also - be the commit hash printed in the paper). So upon publication, we - recommend deleting all of them and running `./project make` to build - them all with the same commit hash. - - * *Copyright as metadata*: people need to know if they can "use" the - dataset (i.e., modify it), or possibly re-distribute it and their - derived products. They also need to know how they can contact the - creator of the datset (who is usually also the copyright owner). So - as another metadata element, also add your name and email-address - (or the name of the person and email of the person who was in charge - of that part of the project), and the copyright license name and - standard link to the fully copyright license. + information (the more the better!) about the dataset as lines starting + with `#`. Based on `reproduce/analysis/config/metadata.conf`, in + `initialize.mk`, Maneage will produce a default set of basic + information for plain-text data and will put it in the + `$(print-general-metadata)` variable. It is thus recommended to print + this variable into your plain-text file before printing the actual + data (so it shows on top of the file). For a real-world example, see + its usage in `reproduce/analysis/make/delete-me.mk` (in the `maneage` + branch). If you are publishing your data in binary formats, please add + all the metadata you see in `$(print-general-metadata)` into each + dataset file (for example keywords in the FITS format). If there are + many files, its easy to define a tiny shell-script to do the job on + each dataset. - **Link to figure datasets in caption**: all the datasets that go into the plots should be uploaded directly to Zenodo so they can be @@ -1201,9 +1186,12 @@ future. initial/final submission to your desired journal. But we'll just add the necessary points for arXiv submission here: - * *Necessary links in comments*: put a link to your project's Git - repository, Zenodo-DOI (this is not your paper's DOI, its the - data/resources DOI), and/or SoftwareHeritage link in the comments. + * *Necessary links in comments*: put a link to your project's Git + repository, Zenodo-DOI (this is not your paper's DOI, its the + data/resources DOI), and/or SoftwareHeritage link in the comments. + + - *Update `metadata.conf`*: Once you have your final arXiv ID (formated + as: `1234.56789`) put it in `reproduce/analysis/config/metadata.conf`. - **Submission to a journal**: different journals accept submissions in different formats, some accept LaTeX, some only want a PDF, or etc. It @@ -1221,6 +1209,33 @@ future. the DOI (so you don't need to upload a new version if you just want to update the metadata). + - **After acceptance (before publication)**: Congratulations on the + acceptance! The main science content of your paper can't be changed any + more, but the paper will now go to the publication editor (for language + and style). Your approval of the final proof is necessary before the + paper is finally published. Use this period to finalize the final + metadata of your project: the journal's DOI. Some journals associate + your paper's DOI during this process. So before approving the final + proof do these steps: + + * Add the Journal DOI in `reproduce/analysis/config/metadata.conf`, + and re-build your final data products, so this important metadata is + added. + + * Once you get the final proof, and if everything is OK for you, + implement all the good language corrections/edits they have made + inside your own copy here and commit it into your project. This will + be the final commit of your project before publication. + + * Submit your final project as a new version to Zenodo (and + arXiv). The Zenodo one is most important because your plots will + link to it and you want the commit hash in the data files that + readers will get from Zenodo to be the same hash as the paper. + + * Tell the journal's publication editor to correct the hash and Zenodo + ID in your final proof confirmation (so the links point to the + correct place). Recall that on every new version upload in Zenodo, + you get a new DOI (or Zenodo ID). diff --git a/reproduce/analysis/config/metadata.conf b/reproduce/analysis/config/metadata.conf index e92a057..aaf2ca0 100644 --- a/reproduce/analysis/config/metadata.conf +++ b/reproduce/analysis/config/metadata.conf @@ -1,6 +1,19 @@ -# Project meta-data that can be used in a project's output datasets and +# Project meta-data that will be used in a project's output datasets and # final paper. Please set the values here and use them in your analysis or -# paper, don't repeat them +# paper, don't repeat them. +# +# These variables are used in 'reproduce/analysis/make/initialize.mk': 1) +# to create a Make variable called 'print-general-metadata'. You can simply +# print this variable's value in any plain-text output. +# +# Why add a Copyright for the data? people need to know if they can "use" +# the dataset (i.e., modify it), or possibly re-distribute it and their +# derived products. They also need to know how they can contact the creator +# of the datset (who is usually also the copyright owner). So take this +# seriously and add your name and email-address (or the name of the person +# and email of the person who was in charge of that part of the project), +# and the copyright license name and standard link to the fully copyright +# license. # # Copyright (C) 2020-2021 Mohammad Akhlaghi # @@ -12,7 +25,9 @@ # Project information metadata-title = The project title goes here -# DOIs and identifiers. +# DOIs and identifiers (don't include fixed URL prefixes like +# 'https://doi.org/' or 'https://arxiv.org/abs'), they will be added +# automatically where necessary. metadata-arxiv = metadata-doi-zenodo = metadata-doi-journal = diff --git a/reproduce/analysis/make/delete-me.mk b/reproduce/analysis/make/delete-me.mk index f275051..c160e51 100644 --- a/reproduce/analysis/make/delete-me.mk +++ b/reproduce/analysis/make/delete-me.mk @@ -48,7 +48,7 @@ $(dm-squared): $(pconfdir)/delete-me-squared-num.conf | $(tex-publish-dir) echo "# Column 2: X_POW2 [arbitrary, f32] The horizontal axis to the power of two." \ >> $@.tmp echo "# " >> $@.tmp - $(call print-copyright, $@.tmp) + $(call print-general-metadata, $@.tmp) # Generate the table of random values. awk 'BEGIN {for(i=1;i<=$(delete-me-squared-num);i+=0.5) \ @@ -107,7 +107,7 @@ $(dm-img-histogram): $(tex-publish-dir)/%-histogram.txt: $(indir)/%.fits \ echo "# " >> $@.tmp awk '/^# Column .:/' $@.data >> $@.tmp echo "# " >> $@.tmp - $(call print-copyright, $@.tmp) + $(call print-general-metadata, $@.tmp) # Add the column numbers in a formatted manner, rename it to the # output and clean up. diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 15a4dbf..744ecbf 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -461,19 +461,27 @@ $(data-publish-dir):; mkdir $@ # Print Copyright statement # ------------------------- # -# This statement can be used in published datasets that are in plain-text -# format. It assumes you have already put the data-specific statements in -# its first argument, it will supplement them with general project links. -print-copyright = \ +# The 'print-general-metadata' can be used to print the general metadata in +# published datasets that are in plain-text format. It should be called +# with make's 'call' function like this (where 'FILENAME' is the name of +# the file it will append this content to): +# +# $(call print-general-metadata, FILENAME) +# +# See 'reproduce/analysis/make/delete-me.mk' (in the Maneage branch) for a +# real-world usage of this variable. +doi-prefix-url = https://doi.org +arxiv-prefix-url = https://arxiv.org/abs +print-general-metadata = \ echo "\# Project title: $(metadata-title)" >> $(1); \ echo "\# Git commit (that produced this dataset): $(project-commit-hash)" >> $(1); \ - echo "\# Project's Git repository: $(metadata-git-repository)" >> $(1); \ + echo "\# Git repository: $(metadata-git-repository)" >> $(1); \ if [ x$(metadata-arxiv) != x ]; then \ - echo "\# Pre-print server: https://arxiv.org/abs/$(metadata-arxiv)" >> $(1); fi; \ + echo "\# Pre-print: $(arxiv-prefix-url)/abs/$(metadata-arxiv)" >> $(1); fi; \ if [ x$(metadata-doi-journal) != x ]; then \ - echo "\# DOI (Journal): $(metadata-doi-journal)" >> $(1); fi; \ + echo "\# DOI (Journal): $(doi-prefix-url)/$(metadata-doi-journal)" >> $(1); fi; \ if [ x$(metadata-doi-zenodo) != x ]; then \ - echo "\# DOI (Zenodo): $(metadata-doi-zenodo)" >> $(1); fi; \ + echo "\# DOI (Zenodo): $(doi-prefix-url)/$(metadata-doi-zenodo)" >> $(1); fi; \ echo "\#" >> $(1); \ echo "\# Copyright (C) $$(date +%Y) $(metadata-copyright-owner)" >> $(1); \ echo "\# Dataset is available under $(metadata-copyright)." >> $(1); \ -- cgit v1.2.1