From be31089199a7697531307930a76ae1cd412a1e88 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Thu, 26 Sep 2019 14:42:39 +0100 Subject: Working project when downloaded from arXiv Until now, we were assuming that the users would just clone the project in Git. But after submitting arXiv:1909.11230, and trying to build directly from the arXiv source, I noticed several problems that wouldn't allow users to build it automatically. So I tried the build step by step and was able to find a fix for the several issues that came up. The scripting parts of the fix were primarily related to the fact that the unpacked arXiv tarball isn't under version control, so some checks had to be put there. Also, we wanted to make it easy to remove the extra files, so an extra `--clean-texdit' option was added to `./project'. Finally, some manual corrections were necessary (prior to running `./project', which are now described in `README.md'. Most of the later steps can be automated and we should do it later, I just don't have enough time now. --- .file-metadata | Bin 6250 -> 6250 bytes README.md | 78 +++++++++++++++++++++++++++++++++- project | 13 ++++++ reproduce/analysis/make/initialize.mk | 6 ++- reproduce/software/make/basic.mk | 31 ++++++++------ 5 files changed, 113 insertions(+), 15 deletions(-) diff --git a/.file-metadata b/.file-metadata index 9c57cfd..dbc185c 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/README.md b/README.md index 7129b07..011c915 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,8 @@ on your system if you have ever built and installed a software from source) and a downloader (Wget or cURL). Note that **Git is not mandatory**: if you don't have Git to run the first command below, go to the URL given in the command on your browser, and download the project's source (there is a -button to download a compressed tarball of the project). +button to download a compressed tarball of the project). If you have +received this source from arXiv, please see the respective section below. ```shell $ git clone XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX @@ -88,6 +89,81 @@ requiring root/administrator permissions. +Source from arXiv +----------------- +If the paper is also published on arXiv, it is highly likely that the +authors also uploaded/published the full reproducible paper template there +along with the LaTeX sources. If you have downloaded (or plan to download) +this source from arXiv, some minor extra steps are necessary: + +1. If the arXiv code for the paper is 1234.56789, then the downloaded + source will be called `1234.56789` (no special identification + suffix). However, it is actually a `.tar.gz` file. So take these steps + to unpack it to see its contents. + + ```shell + $ arxiv=1234.56789 + $ mv $arxiv $arxiv.tar.gz + $ mkdir $arxiv + $ cd $arxiv + $ tar xf ../$arxiv.tar.gz + ``` + +2. arXiv removes the executable flag from the files (for its own + security). So before following the standard procedure of projects + described in the sections above, its necessary to make the necessary + script files executable with the command below: + + ```shell + $ chmod +x project reproduce/software/bash/* reproduce/analysis/bash/* + ``` + +3. Remove extra files. In order to make sure arXiv can build the paper + (resolve conflicts due to different versions of LaTeX packages), it is + sometimes necessary to copy raw LaTeX package files in the tarball + uploaded to arXiv. Later, we will implement a feature to automatically + delete these extra files, but for now, the project's top directory + should only have the following contents (where `reproduce` and `tex` are + directories). You can safely remove any other file/directory. + + ```shell + $ ls + COPYING paper.tex project README-hacking.md README.md reproduce tex + ``` + +4. To build the figures from scratch, please you need to make the following + corrections to the respective source files. + + 4.1: `paper.tex`: uncomment the following line, so it looks like + below. See the comments above it for more information. + + ```shell + \newcommand{\makepdf}{} + ``` + + 4.2: `tex/src/preamble-pgfplots.tex`: set the `tikzsetexternalprefix` + variable to `tikz/`, so it looks like this: + + ```shell + \tikzsetexternalprefix{tikz/} + ``` + +5. In order to let arXiv build the LaTeX paper without bothering to run the + analysis pipeline it was necessary to create and fill the two + `tex/build` and `tex/tikz` subdirectories. But to do a clean build of + the project, it is necessary for these to be symbolic links to the build + directory. So when you are first configuring the project, run it with + `--clean-texdir` (only once is enough, they will be deleted permanently + after that), for example: + + ```shell + $ ./project configure --clean-texdir + ``` + + + + + Copyright information --------------------- diff --git a/project b/project index e7f44a8..383ecb1 100755 --- a/project +++ b/project @@ -31,6 +31,7 @@ build_dir= input_dir= make_targets= software_dir= +clean_texdir=0 existing_conf=0 scriptname="./project" minmapsize=10000000000 @@ -90,6 +91,9 @@ Configure options: -i, --input-dir=STR Directory containing input datasets (optional). -m, --minmapsize=INT [Gnuastro] Minimum number of bytes to use RAM. -s, --software-dir=STR Directory containing necessary software tarballs. + --clean-texdir Remove possibly existing build-time subdirectories + under the project's 'tex/' directory (can happen + when source is from arXiv for example). Configure and Make options: -g, --group=STR Build and run with write permissions for a group. @@ -159,6 +163,8 @@ do -s|--software-dir) software_dir="$2"; check_v "$1" "$software_dir"; shift;shift;; -s=*|--software-dir=*) software_dir="${1#*=}"; check_v "$1" "$software_dir"; shift;; -s*) software_dir=$(echo "$1" | sed -e's/-s//'); check_v "$1" "$software_dir"; shift;; + --clean-texdir) clean_texdir=1; shift;; + --clean-texdir=*) on_off_option_error --clean-texdir;; # Configure and Make options: -g|--group) group="$2"; check_v group "$group"; shift;shift;; @@ -217,6 +223,13 @@ case $operation in # Build the project's software. configure) + # If the user requested, clean the TeX directory from the extra + # (to-be-built) directories that may already be there (and will not + # allow the configuration to complete). + if [ x"$clean_texdir" = x1 ]; then + rm -rf tex/build tex/tikz + fi + # Variables to pass to the configuration script. export jobs=$jobs export host_cc=$host_cc diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index ee9c8c1..918ae06 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -181,7 +181,8 @@ $(lockdir): | $(BDIR); mkdir $@ # we want to ensure that the file is always built in every run: it contains # the project version which may change between two separate runs, even when # no file actually differs. -packagebasename := $(shell echo paper-$$(git describe --dirty --always --long)) +packagebasename := $(shell if [ -d .git ]; then \ + echo paper-$$(git describe --dirty --always --long); else echo NOGIT; fi) packagecontents = $(texdir)/$(packagebasename) .PHONY: all clean dist dist-zip distclean clean-mmap $(packagecontents) \ $(mtexdir)/initialize.tex @@ -342,5 +343,6 @@ dist-zip: $(packagecontents) $(mtexdir)/initialize.tex: | $(mtexdir) # Version of the project. - @v=$$(git describe --dirty --always --long); + @if [ -d .git ]; then v=$$(git describe --dirty --always --long); + else v=NO-GIT; fi echo "\newcommand{\projectversion}{$$v}" > $@ diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 17ff561..b0b757f 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -968,6 +968,11 @@ $(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ # # We want to inform the user if Metastore isn't built, so we don't # continue the call to `gbuild' with an `&&'. + # + # Checking for presence of `.git'. When the project source is + # downloaded from a non-Git source (for example from arXiv), there + # is no `.git' directory to work with. So until we find a better + # solution, avoid the step to to add the Git hooks. current_dir=$$(pwd); \ $(call gbuild, $<, metastore-$(metastore-version), static,, \ NO_XATTR=1 V=1,,pwd,PREFIX=$(idir)); \ @@ -975,18 +980,20 @@ $(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ if [ "x$(needpatchelf)" != x ]; then \ $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/metastore; \ fi; \ - user=$$(whoami); \ - group=$$(groups | awk '{print $$1}'); \ - cd $$current_dir; \ - for f in pre-commit post-checkout; do \ - sed -e's|@USER[@]|'$$user'|g' \ - -e's|@GROUP[@]|'$$group'|g' \ - -e's|@BINDIR[@]|$(ibdir)|g' \ - -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ - reproduce/software/bash/git-$$f > .git/hooks/$$f \ - && chmod +x .git/hooks/$$f \ - && echo "Metastore (forked) $(metastore-version)" > $@; \ - done; \ + if [ -d .git ]; then \ + user=$$(whoami); \ + group=$$(groups | awk '{print $$1}'); \ + cd $$current_dir; \ + for f in pre-commit post-checkout; do \ + sed -e's|@USER[@]|'$$user'|g' \ + -e's|@GROUP[@]|'$$group'|g' \ + -e's|@BINDIR[@]|$(ibdir)|g' \ + -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ + reproduce/software/bash/git-$$f > .git/hooks/$$f \ + && chmod +x .git/hooks/$$f; \ + done; \ + fi \ + && echo "Metastore (forked) $(metastore-version)" > $@; \ else \ echo; echo; echo; \ echo "*****************"; \ -- cgit v1.2.1