diff options
Diffstat (limited to 'reproduce')
| -rw-r--r-- | reproduce/analysis/config/INPUTS.conf | 10 | ||||
| -rw-r--r-- | reproduce/analysis/config/delete-me-squared-num.conf | 9 | ||||
| -rw-r--r-- | reproduce/analysis/config/demo-year.conf | 3 | ||||
| -rw-r--r-- | reproduce/analysis/config/metadata.conf | 10 | ||||
| -rw-r--r-- | reproduce/analysis/make/delete-me.mk | 169 | ||||
| -rw-r--r-- | reproduce/analysis/make/demo-plot.mk | 80 | ||||
| -rw-r--r-- | reproduce/analysis/make/download.mk | 17 | ||||
| -rw-r--r-- | reproduce/analysis/make/format.mk | 86 | ||||
| -rw-r--r-- | reproduce/analysis/make/initialize.mk | 39 | ||||
| -rw-r--r-- | reproduce/analysis/make/paper.mk | 105 | ||||
| -rw-r--r-- | reproduce/analysis/make/top-make.mk | 10 | ||||
| -rw-r--r-- | reproduce/analysis/make/verify.mk | 11 | ||||
| -rw-r--r-- | reproduce/software/config/TARGETS.conf | 9 | ||||
| -rw-r--r-- | reproduce/software/config/texlive-packages.conf | 10 | ||||
| -rwxr-xr-x | reproduce/software/shell/configure.sh | 13 |
15 files changed, 333 insertions, 248 deletions
diff --git a/reproduce/analysis/config/INPUTS.conf b/reproduce/analysis/config/INPUTS.conf index b969945..fd8ac53 100644 --- a/reproduce/analysis/config/INPUTS.conf +++ b/reproduce/analysis/config/INPUTS.conf @@ -47,8 +47,8 @@ -# Demo dataset used in the histogram plot (remove when customizing). -DEMO-DATA = WFPC2ASSNu5780205bx.fits -DEMO-MD5 = a4791e42cd1045892f9c41f11b50bad8 -DEMO-SIZE = 62K -DEMO-URL = https://fits.gsfc.nasa.gov/samples/$(DEMO-DATA) +# Dataset used in this analysis and its checksum for integrity checking. +MK20DATA = menke20.xlsx +MK20MD5 = 8e4eee64791f351fec58680126d558a0 +MK20SIZE = 1.9MB +MK20URL = https://www.biorxiv.org/content/biorxiv/early/2020/01/18/2020.01.15.908111/DC1/embed/media-1.xlsx diff --git a/reproduce/analysis/config/delete-me-squared-num.conf b/reproduce/analysis/config/delete-me-squared-num.conf deleted file mode 100644 index 13068e4..0000000 --- a/reproduce/analysis/config/delete-me-squared-num.conf +++ /dev/null @@ -1,9 +0,0 @@ -# Number of samples in the demonstration analysis (to be deleted). -# -# Copyright (C) 2019-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# -# Copying and distribution of this file, with or without modification, are -# permitted in any medium without royalty provided the copyright notice and -# this notice are preserved. This file is offered as-is, without any -# warranty. -delete-me-squared-num = 50 diff --git a/reproduce/analysis/config/demo-year.conf b/reproduce/analysis/config/demo-year.conf new file mode 100644 index 0000000..429b220 --- /dev/null +++ b/reproduce/analysis/config/demo-year.conf @@ -0,0 +1,3 @@ +# This is the demonstration year showing the number of papers studied +# before 1997. +menke-demo-year = 1996 diff --git a/reproduce/analysis/config/metadata.conf b/reproduce/analysis/config/metadata.conf index e92a057..681c6fa 100644 --- a/reproduce/analysis/config/metadata.conf +++ b/reproduce/analysis/config/metadata.conf @@ -10,14 +10,14 @@ # warranty. # Project information -metadata-title = The project title goes here +metadata-title = Towards Long-term and Archivable Reproducibility # DOIs and identifiers. -metadata-arxiv = -metadata-doi-zenodo = +metadata-arxiv = 2006.03018 +metadata-doi-zenodo = https://doi.org/10.5281/zenodo.4291207 metadata-doi-journal = -metadata-doi = $(metadata-doi-journal) -metadata-git-repository = http://git.maneage.org/project.git +metadata-doi = $(metadata-doi-zenodo) +metadata-git-repository = https://gitlab.com/makhlaghi/maneage-paper # DATA Copyright owner and license information. metadata-copyright-owner = Mohammad Akhlaghi <mohammad@akhlaghi.org> diff --git a/reproduce/analysis/make/delete-me.mk b/reproduce/analysis/make/delete-me.mk deleted file mode 100644 index f275051..0000000 --- a/reproduce/analysis/make/delete-me.mk +++ /dev/null @@ -1,169 +0,0 @@ -# Dummy Makefile to create a random dataset for plotting. -# -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# -# This Makefile is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This Makefile is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this Makefile. If not, see <http://www.gnu.org/licenses/>. - - - - - -# Dummy dataset -# ------------- -# -# Just as a demonstration(!): we will use AWK to generate a table showing X -# and X^2 and draw its plot. -# -# Note that this dataset is directly read by LaTeX to generate a plot, so -# we need to put it in the $(tex-publish-dir) directory. -dm-squared = $(tex-publish-dir)/squared.txt -$(dm-squared): $(pconfdir)/delete-me-squared-num.conf | $(tex-publish-dir) - - # When the plotted values are re-made, it is necessary to also - # delete the TiKZ externalized files so the plot is also re-made by - # PGFPlots. - rm -f $(tikzdir)/delete-me-squared.pdf - - # Write the column metadata in a temporary file name (appending - # '.tmp' to the actual target name). Once all steps are done, it is - # renamed to the final target. We do this because if there is an - # error in the middle, Make will not consider the job to be - # complete and will stop here. - echo "# Data for demonstration plot of default Maneage (MANaging data linEAGE)." > $@.tmp - echo "# It is a simple plot, showing the power of two: y=x^2! " >> $@.tmp - echo "# " >> $@.tmp - echo "# Column 1: X [arbitrary, f32] The horizontal axis numbers." \ - >> $@.tmp - echo "# Column 2: X_POW2 [arbitrary, f32] The horizontal axis to the power of two." \ - >> $@.tmp - echo "# " >> $@.tmp - $(call print-copyright, $@.tmp) - - # Generate the table of random values. - awk 'BEGIN {for(i=1;i<=$(delete-me-squared-num);i+=0.5) \ - printf("%-8.1f%.2f\n", i, i*i); }' >> $@.tmp - - # Write it into the final target - mv $@.tmp $@ - - - - - -# Demo image PDF -# -------------- -# -# For an example image, we'll make a PDF copy of the WFPC II image to -# display in the paper. -dm-histdir = $(texdir)/image-histogram -$(dm-histdir): | $(texdir); mkdir $@ -dm-img-pdf = $(dm-histdir)/wfpc2.pdf -$(dm-img-pdf): $(dm-histdir)/%.pdf: $(indir)/%.fits | $(dm-histdir) - - # When the plotted values are re-made, it is necessary to also - # delete the TiKZ externalized files so the plot is also re-made. - rm -f $(tikzdir)/delete-me-image-histogram.pdf - - # Convert the dataset to a PDF. - astconvertt --colormap=gray --fluxhigh=4 $< -h0 -o$@ - - - - - -# Histogram of demo image -# ----------------------- -# -# For an example plot, we'll show the pixel value histogram also. IMPORTANT -# NOTE: because this histogram contains data that is included in a plot, we -# should publish it, so it will go into the $(tex-publish-dir). -dm-img-histogram = $(tex-publish-dir)/wfpc2-histogram.txt -$(dm-img-histogram): $(tex-publish-dir)/%-histogram.txt: $(indir)/%.fits \ - | $(tex-publish-dir) - - # When the plotted values are re-made, it is necessary to also - # delete the TiKZ externalized files so the plot is also re-made. - rm -f $(tikzdir)/delete-me-image-histogram.pdf - - # Generate the pixel value histogram. - aststatistics --lessthan=5 $< -h0 --histogram -o$@.data - - # Put a two-line description of the dataset, copy the column - # metadata from '$@.data', and add copyright. - echo "# Histogram of example image to demonstrate Maneage (MANaging data linEAGE)." \ - > $@.tmp - echo "# Example image URL: $(DEMO-URL)" >> $@.tmp - echo "# " >> $@.tmp - awk '/^# Column .:/' $@.data >> $@.tmp - echo "# " >> $@.tmp - $(call print-copyright, $@.tmp) - - # Add the column numbers in a formatted manner, rename it to the - # output and clean up. - awk '!/^#/{printf("%-15.4f%d\n", $$1, $$2)}' $@.data >> $@.tmp - mv $@.tmp $@ - rm $@.data - - - - - -# Basic statistics -# ---------------- -# -# This is just as a demonstration on how to get analysic configuration -# parameters from variables defined in `reproduce/analysis/config/'. -dm-img-stats = $(dm-histdir)/wfpc2-stats.txt -$(dm-img-stats): $(dm-histdir)/%-stats.txt: $(indir)/%.fits \ - | $(dm-histdir) - aststatistics $< -h0 --mean --median > $@ - - - - - -# TeX macros -# ---------- -# -# This is how we write the necessary parameters in the final PDF. -# -# NOTE: In LaTeX you cannot use any non-alphabetic character in a variable -# name. -$(mtexdir)/delete-me.tex: $(dm-squared) $(dm-img-pdf) $(dm-img-histogram) \ - $(dm-img-stats) - - # Write the number of random values used. - echo "\newcommand{\deletemenum}{$(delete-me-squared-num)}" > $@ - - # Note that since Make variables start with a `$(', if you want to - # use `$' within the shell (not Make), you have to quote any - # occurance of `$' with another `$'. That is why there are `$$' in - # the AWK command below. - # - # Here, we are first using AWK to find the minimum and maximum - # values, then using it again to read each separately to use in the - # macro definition. - mm=$$(awk 'BEGIN{min=99999; max=-min} - !/^#/{if($$2>max) max=$$2; if($$2<min) min=$$2;} - END{print min, max}' $(dm-squared)); - v=$$(echo "$$mm" | awk '{printf "%.3f", $$1}'); - echo "\newcommand{\deletememin}{$$v}" >> $@ - v=$$(echo "$$mm" | awk '{printf "%.3f", $$2}'); - echo "\newcommand{\deletememax}{$$v}" >> $@ - - # Write the statistics of the demo image as a macro. - mean=$$(awk '{printf("%.2f", $$1)}' $(dm-img-stats)) - echo "\newcommand{\deletemewfpctwomean}{$$mean}" >> $@ - median=$$(awk '{printf("%.2f", $$2)}' $(dm-img-stats)) - echo "\newcommand{\deletemewfpctwomedian}{$$median}" >> $@ diff --git a/reproduce/analysis/make/demo-plot.mk b/reproduce/analysis/make/demo-plot.mk new file mode 100644 index 0000000..dc44c4e --- /dev/null +++ b/reproduce/analysis/make/demo-plot.mk @@ -0,0 +1,80 @@ +# Second step of analysis: +# Data for plot of number/fraction of tools per year. +# +# Copyright (C) 2020 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. See <http://www.gnu.org/licenses/>. + + + + + +# Table for Figure 1C of Menke+20 +# ------------------------------- +a2mk20f1c = $(tex-publish-dir)/tools-per-year.txt +$(a2mk20f1c): $(mk20tab3) | $(tex-publish-dir) + + # Remove the (possibly) produced figure that is created from this + # table: it is created by LaTeX's TiKZ package, and includes + # multiple files with a fixed prefix. + rm -f $(tikzdir)/figure-tools-per-year* + + # Write the column metadata in a temporary file name (appending + # '.tmp' to the actual target name). Once all steps are done, it is + # renamed to the final target. We do this because if there is an + # error in the middle, Make will not consider the job to be + # complete and will stop here. + echo "# Data of plot showing fraction of papers that mentioned software tools" > $@.tmp + echo "# per year to demonstrate the features of Maneage (MANaging data linEAGE)." >> $@.tmp + >> $@.tmp + echo "# Raw data taken from Menke+2020 (https://doi.org/10.1101/2020.01.15.908111)." \ + >> $@.tmp + echo "# " >> $@.tmp + echo "# Column 1: YEAR [count, u16] Publication year of papers." \ + >> $@.tmp + echo "# Column 2: WITH_TOOLS [frac, f32] Fraction of papers mentioning software tools." \ + >> $@.tmp + echo "# Column 3: NUM_PAPERS [count, u32] Total number of papers studied in that year." \ + >> $@.tmp + echo "# " >> $@.tmp + $(call print-copyright, $@.tmp) + + + # Find the maximum number of papers. + awk '!/^#/{all[$$1]+=$$2; id[$$1]+=$$3} \ + END{ for(year in all) \ + printf("%-7d%-10.3f%d\n", year, 100*id[year]/all[year], \ + all[year]) \ + }' $< \ + >> $@.tmp + + # Write it into the final target + mv $@.tmp $@ + + + + + +# Final LaTeX macro +$(mtexdir)/demo-plot.tex: $(a2mk20f1c) $(pconfdir)/demo-year.conf + + # Find the first year (first column of first row) of data. + v=$$(awk '!/^#/ && c==0{c++; print $$1}' $(a2mk20f1c)) + echo "\newcommand{\menkefirstyear}{$$v}" > $@ + + # Find the number of rows in the plotted table. + v=$$(awk '!/^#/{c++} END{print c}' $(a2mk20f1c)) + echo "\newcommand{\menkenumyears}{$$v}" >> $@ + + # Find the number of papers in 1996. + v=$$(awk '$$1==$(menke-demo-year){print $$3}' $(a2mk20f1c)) + echo "\newcommand{\menkenumpapersdemocount}{$$v}" >> $@ + echo "\newcommand{\menkenumpapersdemoyear}{$(menke-demo-year)}" >> $@ diff --git a/reproduce/analysis/make/download.mk b/reproduce/analysis/make/download.mk index 0ea3432..ea70fca 100644 --- a/reproduce/analysis/make/download.mk +++ b/reproduce/analysis/make/download.mk @@ -53,14 +53,14 @@ # progress at every moment. $(indir):; mkdir $@ downloadwrapper = $(bashdir)/download-multi-try -inputdatasets = $(foreach i, wfpc2, $(indir)/$(i).fits) -$(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir) +inputdatasets = $(indir)/menke20.xlsx +$(inputdatasets): $(indir)/%: | $(indir) $(lockdir) # Set the necessary parameters for this input file. - if [ $* = wfpc2 ]; then - localname=$(DEMO-DATA); url=$(DEMO-URL); mdf=$(DEMO-MD5); + if [ $* = menke20.xlsx ]; then + localname=$(MK20DATA); url=$(MK20URL); mdf=$(MK20MD5); else - echo; echo; echo "Not recognized input dataset: '$*.fits'." + echo; echo; echo "Not recognized input dataset: '$*'." echo; echo; exit 1 fi @@ -103,5 +103,8 @@ $(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir) # # It is very important to mention the address where the data were # downloaded in the final report. -$(mtexdir)/download.tex: $(pconfdir)/INPUTS.conf | $(mtexdir) - echo "\\newcommand{\\wfpctwourl}{$(DEMO-URL)}" > $@ +$(mtexdir)/download.tex: $(indir)/menke20.xlsx | $(mtexdir) + echo "\newcommand{\menketwentyxlsxname}{$(MK20DATA)}" > $@ + echo "\newcommand{\menketwentychecksum}{$(MK20MD5)}" >> $@ + echo "\newcommand{\menketwentybytesize}{$(MK20SIZE)}" >> $@ + echo "\newcommand{\menketwentyurl}{$(MK20URL)}" >> $@ diff --git a/reproduce/analysis/make/format.mk b/reproduce/analysis/make/format.mk new file mode 100644 index 0000000..3070e6a --- /dev/null +++ b/reproduce/analysis/make/format.mk @@ -0,0 +1,86 @@ +# First step of analysis: +# Prepare the data, return basic values. +# +# As a demonstration analysis to go with the paper, we use the data from +# Menke 2020 (DOI:10.1101/2020.01.15.908111). This is a relevant paper +# because it provides interesting statistics about tools and methods used +# in scientific papers. +# +# Copyright (C) 2020 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. See <http://www.gnu.org/licenses/>. + + + + +# Save the "Table 3" spreadsheet from the downloaded `.xlsx' file into a +# simple plain-text file that is easy to use. +a1dir = $(BDIR)/analysis1 +mk20tab3 = $(a1dir)/table-3.txt +$(a1dir):; mkdir $@ +$(mk20tab3): $(indir)/menke20.xlsx | $(a1dir) + + # Set a base-name for the table-3 data. + base=$(basename $(notdir $<))-table-3 + + # Unfortunately XLSX I/O only works when the input and output are + # in the directory it is running. So first, we need to switch to + # the input directory, run it, then put our desired output where we + # want and delete the extra files. + topdir=$$(pwd) + cd $(indir) + xlsxio_xlsx2csv $(notdir $<) + cp $(notdir $<)."Table 3 All by journal by year".csv $$base.csv + rm $(notdir $<).*.csv + cd $$topdir + + # Read the necessary information. Note that we are dealing with a + # CSV (comma-separated value) file. But when there are commas in a + # string, quotation signs are put around it. The `FPAT' values is + # fully described in the GNU AWK manual. In short, it ensures that + # if there is a comma in the middle of double-quotes, it doesn't + # count as a delimter. + echo "# Column 1: YEAR [counter, i16] Year of journal's publication." > $@.tmp + echo "# Column 2: NUM_PAPERS [counter, i16] Number of studied papers in that journal." >> $@.tmp + echo "# Column 3: NUM_PAPERS_WITH_TOOLS [counter, i16] Number of papers with an identified tool." >> $@.tmp + echo "# Column 4: NUM_ID_TOOLS [counter, i16] Number of software/tools that were identified." >> $@.tmp + echo "# Column 5: JOURNAL_NAME [string, str150] Name of journal." >> $@.tmp + awk 'NR>1{printf("%-10d%-10d%-10d%-10d %s\n", $$2, $$3, $$3*$$NF, $$(NF-1), $$1)}' \ + FPAT='([^,]+)|("[^"]+")' $(indir)/$$base.csv >> $@.tmp + + # Set the temporary file as the final target. This was done so if + # there is any possible crash in the steps above, this rule is + # re-run (its final target isn't rebuilt). + mv $@.tmp $@ + + + + + +# Main LaTeX macro file +$(mtexdir)/format.tex: $(mk20tab3) + + # Count the total number of papers in their study. + v=$$(awk '!/^#/{c+=$$2} END{print c}' $(mk20tab3)) + echo "\newcommand{\menkenumpapers}{$$v}" > $@ + + # Count how many unique journals there were in the study. Note that + # the `31' comes because we put 10 characters for each numeric + # column and separated the last numeric column from the string + # column with a space. If the number of numeric columns change in + # the future, the `31' also has to change. + v=$$(awk 'BEGIN{FIELDWIDTHS="41 10000"} !/^#/{print $$2}' \ + $(mk20tab3) | uniq | wc -l) + echo "\newcommand{\menkenumjournals}{$$v}" >> $@ + + # Count how many rows the original catalog has. + v=$$(awk '!/^#/{c++} END{print c}' $(mk20tab3)) + echo "\newcommand{\menkenumorigrows}{$$v}" >> $@ diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index a5d5b92..4bae0a4 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -77,21 +77,11 @@ endif # (independent parts of the paper can be added to it independently). To fix # this problem, when we are in a group setting, we'll use the user's ID to # create a separate LaTeX build directory for each user. -# -# The same logic applies to the final paper PDF: each user will create a -# separte final PDF (for example `paper-user1.pdf' and `paper-user2.pdf') -# and no `paper.pdf' will be built. This isn't a problem because -# `initialize.tex' is a .PHONY prerequisite, so the rule to build the final -# paper is always executed (even if it is present and nothing has -# changed). So in terms of over-all efficiency and processing steps, this -# doesn't change anything. ifeq (x$(GROUP-NAME),x) texbtopdir = build -final-paper = paper.pdf else user = $(shell whoami) texbtopdir = build-$(user) -final-paper = paper-$(user).pdf endif texbdir = $(texdir)/$(texbtopdir) tikzdir = $(texbdir)/tikz @@ -129,6 +119,7 @@ curdir := $(shell echo $$(pwd)) # we are also going to overwrite `TEXINPUTS' just before `pdflatex'. .ONESHELL: .SHELLFLAGS = -ec +export TERM=xterm export TEXINPUTS := export CCACHE_DISABLE := 1 export PATH := $(installdir)/bin @@ -219,19 +210,15 @@ project-package-contents = $(texdir)/$(project-package-name) # we want to ensure that the file is always built in every run: it contains # the project version which may change between two separate runs, even when # no file actually differs. -.PHONY: all clean dist dist-zip dist-lzip texclean distclean clean-mmap \ +.PHONY: all clean dist dist-zip dist-lzip texclean distclean \ $(project-package-contents) $(mtexdir)/initialize.tex -# --------- Delete for no Gnuastro --------- -clean-mmap:; rm -f reproduce/config/gnuastro/mmap* -# ------------------------------------------ - texclean: rm *.pdf rm -rf $(BDIR)/tex/build/* mkdir $(BDIR)/tex/build/tikz # 'tikz' is assumed to already exist. -clean: clean-mmap +clean: # Delete the top-level PDF file. rm -f *.pdf @@ -282,10 +269,13 @@ $(project-package-contents): paper.pdf | $(texdir) # (including the bibliography). m=$$dir/Makefile echo "paper.pdf: paper.tex paper.bbl" > $$m - printf "\tpdflatex -shell-escape -halt-on-error paper\n" >> $$m + printf "\tlatex -shell-escape -halt-on-error paper\n" >> $$m + printf "\tdvips paper.dvi\n" >> $$m + printf "\tps2pdf -dNOSAFER paper.ps\n" >> $$m echo "paper.bbl: tex/src/references.tex" >> $$m - printf "\tpdflatex -shell-escape -halt-on-error paper\n" >> $$m - printf "\tbiber paper\n" >> $$m + printf "\tlatex -shell-escape -halt-on-error paper\n" >> $$m + printf "\tbibtex paper\n" >> $$m + printf "\tlatex -shell-escape -halt-on-error paper\n" >> $$m echo ".PHONY: clean" >> $$m echo "clean:" >> $$m printf "\trm -f *.aux *.auxlock *.bbl *.bcf\n" >> $$m @@ -326,6 +316,8 @@ $(project-package-contents): paper.pdf | $(texdir) # be using Bash's extended globbing ('extglob') for excluding this # directory. shopt -s extglob + cp -r tex/img $$dir/tex/img + cp tex/tikz/*.eps $$dir/tex/tikz cp -r tex/build/!($(project-package-name)) $$dir/tex/build # Clean up the $(texdir)/build* directories in the archive (when @@ -353,8 +345,8 @@ $(project-package-contents): paper.pdf | $(texdir) # that may cause problems on the arXiv server. cp tex/build/build/paper.bbl $$dir/ tltopdir=.local/texlive/maneage/texmf-dist/tex/latex - find $$tltopdir/biblatex/ -maxdepth 1 -type f -print0 \ - | xargs -0 cp -t $$dir + #find $$tltopdir/biblatex/ -maxdepth 1 -type f -print0 \ + # | xargs -0 cp -t $$dir # Just in case the package users want to rebuild some of the # figures (manually un-comment the `makepdf' command we commented @@ -492,6 +484,11 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\projecttitle}{$(metadata-title)}" >> $@ echo "\newcommand{\projectversion}{$(project-commit-hash)}" >> $@ + # arXiv/Zenodo identifier (necessary for download link): + echo "\newcommand{\projectarxivid}{$(metadata-arxiv)}" >> $@ + v=$$(echo $(metadata-doi-zenodo) | sed -e's/\./ /g' | awk '{print $$NF}') + echo "\newcommand{\projectzenodoid}{$$v}" >> $@ + # Calculate the latest Maneage commit used to build this # project: # - The project may not have the 'maneage' branch (for example diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk index b5b5b29..00bd3b5 100644 --- a/reproduce/analysis/make/paper.mk +++ b/reproduce/analysis/make/paper.mk @@ -18,6 +18,7 @@ + # LaTeX macros for paper # ---------------------- # @@ -44,7 +45,7 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex # If no PDF is requested, or if LaTeX isn't available, don't # continue to building the final PDF. Otherwise, merge all the TeX # macros into one for building the PDF. - @if [ -f .local/bin/pdflatex ] && [ x"$(pdf-build-final)" = xyes ]; then + @if [ -f .local/bin/latex ] && [ x"$(pdf-build-final)" = xyes ]; then # Put a LaTeX input command for all the necessary macro files. # 'hardware-parameters.tex' is created in 'configure.sh'. @@ -54,6 +55,11 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex echo "\input{tex/build/macros/$$t.tex}" >> $$projecttex done + # Possibly print the appendix in the final PDF. + if [ x"$(separatesupplement)" = x1 ]; then + echo "\newcommand{\separatesupplement}{}" >> $$projecttex + fi + # Possibly highlight the '\new' parts of the text. if [ x"$(highlightnew)" = x1 ]; then echo "\newcommand{\highlightnew}{}" >> $$projecttex @@ -95,24 +101,39 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex # The bibliography # ---------------- # -# We need to run the `biber' program on the output of LaTeX to generate the -# necessary bibliography before making the final paper. So we'll first have -# one run of LaTeX (similar to the `paper.pdf' recipe), then `biber'. +# We need to run the `bibtex' program on the output of LaTeX to generate +# the necessary bibliography before making the final paper. So we'll first +# have one run of LaTeX (similar to the `paper.pdf' recipe), then `biber'. # # NOTE: `$(mtexdir)/project.tex' is an order-only-prerequisite for # `paper.bbl'. This is because we need to run LaTeX in both the `paper.bbl' -# recipe and the `paper.pdf' recipe. But if `tex/src/references.tex' hasn't +# recipe and the `paper.pdf' recipe. But if `tex/src/references.bib' hasn't # been modified, we don't want to re-build the bibliography, only the final # PDF. -$(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ - | $(mtexdir)/project.tex +bbls = $(foreach t,$(subst .pdf,,$(top-pdfs)),$(texbdir)/$(t).bbl) +$(bbls): $(texbdir)/%.bbl: tex/src/references.tex \ + $(mtexdir)/dependencies-bib.tex | $(mtexdir)/project.tex + # If `$(mtexdir)/project.tex' is empty, don't build PDF. @macros=$$(cat $(mtexdir)/project.tex) if [ x"$$macros" != x ]; then + # Unfortunately I can't get bibtex to look into a special + # directory for the references, so we'll copy it into the LaTeX + # building directory. + p=$$(pwd) + if ! [ -L $(texbdir)/references.bib ]; then + ln -sf $$p/tex/src/references.tex $(texbdir)/references.bib + fi + + # Copy the improved IEEE bst file into the build directory. + # The improved bst file provides ArXiv clickable URLs and + # if available, open-access URLs based on the DOIs, with + # closed-access URLs as a fallback, via https://oadoi.org . + ln -sf $$p/tex/src/IEEEtran_openaccess.bst $(texbdir)/ + # We'll run LaTeX first to generate the `.bcf' file (necessary # for `biber') and then run `biber' to generate the `.bbl' file. - p=$$(pwd) export TEXINPUTS=$$p: cd $(texbdir); @@ -122,15 +143,61 @@ $(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ # it. It will be built anyway once this rule is done. rm -f $@ + # Put a link to the main LaTeX source that we want to build. + if [ $* = paper ]; then sdir="$$p" + else sdir="$$p"/tex/src + fi + ln -sf "$$sdir"/$*.tex ./ + # The pdflatex option '-shell-escape' is "normally disallowed for # security reasons" according to the `info pdflatex' manual, but # is enabled here in order to allow the use of PGFPlots. If you - # do not use PGFPlots, then you should remove the `-shell-escape' + # do not use PGFPlots, then you can remove the `-shell-escape' # option for better security. See # https://savannah.nongnu.org/task/?15694 for details. - pdflatex -shell-escape -halt-on-error "$$p"/paper.tex - biber paper + latex -shell-escape -halt-on-error $*.tex + + # When we are building the main paper and the appendices are to + # be built within the main paper's PDF, we need two + # bibliographies: one for the main body, and one for the + # appendix. For this, we use 'multibib'. Multibib creates a + # separate '.aux' file for each bibliography. + bibtex $* + if [ x"$(separatesupplement)" != x1 ]; then + bibtex appendix + fi + + # Hack: tidy up eprint+doi style that didn't work in .bst file. + # TODO (better): read Part 4 of + # http://mirrors.ctan.org/info/bibtex/tamethebeast/ttb_en.pdf + # and fix the .bst style properly. + cp -pv $*.bbl $*-tmp.bbl \ + && sed -e "s/\'/EOLINE/g" $*-tmp.bbl \ + | tr -d '\n' \ + | sed -e 's/\([0-9]\)\( \|EOLINE\)}/\1}/g' \ + | sed -e 's/\([^,]\) *\( \|EOLINE\) *\\eprint/\1, \\eprint/g' \ + | sed -e 's/\([^,]\) *\( \|EOLINE\) *\\doi/\1, \\doi/g' \ + | sed -e 's/EOLINE/\n/g' > $*.bbl + if [ x"$(separatesupplement)" != x1 ]; then + cp -pv appendix.bbl appendix-tmp.bbl \ + && sed -e "s/\'/EOLINE/g" appendix-tmp.bbl \ + | tr -d '\n' \ + | sed -e 's/\([0-9]\)\( \|EOLINE\)}/\1}/g' \ + | sed -e 's/\([^,]\) *\( \|EOLINE\) *\\eprint/\1, \\eprint/g' \ + | sed -e 's/\([^,]\) *\( \|EOLINE\) *\\doi/\1, \\doi/g' \ + | sed -e 's/EOLINE/\n/g' > appendix.bbl + fi + # Paper-specific hacks for reducing very-long author lists. + cp -pv $*.bbl $*-tmp.bbl \ + && sed -e "s/\'/EOLINE/g" $*-tmp.bbl \ + | tr -d '\n' \ + | sed -e 's;, D\..Chong[^{]*Forstag; et al.\\/;' \ + | sed -e 's;, V\..Khodiyar[^{]*Whyte; et al.\\/;' \ + | sed -e 's/EOLINE/\n/g' > $*.bbl + + # The pre-final run of LaTeX after 'paper.bbl' was created. + latex -shell-escape -halt-on-error $*.tex fi @@ -145,7 +212,8 @@ $(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ # to run everything cleanly from there, it is necessary to add the current # directory (top project directory) to the `TEXINPUTS' environment # variable. -paper.pdf: $(mtexdir)/project.tex paper.tex $(texbdir)/paper.bbl +$(top-pdfs): %.pdf: $(mtexdir)/project.tex paper.tex \ + tex/src/appendix-*.tex $(texbdir)/%.bbl # If `$(mtexdir)/project.tex' is empty, don't build the PDF. @macros=$$(cat $(mtexdir)/project.tex) @@ -155,13 +223,20 @@ paper.pdf: $(mtexdir)/project.tex paper.tex $(texbdir)/paper.bbl p=$$(pwd) export TEXINPUTS=$$p: cd $(texbdir) + # See above for a warning and brief discussion on the the # pdflatex option `-shell-escape'. - pdflatex -shell-escape -halt-on-error "$$p"/paper.tex + latex -shell-escape -halt-on-error $*.tex + + # Convert the DVI to PostScript, and the PostScript to PDF. The + # `-dNOSAFER' option to GhostScript allows transparencies in the + # conversion from PostScript to PDF, see + # https://www.ghostscript.com/doc/current/Language.htm#Transparency + dvips $*.dvi + ps2pdf $*.ps # Come back to the top project directory and copy the built PDF # file here. cd "$$p" - cp $(texbdir)/$@ $(final-paper) - + cp $(texbdir)/$*.pdf $@ fi diff --git a/reproduce/analysis/make/top-make.mk b/reproduce/analysis/make/top-make.mk index 026220e..6b055f7 100644 --- a/reproduce/analysis/make/top-make.mk +++ b/reproduce/analysis/make/top-make.mk @@ -64,8 +64,13 @@ include reproduce/software/config/LOCAL.conf # If you are just interested in the processing and don't want to build the # PDF, you can skip the creation of the final PDF by giving a value of # `yes' to `pdf-build-final' in `reproduce/analysis/config/pdf-build.conf'. +ifeq ($(separatesupplement),0) +top-pdfs = paper.pdf +else +top-pdfs = paper.pdf supplement.pdf +endif ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) -all: paper.pdf +all: $(top-pdfs) else all: @if [ "x$(GROUP-NAME)" = x ]; then \ @@ -112,7 +117,8 @@ endif # wild-card like the configuration Makefiles). makesrc = initialize \ download \ - delete-me \ + format \ + demo-plot \ verify \ paper diff --git a/reproduce/analysis/make/verify.mk b/reproduce/analysis/make/verify.mk index 7e16add..6503172 100644 --- a/reproduce/analysis/make/verify.mk +++ b/reproduce/analysis/make/verify.mk @@ -131,16 +131,15 @@ $(mtexdir)/verify.tex: $(foreach s, $(verify-dep), $(mtexdir)/$(s).tex) rm -f $@.tmp # Verify the figure datasets. - $(call verify-txt-no-comments-no-space, \ - $(dm-squared), 6b6d3b0f9c351de53606507b59bca5d1, $@.tmp) - $(call verify-txt-no-comments-no-space, \ - $(dm-img-histogram), b1f9c413f915a1ad96078fee8767b16c, $@.tmp) + $(call verify-txt-no-comments-leading-space, \ + $(a2mk20f1c), 76fc5b13495c4d8e8e6f8d440304cf69) # Verify TeX macros (the values that go into the PDF text). for m in $(verify-check); do file=$(mtexdir)/$$m.tex - if [ $$m == download ]; then s=49e4e9f049aa9da0453a67203d798587 - elif [ $$m == delete-me ]; then s=711e2f7fa1f16ecbeeb3df6bcb4ec705 + if [ $$m == download ]; then s=64da83ee3bfaa236849927cdc001f5d3 + elif [ $$m == format ]; then s=e04d95a539b5540c940bf48994d8d45f + elif [ $$m == demo-plot ]; then s=48bffe6cf8db790c63a33302d20db77f else echo; echo "'$$m' not recognized."; exit 1 fi $(call verify-txt-no-comments-no-space, $$file, $$s, $@.tmp) diff --git a/reproduce/software/config/TARGETS.conf b/reproduce/software/config/TARGETS.conf index bec3adb..95f12f2 100644 --- a/reproduce/software/config/TARGETS.conf +++ b/reproduce/software/config/TARGETS.conf @@ -35,8 +35,11 @@ -# Programs and libraries (for Python modules, add to 'top-level-python'). -top-level-programs = gnuastro +# Programs and libraries. +# +# Ghostscript: to build PDF paper (in particular the `ps2pdf' command). +# XLSXI/O: to read and write XLSX files. +top-level-programs = ghostscript xlsxio # Python libraries/modules. -top-level-python = +top-level-python = diff --git a/reproduce/software/config/texlive-packages.conf b/reproduce/software/config/texlive-packages.conf index 2cca87a..d4900f1 100644 --- a/reproduce/software/config/texlive-packages.conf +++ b/reproduce/software/config/texlive-packages.conf @@ -16,9 +16,7 @@ # the basic installation scheme that we used to install tlmgr, they will be # ignored in the `tlmgr install' command, but will be used later when we # want their versions. -texlive-packages = tex fancyhdr ec newtx fontaxes xkeyval etoolbox xstring \ - xcolor setspace caption footmisc datetime fmtcount \ - titlesec preprint ulem biblatex biber logreq pgf pgfplots \ - fp courier tex-gyre txfonts times csquotes kastrup \ - trimspaces pdftexcmds pdfescape letltxmacro bitset \ - mweights +texlive-typewriter-pkgs = courier inconsolata xkeyval upquote +texlive-packages = times IEEEtran cite xcolor pgfplots ps2eps \ + listings ulem etoolbox multibib \ + $(texlive-typewriter-pkgs) diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 24e8409..219b335 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -1224,6 +1224,19 @@ if ! [ -d "$texbdir" ]; then mkdir "$texbdir"; fi tikzdir="$texbdir"/tikz if ! [ -d "$tikzdir" ]; then mkdir "$tikzdir"; fi +# If 'tex/build' and 'tex/tikz' aren't symbolic links, then we are in the +# tarball (not the Git repository), so we'll give them another name and let +# the script continue normally. +if rm -f tex/build; then + rm -f tex/tikz +else + mv tex/tikz tex/tikz-from-tarball + mv tex/build tex/build-from-tarball +fi + + + + # If 'tex/build' and 'tex/tikz' are symbolic links then 'rm -f' will delete # them and we can continue. However, when the project is being built from |