diff options
Diffstat (limited to 'reproduce/analysis')
| -rwxr-xr-x | reproduce/analysis/bash/download-multi-try | 112 | ||||
| -rw-r--r-- | reproduce/analysis/config/INPUTS.mk | 15 | ||||
| -rw-r--r-- | reproduce/analysis/config/delete-me-num.mk | 2 | ||||
| -rw-r--r-- | reproduce/analysis/config/pdf-build.mk | 21 | ||||
| -rw-r--r-- | reproduce/analysis/make/delete-me.mk | 126 | ||||
| -rw-r--r-- | reproduce/analysis/make/download.mk | 91 | ||||
| -rw-r--r-- | reproduce/analysis/make/initialize.mk | 339 | ||||
| -rw-r--r-- | reproduce/analysis/make/paper.mk | 139 | ||||
| -rw-r--r-- | reproduce/analysis/make/top.mk | 136 |
9 files changed, 981 insertions, 0 deletions
diff --git a/reproduce/analysis/bash/download-multi-try b/reproduce/analysis/bash/download-multi-try new file mode 100755 index 0000000..1fd7497 --- /dev/null +++ b/reproduce/analysis/bash/download-multi-try @@ -0,0 +1,112 @@ +# Attempt downloading multiple times before crashing whole project. From +# the top project directory (for the shebang above), this script must be +# run like this: +# +# $ /path/to/download-multi-try downloader lockfile input-url downloaded-name +# +# NOTE: The `downloader' must contain the option to specify the output name +# in its end. For example "wget -O". Any other option can also be placed in +# the middle. +# +# Due to temporary network problems, a download may fail suddenly, but +# succeed in a second try a few seconds later. Without this script that +# temporary glitch in the network will permanently crash the project and +# it can't continue. The job of this script is to be patient and try the +# download multiple times before crashing the whole project. +# +# LOCK FILE: Since there is ultimately only one network port to the outside +# world, downloading is done much faster in serial, not in parallel. But +# the project's processing may be done in parallel (with multiple threads +# needing to download different files at the same time). Therefore, this +# script uses the `flock' program to only do one download at a time. To +# benefit from it, any call to this script must be given the same lock +# file. +# +# Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This script is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This script is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. See <http://www.gnu.org/licenses/>. + + + + + +# Script settings +# --------------- +# Stop the script if there are any errors. +set -e + + + + + +# Input arguments and necessary sanity checks. +inurl="$3" +outname="$4" +lockfile="$2" +downloader="$1" +if [ "x$downloader" = x ]; then + echo "$0: downloader (first argument) not given."; exit 1; +fi +if [ "x$lockfile" = x ]; then + echo "$0: lock file (second argument) not given."; exit 1; +fi +if [ "x$inurl" = x ]; then + echo "$0: full input URL (third argument) not given."; exit 1; +fi +if [ "x$outname" = x ]; then + echo "$0: output name (fourth argument) not given."; exit 1; +fi + + + + + +# Try downloading multiple times before crashing. +counter=0 +maxcounter=10 +while [ ! -f "$outname" ]; do + + # Increment the counter. We need the `counter=' part here because + # without it the evaluation of arithmetic expression will be like and + # error and the script is set to crash on errors. + counter=$((counter+1)) + + # If we have passed a maximum number of trials, just exit with + # a failed code. + if (( counter > maxcounter )); then + echo + echo "Failed $maxcounter download attempts: $outname" + echo + exit 1 + fi + + # If this isn't the first attempt print a notice and wait a little for + # the next trail. + if (( counter > 1 )); then + tstep=$((counter*5)) + echo "Download trial $counter for '$outname' in $tstep seconds." + sleep $tstep + fi + + # Attempt downloading the file (one-at-a-time). Note that the + # `downloader' ends with the respective option to specify the output + # name. For example "wget -O" (so `outname', that comes after it) will + # be the name of the downloaded file. + flock "$lockfile" bash -c \ + "if ! $downloader $outname $inurl; then rm -f $outname; fi" +done + + + + + +# Return successfully +exit 0 diff --git a/reproduce/analysis/config/INPUTS.mk b/reproduce/analysis/config/INPUTS.mk new file mode 100644 index 0000000..eb38295 --- /dev/null +++ b/reproduce/analysis/config/INPUTS.mk @@ -0,0 +1,15 @@ +# Input files necessary for this project. +# +# This file is read by the configure script and running Makefiles. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice and +# this notice are preserved. This file is offered as-is, without any +# warranty. + +WFPC2IMAGE = WFPC2ASSNu5780205bx.fits +WFPC2MD5 = a4791e42cd1045892f9c41f11b50bad8 +WFPC2SIZE = 62kb +WFPC2URL = https://fits.gsfc.nasa.gov/samples diff --git a/reproduce/analysis/config/delete-me-num.mk b/reproduce/analysis/config/delete-me-num.mk new file mode 100644 index 0000000..17f608c --- /dev/null +++ b/reproduce/analysis/config/delete-me-num.mk @@ -0,0 +1,2 @@ +# Number of samples to create +delete-me-num = 50 diff --git a/reproduce/analysis/config/pdf-build.mk b/reproduce/analysis/config/pdf-build.mk new file mode 100644 index 0000000..3a86ff3 --- /dev/null +++ b/reproduce/analysis/config/pdf-build.mk @@ -0,0 +1,21 @@ +# Make the final PDF? +# ------------------- +# +# During the project's early phases, it is usually not necessary to build +# the PDF file (which makes a lot of output lines on the command-line and +# can make it hard to find the commands and possible errors (and their +# outputs). Also, in some cases, only the produced results may be of +# interest and not the final PDF, so LaTeX (and its necessary packages) may +# not be installed. +# +# If this variable is given any string, a PDF will be made with +# LaTeX. Otherwise, a notice will just printed that for now, no PDF will be +# created. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice and +# this notice are preserved. This file is offered as-is, without any +# warranty. +pdf-build-final = yes diff --git a/reproduce/analysis/make/delete-me.mk b/reproduce/analysis/make/delete-me.mk new file mode 100644 index 0000000..c4cfffe --- /dev/null +++ b/reproduce/analysis/make/delete-me.mk @@ -0,0 +1,126 @@ +# Dummy Makefile to create a random dataset for plotting. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + + + + + +# Dummy dataset +# ------------- +# +# We will use AWK to generate a table showing X and X^2 and draw its plot. +dmdir = $(texdir)/delete-me +dm = $(dmdir)/data.txt +$(dmdir): | $(texdir); mkdir $@ +$(dm): $(pconfdir)/delete-me-num.mk | $(dmdir) + + # When the plotted values are re-made, it is necessary to also + # delete the TiKZ externalized files so the plot is also re-made. + rm -f $(tikzdir)/delete-me.pdf + + # Generate the table of random values. + awk 'BEGIN {for(i=1;i<=$(delete-me-num);i+=0.5) print i, i*i; }' > $@ + + + + + +# WFPC2 image PDF +# ----------------- +# +# For an example image, we'll make a PDF copy of the WFPC II image to +# display in the paper. +dddemodir = $(texdir)/delete-me-demo +$(dddemodir): | $(texdir); mkdir $@ +demopdf = $(dddemodir)/wfpc2.pdf +$(demopdf): $(dddemodir)/%.pdf: $(indir)/%.fits | $(dddemodir) + + # When the plotted values are re-made, it is necessary to also + # delete the TiKZ externalized files so the plot is also re-made. + rm -f $(tikzdir)/delete-me-wfpc2.pdf + + # Convert the dataset to a PDF. + astconvertt --colormap=gray --fluxhigh=4 $< -h0 -o$@ + + + + + +# Histogram of WFPC2 image +# ------------------------ +# +# For an example plot, we'll show the pixel value histogram also. +histogram = $(dddemodir)/wfpc2-hist.txt +$(histogram): $(dddemodir)/%-hist.txt: $(indir)/%.fits | $(dddemodir) + + # When the plotted values are re-made, it is necessary to also + # delete the TiKZ externalized files so the plot is also re-made. + rm -f $(tikzdir)/delete-me-wfpc2.pdf + + # Generate the pixel value distribution + aststatistics --lessthan=5 $< -h0 --histogram -o$@ + + + + + +# Basic statistics +# ---------------- +# +# This is just as a demonstration on how to get analysic configuration +# parameters from variables defined in `reproduce/analysis/config/'. +stats = $(dddemodir)/wfpc2-stats.txt +$(stats): $(dddemodir)/%-stats.txt: $(indir)/%.fits | $(dddemodir) + aststatistics $< -h0 --mean --median > $@ + + + + + +# TeX macros +# ---------- +# +# This is how we write the necessary parameters in the final PDF. +# +# NOTE: In LaTeX you cannot use any non-alphabetic character in a variable +# name. +$(mtexdir)/delete-me.tex: $(dm) $(demopdf) $(histogram) $(stats) + + # Write the number of random values used. + echo "\newcommand{\deletemenum}{$(delete-me-num)}" > $@ + + # Note that since Make variables start with a `$(', if you want to + # use `$' within the shell (not Make), you have to quote any + # occurance of `$' with another `$'. That is why there are `$$' in + # the AWK command below. + # + # Here, we are first using AWK to find the minimum and maximum + # values, then using it again to read each separately to use in the + # macro definition. + mm=$$(awk 'BEGIN{min=99999; max=-min} + {if($$2>max) max=$$2; if($$2<min) min=$$2;} + END{print min, max}' $(dm)); + v=$$(echo "$$mm" | awk '{printf "%.3f", $$1}'); + echo "\newcommand{\deletememin}{$$v}" >> $@ + v=$$(echo "$$mm" | awk '{printf "%.3f", $$2}'); + echo "\newcommand{\deletememax}{$$v}" >> $@ + + # Write the statistics of the WFPC2 image as a macro. + mean=$$(awk '{printf("%.2f", $$1)}' $(stats)) + echo "\newcommand{\deletemewfpctwomean}{$$mean}" >> $@ + median=$$(awk '{printf("%.2f", $$2)}' $(stats)) + echo "\newcommand{\deletemewfpctwomedian}{$$median}" >> $@ diff --git a/reproduce/analysis/make/download.mk b/reproduce/analysis/make/download.mk new file mode 100644 index 0000000..a721863 --- /dev/null +++ b/reproduce/analysis/make/download.mk @@ -0,0 +1,91 @@ +# Download all the necessary inputs if they are not already present. +# +# Since most systems only have one input/connection into the network, +# downloading is essentially a serial (not parallel) operation. so the +# recipes in this Makefile all use a single file lock to have one download +# script running at every instant. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. See <http://www.gnu.org/licenses/>. + + + + + +# Download input data +# -------------------- +# +# The input dataset properties are defined in `$(pconfdir)/INPUTS.mk'. For +# this template we only have one dataset to enable easy processing, so all +# the extra checks in this rule may seem redundant. +# +# In a real project, you will need more than one dataset. In that case, +# just add them to the target list and add an `elif' statement to define it +# in the recipe. +# +# Files in a server usually have very long names, which are mainly designed +# for helping in data-base management and being generic. Since Make uses +# file names to identify which rule to execute, and the scope of this +# research project is much less than the generic survey/dataset, it is +# easier to have a simple/short name for the input dataset and work with +# that. In the first condition of the recipe below, we connect the short +# name with the raw database name of the dataset. +# +# Download lock file: Most systems have a single connection to the +# internet, therefore downloading is inherently done in series. As a +# result, when more than one dataset is necessary for download, if they are +# done in parallel, the speed will be slower than downloading them in +# series. We thus use the `flock' program to tie/lock the downloading +# process with a file and make sure that only one downloading event is in +# progress at every moment. +$(indir):; mkdir $@ +downloadwrapper = $(bashdir)/download-multi-try +inputdatasets = $(foreach i, wfpc2, $(indir)/$(i).fits) +$(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir) + + # Set the necessary parameters for this input file. + if [ $* = wfpc2 ]; then + origname=$(WFPC2IMAGE); url=$(WFPC2URL); mdf=$(WFPC2MD5); + else + echo; echo; echo "Not recognized input dataset: '$*.fits'." + echo; echo; exit 1 + fi + + # Download (or make the link to) the input dataset. + if [ -f $(INDIR)/$$origname ]; then + ln -s $(INDIR)/$$origname $@ + else + touch $(lockdir)/download + $(downloadwrapper) "wget --no-use-server-timestamps -O" \ + $(lockdir)/download $$url/$$origname $@ + fi + + # Check the md5 sum to see if this is the proper dataset. + sum=$$(md5sum $@ | awk '{print $$1}') + if [ $$sum != $$mdf ]; then + wrongname=$(dir $@)/wrong-$(notdir $@) + mv $@ $$wrongname + echo; echo; echo "Wrong MD5 checksum for '$$origname' in $$wrongname" + echo; echo; exit 1 + fi + + + + + +# Final TeX macro +# --------------- +# +# It is very important to mention the address where the data were +# downloaded in the final report. +$(mtexdir)/download.tex: $(pconfdir)/INPUTS.mk | $(mtexdir) + echo "\\newcommand{\\wfpctwourl}{$(WFPC2URL)}" > $@ diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk new file mode 100644 index 0000000..a034494 --- /dev/null +++ b/reproduce/analysis/make/initialize.mk @@ -0,0 +1,339 @@ +# Project initialization. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + + + + + +# High-level directory definitions +# -------------------------------- +# +# Basic directories that are used throughout the project. +# +# Locks are used to make sure that an operation is done in series not in +# parallel (even if Make is run in parallel with the `-j' option). The most +# common case is downloads which are better done in series and not in +# parallel. Also, some programs may not be thread-safe, therefore it will +# be necessary to put a lock on them. This project uses the `flock' program +# to achieve this. +texdir = $(BDIR)/tex +lockdir = $(BDIR)/locks +indir = $(BDIR)/inputs +mtexdir = $(texdir)/macros +bashdir = reproduce/analysis/bash +pconfdir = reproduce/analysis/config +installdir = $(BDIR)/software/installed +# --------- Delete for no Gnuastro --------- +gconfdir = reproduce/software/config/gnuastro +# ------------------------------------------ + + + + + +# TeX build directory +# ------------------ +# +# In scenarios where multiple users are working on the project +# simultaneously, they can't all build the final paper together, there will +# be conflicts! It is possible to manage the working on the analysis, so no +# conflict is caused in that phase, but it would be very slow to only let +# one of the project members to build the paper at every instance +# (independent parts of the paper can be added to it independently). To fix +# this problem, when we are in a group setting, we'll use the user's ID to +# create a separate LaTeX build directory for each user. +# +# The same logic applies to the final paper PDF: each user will create a +# separte final PDF (for example `paper-user1.pdf' and `paper-user2.pdf') +# and no `paper.pdf' will be built. This isn't a problem because +# `initialize.tex' is a .PHONY prerequisite, so the rule to build the final +# paper is always executed (even if it is present and nothing has +# changed). So in terms of over-all efficiency and processing steps, this +# doesn't change anything. +ifeq (x$(GROUP-NAME),x) +texbdir = $(texdir)/build +final-paper = paper.pdf +else +user = $(shell whoami) +texbdir = $(texdir)/build-$(user) +final-paper = paper-$(user).pdf +endif +tikzdir = $(texbdir)/tikz + + + + + +# Original system environment +# --------------------------- +# +# Before defining the local sub-environment here, we'll need to save the +# system's environment for some scenarios (for example after `clean'ing the +# built programs). +sys-path := $(PATH) +sys-rm := $(shell which rm) +curdir := $(shell echo $$(pwd)) + + + + + +# High level environment +# ---------------------- +# +# We want the full recipe to be executed in one call to the shell. Also we +# want Make to run the specific version of Bash that we have installed +# during `./configure' time. +# +# Regarding the directories, this project builds its major dependencies +# itself and doesn't use the local system's default tools. With these +# environment variables, we are setting it to prefer the software we have +# build here. +.ONESHELL: +.SHELLFLAGS = -ec +export CCACHE_DISABLE := 1 +export PATH := $(installdir)/bin +export LD_LIBRARY_PATH := $(installdir)/lib +export LDFLAGS := -L$(installdir)/lib +export SHELL := $(installdir)/bin/bash +export CPPFLAGS := -I$(installdir)/include + + + + + +# Python enviroment +# ----------------- +# +# The main Python environment variable is `PYTHONPATH'. However, so far we +# have found several other Python-related environment variables on some +# systems which might interfere. To be safe, we are removing all their +# values. +export PYTHONPATH := $(installdir)/lib/python/site-packages +export PYTHONPATH3 := $(PYTHONPATH) +export _LMFILES_ := +export PYTHONPATH2 := +export LOADEDMODULES := +export MPI_PYTHON_SITEARCH := +export MPI_PYTHON2_SITEARCH := +export MPI_PYTHON3_SITEARCH := + + + + + +# High-level level directories +# ---------------------------- +# +# These are just the top-level directories for all the separate steps. The +# directories (or possible sub-directories) for individual steps will be +# defined and added within their own Makefiles. +# +# The `.SUFFIXES' rule with no prerequisite is defined to eliminate all the +# default implicit rules. The default implicit rules are to do with +# programming (for example converting `.c' files to `.o' files). The +# problem they cause is when you want to debug the make command with `-d' +# option: they add too many extra checks that make it hard to find what you +# are looking for in the outputs. +.SUFFIXES: +$(lockdir): | $(BDIR); mkdir $@ +$(texbdir): | $(texdir); mkdir $@ +$(tikzdir): | $(texbdir); mkdir $@ && ln -fs $@ tex/tikz + + + + + +# High-level Makefile management +# ------------------------------ +# +# About `.PHONY': these are targets that must be built even if a file with +# their name exists. +# +# Only `$(mtexdir)/initialize.tex' corresponds to a file. This is because +# we want to ensure that the file is always built in every run: it contains +# the project version which may change between two separate runs, even when +# no file actually differs. +packagebasename := $(shell echo paper-$$(git describe --dirty --always)) +packagecontents = $(texdir)/$(packagebasename) +.PHONY: all clean dist dist-zip distclean clean-mmap $(packagecontents) \ + $(mtexdir)/initialize.tex + +# --------- Delete for no Gnuastro --------- +clean-mmap:; rm -f reproduce/config/gnuastro/mmap* +# ------------------------------------------ + +clean: clean-mmap + # Delete the top-level PDF file. + rm -f *.pdf + + # Delete all the built outputs except the dependency + # programs. We'll use Bash's extended options builtin (`shopt') to + # enable "extended glob" (for listing of files). It allows extended + # features like ignoring the listing of a file with `!()' that we + # are using afterwards. + shopt -s extglob + rm -rf $(BDIR)/!(dependencies) + +distclean: clean + # We'll be deleting the built environent programs and just need the + # `rm' program. So for this recipe, we'll use the host system's + # `rm', not our own. + $(sys-rm) -rf $(BDIR) reproduce/build + $(sys-rm) -f Makefile .gnuastro .local + $(sys-rm) -f $(pconfdir)/LOCAL.mk $(gconfdir)/gnuastro-local.conf + + + + + +# Packaging rules +# --------------- +# +# With the rules in this section, you can package the project in a state +# that is ready for building the final PDF with LaTeX. This is useful for +# collaborators who only want to contribute to the text of your project, +# without having to worry about the technicalities of the analysis. +$(packagecontents): | $(texdir) + + # Set up the output directory, delete it if it exists and remake it + # to fill with new contents. + dir=$(texdir)/$(packagebasename) + rm -rf $$dir + mkdir $$dir + + # Build a small Makefile to help in automatizing the paper building + # (including the bibliography). + m=$$dir/Makefile + echo "paper.pdf: paper.tex paper.bbl" > $$m + printf "\tpdflatex -shell-escape -halt-on-error paper\n" >> $$m + echo "paper.bbl: tex/src/references.tex" >> $$m + printf "\tpdflatex -shell-escape -halt-on-error paper\n" >> $$m + printf "\tbiber paper\n" >> $$m + echo ".PHONY: clean" >> $$m + echo "clean:" >> $$m + printf "\trm -f *.aux *.auxlock *.bbl *.bcf\n" >> $$m + printf "\trm -f *.blg *.log *.out *.run.xml\n" >> $$m + + # Copy the top-level contents into it. + cp configure COPYING for-group README.md README-hacking.md $$dir/ + + # Build the top-level directories. + mkdir $$dir/reproduce $$dir/tex $$dir/tex/tikz $$dir/tex/build + + # Copy all the necessary `reproduce' and `tex' contents. + shopt -s extglob + cp -r tex/src $$dir/tex/src + cp tex/tikz/*.pdf $$dir/tex/tikz + cp -r reproduce/ $$dir/reproduce + cp -r tex/build/!($(packagebasename)) $$dir/tex/build + + # Clean up un-necessary/local files: 1) the $(texdir)/build* + # directories (when building in a group structure, there will be + # `build-user1', `build-user2' and etc), are just temporary LaTeX + # build files and don't have any relevant/hand-written files in + # them. 2) The `LOCAL.mk' and `gnuastro-local.conf' files just have + # this machine's local settings and are irrelevant for anyone else. + rm -rf $$dir/tex/build/build* + rm $$dir/reproduce/software/config/installation/LOCAL.mk + rm $$dir/reproduce/software/config/gnuastro/gnuastro-local.conf + + # PROJECT SPECIFIC: under this comment, copy any other file for + # packaging, or remove any of the copied files above to suite your + # project. + + # Since the packaging is mainly intended for high-level building of + # the PDF with LaTeX, we'll comment the `makepdf' LaTeX macro in + # the paper. + sed -e's|\\newcommand{\\makepdf}{}|%\\newcommand{\\makepdf}{}|' \ + paper.tex > $$dir/paper.tex + + # Just in case the package users want to rebuild some of the + # figures (manually un-comments the `makepdf' command we commented + # above), correct the TikZ external directory, so the figures can + # be rebuilt. + pgfsettings="$$dir/tex/src/preamble-pgfplots.tex" + sed -e's|{tikz/}|{tex/tikz/}|' $$pgfsettings > $$pgfsettings.new + mv $$pgfsettings.new $$pgfsettings + + # Clean temporary (currently those ending in `~') files. + cd $(texdir) + find $(packagebasename) -name \*~ -delete + +# Package into `.tar.gz'. +dist: $(packagecontents) + curdir=$$(pwd) + cd $(texdir) + tar -cf $(packagebasename).tar $(packagebasename) + gzip -f --best $(packagebasename).tar + cd $$curdir + mv $(texdir)/$(packagebasename).tar.gz ./ + +# Package into `.zip'. +dist-zip: $(packagecontents) + curdir=$$(pwd) + cd $(texdir) + zip -q -r $(packagebasename).zip $(packagebasename) + cd $$curdir + mv $(texdir)/$(packagebasename).zip ./ + + + + + +# Check the version of programs which write their version +# ------------------------------------------------------- +pvcheck = prog="$(strip $(1))"; \ + ver="$(strip $(2))"; \ + name="$(strip $(3))"; \ + macro="$(strip $(4))"; \ + verop="$(strip $(5))"; \ + if [ "x$$verop" = x ]; then V="--version"; else V=$$verop; fi; \ + v=$$($$prog $$V | awk '/'$$ver'/{print "y"; exit 0}'); \ + if [ x$$v != xy ]; then \ + echo; echo "PROJECT ERROR: Not running $$name $$ver"; echo; \ + exit 1; \ + fi; \ + echo "\newcommand{\\$$macro}{$$ver}" >> $@ + +lvcheck = idir=$(BDIR)/dependencies/installed/include; \ + f="$$idir/$(strip $(1))"; \ + ver="$(strip $(2))"; \ + name="$(strip $(3))"; \ + macro="$(strip $(4))"; \ + v=$$(awk '/^\#/&&/define/&&/'$$ver'/{print "y";exit 0}' $$f); \ + if [ x$$v != xy ]; then \ + echo; echo "PROJECT ERROR: Not linking with $$name $$ver"; \ + echo; exit 1; \ + fi; \ + echo "\newcommand{\\$$macro}{$$ver}" >> $@ + + + + +# Project initialization results +# ------------------------------ +# +# This file will store some basic info about the project that is necessary +# for the final PDF. Since these are not version controlled, it must be +# calculated everytime the project is run. So even though this file +# actually exists, it is also aded as a `.PHONY' target above. +$(mtexdir)/initialize.tex: | $(mtexdir) + + # Version of the project. + @v=$$(git describe --dirty --always); + echo "\newcommand{\projectversion}{$$v}" > $@ diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk new file mode 100644 index 0000000..5378ee3 --- /dev/null +++ b/reproduce/analysis/make/paper.mk @@ -0,0 +1,139 @@ +# Build the final PDF paper/report. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + + + + +# LaTeX macros for paper +# ---------------------- +# +# To report the input settings and results, the final report's PDF (final +# target of this project) uses macros generated from various steps of the +# project. All these macros are defined through `$(mtexdir)/project.tex'. +# +# `$(mtexdir)/project.tex' is actually just a combination of separate files +# that keep the LaTeX macros related to each workhorse Makefile (in +# `reproduce/src/make/*.mk'). Those individual macros are pre-requisites to +# `$(mtexdir)/project.tex'. The only workhorse Makefile that doesn't need +# to produce LaTeX macros is this Makefile (`reproduce/src/make/paper.mk'). +# +# This file is thus the interface between the processing scripts and the +# final PDF: when we get to this point, all the processing has been +# completed. +# +# Note that if you don't want the final PDF and just want the processing +# and file outputs, you can remove the value of `pdf-build-final' in +# `reproduce/analysis/config/pdf-build.mk'. +$(mtexdir)/project.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s).tex) + + # If no PDF is requested, or if LaTeX isn't available, don't + # continue to building the final PDF. Otherwise, merge all the TeX + # macros into one for building the PDF. + @if [ -f .local/bin/pdflatex ] && [ x"$(pdf-build-final)" != x ]; then + + # Put a LaTeX input command for all the necessary macro files. + rm -f $(mtexdir)/project.tex + for t in $(subst paper,,$(makesrc)); do + echo "\input{tex/build/macros/$$t.tex}" >> $(mtexdir)/project.tex + done + else + echo + echo "-----" + echo "The processing has COMPLETED SUCCESSFULLY! But the final " + echo "LaTeX-built PDF paper will not be built." + echo + if [ x$(more-on-building-pdf) = x1 ]; then + echo "To do so, make sure you have LaTeX within the project (you" + echo "can check by running './.local/bin/latex --version'), _AND_" + echo "make sure that the 'pdf-build-final' variable has a value." + echo "'pdf-build-final' is defined in: " + echo "'reproduce/analysis/config/pdf-build.mk'." + echo + echo "If you don't have LaTeX within the project, please re-run" + echo "'./configure' when you have internet access. To speed it up," + echo "you can keep the previous configuration files (answer 'n'" + echo "when it asks about re-writing previous configuration files)." + else + echo "For more, run './.local/bin/make more-on-building-pdf=1'" + fi + echo + echo "" > $@ + fi + + + + + +# The bibliography +# ---------------- +# +# We need to run the `biber' program on the output of LaTeX to generate the +# necessary bibliography before making the final paper. So we'll first have +# one run of LaTeX (similar to the `paper.pdf' recipe), then `biber'. +# +# NOTE: `$(mtexdir)/project.tex' is an order-only-prerequisite for +# `paper.bbl'. This is because we need to run LaTeX in both the `paper.bbl' +# recipe and the `paper.pdf' recipe. But if `tex/src/references.tex' hasn't +# been modified, we don't want to re-build the bibliography, only the final +# PDF. +$(texbdir)/paper.bbl: tex/src/references.tex \ + | $(tikzdir) $(texbdir) $(mtexdir)/project.tex + # If `$(mtexdir)/project.tex' is empty, don't build PDF. + @macros=$$(cat $(mtexdir)/project.tex) + if [ x"$$macros" != x ]; then + + # We'll run LaTeX first to generate the `.bcf' file (necessary + # for `biber') and then run `biber' to generate the `.bbl' file. + p=$$(pwd); + export TEXINPUTS=$$p:$$TEXINPUTS; + cd $(texbdir); + pdflatex -shell-escape -halt-on-error $$p/paper.tex; + biber paper + + fi + + + + + +# The final paper +# --------------- +# +# Run LaTeX in the `$(texbdir)' directory so all the intermediate and +# auxiliary files stay there and keep the top directory clean. To be able +# to run everything cleanly from there, it is necessary to add the current +# directory (top project directory) to the `TEXINPUTS' environment +# variable. +paper.pdf: $(mtexdir)/project.tex paper.tex $(texbdir)/paper.bbl \ + | $(tikzdir) $(texbdir) + + # If `$(mtexdir)/project.tex' is empty, don't build the PDF. + @macros=$$(cat $(mtexdir)/project.tex) + if [ x"$$macros" != x ]; then + + # Go into the top TeX build directory and make the paper. + p=$$(pwd) + export TEXINPUTS=$$p:$$TEXINPUTS + cd $(texbdir) + pdflatex -shell-escape -halt-on-error $$p/paper.tex + + # Come back to the top project directory and copy the built PDF + # file here. + cd $$p + cp $(texbdir)/$@ $(final-paper) + + fi diff --git a/reproduce/analysis/make/top.mk b/reproduce/analysis/make/top.mk new file mode 100644 index 0000000..a85c530 --- /dev/null +++ b/reproduce/analysis/make/top.mk @@ -0,0 +1,136 @@ +# Top-level Makefile (first to be loaded). +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + + + + + +# Load the local configuration (created after running `./configure'). +include reproduce/software/config/installation/LOCAL.mk + + + + + +# Ultimate target of this project +# ------------------------------- +# +# The final paper/report (`paper.pdf') is the main target of this +# project. As defined in the Make paradigm, it must be the first target +# that Make encounters (immediately after loading the local configuration +# settings, necessary for a group building scenario mentioned next). +# +# +# Group build +# ----------- +# +# This project can also be configured to have a shared build directory +# between multiple users. In this scenario, many users (on a server) can +# have their own/separate version controlled project source, but share the +# same build outputs (in a common directory). This will allow a group to +# work separately, on parallel parts of the analysis that don't +# interfere. It is thus very useful in cases were special storage +# requirements or CPU power is necessary and its not possible/efficient for +# each user to have a fully separate copy of the build directory. +# +# Controlling this requires two variables that are available at this stage: +# +# - `GROUP-NAME': from `LOCAL.mk' (which was built by `./configure'). +# - `reproducible_paper_group_name': from the `./for-group' script (if it +# was used to call Make). +# +# The analysis is only done when both have the same group name. Note that +# when the project isn't being built for a group, both variables will be an +# empty string. +# +# +# Only processing, no LaTeX PDF +# ----------------------------- +# +# If you are just interested in the processing and don't want to build the +# PDF, you can skip the creatation of the final PDF by removing the value +# of `pdf-build-final' in `reproduce/analysis/config/pdf-build.mk'. +ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) +all: paper.pdf +else +all: + @if [ "x$(GROUP-NAME)" = x ]; then \ + echo "Project is NOT configured for groups, please run"; \ + echo " $$ .local/bin/make"; \ + else \ + echo "Project is configured for groups, please run"; \ + echo " $$ ./for-group $(GROUP-NAME) make -j8"; \ + fi +endif + + + + + +# Define source Makefiles +# ----------------------- +# +# To keep things clean, managable and readable, each set of operations +# is (and must be) classified (modularized) by context into separate +# Makefiles: the more the better. These modular steps are then +# included in this top-level Makefile through the `include' command of +# the next step. Each Makefile should also produce a LaTeX macro file +# with the same fixed name (used to keep all the parameters and +# relevant outputs of the steps in it for the final paper). +# +# In the rare case that no special LaTeX macros are necessary in a +# workhorse Makefile, you can simply make an empty file with `touch +# $@'. This will not add any lines to the final combined LaTeX macros +# file, but will create the file that is a prerequisite to the final +# paper generation. +# +# To (significantly) help in readability, this top-level Makefile should be +# the only one in charge of including Makefiles. So if you care about easy +# maintainence and understandability (even for your self, in one year! It +# is VERY IMPORTANT and as a scientist, you MUST care about it!), do not +# include Makefiles from any other Makefile. +# +# IMPORTANT NOTE: order matters in the inclusion of the processing +# Makefiles. As the project grows, some Makefiles will define +# variables/dependencies that later Makefiles need. Therefore we are using +# a `foreach' loop in the next step to explicitly request loading them in +# the same order that they are defined here (we aren't just using a +# wild-card like the configuration Makefiles). +makesrc = initialize \ + download \ + delete-me \ + paper + + + + + +# Include all analysis Makefiles +# ------------------------------ +# +# 1) All the analysis configuration-Makefiles (Makefiles that only define +# variables with no rules or order). +# +# 2) From the software configuration-Makefiles, we only include the one +# containing software versions, just incase its necessary to +# use/report outside of the acknowledgments section of the paper. +# +# 3) Finally, we'll import all the analysis workhorse-Makefiles which +# contain rules to actually do this project's processing. +include reproduce/analysis/config/*.mk +include reproduce/software/config/installation/versions.mk +include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk) |