diff options
Diffstat (limited to 'reproduce/analysis/make')
| -rw-r--r-- | reproduce/analysis/make/delete-me.mk | 64 | ||||
| -rw-r--r-- | reproduce/analysis/make/download.mk | 107 | ||||
| -rw-r--r-- | reproduce/analysis/make/initialize.mk | 421 | ||||
| -rw-r--r-- | reproduce/analysis/make/paper.mk | 83 | ||||
| -rw-r--r-- | reproduce/analysis/make/prepare.mk | 61 | ||||
| -rw-r--r-- | reproduce/analysis/make/top-make.mk | 21 | ||||
| -rw-r--r-- | reproduce/analysis/make/top-prepare.mk | 21 | ||||
| -rw-r--r-- | reproduce/analysis/make/verify.mk | 45 |
8 files changed, 453 insertions, 370 deletions
diff --git a/reproduce/analysis/make/delete-me.mk b/reproduce/analysis/make/delete-me.mk index c160e51..325280d 100644 --- a/reproduce/analysis/make/delete-me.mk +++ b/reproduce/analysis/make/delete-me.mk @@ -1,6 +1,6 @@ # Dummy Makefile to create a random dataset for plotting. # -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -30,16 +30,16 @@ dm-squared = $(tex-publish-dir)/squared.txt $(dm-squared): $(pconfdir)/delete-me-squared-num.conf | $(tex-publish-dir) - # When the plotted values are re-made, it is necessary to also - # delete the TiKZ externalized files so the plot is also re-made by - # PGFPlots. +# When the plotted values are re-made, it is necessary to also delete +# the TiKZ externalized files so the plot is also re-made by +# PGFPlots. rm -f $(tikzdir)/delete-me-squared.pdf - # Write the column metadata in a temporary file name (appending - # '.tmp' to the actual target name). Once all steps are done, it is - # renamed to the final target. We do this because if there is an - # error in the middle, Make will not consider the job to be - # complete and will stop here. +# Write the column metadata in a temporary file name (appending +# '.tmp' to the actual target name). Once all steps are done, it is +# renamed to the final target. We do this because if there is an +# error in the middle, Make will not consider the job to be complete +# and will stop here. echo "# Data for demonstration plot of default Maneage (MANaging data linEAGE)." > $@.tmp echo "# It is a simple plot, showing the power of two: y=x^2! " >> $@.tmp echo "# " >> $@.tmp @@ -50,11 +50,11 @@ $(dm-squared): $(pconfdir)/delete-me-squared-num.conf | $(tex-publish-dir) echo "# " >> $@.tmp $(call print-general-metadata, $@.tmp) - # Generate the table of random values. +# Generate the table of random values. awk 'BEGIN {for(i=1;i<=$(delete-me-squared-num);i+=0.5) \ printf("%-8.1f%.2f\n", i, i*i); }' >> $@.tmp - # Write it into the final target +# Write it into the final target mv $@.tmp $@ @@ -71,11 +71,11 @@ $(dm-histdir): | $(texdir); mkdir $@ dm-img-pdf = $(dm-histdir)/wfpc2.pdf $(dm-img-pdf): $(dm-histdir)/%.pdf: $(indir)/%.fits | $(dm-histdir) - # When the plotted values are re-made, it is necessary to also - # delete the TiKZ externalized files so the plot is also re-made. +# When the plotted values are re-made, it is necessary to also +# delete the TiKZ externalized files so the plot is also re-made. rm -f $(tikzdir)/delete-me-image-histogram.pdf - # Convert the dataset to a PDF. +# Convert the dataset to a PDF. astconvertt --colormap=gray --fluxhigh=4 $< -h0 -o$@ @@ -92,15 +92,15 @@ dm-img-histogram = $(tex-publish-dir)/wfpc2-histogram.txt $(dm-img-histogram): $(tex-publish-dir)/%-histogram.txt: $(indir)/%.fits \ | $(tex-publish-dir) - # When the plotted values are re-made, it is necessary to also - # delete the TiKZ externalized files so the plot is also re-made. +# When the plotted values are re-made, it is necessary to also delete +# the TiKZ externalized files so the plot is also re-made. rm -f $(tikzdir)/delete-me-image-histogram.pdf - # Generate the pixel value histogram. +# Generate the pixel value histogram. aststatistics --lessthan=5 $< -h0 --histogram -o$@.data - # Put a two-line description of the dataset, copy the column - # metadata from '$@.data', and add copyright. +# Put a two-line description of the dataset, copy the column metadata +# from '$@.data', and add copyright. echo "# Histogram of example image to demonstrate Maneage (MANaging data linEAGE)." \ > $@.tmp echo "# Example image URL: $(DEMO-URL)" >> $@.tmp @@ -109,8 +109,8 @@ $(dm-img-histogram): $(tex-publish-dir)/%-histogram.txt: $(indir)/%.fits \ echo "# " >> $@.tmp $(call print-general-metadata, $@.tmp) - # Add the column numbers in a formatted manner, rename it to the - # output and clean up. +# Add the column numbers in a formatted manner, rename it to the +# output and clean up. awk '!/^#/{printf("%-15.4f%d\n", $$1, $$2)}' $@.data >> $@.tmp mv $@.tmp $@ rm $@.data @@ -123,7 +123,7 @@ $(dm-img-histogram): $(tex-publish-dir)/%-histogram.txt: $(indir)/%.fits \ # ---------------- # # This is just as a demonstration on how to get analysic configuration -# parameters from variables defined in `reproduce/analysis/config/'. +# parameters from variables defined in 'reproduce/analysis/config/'. dm-img-stats = $(dm-histdir)/wfpc2-stats.txt $(dm-img-stats): $(dm-histdir)/%-stats.txt: $(indir)/%.fits \ | $(dm-histdir) @@ -143,17 +143,17 @@ $(dm-img-stats): $(dm-histdir)/%-stats.txt: $(indir)/%.fits \ $(mtexdir)/delete-me.tex: $(dm-squared) $(dm-img-pdf) $(dm-img-histogram) \ $(dm-img-stats) - # Write the number of random values used. +# Write the number of random values used. echo "\newcommand{\deletemenum}{$(delete-me-squared-num)}" > $@ - # Note that since Make variables start with a `$(', if you want to - # use `$' within the shell (not Make), you have to quote any - # occurance of `$' with another `$'. That is why there are `$$' in - # the AWK command below. - # - # Here, we are first using AWK to find the minimum and maximum - # values, then using it again to read each separately to use in the - # macro definition. +# Note that since Make variables start with a '$(', if you want to +# use '$' within the shell (not Make), you have to quote any +# occurance of '$' with another '$'. That is why there are '$$' in +# the AWK command below. +# +# Here, we are first using AWK to find the minimum and maximum +# values, then using it again to read each separately to use in the +# macro definition. mm=$$(awk 'BEGIN{min=99999; max=-min} !/^#/{if($$2>max) max=$$2; if($$2<min) min=$$2;} END{print min, max}' $(dm-squared)); @@ -162,7 +162,7 @@ $(mtexdir)/delete-me.tex: $(dm-squared) $(dm-img-pdf) $(dm-img-histogram) \ v=$$(echo "$$mm" | awk '{printf "%.3f", $$2}'); echo "\newcommand{\deletememax}{$$v}" >> $@ - # Write the statistics of the demo image as a macro. +# Write the statistics of the demo image as a macro. mean=$$(awk '{printf("%.2f", $$1)}' $(dm-img-stats)) echo "\newcommand{\deletemewfpctwomean}{$$mean}" >> $@ median=$$(awk '{printf("%.2f", $$2)}' $(dm-img-stats)) diff --git a/reproduce/analysis/make/download.mk b/reproduce/analysis/make/download.mk deleted file mode 100644 index 0ea3432..0000000 --- a/reproduce/analysis/make/download.mk +++ /dev/null @@ -1,107 +0,0 @@ -# Download all the necessary inputs if they are not already present. -# -# Since most systems only have one input/connection into the network, -# downloading is essentially a serial (not parallel) operation. so the -# recipes in this Makefile all use a single file lock to have one download -# script running at every instant. -# -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# -# This Makefile is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This Makefile is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this Makefile. If not, see <http://www.gnu.org/licenses/>. - - - - - -# Download input data -# -------------------- -# -# The input dataset properties are defined in -# `$(pconfdir)/INPUTS.conf'. For this template we only have one dataset to -# enable easy processing, so all the extra checks in this rule may seem -# redundant. -# -# In a real project, you will need more than one dataset. In that case, -# just add them to the target list and add an `elif' statement to define it -# in the recipe. -# -# Files in a server usually have very long names, which are mainly designed -# for helping in data-base management and being generic. Since Make uses -# file names to identify which rule to execute, and the scope of this -# research project is much less than the generic survey/dataset, it is -# easier to have a simple/short name for the input dataset and work with -# that. In the first condition of the recipe below, we connect the short -# name with the raw database name of the dataset. -# -# Download lock file: Most systems have a single connection to the -# internet, therefore downloading is inherently done in series. As a -# result, when more than one dataset is necessary for download, if they are -# done in parallel, the speed will be slower than downloading them in -# series. We thus use the `flock' program to tie/lock the downloading -# process with a file and make sure that only one downloading event is in -# progress at every moment. -$(indir):; mkdir $@ -downloadwrapper = $(bashdir)/download-multi-try -inputdatasets = $(foreach i, wfpc2, $(indir)/$(i).fits) -$(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir) - - # Set the necessary parameters for this input file. - if [ $* = wfpc2 ]; then - localname=$(DEMO-DATA); url=$(DEMO-URL); mdf=$(DEMO-MD5); - else - echo; echo; echo "Not recognized input dataset: '$*.fits'." - echo; echo; exit 1 - fi - - # Download (or make the link to) the input dataset. If the file - # exists in `INDIR', it may be a symbolic link to some other place - # in the filesystem. To avoid too many links when using these files - # during processing, we'll use `readlink -f' so the link we make - # here points to the final file directly (note that `readlink' is - # part of GNU Coreutils). If its not a link, the `readlink' part - # has no effect. - unchecked=$@.unchecked - if [ -f $(INDIR)/$$localname ]; then - ln -fs $$(readlink -f $(INDIR)/$$localname) $$unchecked - else - touch $(lockdir)/download - $(downloadwrapper) "wget --no-use-server-timestamps -O" \ - $(lockdir)/download $$url $$unchecked - fi - - # Check the md5 sum to see if this is the proper dataset. - sum=$$(md5sum $$unchecked | awk '{print $$1}') - if [ $$sum = $$mdf ]; then - mv $$unchecked $@ - echo "Integrity confirmed, using $@ in this project." - else - echo; echo; - echo "Wrong MD5 checksum for input file '$$localname':" - echo " File location: $$unchecked"; \ - echo " Expected MD5 checksum: $$mdf"; \ - echo " Calculated MD5 checksum: $$sum"; \ - echo; exit 1 - fi - - - - - -# Final TeX macro -# --------------- -# -# It is very important to mention the address where the data were -# downloaded in the final report. -$(mtexdir)/download.tex: $(pconfdir)/INPUTS.conf | $(mtexdir) - echo "\\newcommand{\\wfpctwourl}{$(DEMO-URL)}" > $@ diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 744ecbf..9e8db4a 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -1,6 +1,6 @@ # Project initialization. # -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,10 +25,10 @@ # Basic directories that are used throughout the project. # # Locks are used to make sure that an operation is done in series not in -# parallel (even if Make is run in parallel with the `-j' option). The most +# parallel (even if Make is run in parallel with the '-j' option). The most # common case is downloads which are better done in series and not in # parallel. Also, some programs may not be thread-safe, therefore it will -# be necessary to put a lock on them. This project uses the `flock' program +# be necessary to put a lock on them. This project uses the 'flock' program # to achieve this. # # To help with modularity and clarity of the build directory (not mixing @@ -43,7 +43,7 @@ bsdir=$(BDIR)/software texdir = $(badir)/tex lockdir = $(bsdir)/locks indir = $(badir)/inputs -prepdir = $(padir)/prepare +prepdir = $(badir)/prepare mtexdir = $(texdir)/macros installdir = $(bsdir)/installed bashdir = reproduce/analysis/bash @@ -56,19 +56,26 @@ pconfdir = reproduce/analysis/config # Preparation phase # ----------------- # -# This Makefile is loaded both for the `prepare' phase and the `make' +# This Makefile is loaded both for the 'prepare' phase and the 'make' # phase. But the preparation files should be dealt with differently -# (depending on the phase). In the `prepare' phase, the main directory -# should be created, and in the `make' phase, its contents should be +# (depending on the phase). In the 'prepare' phase, the main directory +# should be created, and in the 'make' phase, its contents should be # loaded. # -# If you don't need any preparation, please simply comment these lines. +# If your project doesn't need any preparation, you can ignore this. +# +# The '-' behind the include commands is used for adding the files only if +# it is possible (they exist). This is necessary because sometimes the user +# will have only '*.conf' or '*.mk' files, or with 'make clean' (where the +# preparation Makefile may call initialize.mk before the main +# 'top-make.mk'). If the '-' is not used, Make will complain about not +# finding these files. ifeq (x$(project-phase),xprepare) $(prepdir):; mkdir $@ else -include $(bsdir)/preparation-done.mk +-include $(bsdir)/preparation-done.mk ifeq (x$(include-prepare-results),xyes) -include $(prepdir)/*.mk +-include $(prepdir)/*.mk $(prepdir)/*.conf endif endif @@ -89,9 +96,9 @@ endif # create a separate LaTeX build directory for each user. # # The same logic applies to the final paper PDF: each user will create a -# separte final PDF (for example `paper-user1.pdf' and `paper-user2.pdf') -# and no `paper.pdf' will be built. This isn't a problem because -# `initialize.tex' is a .PHONY prerequisite, so the rule to build the final +# separte final PDF (for example 'paper-user1.pdf' and 'paper-user2.pdf') +# and no 'paper.pdf' will be built. This isn't a problem because +# 'initialize.tex' is a .PHONY prerequisite, so the rule to build the final # paper is always executed (even if it is present and nothing has # changed). So in terms of over-all efficiency and processing steps, this # doesn't change anything. @@ -114,7 +121,7 @@ tikzdir = $(texbdir)/tikz # --------------------------- # # Before defining the local sub-environment here, we'll need to save the -# system's environment for some scenarios (for example after `clean'ing the +# system's environment for some scenarios (for example after 'clean'ing the # built programs). curdir := $(shell echo $$(pwd)) @@ -127,21 +134,21 @@ curdir := $(shell echo $$(pwd)) # # We want the full recipe to be executed in one call to the shell. Also we # want Make to run the specific version of Bash that we have installed -# during `./project configure' time. +# during './project configure' time. # # Regarding the directories, this project builds its major dependencies # itself and doesn't use the local system's default tools. With these # environment variables, we are setting it to prefer the software we have # build here. # -# `TEXINPUTS': we have to remove all possible user-specified directories to -# avoid conflicts with existing TeX Live solutions. Later (in `paper.mk'), -# we are also going to overwrite `TEXINPUTS' just before `pdflatex'. +# 'TEXINPUTS': we have to remove all possible user-specified directories to +# avoid conflicts with existing TeX Live solutions. Later (in 'paper.mk'), +# we are also going to overwrite 'TEXINPUTS' just before 'pdflatex'. .ONESHELL: -.SHELLFLAGS = -ec export TEXINPUTS := export CCACHE_DISABLE := 1 export PATH := $(installdir)/bin +.SHELLFLAGS = --noprofile --norc -ec export LDFLAGS := -L$(installdir)/lib export SHELL := $(installdir)/bin/bash export CPPFLAGS := -I$(installdir)/include @@ -153,26 +160,28 @@ export LD_LIBRARY_PATH := $(installdir)/lib # will be empty. export CPATH := $(SYS_CPATH) -# RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is +# RPATH is automatically written in macOS, so 'DYLD_LIBRARY_PATH' is # ultimately redundant. But on some systems, even having a single value # causes crashs (see bug #56682). So we'll just give it no value at all. export DYLD_LIBRARY_PATH := -# OpenMPI can depend on an existing `ssh' or `rsh' binary. However, because +# OpenMPI can depend on an existing 'ssh' or 'rsh' binary. However, because # of security reasons, its best to not install them, disable any # remote-shell accesss through this environment variable. export OMPI_MCA_plm_rsh_agent=/bin/false # Recipe startup script. export PROJECT_STATUS := make -export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh +export BASH_ENV := $(curdir)/reproduce/software/shell/bashrc.sh + + # Python enviroment # ----------------- # -# The main Python environment variable is `PYTHONPATH'. However, so far we +# The main Python environment variable is 'PYTHONPATH'. However, so far we # have found several other Python-related environment variables on some # systems which might interfere. To be safe, we are removing all their # values. @@ -196,10 +205,10 @@ export MPI_PYTHON3_SITEARCH := # directories (or possible sub-directories) for individual steps will be # defined and added within their own Makefiles. # -# The `.SUFFIXES' rule with no prerequisite is defined to eliminate all the +# The '.SUFFIXES' rule with no prerequisite is defined to eliminate all the # default implicit rules. The default implicit rules are to do with -# programming (for example converting `.c' files to `.o' files). The -# problem they cause is when you want to debug the make command with `-d' +# programming (for example converting '.c' files to '.o' files). The +# problem they cause is when you want to debug the make command with '-d' # option: they add too many extra checks that make it hard to find what you # are looking for in the outputs. .SUFFIXES: @@ -209,10 +218,19 @@ $(lockdir): | $(bsdir); mkdir $@ -# Version and distribution tarball definitions -project-commit-hash := $(shell if [ -d .git ]; then \ - echo $$(git describe --dirty --always --long); else echo NOGIT; fi) -project-package-name := maneaged-$(project-commit-hash) +# Version and distribution tarball definitions. +# +# We need to export 'LD_LIBRARY_PATH' before calling 'git' because we the +# default export of 'LD_LIBRARY_PATH' doesn't take effect at this point +# (only within the recipes). Its also safe to directly use the 'git' +# executable using its absolute location (and not rely on 'PATH' at this +# stage). +project-commit-hash := $(shell \ + if [ -d .git ]; then \ + export LD_LIBRARY_PATH="$(installdir)/lib"; \ + echo $$($(installdir)/bin/git describe --dirty --always --long); \ + else echo NOGIT; fi) +project-package-name = maneaged-$(project-commit-hash) project-package-contents = $(texdir)/$(project-package-name) @@ -222,10 +240,10 @@ project-package-contents = $(texdir)/$(project-package-name) # High-level Makefile management # ------------------------------ # -# About `.PHONY': these are targets that must be built even if a file with +# About '.PHONY': these are targets that must be built even if a file with # their name exists. # -# Only `$(mtexdir)/initialize.tex' corresponds to a file. This is because +# Only '$(mtexdir)/initialize.tex' corresponds to a file. This is because # we want to ensure that the file is always built in every run: it contains # the project version which may change between two separate runs, even when # no file actually differs. @@ -238,14 +256,20 @@ texclean: mkdir $(texdir)/build/tikz # 'tikz' is assumed to already exist. clean: - # Delete the top-level PDF file. +# Delete the top-level PDF file. rm -f *.pdf - # Delete all the built outputs except the dependency - # programs. We'll use Bash's extended options builtin (`shopt') to - # enable "extended glob" (for listing of files). It allows extended - # features like ignoring the listing of a file with `!()' that we - # are using afterwards. +# Delete possible LaTeX output in top directory. This can happen when +# the user has run LaTeX with applications other than maneage. For +# example, when opening 'paper.tex' file with 'texstudio' and +# executing 'build'. + rm -f *.aux *.log *.synctex *.auxlock *.dvi *.out *.run.xml *.bcf + +# Delete all the built outputs except the dependency programs. We'll +# use Bash's extended options builtin ('shopt') to enable "extended +# glob" (for listing of files). It allows extended features like +# ignoring the listing of a file with '!()' that we are using +# afterwards. shopt -s extglob rm -rf $(texdir)/macros/!(dependencies.tex|dependencies-bib.tex|hardware-parameters.tex) rm -rf $(badir)/!(tex) $(texdir)/!(macros|$(texbtopdir)) @@ -253,14 +277,13 @@ clean: rm -rf $(bsdir)/preparation-done.mk distclean: clean - # Without cleaning the Git hooks, we won't be able to easily - # commit or checkout after this task is done. So we'll remove them - # first. +# Without cleaning the Git hooks, we won't be able to easily commit +# or checkout after this task is done. So we'll remove them first. rm -f .git/hooks/post-checkout .git/hooks/pre-commit - # We'll be deleting the built environent programs and just need the - # `rm' program. So for this recipe, we'll use the host system's - # `rm', not our own. +# We'll be deleting the built environent programs and just need the +# 'rm' program. So for this recipe, we'll use the host system's 'rm', +# not our own. $$sys_rm -rf $(BDIR) $$sys_rm -f .local .build $(pconfdir)/LOCAL.conf @@ -277,14 +300,14 @@ distclean: clean # without having to worry about the technicalities of the analysis. $(project-package-contents): paper.pdf | $(texdir) - # Set up the output directory, delete it if it exists and remake it - # to fill with new contents. +# Set up the output directory, delete it if it exists and remake it +# to fill with new contents. dir=$@ rm -rf $$dir mkdir $$dir - # Build a small Makefile to help in automatizing the paper building - # (including the bibliography). +# Build a small Makefile to help in automatizing the paper building +# (including the bibliography). m=$$dir/Makefile echo "paper.pdf: paper.tex paper.bbl" > $$m printf "\tpdflatex -shell-escape -halt-on-error paper\n" >> $$m @@ -296,93 +319,90 @@ $(project-package-contents): paper.pdf | $(texdir) printf "\trm -f *.aux *.auxlock *.bbl *.bcf\n" >> $$m printf "\trm -f *.blg *.log *.out *.run.xml\n" >> $$m - # Copy the top-level contents (see next step for `paper.tex'). +# Copy the top-level contents (see next step for 'paper.tex'). cp COPYING project README.md README-hacking.md $$dir/ - # Since the packaging is mainly intended for high-level building of - # the PDF with LaTeX, we'll comment the `makepdf' LaTeX macro in - # the paper. This will disable usage of TiKZ. +# Since the packaging is mainly intended for high-level building of +# the PDF with LaTeX, we'll comment the 'makepdf' LaTeX macro in the +# paper. This will disable usage of TiKZ. sed -e's|\\newcommand{\\makepdf}{}|%\\newcommand{\\makepdf}{}|' \ paper.tex > $$dir/paper.tex - # Copy ONLY the version-controlled files in 'reproduce' and - # 'tex/src'. This is important because files like 'LOCAL.conf' (in - # 'reproduce/software/config') should not be archived, they contain - # information about the host computer and are irrelevant for - # others. Also some project authors may have temporary files here - # that are not under version control and thus shouldn't be archived - # (although this is bad practice, but that is up to the user). - # - # To keep the sub-directory structure, we are packaging the files - # with Tar, piping it, and unpacking it in the archive - # directory. So afterwards we need to come back to the current - # directory. +# Copy ONLY the version-controlled files in 'reproduce' and +# 'tex/src'. This is important because files like 'LOCAL.conf' (in +# 'reproduce/software/config') should not be archived, they contain +# information about the host computer and are irrelevant for +# others. Also some project authors may have temporary files here +# that are not under version control and thus shouldn't be archived +# (although this is bad practice, but that is up to the user). +# +# To keep the sub-directory structure, we are packaging the files +# with Tar, piping it, and unpacking it in the archive directory. So +# afterwards we need to come back to the current directory. tar -c -f - $$(git ls-files reproduce tex/src) \ | (cd $$dir ; tar -x -f -) cd $(curdir) - # Build the other two subdirectories of 'tex/' that we need in the - # archive (in the actual project, these are symbolic links to the - # build directory). +# Build the other two subdirectories of 'tex/' that we need in the +# archive (in the actual project, these are symbolic links to the +# build directory). mkdir $$dir/tex/tikz $$dir/tex/build - # Copy the 'tex/build' directory into the archive (excluding the - # temporary archive directory that we are now copying to). We will - # be using Bash's extended globbing ('extglob') for excluding this - # directory. +# Copy the 'tex/build' directory into the archive (excluding the +# temporary archive directory that we are now copying to). We will be +# using Bash's extended globbing ('extglob') for excluding this +# directory. shopt -s extglob cp -r tex/build/!($(project-package-name)) $$dir/tex/build - # Clean up the $(texdir)/build* directories in the archive (when - # building in a group structure, there will be `build-user1', - # `build-user2' and etc). These are just temporary LaTeX build - # files and don't have any relevant/hand-written files in them. +# Clean up the $(texdir)/build* directories in the archive (when +# building in a group structure, there will be 'build-user1', +# 'build-user2' and etc). These are just temporary LaTeX build files +# and don't have any relevant/hand-written files in them. rm -rf $$dir/tex/build/build* - # If the project has any PDFs in its 'tex/tikz' directory (TiKZ or - # PGFPlots was used to generate them), copy them too. +# If the project has any PDFs in its 'tex/tikz' directory (TiKZ or +# PGFPlots was used to generate them), copy them too. if ls tex/tikz/*.pdf &> /dev/null; then cp tex/tikz/*.pdf $$dir/tex/tikz fi - # When submitting to places like arXiv, they will just run LaTeX - # once and won't run `biber'. So we need to also keep the `.bbl' - # file into the distributing tarball. However, BibLaTeX is - # particularly sensitive to versioning (a `.bbl' file has to be - # read by the same BibLaTeX version that created it). This is hard - # to do with non-up-to-date places like arXiv. Therefore, we thus - # just copy the whole of BibLaTeX's source (the version we are - # using) into the top tarball directory. In this way, arXiv's LaTeX - # engine will use the same BibLaTeX version to interpret the `.bbl' - # file. TIP: you can use the same strategy for other LaTeX packages - # that may cause problems on the arXiv server. +# When submitting to places like arXiv, they will just run LaTeX once +# and won't run 'biber'. So we need to also keep the '.bbl' file into +# the distributing tarball. However, BibLaTeX is particularly +# sensitive to versioning (a '.bbl' file has to be read by the same +# BibLaTeX version that created it). This is hard to do with +# non-up-to-date places like arXiv. Therefore, we thus just copy the +# whole of BibLaTeX's source (the version we are using) into the top +# tarball directory. In this way, arXiv's LaTeX engine will use the +# same BibLaTeX version to interpret the '.bbl' file. TIP: you can +# use the same strategy for other LaTeX packages that may cause +# problems on the arXiv server. cp tex/build/build/paper.bbl $$dir/ tltopdir=.local/texlive/maneage/texmf-dist/tex/latex find $$tltopdir/biblatex/ -maxdepth 1 -type f -print0 \ | xargs -0 cp -t $$dir - # Just in case the package users want to rebuild some of the - # figures (manually un-comment the `makepdf' command we commented - # above), correct the TikZ external directory, so the figures can - # be rebuilt. +# Just in case the package users want to rebuild some of the figures +# (manually un-comment the 'makepdf' command we commented above), +# correct the TikZ external directory, so the figures can be rebuilt. pgfsettings="$$dir/tex/src/preamble-pgfplots.tex" sed -e's|{tikz/}|{tex/tikz/}|' $$pgfsettings > $$pgfsettings.new mv $$pgfsettings.new $$pgfsettings - # PROJECT SPECIFIC - # ---------------- - # Put any project-specific distribution steps here. +# PROJECT SPECIFIC +# ---------------- +# Put any project-specific distribution steps here. - # ---------------- +# ---------------- - # Clean temporary files that may have been created by text editors. +# Clean temporary files that may have been created by text editors. cd $(texdir) find $(project-package-name) -name \*~ -delete find $(project-package-name) -name \*.swp -delete -# Package into `.tar.gz' or '.tar.lz'. +# Package into '.tar.gz' or '.tar.lz'. dist dist-lzip: $(project-package-contents) - curdir=$$(pwd) cd $(texdir) tar -cf $(project-package-name).tar $(project-package-name) if [ $@ = dist ]; then @@ -393,21 +413,19 @@ dist dist-lzip: $(project-package-contents) lzip -f --best $(project-package-name).tar fi rm -rf $(project-package-name) - cd $$curdir + cd $(curdir) mv $(texdir)/$(project-package-name).tar.$$suffix ./ -# Package into `.zip'. +# Package into '.zip'. dist-zip: $(project-package-contents) - curdir=$$(pwd) cd $(texdir) zip -q -r $(project-package-name).zip $(project-package-name) rm -rf $(project-package-name) - cd $$curdir + cd $(curdir) mv $(texdir)/$(project-package-name).zip ./ # Package the software tarballs. dist-software: - curdir=$$(pwd) dirname=software-$(project-commit-hash) cd $(bsdir) if [ -d $$dirname ]; then rm -rf $$dirname; fi @@ -416,13 +434,169 @@ dist-software: tar -cf $$dirname.tar $$dirname gzip -f --best $$dirname.tar rm -rf $$dirname - cd $$curdir + cd $(curdir) mv $(bsdir)/$$dirname.tar.gz ./ +# Import input data +# ----------------- +# +# The list files to be imported (downloaded from a server, or linked from a +# local location), are listed in 'reproduce/analysis/config/INPUTS.conf' +# along with their URLs and verification checksums. In most cases, you will +# not need to edit this rule. Simply follow the instructions at the top of +# 'INPUTS.conf' and set the variables names according to the described +# standards and everything should be fine. +# +# TECHNICAL NOTE on the '$(foreach, n ...)' loop of 'inputdatasets': we are +# using several (relatively complex!) features particular to Make: In GNU +# Make, '.VARIABLES' "... expands to a list of the names of all global +# variables defined so far" (from the "Other Special Variables" section of +# the GNU Make manual). Assuming that the pattern 'INPUT-%-sha256' is only +# used for input files, we find all the variables that contain the input +# file name (the '%' is the filename). Finally, using the +# pattern-substitution function ('patsubst'), we remove the fixed string at +# the start and end of the variable name. +# +# Download lock file: Most systems have a single connection to the +# internet, therefore downloading is inherently done in series. As a +# result, when more than one dataset is necessary for download, if they are +# done in parallel, the speed will be slower than downloading them in +# series. We thus use the 'flock' program to tie/lock the downloading +# process with a file and make sure that only one downloading event is in +# progress at every moment. +$(indir):; mkdir $@ +downloadwrapper = $(bashdir)/download-multi-try +inputdatasets := $(foreach i, \ + $(patsubst INPUT-%-sha256,%, \ + $(filter INPUT-%-sha256,$(.VARIABLES))) \ + $(patsubst INPUT-%-fitsdatasum,%, \ + $(filter INPUT-%-fitsdatasum,$(.VARIABLES))), \ + $(indir)/$(i)) +$(inputdatasets): $(indir)/%: | $(indir) $(lockdir) + +# Starting rule with '@': In case there is a username or password +# given for the database, we don't want the values to be printed in +# the terminal as the pipeline is running. We are therefore starting +# this recipe with an '@' (so Make doesn't print the used +# commands). To help the user know what is happening (in case they +# can't tell from the Wget outputs), we'll just start the recipe with +# a notice on what is being imported. + @echo "Importing $@" + +# If a username or password has been provided, add them to the WGET +# command. The two variables are defined in the local configuation +# file 'reproduce/software/config/LOCAL.conf' that is not under +# version control. Different servers may use different authentication +# formats. If the default one doesn't work for your server, comment +# it and uncomment the one that works. If your serve needs a +# different kind of authentication format, please add it yourself. In +# case you need a new format, we encourage you to send the format to +# us using the link below: +# https://savannah.nongnu.org/support/?group=reproduce&func=additem + authopt="" + if [ x"$(DATABASEAUTHTYPE)" != x ]; then + case "$(DATABASEAUTHTYPE)" in + +# Format: '--user=XXXX --password=YYYY' + userpass) + if [ x'$(DATABASEUSER)' != x ]; then + authopt="--user='$(DATABASEUSER)'"; fi + if [ x'$(DATABASEPASS)' != x ]; then + authopt="$$authopt --password='$(DATABASEPASS)'"; fi + ;; + +# Format: --post-data 'username=XXXX&password=YYYY' + postdata) + if [ x'$(DATABASEUSER)' != x ]; then + authopt="--post-data 'username=$(DATABASEUSER)"; fi + if [ x'$(DATABASEPASS)' != x ]; then + authopt="$$authopt""&password=$(DATABASEPASS)'"; + else authopt="$$authopt'" # To close the single quote + fi + ;; + +# Unrecognized format. + *) + echo "Maneage: 'DATABASEAUTHTYPE' format not recognized! Please see the description of this variable in 'reproduce/software/config/LOCAL.conf' for the acceptable values."; exit 1;; + esac + fi + +# Download (or make the link to) the input dataset. If the file +# exists in 'INDIR', it may be a symbolic link to some other place in +# the filesystem. To avoid too many links when using these files +# during processing, we'll use 'readlink -f' so the link we make here +# points to the final file directly (note that 'readlink' is part of +# GNU Coreutils). If its not a link, the 'readlink' part has no +# effect. + unchecked=$@.unchecked + if [ -f $(INDIR)/$* ]; then + ln -fs $$(readlink -f $(INDIR)/$*) $$unchecked + else + touch $(lockdir)/download + $(downloadwrapper) "wget $$authopt --no-use-server-timestamps -O" \ + $(lockdir)/download "$(INPUT-$*-url)" $$unchecked + fi + +# Set the checksum related variables. + if [ x"$(INPUT-$*-sha256)" != x ]; then + suffix=sha256 + sumin=$(INPUT-$*-sha256) + verifname="SHA256 checksum" + sum=$$(sha256sum $$unchecked | awk '{print $$1}') + elif [ x"$(INPUT-$*-fitsdatasum)" != x ]; then + suffix=fitsdatasum + sumin=$(INPUT-$*-fitsdatasum) + verifname="FITS standard DATASUM" + if [ x"$(INPUT-$*-fitshdu)" = x ]; then hdu=1; + else hdu="$(INPUT-$*-fitshdu)"; fi + sum=$$(astfits $$unchecked -h$$hdu --datasum | awk '{print $$1}') + else + echo "$@: checksum for verifyication not recognized!"; exit 1 + fi + +# Verify the input. + if [ $$sum = $$sumin ]; then + mv $$unchecked $@ + echo "Integrity confirmed, using $@ in this project." + +# Checksums didn't match. + else + +# The user has asked to update the checksum in 'INPUTS.conf'. + if [ $$sumin = "--auto-replace--" ]; then + +# Put the updated 'INPUTS.conf' in a temporary file. + inputstmp=$@.inputs + awk '{if($$1 == "INPUT-$*-'$$suffix'") \ + $$3="'$$sum'"; print}' \ + $(pconfdir)/INPUTS.conf > $$inputstmp + +# Update the INPUTS.conf, but not in parallel (using the +# file-lock feature of 'flock'). + touch $(lockdir)/inputs-update + flock $(lockdir)/inputs-update \ + sh -c "mv $$inputstmp $(pconfdir)/INPUTS.conf" + mv $$unchecked $@ + +# Error on non-matching checksums. + else + echo; echo; + echo "Wrong $$verifname for input file '$*':" + echo " File location: $$unchecked"; \ + echo " Expected $$verifname: $$sumin"; \ + echo " Calculated $$verifname: $$sum"; \ + echo; exit 1 + fi + fi + + + + + # Directory containing to-be-published datasets # --------------------------------------------- # @@ -497,13 +671,13 @@ print-general-metadata = \ # This file will store some basic info about the project that is necessary # for the final PDF. Since these are not version controlled, it must be # calculated everytime the project is run. So even though this file -# actually exists, it is also aded as a `.PHONY' target above. +# actually exists, it is also aded as a '.PHONY' target above. $(mtexdir)/initialize.tex: | $(mtexdir) - # Version and title of project. About the starting '@': since these - # commands are run every time with './project make', it is annoying - # to print them on the standard output every time. With the '@', - # make will not print the commands that it runs in this recipe. +# Version and title of project. About the starting '@': since these +# commands are run every time with './project make', it is annoying +# to print them on the standard output every time. With the '@', make +# will not print the commands that it runs in this recipe. @d=$$(git show -s --format=%aD HEAD | awk '{print $$2, $$3, $$4}') echo "\newcommand{\projectdate}{$$d}" > $@ echo "\newcommand{\projecttitle}{$(metadata-title)}" >> $@ @@ -511,16 +685,15 @@ $(mtexdir)/initialize.tex: | $(mtexdir) echo "\newcommand{\projectgitrepo}{$(metadata-git-repository)}" >> $@ echo "\newcommand{\projectcopyrightowner}{$(metadata-copyright-owner)}" >> $@ - # Calculate the latest Maneage commit used to build this - # project: - # - The project may not have the 'maneage' branch (for example - # after cloning from a fork that didn't include it!). In this - # case, we'll print a descriptive warning, telling the user what - # should be done (reporting the last merged commit and its date - # is very useful for the future). - # - The '--dirty' option (used in 'project-commit-hash') isn't - # applicable to "commit-ishes" (direct quote from Git's error - # message!). +# Calculate the latest Maneage commit used to build this project: +# - The project may not have the 'maneage' branch (for example +# after cloning from a fork that didn't include it!). In this +# case, we'll print a descriptive warning, telling the user what +# should be done (reporting the last merged commit and its date +# is very useful for the future). +# - The '--dirty' option (used in 'project-commit-hash') isn't +# applicable to "commit-ishes" (direct quote from Git's error +# message!). if git log maneage -1 &> /dev/null; then c=$$(git merge-base HEAD maneage) v=$$(git describe --always --long $$c) @@ -541,3 +714,11 @@ $(mtexdir)/initialize.tex: | $(mtexdir) fi echo "\newcommand{\maneagedate}{$$d}" >> $@ echo "\newcommand{\maneageversion}{$$v}" >> $@ + +# ----------------- delete the lines below this ------------------- +# These lines are only intended for the default template's output, to +# demonstrate that is it important to put links in the PDF (for +# showing where your input data came from). Remove these lines as +# part of the initial customization of Maneage for your project. + echo "\\newcommand{\\wfpctwourl}{$(INPUT-wfpc2.fits-url)}" >> $@ +# ----------------- delete the lines above this ------------------- diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk index b5b5b29..791108b 100644 --- a/reproduce/analysis/make/paper.mk +++ b/reproduce/analysis/make/paper.mk @@ -1,6 +1,6 @@ # Build the final PDF paper/report. # -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,14 +23,14 @@ # # To report the input settings and results, the final report's PDF (final # target of this project) uses macros generated from various steps of the -# project. All these macros are defined through `$(mtexdir)/project.tex'. +# project. All these macros are defined through '$(mtexdir)/project.tex'. # -# `$(mtexdir)/project.tex' is actually just a combination of separate files +# '$(mtexdir)/project.tex' is actually just a combination of separate files # that keep the LaTeX macros related to each workhorse Makefile (in -# `reproduce/src/make/*.mk'). Those individual macros are pre-requisites to -# `$(mtexdir)/verify.tex' which will check them before starting to build +# 'reproduce/src/make/*.mk'). Those individual macros are pre-requisites to +# '$(mtexdir)/verify.tex' which will check them before starting to build # the paper. The only workhorse Makefile that doesn't need to produce LaTeX -# macros is this Makefile (`reproduce/src/make/paper.mk'). +# macros is this Makefile ('reproduce/src/make/paper.mk'). # # This file is thus the interface between the analysis/processing steps and # the final PDF: when we get to this point, all the processing has been @@ -38,33 +38,33 @@ # # Note that if you don't want the final PDF and just want the processing # and file outputs, you can give any value other than 'yes' to -# 'pdf-build-final' in `reproduce/analysis/config/pdf-build.conf'. +# 'pdf-build-final' in 'reproduce/analysis/config/pdf-build.conf'. $(mtexdir)/project.tex: $(mtexdir)/verify.tex - # If no PDF is requested, or if LaTeX isn't available, don't - # continue to building the final PDF. Otherwise, merge all the TeX - # macros into one for building the PDF. +# If no PDF is requested, or if LaTeX isn't available, don't continue +# to building the final PDF. Otherwise, merge all the TeX macros into +# one for building the PDF. @if [ -f .local/bin/pdflatex ] && [ x"$(pdf-build-final)" = xyes ]; then - # Put a LaTeX input command for all the necessary macro files. - # 'hardware-parameters.tex' is created in 'configure.sh'. +# Put a LaTeX input command for all the necessary macro files. +# 'hardware-parameters.tex' is created in 'configure.sh'. projecttex=$(mtexdir)/project.tex rm -f $$projecttex for t in $(subst paper,,$(makesrc)) hardware-parameters; do echo "\input{tex/build/macros/$$t.tex}" >> $$projecttex done - # Possibly highlight the '\new' parts of the text. +# Possibly highlight the '\new' parts of the text. if [ x"$(highlightnew)" = x1 ]; then echo "\newcommand{\highlightnew}{}" >> $$projecttex fi - # Possibly show the text within '\tonote'. +# Possibly show the text within '\tonote'. if [ x"$(highlightnotes)" = x1 ]; then echo "\newcommand{\highlightnotes}{}" >> $$projecttex fi - # The paper shouldn't be built. +# The paper shouldn't be built. else echo echo "-----" @@ -95,39 +95,39 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex # The bibliography # ---------------- # -# We need to run the `biber' program on the output of LaTeX to generate the +# We need to run the 'biber' program on the output of LaTeX to generate the # necessary bibliography before making the final paper. So we'll first have -# one run of LaTeX (similar to the `paper.pdf' recipe), then `biber'. +# one run of LaTeX (similar to the 'paper.pdf' recipe), then 'biber'. # -# NOTE: `$(mtexdir)/project.tex' is an order-only-prerequisite for -# `paper.bbl'. This is because we need to run LaTeX in both the `paper.bbl' -# recipe and the `paper.pdf' recipe. But if `tex/src/references.tex' hasn't +# NOTE: '$(mtexdir)/project.tex' is an order-only-prerequisite for +# 'paper.bbl'. This is because we need to run LaTeX in both the 'paper.bbl' +# recipe and the 'paper.pdf' recipe. But if 'tex/src/references.tex' hasn't # been modified, we don't want to re-build the bibliography, only the final # PDF. $(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ | $(mtexdir)/project.tex - # If `$(mtexdir)/project.tex' is empty, don't build PDF. +# If '$(mtexdir)/project.tex' is empty, don't build PDF. @macros=$$(cat $(mtexdir)/project.tex) if [ x"$$macros" != x ]; then - # We'll run LaTeX first to generate the `.bcf' file (necessary - # for `biber') and then run `biber' to generate the `.bbl' file. +# We'll run LaTeX first to generate the '.bcf' file (necessary for +# 'biber') and then run 'biber' to generate the '.bbl' file. p=$$(pwd) export TEXINPUTS=$$p: cd $(texbdir); - # Delete any possibly existing target (a '.bbl' file) to avoid - # complications with LaTeX being run before the command that - # generates it. Otherwise users will have to manually delete - # it. It will be built anyway once this rule is done. +# Delete any possibly existing target (a '.bbl' file) to avoid +# complications with LaTeX being run before the command that +# generates it. Otherwise users will have to manually delete it. It +# will be built anyway once this rule is done. rm -f $@ - # The pdflatex option '-shell-escape' is "normally disallowed for - # security reasons" according to the `info pdflatex' manual, but - # is enabled here in order to allow the use of PGFPlots. If you - # do not use PGFPlots, then you should remove the `-shell-escape' - # option for better security. See - # https://savannah.nongnu.org/task/?15694 for details. +# The pdflatex option '-shell-escape' is "normally disallowed for +# security reasons" according to the 'info pdflatex' manual, but is +# enabled here in order to allow the use of PGFPlots. If you do not +# use PGFPlots, then you should remove the '-shell-escape' option +# for better security. See https://savannah.nongnu.org/task/?15694 +# for details. pdflatex -shell-escape -halt-on-error "$$p"/paper.tex biber paper @@ -140,27 +140,28 @@ $(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ # The final paper # --------------- # -# Run LaTeX in the `$(texbdir)' directory so all the intermediate and +# Run LaTeX in the '$(texbdir)' directory so all the intermediate and # auxiliary files stay there and keep the top directory clean. To be able # to run everything cleanly from there, it is necessary to add the current -# directory (top project directory) to the `TEXINPUTS' environment +# directory (top project directory) to the 'TEXINPUTS' environment # variable. paper.pdf: $(mtexdir)/project.tex paper.tex $(texbdir)/paper.bbl - # If `$(mtexdir)/project.tex' is empty, don't build the PDF. +# If '$(mtexdir)/project.tex' is empty, don't build the PDF. @macros=$$(cat $(mtexdir)/project.tex) if [ x"$$macros" != x ]; then - # Go into the top TeX build directory and make the paper. +# Go into the top TeX build directory and make the paper. p=$$(pwd) export TEXINPUTS=$$p: cd $(texbdir) - # See above for a warning and brief discussion on the the - # pdflatex option `-shell-escape'. + +# See above for a warning and brief discussion on the the pdflatex +# option '-shell-escape'. pdflatex -shell-escape -halt-on-error "$$p"/paper.tex - # Come back to the top project directory and copy the built PDF - # file here. +# Come back to the top project directory and copy the built PDF +# file here. cd "$$p" cp $(texbdir)/$@ $(final-paper) diff --git a/reproduce/analysis/make/prepare.mk b/reproduce/analysis/make/prepare.mk index d0b61d9..36f5294 100644 --- a/reproduce/analysis/make/prepare.mk +++ b/reproduce/analysis/make/prepare.mk @@ -1,6 +1,6 @@ -# Basic preparations, called by `./project prepare'. +# Basic preparations, called by './project make'. # -# Copyright (C) 2019-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,30 +21,41 @@ # Final-target # -# Without this file, `./project make' won't work. -prepare-dep = $(subst prepare, ,$(makesrc)) +# Without this file, './project make' won't work. +# +# We need to remove the 'prepare' word from the list of 'makesrc'. +prepare-dep = $(filter-out prepare, $(makesrc)) $(bsdir)/preparation-done.mk: \ $(foreach s, $(prepare-dep), $(mtexdir)/$(s).tex) - # If you need to add preparations define targets above to do the - # preparations, then set the value below to `yes'. Recall that just - # like `./project make', before loading this file, `./project - # prepare' loads loads `initialize.mk' and `download.mk', so you - # can safely assume everything that is defined there in the - # preparation phase also. - # - # TIP: the targets can actually be automatically generated - # Makefiles that are used by `./project make'. They can include - # variables, or automatically generated rules. Just make sure that - # those Makefiles aren't written in the source directory. Even - # though they are Makefiles, they are automatically built, so they - # don't belong in the source. `$(prepdir)' has been defined for - # this purpose (see `initialize.mk'), we recommend that you put all - # automatically generated Makefiles under this directory. In the - # `make' phase, `initialize.mk' will automatically load all the - # `*.mk' files. If you need to load your generated - # configuration-makefiles before automatically generated Makefiles - # containing rules, you can use some naming convension like - # `conf-*.mk' and `rule-*.mk', or you can put them in - # subdirectories. +# If you need to add preparations (mainly automatically generated +# configuration files or Makefiles to simplify the './project make' +# phase) take the following step: +# +# 1. Add prerequisites to this target ('preparation-done.mk'). Try +# to avoid any kind of analysis in this (preparation) phase! +# Preparation should ideally only involve automatic creation of +# configuration files or Makefile that will be loaded into the +# analysis phase (from 'top-make.mk'). +# +# 2. Set the value of 'include-prepare-results' (defined below) to +# 'yes'. If it is kept to the default 'no', then your +# prepartion outputs will not be automatically +# generated. Recall that just like 'top-make.mk', +# 'top-prepare.mk' also loads 'initialize.mk' before everything +# else. So you can safely assume everything that is defined in +# 'initialize.mk' to be available in the preparation phase +# also. +# +# TIP: the targets can actually be automatically generated Makefiles +# that are used by './project make'. They can include variables, or +# automatically generated rules. Just make sure that those Makefiles +# aren't written in the source directory (only hand-written files +# should be in your source). Even though they are Makefiles, they are +# automatically built, so they don't belong in the +# source. '$(prepdir)' has been defined for this purpose (see +# 'initialize.mk'), we recommend that you put all automatically +# generated configuration files or Makefiles under this directory. In +# the 'make' phase, 'initialize.mk' will automatically load all the +# '*.mk' files there. @echo "include-prepare-results = no" > $@ diff --git a/reproduce/analysis/make/top-make.mk b/reproduce/analysis/make/top-make.mk index 596cc0d..460433b 100644 --- a/reproduce/analysis/make/top-make.mk +++ b/reproduce/analysis/make/top-make.mk @@ -1,6 +1,6 @@ # Top-level Makefile (first to be loaded). # -# Copyright (C) 2018-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,7 +20,7 @@ # Load the local configuration (created after running -# `./project configure'). +# './project configure'). include reproduce/software/config/LOCAL.conf @@ -30,7 +30,7 @@ include reproduce/software/config/LOCAL.conf # Ultimate target of this project # ------------------------------- # -# The final paper/report (`paper.pdf') is the main target of this +# The final paper/report ('paper.pdf') is the main target of this # project. As defined in the Make paradigm, it must be the first target # that Make encounters (immediately after loading the local configuration # settings, necessary for a group building scenario mentioned next). @@ -50,8 +50,8 @@ include reproduce/software/config/LOCAL.conf # # Controlling this requires two variables that are available at this stage: # -# - `GROUP-NAME': from `LOCAL.conf' (which was built by `./project configure'). -# - `maneage_group_name': value to the `--group' option. +# - 'GROUP-NAME': from 'LOCAL.conf' (which was built by './project configure'). +# - 'maneage_group_name': value to the '--group' option. # # The analysis is only done when both have the same group name. Note that # when the project isn't being built for a group, both variables will be an @@ -63,7 +63,7 @@ include reproduce/software/config/LOCAL.conf # # If you are just interested in the processing and don't want to build the # PDF, you can skip the creation of the final PDF by giving a value of -# `yes' to `pdf-build-final' in `reproduce/analysis/config/pdf-build.conf'. +# 'yes' to 'pdf-build-final' in 'reproduce/analysis/config/pdf-build.conf'. ifeq (x$(maneage_group_name),x$(GROUP-NAME)) all: paper.pdf else @@ -87,13 +87,13 @@ endif # To keep things clean, managable and readable, each set of operations # is (and must be) classified (modularized) by context into separate # Makefiles: the more the better. These modular steps are then -# included in this top-level Makefile through the `include' command of +# included in this top-level Makefile through the 'include' command of # the next step. Each Makefile should also produce a LaTeX macro file # with the same fixed name (used to keep all the parameters and # relevant outputs of the steps in it for the final paper). # # In the rare case that no special LaTeX macros are necessary in a -# workhorse Makefile, you can simply make an empty file with `touch +# workhorse Makefile, you can simply make an empty file with 'touch # $@'. This will not add any lines to the final combined LaTeX macros # file, but will create the file that is a prerequisite to the final # paper generation. @@ -107,11 +107,10 @@ endif # IMPORTANT NOTE: order matters in the inclusion of the processing # Makefiles. As the project grows, some Makefiles will define # variables/dependencies that later Makefiles need. Therefore we are using -# a `foreach' loop in the next step to explicitly request loading them in +# a 'foreach' loop in the next step to explicitly request loading them in # the same order that they are defined here (we aren't just using a # wild-card like the configuration Makefiles). makesrc = initialize \ - download \ delete-me \ verify \ paper @@ -130,7 +129,7 @@ makesrc = initialize \ # contain rules to actually do this project's processing. # # But before that, we need to identify the phase for the Makefiles that are -# run both in `./project prepare' and `./project make'. +# run both in './project prepare' and './project make'. project-phase = make include reproduce/analysis/config/*.conf include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk) diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk index fb5700e..930c2a9 100644 --- a/reproduce/analysis/make/top-prepare.mk +++ b/reproduce/analysis/make/top-prepare.mk @@ -1,10 +1,10 @@ # Do basic preparations to optimize the project's running. # -# NOTE: This file is very similar to `top-make.mk', so the large comments +# NOTE: This file is very similar to 'top-make.mk', so the large comments # are not included here. Please see that file for thorough comments on each # step. # -# Copyright (C) 2019-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,7 +24,7 @@ # Load the local configuration (created after running -# `./project configure'). +# './project configure'). include reproduce/software/config/LOCAL.conf @@ -34,7 +34,7 @@ include reproduce/software/config/LOCAL.conf # Ultimate target of this project # ------------------------------- # -# See `top-make.mk' for complete explanation. +# See 'top-make.mk' for complete explanation. ifeq (x$(maneage_group_name),x$(GROUP-NAME)) all: $(BDIR)/software/preparation-done.mk @echo "Project preparation is complete."; @@ -57,14 +57,13 @@ endif # Define source Makefiles # ----------------------- # -# See `top-make.mk' for complete explanation. +# See 'top-make.mk' for complete explanation. # -# To ensure that `prepare' and `make' have the same basic definitions and -# environment and that all `downloads' are managed in one place, both -# `./project prepare' and `./project make' will first read `initialize.mk' -# and `downloads.mk'. +# To ensure that 'prepare' and 'make' have the same basic definitions and +# environment and that all 'downloads' are managed in one place, both +# './project prepare' and './project make' will first read 'initialize.mk' +# and 'downloads.mk'. makesrc = initialize \ - download \ prepare @@ -74,7 +73,7 @@ makesrc = initialize \ # Include all analysis Makefiles # ------------------------------ # -# See `top-make.mk' for complete explanation. +# See 'top-make.mk' for complete explanation. project-phase = prepare include reproduce/analysis/config/*.conf include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk) diff --git a/reproduce/analysis/make/verify.mk b/reproduce/analysis/make/verify.mk index 7e16add..aa026b5 100644 --- a/reproduce/analysis/make/verify.mk +++ b/reproduce/analysis/make/verify.mk @@ -1,6 +1,6 @@ # Verify the project outputs before building the paper. # -# Copyright (C) 2020-2021 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2020-2023 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -94,7 +94,7 @@ verify-txt-no-comments-no-space = \ # (generated in various stages of the analysis. # # Since each analysis step's data files are already prerequisites of their -# respective TeX macro file, its enough for `verify.tex' to depend on the +# respective TeX macro file, its enough for 'verify.tex' to depend on the # final TeX macro. # # USEFUL TIP: during the early phases of your research (when you are @@ -103,50 +103,49 @@ verify-txt-no-comments-no-space = \ # # Here is a description of the variables defined here. # -# verify-dep: The major step dependencies of `verify.tex', this includes +# verify-dep: The major step dependencies of 'verify.tex', this includes # all the steps that must be finished before it. # # verify-changes: The files whose contents are important. This is -# essentially the same as `verify-dep', but it has removed -# the `initialize' step (which is information about the +# essentially the same as 'verify-dep', but it has removed +# the 'initialize' step (which is information about the # pipeline, not the results). -verify-dep = $(subst verify,,$(subst paper,,$(makesrc))) +verify-dep = $(filter-out verify paper, $(makesrc)) verify-check = $(subst initialize,,$(verify-dep)) $(mtexdir)/verify.tex: $(foreach s, $(verify-dep), $(mtexdir)/$(s).tex) - # Make sure that verification is actually requested, the '@' at the - # start of the recipe is added so Make doesn't print the commands - # on the standard output because this recipe is run on every call - # to the project and can be annoying (get mixed in the middle of - # the analysis outputs or the LaTeX outputs). +# Make sure that verification is actually requested, the '@' at the +# start of the recipe is added so Make doesn't print the commands on +# the standard output because this recipe is run on every call to the +# project and can be annoying (get mixed in the middle of the +# analysis outputs or the LaTeX outputs). @if [ x"$(verify-outputs)" = xyes ]; then - # Make sure the temporary output doesn't exist (because we want - # to append to it). We are making a temporary output target so if - # there is a crash in the middle, Make will not continue. If we - # write in the final target progressively, the file will exist, - # and its date will be more recent than all prerequisites, so - # next time the project is run, Make will continue and ignore the - # rest of the checks. +# Make sure the temporary output doesn't exist (because we want to +# append to it). We are making a temporary output target so if +# there is a crash in the middle, Make will not continue. If we +# write in the final target progressively, the file will exist, and +# its date will be more recent than all prerequisites, so next time +# the project is run, Make will continue and ignore the rest of the +# checks. rm -f $@.tmp - # Verify the figure datasets. +# Verify the figure datasets. $(call verify-txt-no-comments-no-space, \ $(dm-squared), 6b6d3b0f9c351de53606507b59bca5d1, $@.tmp) $(call verify-txt-no-comments-no-space, \ $(dm-img-histogram), b1f9c413f915a1ad96078fee8767b16c, $@.tmp) - # Verify TeX macros (the values that go into the PDF text). +# Verify TeX macros (the values that go into the PDF text). for m in $(verify-check); do file=$(mtexdir)/$$m.tex - if [ $$m == download ]; then s=49e4e9f049aa9da0453a67203d798587 - elif [ $$m == delete-me ]; then s=711e2f7fa1f16ecbeeb3df6bcb4ec705 + if [ $$m == delete-me ]; then s=711e2f7fa1f16ecbeeb3df6bcb4ec705 else echo; echo "'$$m' not recognized."; exit 1 fi $(call verify-txt-no-comments-no-space, $$file, $$s, $@.tmp) done - # Move temporary file to final target. +# Move temporary file to final target. mv $@.tmp $@ else echo "% Verification was DISABLED!" > $@ |