diff options
Diffstat (limited to 'reproduce/analysis')
| -rwxr-xr-x | reproduce/analysis/bash/download-multi-try.sh (renamed from reproduce/analysis/bash/download-multi-try) | 19 | ||||
| -rw-r--r-- | reproduce/analysis/config/INPUTS.conf | 118 | ||||
| -rw-r--r-- | reproduce/analysis/config/delete-me-squared-num.conf | 2 | ||||
| -rw-r--r-- | reproduce/analysis/config/metadata.conf | 2 | ||||
| -rw-r--r-- | reproduce/analysis/config/pdf-build.conf | 2 | ||||
| -rw-r--r-- | reproduce/analysis/config/verify-outputs.conf | 2 | ||||
| -rw-r--r-- | reproduce/analysis/make/delete-me.mk | 2 | ||||
| -rw-r--r-- | reproduce/analysis/make/initialize.mk | 150 | ||||
| -rw-r--r-- | reproduce/analysis/make/paper.mk | 49 | ||||
| -rw-r--r-- | reproduce/analysis/make/prepare.mk | 4 | ||||
| -rw-r--r-- | reproduce/analysis/make/top-make.mk | 8 | ||||
| -rw-r--r-- | reproduce/analysis/make/top-prepare.mk | 10 | ||||
| -rw-r--r-- | reproduce/analysis/make/verify.mk | 2 |
13 files changed, 259 insertions, 111 deletions
diff --git a/reproduce/analysis/bash/download-multi-try b/reproduce/analysis/bash/download-multi-try.sh index 994a8fa..bea88d5 100755 --- a/reproduce/analysis/bash/download-multi-try +++ b/reproduce/analysis/bash/download-multi-try.sh @@ -1,14 +1,19 @@ -#!/bin/sh +#!/usr/bin/env sh # # Attempt downloading multiple times before crashing whole project. From # the top project directory (for the shebang above), this script must be # run like this: # -# $ /path/to/download-multi-try downloader lockfile input-url downloaded-name +# $ $SHELL /path/to/download-multi-try.sh downloader lockfile \ +# input-url downloaded-name # -# NOTE: The 'downloader' must contain the option to specify the output name -# in its end. For example "wget -O". Any other option can also be placed in -# the middle. +# NOTE: +# - This script doesn't have a Shebang because in different stages it +# should be built with different shells ('/bin/sh' before Maneage +# installs its own shell and afterwards with Maneage's own shell). +# - The 'downloader' must contain the option to specify the output name +# in its end. For example "wget -O". Any other option can also be placed in +# the middle. # # Due to temporary network problems, a download may fail suddenly, but # succeed in a second try a few seconds later. Without this script that @@ -26,7 +31,7 @@ # reason, you don't want to use a lock file, set the 'lockfile' name to # 'nolock'. # -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -123,7 +128,7 @@ while [ ! -f "$outname" ]; do else # Try downloading from the requested URL. flock "$lockfile" sh -c \ - "if ! $downloader $outname $inurl; then rm -f $outname; fi" + "if ! $downloader $outname \"$inurl\"; then rm -f $outname; fi" fi # If the download failed, try the backup server(s). diff --git a/reproduce/analysis/config/INPUTS.conf b/reproduce/analysis/config/INPUTS.conf index 75e24de..5860806 100644 --- a/reproduce/analysis/config/INPUTS.conf +++ b/reproduce/analysis/config/INPUTS.conf @@ -1,9 +1,10 @@ # This project's input file information (metadata). # # For each input (external) data file that is used within the project, -# three variables are suggested here (two of them are mandatory). These -# variables will be used by 'reproduce/analysis/make/download.mk' to import -# the dataset into the project (within the build directory): +# three variables are suggested here (only the verification variable is +# strictly mandatory). These variables will be used by the download rule of +# 'reproduce/analysis/make/initialize.mk' to import the dataset into the +# project (within the build directory): # # - If the file already exists locally in '$(INDIR)' (the optional input # directory that may have been specified at configuration time with @@ -12,27 +13,53 @@ # files are large. # # - If the file doesn't exist in '$(INDIR)', or no input directory was -# specified at configuration time, then the file is downloaded from a -# specific URL. +# specified at configuration time, then the file is downloaded from the +# specified URL for that dataset. # # In both cases, before placing the file (or its link) in the build -# directory, 'reproduce/analysis/make/download.mk' will check the SHA256 -# checksum of the dataset and if it differs from the pre-defined value (set -# for that file, here), it will abort (since this is not the intended -# dataset). -# -# Therefore, the two variables specifying the URL and SHA256 checksum of -# the file are MANDATORY. The third variable (INPUT-%-size) showing the -# human-readable size of the file (from 'ls -lh') is optional (but -# recommended: because it gives future scientists to get a feeling of the -# volume of data they need to input: will become important if the -# size/number of files is large). +# directory, the download rule of 'reproduce/analysis/make/initialize.mk' +# will check the verification of the dataset and if it differs from the +# pre-defined value (set for that file, here), it will abort (since this is +# not the intended dataset). # +# Verification (two modes) +# ------------------------ +# - SHA256 checksum. This will check the full contents of the file, and +# is generic to any data format. However, if the server inserts custom +# headers like the query date or query code and etc, this form of +# validation is not useful: because every download will have different +# headers. In such cases, you should use the other verification methods +# below. In other words, this method is only good for files that are +# "static" on the server (and left there unchanged). If the file is +# generated at request time, the server usually inserts custom run-time +# dependent headers; making it impossible to verify with an SHA +# checksum of the whole file. +# - The FITS Standard's 'DATASUM' (which will only check the data, not +# the headers). According to the FITS standard, this sum ignores all +# headers, and is only calculated on a HDU's data. By default, this +# will require Gnuastro (which can easily calculate and return the +# value on the command-line), and it assumes HDU number 1 (counting +# from 0). You can modify the defaults by modifying the rule in +# 'reproduce/analysis/make/initialize.mk'. +# +# Automatic writing of verification +# --------------------------------- +# In case you would like Maneage to find the checksum upon downloading, put +# the string '--auto-replace--' instead of a checksum. This can be helpful +# for large datasets; where downloading only for adding the checksum is not +# easy/possible and can be buggy. In this scenario, upon downloading the +# file its checksum will be calculated and will be replaced with the +# '--auto-replace--' in this file. But since this file is under version +# control, be sure to commit all the updated checksums after your downloads +# are finished! +# +# Variable description +# -------------------- # The naming convension is critical for the input files to be properly -# imported into the project. In the patterns below, the '%' is the full -# file name (including its suffix): for example in the demo input of this -# file in the 'maneage' branch, we have 'INPUT-wfpc2.fits-sha256': -# therefore, the input file (within the project's '$(indir)') is called +# imported into Maneage. In the patterns below, the '%' is the full file +# name (including its suffix): for example in the demo input of this file +# in the 'maneage' branch, we have 'INPUT-wfpc2.fits-sha256': therefore, +# the input file (within the project's '$(indir)') is called # 'wfpc2.fits'. This allows you to simply set '$(indir)/wfpc2.fits' as the # pre-requisite of any recipe that needs the input file: you will rarely # (if at all!) need to use these variables directly. @@ -40,23 +67,18 @@ # INPUT-%-sha256: The sha256 checksum of the file. You can generate the # SHA256 checksum of a file with the 'sha256sum FILENAME' # command (where 'FILENAME' is the name of your -# file). this is very important for an automatic -# verification of the file: that it hasn't changed -# between different runs of the project (locally or in -# the URL). There are more robust checksum algorithms -# like the 'SHA' standards. -# -# AUTOMATIC CHEKSUM CALCULATION: In case you would like -# Maneage to find the checksum upon downloading, put the -# string '--auto-replace--' instead of a checksum. This -# can be helpful for large datasets; where downloading -# only for adding the checksum is not easy/possible and -# can be buggy. In this scenario, upon downloading the -# file its checksum will be calculated and will be -# replaced with the '--auto-replace--' in this file. But -# since this file is under version control, be sure to -# commit all the updated checksums after your downloads -# are finished! +# file). Don't use this if you give the 'fitsdatasum' +# keyvalue. +# +# INPUT-%-fitsdatasum: The FITS standard DATASUM value for the HDU given +# to '-fitshdu' (below) of the FITS file (counting +# from 0). Don't use this if you give the 'sha256' +# keyword. +# +# INPUT-%-fitshdu: The HDU identifier (counter from 0, or name) to use +# for the verification. This is only relevant in the +# 'fitsdatasum' verification method and optional (if not +# given, HDU number 1 is used; counting from 0). # # INPUT-%-url: The URL to download the file if it is not available # locally. It can happen that during the first phases of @@ -70,13 +92,20 @@ # good feeling of the necessary network and storage # capacity that is necessary to start the project. # +# Therefore, the the verification variable is MANDATORY in any case. The +# variable with a URL is only necessary if you do not have the file +# locally. However, The size variable is optional (but recommended: because +# it gives future scientists a feeling of the volume of data they need to +# input to run your project: will become important if the size/number of +# files is large). +# # The input dataset's name (that goes into the '%') can be different from # the URL's file name (last component of the URL, after the last '/'). Just # note that it is assumed that the local copy (outside of your project) is # also called '%' (if your local copy of the input dataset and the only # repository names are the same, be sure to set '%' accordingly). # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice and @@ -87,7 +116,18 @@ -# Demo dataset used in the histogram plot (remove when customizing). +# Demo dataset used in the histogram plot +# --------------------------------------- +# +# Remove this part while you are entering your project's datasets. +# +# Since the demonstration dataset is a FITS file, we have also added the +# two '$(INPUT-%-fits*)' variables as a demonstration. But they are +# commented because the SHA256 method is also possible for this file (its +# not generated on the server at query time; it is a static file on the +# server). INPUT-wfpc2.fits-size = 62K INPUT-wfpc2.fits-url = https://fits.gsfc.nasa.gov/samples/WFPC2ASSNu5780205bx.fits INPUT-wfpc2.fits-sha256 = 9851bc2bf9a42008ea606ec532d04900b60865daaff2f233e5c8565dac56ad5f +#INPUT-wfpc2.fits-fitshdu = 0 +#INPUT-wfpc2.fits-fitsdatasum = 2218330266 diff --git a/reproduce/analysis/config/delete-me-squared-num.conf b/reproduce/analysis/config/delete-me-squared-num.conf index 4df2101..ba8c960 100644 --- a/reproduce/analysis/config/delete-me-squared-num.conf +++ b/reproduce/analysis/config/delete-me-squared-num.conf @@ -1,6 +1,6 @@ # Number of samples in the demonstration analysis (to be deleted). # -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice and diff --git a/reproduce/analysis/config/metadata.conf b/reproduce/analysis/config/metadata.conf index 0241136..1ab7de1 100644 --- a/reproduce/analysis/config/metadata.conf +++ b/reproduce/analysis/config/metadata.conf @@ -15,7 +15,7 @@ # and the copyright license name and standard link to the fully copyright # license. # -# Copyright (C) 2020-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2020-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice and diff --git a/reproduce/analysis/config/pdf-build.conf b/reproduce/analysis/config/pdf-build.conf index a57b529..7821306 100644 --- a/reproduce/analysis/config/pdf-build.conf +++ b/reproduce/analysis/config/pdf-build.conf @@ -12,7 +12,7 @@ # LaTeX. Otherwise, a notice will just printed that, no PDF will be # created. # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice and diff --git a/reproduce/analysis/config/verify-outputs.conf b/reproduce/analysis/config/verify-outputs.conf index 37fc43c..031085d 100644 --- a/reproduce/analysis/config/verify-outputs.conf +++ b/reproduce/analysis/config/verify-outputs.conf @@ -1,6 +1,6 @@ # To enable verification of output datasets set this variable to 'yes'. # -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # Copying and distribution of this file, with or without modification, are # permitted in any medium without royalty provided the copyright notice and diff --git a/reproduce/analysis/make/delete-me.mk b/reproduce/analysis/make/delete-me.mk index f4c8600..a20abc6 100644 --- a/reproduce/analysis/make/delete-me.mk +++ b/reproduce/analysis/make/delete-me.mk @@ -1,6 +1,6 @@ # Dummy Makefile to create a random dataset for plotting. # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index eb5f2ff..1f2158a 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -1,6 +1,6 @@ # Project initialization. # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -41,7 +41,7 @@ bsdir=$(BDIR)/software # Derived directories (the locks directory can be shared with software # which already has this directory.). texdir = $(badir)/tex -lockdir = $(bsdir)/locks +lockdir = $(badir)/.locks indir = $(badir)/inputs prepdir = $(badir)/prepare mtexdir = $(texdir)/macros @@ -63,15 +63,18 @@ pconfdir = reproduce/analysis/config # loaded. # # If your project doesn't need any preparation, you can ignore this. +# +# The '-' behind the include commands is used for adding the files only if +# it is possible (they exist). This is necessary because sometimes the user +# will have only '*.conf' or '*.mk' files, or with 'make clean' (where the +# preparation Makefile may call initialize.mk before the main +# 'top-make.mk'). If the '-' is not used, Make will complain about not +# finding these files. ifeq (x$(project-phase),xprepare) $(prepdir):; mkdir $@ else -include $(bsdir)/preparation-done.mk +-include $(badir)/preparation-done.mk ifeq (x$(include-prepare-results),xyes) -# The '-' behind the include is The '-' is used for adding the files only -# if it is possible (they exist). This is necessary because sometimes the -# user will have only '*.conf' or '*.mk' files. So, if the '-' is not used, -# Make will complain about not finding these files. -include $(prepdir)/*.mk $(prepdir)/*.conf endif endif @@ -227,7 +230,7 @@ project-commit-hash := $(shell \ export LD_LIBRARY_PATH="$(installdir)/lib"; \ echo $$($(installdir)/bin/git describe --dirty --always --long); \ else echo NOGIT; fi) -project-package-name := maneaged-$(project-commit-hash) +project-package-name = maneaged-$(project-commit-hash) project-package-contents = $(texdir)/$(project-package-name) @@ -262,16 +265,8 @@ clean: # executing 'build'. rm -f *.aux *.log *.synctex *.auxlock *.dvi *.out *.run.xml *.bcf -# Delete all the built outputs except the dependency programs. We'll -# use Bash's extended options builtin ('shopt') to enable "extended -# glob" (for listing of files). It allows extended features like -# ignoring the listing of a file with '!()' that we are using -# afterwards. - shopt -s extglob - rm -rf $(texdir)/macros/!(dependencies.tex|dependencies-bib.tex|hardware-parameters.tex) - rm -rf $(badir)/!(tex) $(texdir)/!(macros|$(texbtopdir)) - rm -rf $(texdir)/build/!(tikz) $(texdir)/build/tikz/* - rm -rf $(bsdir)/preparation-done.mk +# Delete the full 'badir' (containing all analysis outputs). + rm -rf $(badir) distclean: clean # Without cleaning the Git hooks, we won't be able to easily commit @@ -282,7 +277,7 @@ distclean: clean # 'rm' program. So for this recipe, we'll use the host system's 'rm', # not our own. $$sys_rm -rf $(BDIR) - $$sys_rm -f .local .build $(pconfdir)/LOCAL.conf + $$sys_rm -f .local .build @@ -438,13 +433,15 @@ dist-software: -# Download input data -# -------------------- +# Import input data +# ----------------- # -# 'reproduce/analysis/config/INPUTS.conf' contains the input dataset -# properties. In most cases, you will not need to edit this rule. Simply -# follow the instructions of 'INPUTS.conf' and set the variables names -# according to the described standards and everything should be fine. +# The list files to be imported (downloaded from a server, or linked from a +# local location), are listed in 'reproduce/analysis/config/INPUTS.conf' +# along with their URLs and verification checksums. In most cases, you will +# not need to edit this rule. Simply follow the instructions at the top of +# 'INPUTS.conf' and set the variables names according to the described +# standards and everything should be fine. # # TECHNICAL NOTE on the '$(foreach, n ...)' loop of 'inputdatasets': we are # using several (relatively complex!) features particular to Make: In GNU @@ -464,17 +461,64 @@ dist-software: # process with a file and make sure that only one downloading event is in # progress at every moment. $(indir):; mkdir $@ -downloadwrapper = $(bashdir)/download-multi-try -inputdatasets = $(foreach i, \ - $(patsubst INPUT-%-sha256,%, \ - $(filter INPUT-%-sha256,$(.VARIABLES))), \ - $(indir)/$(i)) +downloadwrapper = $(bashdir)/download-multi-try.sh +inputdatasets := $(foreach i, \ + $(patsubst INPUT-%-sha256,%, \ + $(filter INPUT-%-sha256,$(.VARIABLES))) \ + $(patsubst INPUT-%-fitsdatasum,%, \ + $(filter INPUT-%-fitsdatasum,$(.VARIABLES))), \ + $(indir)/$(i)) $(inputdatasets): $(indir)/%: | $(indir) $(lockdir) -# Set the necessary parameters for this input file as shell variables -# (to help in readability). - url=$(INPUT-$*-url) - sha=$(INPUT-$*-sha256) +# Starting rule with '@': In case there is a username or password +# given for the database, we don't want the values to be printed in +# the terminal as the pipeline is running. We are therefore starting +# this recipe with an '@' (so Make doesn't print the used +# commands). To help the user know what is happening (in case they +# can't tell from the Wget outputs), we'll just start the recipe with +# a notice on what is being imported. + @echo "Importing $@" + +# If a username or password has been provided, add them to the WGET +# command. The two variables are defined in the local configuation +# file 'reproduce/software/config/LOCAL.conf' that is not under +# version control. Different servers may use different authentication +# formats. If the default one doesn't work for your server, comment +# it and uncomment the one that works. If your serve needs a +# different kind of authentication format, please add it yourself. In +# case you need a new format, we encourage you to send the format to +# us using the link below: +# https://savannah.nongnu.org/support/?group=reproduce&func=additem + authopt="" + if [ x"$(DATABASEAUTHTYPE)" != x ]; then + case "$(DATABASEAUTHTYPE)" in + +# Format: '--user=XXXX --password=YYYY' + userpass) + if [ x'$(DATABASEUSER)' != x ]; then + authopt="--user='$(DATABASEUSER)'"; fi + if [ x'$(DATABASEPASS)' != x ]; then + authopt="$$authopt --password='$(DATABASEPASS)'"; fi + ;; + +# Format: --post-data 'username=XXXX&password=YYYY' + postdata) + if [ x'$(DATABASEUSER)' != x ]; then + authopt="--post-data 'username=$(DATABASEUSER)"; fi + if [ x'$(DATABASEPASS)' != x ]; then + authopt="$$authopt""&password=$(DATABASEPASS)'"; + else authopt="$$authopt'" # To close the single quote + fi + ;; + +# Unrecognized format. + *) + printf "Maneage: 'DATABASEAUTHTYPE' format not recognized! " + printf "Please see the description of this variable in " + printf "'$(bsdir)/config/LOCAL.conf' for the acceptable " + printf "values."; exit 1;; + esac + fi # Download (or make the link to) the input dataset. If the file # exists in 'INDIR', it may be a symbolic link to some other place in @@ -488,13 +532,29 @@ $(inputdatasets): $(indir)/%: | $(indir) $(lockdir) ln -fs $$(readlink -f $(INDIR)/$*) $$unchecked else touch $(lockdir)/download - $(downloadwrapper) "wget --no-use-server-timestamps -O" \ - $(lockdir)/download $$url $$unchecked + $(downloadwrapper) "wget $$authopt --no-use-server-timestamps -O" \ + $(lockdir)/download "$(INPUT-$*-url)" $$unchecked + fi + +# Set the checksum related variables. + if [ x"$(INPUT-$*-sha256)" != x ]; then + suffix=sha256 + sumin=$(INPUT-$*-sha256) + verifname="SHA256 checksum" + sum=$$(sha256sum $$unchecked | awk '{print $$1}') + elif [ x"$(INPUT-$*-fitsdatasum)" != x ]; then + suffix=fitsdatasum + sumin=$(INPUT-$*-fitsdatasum) + verifname="FITS standard DATASUM" + if [ x"$(INPUT-$*-fitshdu)" = x ]; then hdu=1; + else hdu="$(INPUT-$*-fitshdu)"; fi + sum=$$(astfits $$unchecked -h$$hdu --datasum | awk '{print $$1}') + else + echo "$@: checksum for verifyication not recognized!"; exit 1 fi -# Check the checksum to see if this is the proper dataset. - sum=$$(sha256sum $$unchecked | awk '{print $$1}') - if [ $$sum = $$sha ]; then +# Verify the input. + if [ $$sum = $$sumin ]; then mv $$unchecked $@ echo "Integrity confirmed, using $@ in this project." @@ -502,11 +562,11 @@ $(inputdatasets): $(indir)/%: | $(indir) $(lockdir) else # The user has asked to update the checksum in 'INPUTS.conf'. - if [ $$sha = "--auto-replace--" ]; then + if [ $$sumin = "--auto-replace--" ]; then # Put the updated 'INPUTS.conf' in a temporary file. inputstmp=$@.inputs - awk '{if($$1 == "INPUT-$*-sha256") \ + awk '{if($$1 == "INPUT-$*-'$$suffix'") \ $$3="'$$sum'"; print}' \ $(pconfdir)/INPUTS.conf > $$inputstmp @@ -520,10 +580,10 @@ $(inputdatasets): $(indir)/%: | $(indir) $(lockdir) # Error on non-matching checksums. else echo; echo; - echo "Wrong SHA256 checksum for input file '$*':" + echo "Wrong $$verifname for input file '$*':" echo " File location: $$unchecked"; \ - echo " Expected SHA256 checksum: $$sha"; \ - echo " Calculated SHA256 checksum: $$sum"; \ + echo " Expected $$verifname: $$sumin"; \ + echo " Calculated $$verifname: $$sum"; \ echo; exit 1 fi fi @@ -607,7 +667,7 @@ print-general-metadata = \ # for the final PDF. Since these are not version controlled, it must be # calculated everytime the project is run. So even though this file # actually exists, it is also aded as a '.PHONY' target above. -$(mtexdir)/initialize.tex: | $(mtexdir) +$(mtexdir)/initialize.tex: # Version and title of project. About the starting '@': since these # commands are run every time with './project make', it is annoying diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk index 740dc7d..b1b794c 100644 --- a/reproduce/analysis/make/paper.mk +++ b/reproduce/analysis/make/paper.mk @@ -1,6 +1,6 @@ # Build the final PDF paper/report. # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ + # LaTeX macros for paper # ---------------------- # @@ -92,6 +93,38 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex +# TeX build directory +# ------------------- +# +# If built in a group scenario, the TeX build directory must be separate +# for each member (so they can work on their relevant parts of the paper +# without conflicting with each other). +ifeq ($(strip $(maneage_group_name)),) +texbdir:=$(texdir)/build +else +texbdir:=$(texdir)/build-$(shell whoami) +endif +tikzdir:=$(texbdir)/tikz + + + + + +# Software info in TeX +# -------------------- +# +# The information of the installed software is placed in the +# '.build/software' directory (which the TeX build should not depend +# on). Therefore, we should copy those macros here in the LaTeX build +# directory, so the TeX directory is completely independent from each +# other. +$(mtexdir)/dependencies.tex: $(bsdir)/config/dependencies.tex + cp $(bsdir)/config/*.tex $(mtexdir)/ + + + + + # The bibliography # ---------------- # @@ -104,8 +137,9 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex # recipe and the 'paper.pdf' recipe. But if 'tex/src/references.tex' hasn't # been modified, we don't want to re-build the bibliography, only the final # PDF. -$(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ +$(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies.tex \ | $(mtexdir)/project.tex + # If '$(mtexdir)/project.tex' is empty, don't build PDF. @macros=$$(cat $(mtexdir)/project.tex) if [ x"$$macros" != x ]; then @@ -128,9 +162,13 @@ $(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ # use PGFPlots, then you should remove the '-shell-escape' option # for better security. See https://savannah.nongnu.org/task/?15694 # for details. +# +# We need the modification to 'LD_LIBRARY_PATH' because we do not +# build LaTeX from source and it uses '/bin/sh' (among other +# possible system-wide things). + export LD_LIBRARY_PATH="$(sys_library_sh_path):$$LD_LIBRARY_PATH" pdflatex -shell-escape -halt-on-error "$$p"/paper.tex biber paper - fi @@ -158,6 +196,11 @@ paper.pdf: $(mtexdir)/project.tex paper.tex $(texbdir)/paper.bbl # See above for a warning and brief discussion on the the pdflatex # option '-shell-escape'. +# +# We need the modification to 'LD_LIBRARY_PATH' because we do not +# build LaTeX from source and it uses '/bin/sh' (among other +# possible system-wide things). + export LD_LIBRARY_PATH="$(sys_library_sh_path):$$LD_LIBRARY_PATH" pdflatex -shell-escape -halt-on-error "$$p"/paper.tex # Come back to the top project directory and copy the built PDF diff --git a/reproduce/analysis/make/prepare.mk b/reproduce/analysis/make/prepare.mk index ebc2b4d..ffb2a3c 100644 --- a/reproduce/analysis/make/prepare.mk +++ b/reproduce/analysis/make/prepare.mk @@ -1,6 +1,6 @@ # Basic preparations, called by './project make'. # -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,7 +25,7 @@ # # We need to remove the 'prepare' word from the list of 'makesrc'. prepare-dep = $(filter-out prepare, $(makesrc)) -$(bsdir)/preparation-done.mk: \ +$(badir)/preparation-done.mk: \ $(foreach s, $(prepare-dep), $(mtexdir)/$(s).tex) # If you need to add preparations (mainly automatically generated diff --git a/reproduce/analysis/make/top-make.mk b/reproduce/analysis/make/top-make.mk index d6e3822..e87aed8 100644 --- a/reproduce/analysis/make/top-make.mk +++ b/reproduce/analysis/make/top-make.mk @@ -1,6 +1,6 @@ # Top-level Makefile (first to be loaded). # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,9 +19,9 @@ -# Load the local configuration (created after running -# './project configure'). -include reproduce/software/config/LOCAL.conf +# Load the local configuration (created after running './project +# configure'). +include .build/software/config/LOCAL.conf diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk index 28dfc4a..d2d1c14 100644 --- a/reproduce/analysis/make/top-prepare.mk +++ b/reproduce/analysis/make/top-prepare.mk @@ -4,7 +4,7 @@ # are not included here. Please see that file for thorough comments on each # step. # -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,9 +23,9 @@ -# Load the local configuration (created after running -# './project configure'). -include reproduce/software/config/LOCAL.conf +# Load the local configuration (created after running './project +# configure'). +include .build/software/config/LOCAL.conf @@ -36,7 +36,7 @@ include reproduce/software/config/LOCAL.conf # # See 'top-make.mk' for complete explanation. ifeq (x$(maneage_group_name),x$(GROUP-NAME)) -all: $(BDIR)/software/preparation-done.mk +all: $(BDIR)/analysis/preparation-done.mk @echo "Project preparation is complete."; else all: diff --git a/reproduce/analysis/make/verify.mk b/reproduce/analysis/make/verify.mk index 25b3bec..c74f8ca 100644 --- a/reproduce/analysis/make/verify.mk +++ b/reproduce/analysis/make/verify.mk @@ -1,6 +1,6 @@ # Verify the project outputs before building the paper. # -# Copyright (C) 2020-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2020-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by |