aboutsummaryrefslogtreecommitdiff
path: root/reproduce/src/make/initialize.mk
blob: ff06dcbd774e4915b0a065426e3c5e9aed03392d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
# Initialize the reproduction pipeline.
#
# Original author:
#     Mohammad Akhlaghi <mohammad@akhlaghi.org>
# Contributing author(s):
#     Your name <your@email.address>
# Copyright (C) 2018-2019, Your Name.
#
# This Makefile is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This Makefile is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.
#
# A copy of the GNU General Public License is available at
# <http://www.gnu.org/licenses/>.





# High-level directory definitions
# --------------------------------
#
# Basic directories that are used throughout the whole pipeline.
#
# Locks are used to make sure that an operation is done in series not in
# parallel (even if Make is run in parallel with the `-j' option). The most
# common case is downloads which are better done in series and not in
# parallel. Also, some programs may not be thread-safe, therefore it will
# be necessary to put a lock on them. This pipeline uses the `flock'
# program to achieve this.
texdir      = $(BDIR)/tex
srcdir      = reproduce/src
lockdir     = $(BDIR)/locks
indir       = $(BDIR)/inputs
mtexdir     = $(texdir)/macros
pconfdir    = reproduce/config/pipeline
installdir  = $(BDIR)/dependencies/installed
# --------- Delete for no Gnuastro ---------
gconfdir    = reproduce/config/gnuastro
# ------------------------------------------





# TeX build directory
# ------------------
#
# In scenarios where multiple users are working on the pipeline
# simultaneously, they can't all build the final paper together, there will
# be conflicts! It is possible to manage the working on the analysis, so no
# conflict is caused in that phase, but it would be very slow to only let
# one of the project members to build the paper at every instance
# (independent parts of the paper can be added to it independently). To fix
# this problem, when we are in a group setting, we'll use the user's ID to
# create a separate LaTeX build directory for each user.
#
# The same logic applies to the final paper PDF: each user will create a
# separte final PDF (for example `paper-user1.pdf' and `paper-user2.pdf')
# and no `paper.pdf' will be built. This isn't a problem because
# `initialize.tex' is a .PHONY prerequisite, so the rule to build the final
# paper is always executed (even if it is present and nothing has
# changed). So in terms of over-all efficiency and processing steps, this
# doesn't change anything.
ifeq (x$(GROUP-NAME),x)
texbdir     = $(texdir)/build
final-paper = paper.pdf
else
user        = $(shell whoami)
texbdir     = $(texdir)/build-$(user)
final-paper = paper-$(user).pdf
endif
tikzdir     = $(texbdir)/tikz





# Original system environment
# ---------------------------
#
# Before defining the local sub-environment here, we'll need to save the
# system's environment for some scenarios (for example after `clean'ing the
# built programs).
sys-path := $(PATH)
sys-rm   := $(shell which rm)
curdir   := $(shell echo $$(pwd))





# High level environment
# ----------------------
#
# We want the full recipe to be executed in one call to the shell. Also we
# want Make to run the specific version of Bash that we have installed
# during `./configure' time.
#
# Regarding the directories, this pipeline builds its major dependencies
# itself and doesn't use the local system's default tools. With these
# environment variables, we are setting it to prefer the software we have
# build here.
.ONESHELL:
.SHELLFLAGS             = -ec
export CCACHE_DISABLE  := 1
export PATH            := $(installdir)/bin
export LD_LIBRARY_PATH := $(installdir)/lib
export LDFLAGS         := -L$(installdir)/lib
export SHELL           := $(installdir)/bin/bash
export CPPFLAGS        := -I$(installdir)/include





# High-level level directories
# ----------------------------
#
# These are just the top-level directories for all the separate steps. The
# directories (or possible sub-directories) for individual steps will be
# defined and added within their own Makefiles.
#
# IMPORTANT NOTE for $(BDIR)'s dependency: it only depends on the existance
# (not the time-stamp) of `$(pconfdir)/LOCAL.mk'. So the user can make any
# changes within that file and if they don't affect the pipeline. For
# example a change of the top $(BDIR) name, while the contents are the same
# as before.
#
# The `.SUFFIXES' rule with no prerequisite is defined to eliminate all the
# default implicit rules. The default implicit rules are to do with
# programming (for example converting `.c' files to `.o' files). The
# problem they cause is when you want to debug the make command with `-d'
# option: they add too many extra checks that make it hard to find what you
# are looking for in this pipeline.
.SUFFIXES:
$(texdir) $(lockdir): | $(BDIR); mkdir $@
$(mtexdir) $(texbdir): | $(texdir); mkdir $@
$(tikzdir): | $(texbdir); mkdir $@ && ln -s $(tikzdir) tex/tikz





# High-level Makefile management
# ------------------------------
#
# About `.PHONY': these are targets that must be built even if a file with
# their name exists.
#
# Only `$(mtexdir)/initialize.tex' corresponds to a file. This is because
# we want to ensure that the file is always built in every run: it contains
# the pipeline version which may change between two separate runs, even
# when no file actually differs.
packagebasename := $(shell echo paper-$$(git describe --dirty --always))
packagecontents = $(texdir)/$(packagebasename)
.PHONY: all clean dist dist-zip distclean clean-mmap $(packagecontents) \
        $(mtexdir)/initialize.tex

# --------- Delete for no Gnuastro ---------
clean-mmap:; rm -f reproduce/config/gnuastro/mmap*
# ------------------------------------------

clean: clean-mmap
        # Delete the top-level PDF file.
	rm -f *.pdf

        # Delete all the built outputs except the dependency
        # programs. We'll use Bash's extended options builtin (`shopt') to
        # enable "extended glob" (for listing of files). It allows extended
        # features like ignoring the listing of a file with `!()' that we
        # are using afterwards.
	shopt -s extglob
	rm -rf $(BDIR)/!(dependencies)

distclean: clean
        # We'll be deleting the built environent programs and just need the
        # `rm' program. So for this recipe, we'll use the host system's
        # `rm', not our own.
	$(sys-rm) -rf $(BDIR) reproduce/build
	$(sys-rm) -f Makefile .gnuastro .local
	$(sys-rm) -f $(pconfdir)/LOCAL.mk $(gconfdir)/gnuastro-local.conf





# Packaging rules
# ---------------
#
# With the rules in this section, you can package the project in a state
# that is ready for building the final PDF with LaTeX. This is useful for
# collaborators who only want to contribute to the text of your project,
# without having to worry about the technicalities of the analysis.
$(packagecontents): | $(texdir)

        # Set up the output directory, delete it if it exists and remake it
        # to fill with new contents.
	dir=$(texdir)/$(packagebasename)
	rm -rf $$dir
	mkdir $$dir

        # Build a small Makefile to help in automatizing the paper building
        # (including the bibliography).
	m=$$dir/Makefile
	echo   "paper.pdf: paper.tex paper.bbl"                   > $$m
	printf "\tpdflatex -shell-escape -halt-on-error paper\n" >> $$m
	echo   "paper.bbl: tex/src/references.tex"               >> $$m
	printf "\tpdflatex -shell-escape -halt-on-error paper\n" >> $$m
	printf "\tbiber paper\n"                                 >> $$m
	echo   ".PHONY: clean"                                   >> $$m
	echo   "clean:"                                          >> $$m
	printf "\trm -f *.aux *.auxlock *.bbl *.bcf\n"           >> $$m
	printf "\trm -f *.blg *.log *.out *.run.xml\n"           >> $$m

        # Copy the top-level contents into it.
	cp configure COPYING for-group README.md README-hacking.md $$dir/

        # Build the top-level directories.
	mkdir $$dir/reproduce $$dir/tex $$dir/tex/tikz $$dir/tex/pipeline

        # Copy all the `reproduce' contents except for the `build' symbolic
        # link.
	shopt -s extglob
	cp -r tex/src                            $$dir/tex/src
	cp tex/tikz/*.pdf                        $$dir/tex/tikz
	cp -r reproduce/!(build)                 $$dir/reproduce
	cp -r tex/pipeline/!($(packagebasename)) $$dir/tex/pipeline

        # Clean up un-necessary/local files: 1) the $(texdir)/build*
        # directories (when building in a group structure, there will be
        # `build-user1', `build-user2' and etc), are just temporary LaTeX
        # build files and don't have any relevant/hand-written files in
        # them. 2) The `LOCAL.mk' and `gnuastro-local.conf' files just have
        # this machine's local settings and are irrelevant for anyone else.
	rm -rf $$dir/tex/pipeline/build*
	rm $$dir/reproduce/config/pipeline/LOCAL.mk
	rm $$dir/reproduce/config/gnuastro/gnuastro-local.conf

        # PIPELINE SPECIFIC: under this comment, copy any other file for
        # packaging, or remove any of the copied files above to suite your
        # project.

        # Since the packaging is mainly intended for high-level building of
        # the PDF with LaTeX, we'll comment the `makepdf' LaTeX macro in
        # the paper.
	sed -e's|\\newcommand{\\makepdf}{}|%\\newcommand{\\makepdf}{}|' \
	    paper.tex > $$dir/paper.tex

        # Just in case the package users want to rebuild some of the
        # figures (manually un-comments the `makepdf' command we commented
        # above), correct the TikZ external directory, so the figures can
        # be rebuilt.
	pgfsettings="$$dir/tex/src/preamble-pgfplots.tex"
	sed -e's|{tikz/}|{tex/tikz/}|' $$pgfsettings > $$pgfsettings.new
	mv $$pgfsettings.new $$pgfsettings

        # Clean temporary (currently those ending in `~') files.
	cd $(texdir)
	find $(packagebasename) -name \*~ -delete

# Package into `.tar.gz'.
dist: $(packagecontents)
	curdir=$$(pwd)
	cd $(texdir)
	tar -cf $(packagebasename).tar $(packagebasename)
	gzip -f --best $(packagebasename).tar
	cd $$curdir
	mv $(texdir)/$(packagebasename).tar.gz ./

# Package into `.zip'.
dist-zip: $(packagecontents)
	curdir=$$(pwd)
	cd $(texdir)
	zip -q -r $(packagebasename).zip $(packagebasename)
	cd $$curdir
	mv $(texdir)/$(packagebasename).zip ./





# Check the version of programs which write their version
# -------------------------------------------------------
pvcheck = prog="$(strip $(1))";                                          \
	  ver="$(strip $(2))";                                           \
	  name="$(strip $(3))";                                          \
	  macro="$(strip $(4))";                                         \
	  verop="$(strip $(5))";                                         \
	  if [ "x$$verop" = x ]; then V="--version"; else V=$$verop; fi; \
	  v=$$($$prog $$V | awk '/'$$ver'/{print "y"; exit 0}');         \
	  if [ x$$v != xy ]; then                                        \
	    echo; echo "PIPELINE ERROR: Not running $$name $$ver"; echo; \
	    exit 1;                                                      \
	  fi;                                                            \
	  echo "\newcommand{\\$$macro}{$$ver}" >> $@

lvcheck = idir=$(BDIR)/dependencies/installed/include;                   \
	  f="$$idir/$(strip $(1))";                                      \
	  ver="$(strip $(2))";                                           \
	  name="$(strip $(3))";                                          \
	  macro="$(strip $(4))";                                         \
	  v=$$(awk '/^\#/&&/define/&&/'$$ver'/{print "y";exit 0}' $$f);  \
	  if [ x$$v != xy ]; then                                        \
	    echo; echo "PIPELINE ERROR: Not linking with $$name $$ver";  \
	    echo; exit 1;                                                \
	  fi;                                                            \
	  echo "\newcommand{\\$$macro}{$$ver}" >> $@





# Pipeline initialization results
# -------------------------------
#
# This file will store some basic info about the pipeline that is necessary
# for the final PDF. Since these are not version controlled, it must be
# calculated everytime the pipeline is run. So even though this file
# actually exists, it is also aded as a `.PHONY' target above.
$(mtexdir)/initialize.tex: | $(mtexdir)

        # Version of the pipeline and build directory (for LaTeX inputs).
	@v=$$(git describe --dirty --always);
	echo "\newcommand{\pipelineversion}{$$v}"  > $@

        # Versions of programs (same order as 'dependency-versions.mk'),
        # ordered alphabetically (by their executable name).
        # --------- Delete for no Gnuastro ---------
	$(call pvcheck, astnoisechisel, $(gnuastro-version), Gnuastro, \
                        gnuastroversion)
        # ------------------------------------------
	$(call pvcheck, awk, $(gawk-version), GNU AWK, gawkversion)
	$(call pvcheck, bash, $(bash-version), GNU Bash, bashversion)
	$(call pvcheck, cmake, $(cmake-version), CMake, cmakeversion)
	$(call pvcheck, curl, $(curl-version), cURL, curlversion)
	$(call pvcheck, diff, $(diffutils-version), GNU Diffutils,     \
	                diffutilsversion)
	$(call pvcheck, find, $(findutils-version), GNU Findutils,     \
	                findutilsversion)
	$(call pvcheck, git, $(git-version), Git, gitversion)
	$(call pvcheck, glibtool, $(libtool-version), GNU Libtool,     \
	                libtoolversion)
	$(call pvcheck, grep, $(grep-version), GNU Grep, grepversion)
	$(call pvcheck, gs, $(ghostscript-version), GPL Ghostscript,   \
	                ghostscriptversion)
	$(call pvcheck, gzip, $(gzip-version), GNU Gzip, gzipversion)
	$(call pvcheck, ls, $(coreutils-version), GNU Coreutils,       \
	                coreutilsversion)
	$(call pvcheck, lzip, $(lzip-version), Lzip, lzipversion)
	$(call pvcheck, make, $(make-version), GNU Make, makeversion)
	$(call pvcheck, metastore, $(metastore-version), Metastore,    \
	                metastoreversion)
	$(call pvcheck, pkg-config, $(pkgconfig-version), pkg-config,  \
	                pkgconfigversion)
	$(call pvcheck, sed, $(sed-version), GNU SED, sedversion)
	$(call pvcheck, tar, $(tar-version), GNU Tar, tarversion)
	$(call pvcheck, unzip, $(unzip-version), Unzip, unzipversion, -v)
	$(call pvcheck, wget, $(wget-version), GNU Wget, wgetversion)
	$(call pvcheck, which, $(which-version), GNU Which, whichversion)
	$(call pvcheck, xz, $(xz-version), XZ Utils, xzversion)
	$(call pvcheck, zip, $(zip-version), Zip, zipversion, -v)

        # Bzip2 prints its version in standard error, not standard output!
	echo "" | bzip2 --version &> $@_bzip2_ver;
	v=$$(awk 'NR==1 && /'$(bzip2-version)'/{print "y"; exit 0}' \
	         $@_bzip2_ver);
	if [ x$$v != xy ]; then
	  echo; echo "PIPELINE ERROR: Not running Bzip2 $(bzip2-version)";
	  echo; exit 1;
	fi;
	rm $@_bzip2_ver
	echo "\newcommand{\\bziptwoversion}{$(bzip2-version)}" >> $@

        # Unfortunately we couldn't find a way to retrieve the version of
        # the discoteq `flock' that we are using here. So we'll just repot
        # the version we downloaded and installed.
	echo "\newcommand{\\flockversion}{$(flock-version)}" >> $@





        # Versions of libraries.
	$(call lvcheck, fitsio.h, $(cfitsio-version), CFITSIO, cfitsioversion)
	$(call lvcheck, gsl/gsl_version.h, $(gsl-version),  \
	                GNU Scientific Library (GSL), gslversion)
	$(call lvcheck, git2/version.h, $(libgit2-version), Libgit2, \
	                libgitwoversion)
	$(call lvcheck, openssl/opensslv.h, $(openssl-version), OpenSSL, \
	                opensslversion)
	$(call lvcheck, ncursesw/curses.h, $(ncurses-version), GNU NCURSES, \
	                ncursesversion)
	$(call lvcheck, readline/readline.h, $(readline-version), GNU Readline, \
	                readlineversion)
	$(call lvcheck, tiffvers.h, $(libtiff-version), Libtiff, \
	                libtiffversion)
	$(call lvcheck, wcslib/wcsconfig.h, $(wcslib-version), WCSLIB, \
	                wcslibversion)
	$(call lvcheck, zlib.h, $(zlib-version), zlib, zlibversion)

        # Problematic libraries:
        # - libjpeg not  yet checked.
        # - libbsd has no version string in its headers.
	echo "\newcommand{\\libbsdversion}{$(libbsd-version)}"   >> $@
	echo "\newcommand{\\libjpegversion}{$(libjpeg-version)}" >> $@

        # TeX package versions
	cat $(BDIR)/dependencies/texlive-versions.tex >> $@