aboutsummaryrefslogtreecommitdiff
path: root/reproduce
diff options
context:
space:
mode:
authorMohammad Akhlaghi <mohammad@akhlaghi.org>2019-04-15 01:47:58 +0100
committerMohammad Akhlaghi <mohammad@akhlaghi.org>2019-04-15 02:24:09 +0100
commit313b936b502d22b6a2ff43f560dee0bb51fd01d0 (patch)
tree70f884b91b393be4d3c6b7cfaeaf3412900bd16f /reproduce
parent4722ea598edd6b630227404c48c1c09ac527e9b8 (diff)
New architecture to separate software-building and analysis steps
Until now, the software building and analysis steps of the pipeline were intertwined. However, these steps (of how to build a software, and how to use it) are logically completely independent. Therefore with this commit, the pipeline now has a new architecture (particularly in the `reproduce' directory) to emphasize this distinction: The `reproduce' directory now has the two `software' and `analysis' subdirectories and the respective parts of the previous architecture have been broken up between these two based on their function. There is also no more `src' directory. The `config' directory for software and analysis is now mixed with the language-specific directories. Also, some of the software versions were also updated after some checks with their webpages. This new architecture will allow much more focused work on each part of the pipeline (to install the software and to run them for an analysis).
Diffstat (limited to 'reproduce')
-rwxr-xr-xreproduce/analysis/bash/download-multi-try (renamed from reproduce/src/bash/download-multi-try)0
-rw-r--r--reproduce/analysis/config/INPUTS.mk (renamed from reproduce/config/pipeline/INPUTS.mk)0
-rw-r--r--reproduce/analysis/config/delete-me-num.mk (renamed from reproduce/config/pipeline/delete-me-num.mk)0
-rw-r--r--reproduce/analysis/config/pdf-build.mk (renamed from reproduce/config/pipeline/pdf-build.mk)0
-rw-r--r--reproduce/analysis/make/delete-me.mk (renamed from reproduce/src/make/delete-me.mk)2
-rw-r--r--reproduce/analysis/make/download.mk (renamed from reproduce/src/make/download.mk)2
-rw-r--r--reproduce/analysis/make/initialize.mk (renamed from reproduce/src/make/initialize.mk)26
-rw-r--r--reproduce/analysis/make/paper.mk (renamed from reproduce/src/make/paper.mk)39
-rw-r--r--reproduce/analysis/make/top.mk (renamed from reproduce/src/make/top.mk)27
-rw-r--r--reproduce/software/bash/git-post-checkout (renamed from reproduce/src/bash/git-post-checkout)0
-rw-r--r--reproduce/software/bash/git-pre-commit (renamed from reproduce/src/bash/git-pre-commit)0
-rw-r--r--reproduce/software/bibtex/astropy.tex97
-rw-r--r--reproduce/software/bibtex/cython.tex21
-rw-r--r--reproduce/software/bibtex/fftw.tex19
-rw-r--r--reproduce/software/bibtex/gnuastro.tex24
-rw-r--r--reproduce/software/bibtex/matplotlib.tex22
-rw-r--r--reproduce/software/bibtex/mpi4py.tex20
-rw-r--r--reproduce/software/bibtex/numpy.tex23
-rw-r--r--reproduce/software/bibtex/scipy.tex34
-rw-r--r--reproduce/software/config/gnuastro/astbuildprog.conf (renamed from reproduce/config/gnuastro/astbuildprog.conf)0
-rw-r--r--reproduce/software/config/gnuastro/astconvertt.conf (renamed from reproduce/config/gnuastro/astconvertt.conf)0
-rw-r--r--reproduce/software/config/gnuastro/aststatistics.conf (renamed from reproduce/config/gnuastro/aststatistics.conf)0
-rw-r--r--reproduce/software/config/gnuastro/gnuastro.conf (renamed from reproduce/config/gnuastro/gnuastro.conf)0
-rw-r--r--reproduce/software/config/installation/LOCAL.mk.in (renamed from reproduce/config/pipeline/LOCAL.mk.in)0
-rw-r--r--reproduce/software/config/installation/numpy-scipy.cfg (renamed from reproduce/config/pipeline/dependency-numpy-scipy.cfg)0
-rw-r--r--reproduce/software/config/installation/texlive.conf (renamed from reproduce/config/pipeline/texlive.conf)0
-rw-r--r--reproduce/software/config/installation/texlive.mk (renamed from reproduce/config/pipeline/dependency-texlive.mk)0
-rw-r--r--reproduce/software/config/installation/versions.mk (renamed from reproduce/config/pipeline/dependency-versions.mk)14
-rwxr-xr-xreproduce/software/make/atlas-multiple.mk (renamed from reproduce/src/make/dependencies-atlas-multiple.mk)0
-rwxr-xr-xreproduce/software/make/atlas-single.mk (renamed from reproduce/src/make/dependencies-atlas-single.mk)0
-rw-r--r--reproduce/software/make/basic.mk (renamed from reproduce/src/make/dependencies-basic.mk)124
-rw-r--r--reproduce/software/make/build-rules.mk (renamed from reproduce/src/make/dependencies-build-rules.mk)0
-rw-r--r--reproduce/software/make/high-level.mk (renamed from reproduce/src/make/dependencies.mk)126
-rw-r--r--reproduce/software/make/python.mk (renamed from reproduce/src/make/dependencies-python.mk)4
34 files changed, 456 insertions, 168 deletions
diff --git a/reproduce/src/bash/download-multi-try b/reproduce/analysis/bash/download-multi-try
index 1fd7497..1fd7497 100755
--- a/reproduce/src/bash/download-multi-try
+++ b/reproduce/analysis/bash/download-multi-try
diff --git a/reproduce/config/pipeline/INPUTS.mk b/reproduce/analysis/config/INPUTS.mk
index eb38295..eb38295 100644
--- a/reproduce/config/pipeline/INPUTS.mk
+++ b/reproduce/analysis/config/INPUTS.mk
diff --git a/reproduce/config/pipeline/delete-me-num.mk b/reproduce/analysis/config/delete-me-num.mk
index 17f608c..17f608c 100644
--- a/reproduce/config/pipeline/delete-me-num.mk
+++ b/reproduce/analysis/config/delete-me-num.mk
diff --git a/reproduce/config/pipeline/pdf-build.mk b/reproduce/analysis/config/pdf-build.mk
index 3a86ff3..3a86ff3 100644
--- a/reproduce/config/pipeline/pdf-build.mk
+++ b/reproduce/analysis/config/pdf-build.mk
diff --git a/reproduce/src/make/delete-me.mk b/reproduce/analysis/make/delete-me.mk
index 701a316..c4cfffe 100644
--- a/reproduce/src/make/delete-me.mk
+++ b/reproduce/analysis/make/delete-me.mk
@@ -82,7 +82,7 @@ $(histogram): $(dddemodir)/%-hist.txt: $(indir)/%.fits | $(dddemodir)
# ----------------
#
# This is just as a demonstration on how to get analysic configuration
-# parameters from variables defined in `reproduce/config/pipeline'.
+# parameters from variables defined in `reproduce/analysis/config/'.
stats = $(dddemodir)/wfpc2-stats.txt
$(stats): $(dddemodir)/%-stats.txt: $(indir)/%.fits | $(dddemodir)
aststatistics $< -h0 --mean --median > $@
diff --git a/reproduce/src/make/download.mk b/reproduce/analysis/make/download.mk
index dfc49da..a721863 100644
--- a/reproduce/src/make/download.mk
+++ b/reproduce/analysis/make/download.mk
@@ -48,7 +48,7 @@
# process with a file and make sure that only one downloading event is in
# progress at every moment.
$(indir):; mkdir $@
-downloadwrapper = $(srcdir)/bash/download-multi-try
+downloadwrapper = $(bashdir)/download-multi-try
inputdatasets = $(foreach i, wfpc2, $(indir)/$(i).fits)
$(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir)
diff --git a/reproduce/src/make/initialize.mk b/reproduce/analysis/make/initialize.mk
index cd533f2..a034494 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/analysis/make/initialize.mk
@@ -31,14 +31,14 @@
# be necessary to put a lock on them. This project uses the `flock' program
# to achieve this.
texdir = $(BDIR)/tex
-srcdir = reproduce/src
lockdir = $(BDIR)/locks
indir = $(BDIR)/inputs
mtexdir = $(texdir)/macros
-pconfdir = reproduce/config/pipeline
-installdir = $(BDIR)/dependencies/installed
+bashdir = reproduce/analysis/bash
+pconfdir = reproduce/analysis/config
+installdir = $(BDIR)/software/installed
# --------- Delete for no Gnuastro ---------
-gconfdir = reproduce/config/gnuastro
+gconfdir = reproduce/software/config/gnuastro
# ------------------------------------------
@@ -233,16 +233,14 @@ $(packagecontents): | $(texdir)
cp configure COPYING for-group README.md README-hacking.md $$dir/
# Build the top-level directories.
- mkdir $$dir/reproduce $$dir/tex $$dir/tex/tikz $$dir/tex/pipeline
+ mkdir $$dir/reproduce $$dir/tex $$dir/tex/tikz $$dir/tex/build
- # Copy all the `reproduce' contents except for the `build' symbolic
- # link.
+ # Copy all the necessary `reproduce' and `tex' contents.
shopt -s extglob
cp -r tex/src $$dir/tex/src
cp tex/tikz/*.pdf $$dir/tex/tikz
- cp -r reproduce/!(build) $$dir/reproduce
- cp -r tex/pipeline/!($(packagebasename)) $$dir/tex/pipeline
- cp -r tex/dependencies $$dir/tex/dependencies
+ cp -r reproduce/ $$dir/reproduce
+ cp -r tex/build/!($(packagebasename)) $$dir/tex/build
# Clean up un-necessary/local files: 1) the $(texdir)/build*
# directories (when building in a group structure, there will be
@@ -250,9 +248,9 @@ $(packagecontents): | $(texdir)
# build files and don't have any relevant/hand-written files in
# them. 2) The `LOCAL.mk' and `gnuastro-local.conf' files just have
# this machine's local settings and are irrelevant for anyone else.
- rm -rf $$dir/tex/pipeline/build*
- rm $$dir/reproduce/config/pipeline/LOCAL.mk
- rm $$dir/reproduce/config/gnuastro/gnuastro-local.conf
+ rm -rf $$dir/tex/build/build*
+ rm $$dir/reproduce/software/config/installation/LOCAL.mk
+ rm $$dir/reproduce/software/config/gnuastro/gnuastro-local.conf
# PROJECT SPECIFIC: under this comment, copy any other file for
# packaging, or remove any of the copied files above to suite your
@@ -338,4 +336,4 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
# Version of the project.
@v=$$(git describe --dirty --always);
- echo "\newcommand{\pipelineversion}{$$v}" > $@
+ echo "\newcommand{\projectversion}{$$v}" > $@
diff --git a/reproduce/src/make/paper.mk b/reproduce/analysis/make/paper.mk
index 0c42bee..5378ee3 100644
--- a/reproduce/src/make/paper.mk
+++ b/reproduce/analysis/make/paper.mk
@@ -23,35 +23,32 @@
#
# To report the input settings and results, the final report's PDF (final
# target of this project) uses macros generated from various steps of the
-# project. All these macros are defined in `$(mtexdir)/pipeline.tex'.
+# project. All these macros are defined through `$(mtexdir)/project.tex'.
#
-# `$(mtexdir)/pipeline.tex' is actually just a combination of separate
-# files that keep the LaTeX macros related to each workhorse Makefile (in
+# `$(mtexdir)/project.tex' is actually just a combination of separate files
+# that keep the LaTeX macros related to each workhorse Makefile (in
# `reproduce/src/make/*.mk'). Those individual macros are pre-requisites to
-# `$(mtexdir)/pipeline.tex'. The only workhorse Makefile that doesn't need
+# `$(mtexdir)/project.tex'. The only workhorse Makefile that doesn't need
# to produce LaTeX macros is this Makefile (`reproduce/src/make/paper.mk').
#
# This file is thus the interface between the processing scripts and the
# final PDF: when we get to this point, all the processing has been
# completed.
#
-# Note that if you don't want the final PDF and just want the
-# processing and file outputs, you can remove the value of
-# `pdf-build-final' in `reproduce/config/pipeline/pdf-build.mk'.
-$(mtexdir)/pipeline.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s).tex)
+# Note that if you don't want the final PDF and just want the processing
+# and file outputs, you can remove the value of `pdf-build-final' in
+# `reproduce/analysis/config/pdf-build.mk'.
+$(mtexdir)/project.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s).tex)
# If no PDF is requested, or if LaTeX isn't available, don't
# continue to building the final PDF. Otherwise, merge all the TeX
# macros into one for building the PDF.
@if [ -f .local/bin/pdflatex ] && [ x"$(pdf-build-final)" != x ]; then
- # First make sure the `tex/pipeline' symbolic link exists.
- if [ ! -e tex/pipeline ]; then ln -s $(texdir) tex/pipeline; fi
-
# Put a LaTeX input command for all the necessary macro files.
- rm -f $(mtexdir)/pipeline.tex
+ rm -f $(mtexdir)/project.tex
for t in $(subst paper,,$(makesrc)); do
- echo "\input{tex/pipeline/macros/$$t.tex}" >> $(mtexdir)/pipeline.tex
+ echo "\input{tex/build/macros/$$t.tex}" >> $(mtexdir)/project.tex
done
else
echo
@@ -64,7 +61,7 @@ $(mtexdir)/pipeline.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s
echo "can check by running './.local/bin/latex --version'), _AND_"
echo "make sure that the 'pdf-build-final' variable has a value."
echo "'pdf-build-final' is defined in: "
- echo "'reproduce/config/pipeline/pdf-build.mk'."
+ echo "'reproduce/analysis/config/pdf-build.mk'."
echo
echo "If you don't have LaTeX within the project, please re-run"
echo "'./configure' when you have internet access. To speed it up,"
@@ -88,15 +85,15 @@ $(mtexdir)/pipeline.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s
# necessary bibliography before making the final paper. So we'll first have
# one run of LaTeX (similar to the `paper.pdf' recipe), then `biber'.
#
-# NOTE: `$(mtexdir)/pipeline.tex' is an order-only-prerequisite for
+# NOTE: `$(mtexdir)/project.tex' is an order-only-prerequisite for
# `paper.bbl'. This is because we need to run LaTeX in both the `paper.bbl'
# recipe and the `paper.pdf' recipe. But if `tex/src/references.tex' hasn't
# been modified, we don't want to re-build the bibliography, only the final
# PDF.
$(texbdir)/paper.bbl: tex/src/references.tex \
- | $(tikzdir) $(texbdir) $(mtexdir)/pipeline.tex
- # If `$(mtexdir)/pipeline.tex' is empty, don't build PDF.
- @macros=$$(cat $(mtexdir)/pipeline.tex)
+ | $(tikzdir) $(texbdir) $(mtexdir)/project.tex
+ # If `$(mtexdir)/project.tex' is empty, don't build PDF.
+ @macros=$$(cat $(mtexdir)/project.tex)
if [ x"$$macros" != x ]; then
# We'll run LaTeX first to generate the `.bcf' file (necessary
@@ -121,11 +118,11 @@ $(texbdir)/paper.bbl: tex/src/references.tex \
# to run everything cleanly from there, it is necessary to add the current
# directory (top project directory) to the `TEXINPUTS' environment
# variable.
-paper.pdf: $(mtexdir)/pipeline.tex paper.tex $(texbdir)/paper.bbl \
+paper.pdf: $(mtexdir)/project.tex paper.tex $(texbdir)/paper.bbl \
| $(tikzdir) $(texbdir)
- # If `$(mtexdir)/pipeline.tex' is empty, don't build the PDF.
- @macros=$$(cat $(mtexdir)/pipeline.tex)
+ # If `$(mtexdir)/project.tex' is empty, don't build the PDF.
+ @macros=$$(cat $(mtexdir)/project.tex)
if [ x"$$macros" != x ]; then
# Go into the top TeX build directory and make the paper.
diff --git a/reproduce/src/make/top.mk b/reproduce/analysis/make/top.mk
index 763dbd7..a85c530 100644
--- a/reproduce/src/make/top.mk
+++ b/reproduce/analysis/make/top.mk
@@ -20,7 +20,7 @@
# Load the local configuration (created after running `./configure').
-include reproduce/config/pipeline/LOCAL.mk
+include reproduce/software/config/installation/LOCAL.mk
@@ -63,7 +63,7 @@ include reproduce/config/pipeline/LOCAL.mk
#
# If you are just interested in the processing and don't want to build the
# PDF, you can skip the creatation of the final PDF by removing the value
-# of `pdf-build-final' in `reproduce/config/pipeline/pdf-build.mk'.
+# of `pdf-build-final' in `reproduce/analysis/config/pdf-build.mk'.
ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME))
all: paper.pdf
else
@@ -119,17 +119,18 @@ makesrc = initialize \
-# Include all Makefiles
-# ---------------------
+# Include all analysis Makefiles
+# ------------------------------
#
-# We have two classes of Makefiles, separated by context and their location:
+# 1) All the analysis configuration-Makefiles (Makefiles that only define
+# variables with no rules or order).
#
-# 1) First, we'll include all the configuration-Makefiles. These
-# Makefiles only define variables with no rules or order. We just
-# won't include `LOCAL.mk' because it has already been included
-# above.
+# 2) From the software configuration-Makefiles, we only include the one
+# containing software versions, just incase its necessary to
+# use/report outside of the acknowledgments section of the paper.
#
-# 2) Then, we'll import the workhorse-Makefiles which contain rules to
-# actually do this project's processing.
-include $(filter-out %LOCAL.mk, reproduce/config/pipeline/*.mk)
-include $(foreach s,$(makesrc), reproduce/src/make/$(s).mk)
+# 3) Finally, we'll import all the analysis workhorse-Makefiles which
+# contain rules to actually do this project's processing.
+include reproduce/analysis/config/*.mk
+include reproduce/software/config/installation/versions.mk
+include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk)
diff --git a/reproduce/src/bash/git-post-checkout b/reproduce/software/bash/git-post-checkout
index 9552f01..9552f01 100644
--- a/reproduce/src/bash/git-post-checkout
+++ b/reproduce/software/bash/git-post-checkout
diff --git a/reproduce/src/bash/git-pre-commit b/reproduce/software/bash/git-pre-commit
index dbe0ecc..dbe0ecc 100644
--- a/reproduce/src/bash/git-pre-commit
+++ b/reproduce/software/bash/git-pre-commit
diff --git a/reproduce/software/bibtex/astropy.tex b/reproduce/software/bibtex/astropy.tex
new file mode 100644
index 0000000..450212d
--- /dev/null
+++ b/reproduce/software/bibtex/astropy.tex
@@ -0,0 +1,97 @@
+%% Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org>
+%%
+%% Copying and distribution of this file, with or without modification,
+%% are permitted in any medium without royalty provided the copyright
+%% notice and this notice are preserved. This file is offered as-is,
+%% without any warranty.
+
+@ARTICLE{astropy2018,
+ author = {{Astropy Collaboration} and {Price-Whelan}, A.~M. and {Sip{\H o}cz}, B.~M. and
+ {G{\"u}nther}, H.~M. and {Lim}, P.~L. and {Crawford}, S.~M. and
+ {Conseil}, S. and {Shupe}, D.~L. and {Craig}, M.~W. and {Dencheva}, N. and
+ {Ginsburg}, A. and {VanderPlas}, J.~T. and {Bradley}, L.~D. and
+ {P{\'e}rez-Su{\'a}rez}, D. and {de Val-Borro}, M. and {Aldcroft}, T.~L. and
+ {Cruz}, K.~L. and {Robitaille}, T.~P. and {Tollerud}, E.~J. and
+ {Ardelean}, C. and {Babej}, T. and {Bach}, Y.~P. and {Bachetti}, M. and
+ {Bakanov}, A.~V. and {Bamford}, S.~P. and {Barentsen}, G. and
+ {Barmby}, P. and {Baumbach}, A. and {Berry}, K.~L. and {Biscani}, F. and
+ {Boquien}, M. and {Bostroem}, K.~A. and {Bouma}, L.~G. and {Brammer}, G.~B. and
+ {Bray}, E.~M. and {Breytenbach}, H. and {Buddelmeijer}, H. and
+ {Burke}, D.~J. and {Calderone}, G. and {Cano Rodr{\'{\i}}guez}, J.~L. and
+ {Cara}, M. and {Cardoso}, J.~V.~M. and {Cheedella}, S. and {Copin}, Y. and
+ {Corrales}, L. and {Crichton}, D. and {D'Avella}, D. and {Deil}, C. and
+ {Depagne}, {\'E}. and {Dietrich}, J.~P. and {Donath}, A. and
+ {Droettboom}, M. and {Earl}, N. and {Erben}, T. and {Fabbro}, S. and
+ {Ferreira}, L.~A. and {Finethy}, T. and {Fox}, R.~T. and {Garrison}, L.~H. and
+ {Gibbons}, S.~L.~J. and {Goldstein}, D.~A. and {Gommers}, R. and
+ {Greco}, J.~P. and {Greenfield}, P. and {Groener}, A.~M. and
+ {Grollier}, F. and {Hagen}, A. and {Hirst}, P. and {Homeier}, D. and
+ {Horton}, A.~J. and {Hosseinzadeh}, G. and {Hu}, L. and {Hunkeler}, J.~S. and
+ {Ivezi{\'c}}, {\v Z}. and {Jain}, A. and {Jenness}, T. and {Kanarek}, G. and
+ {Kendrew}, S. and {Kern}, N.~S. and {Kerzendorf}, W.~E. and
+ {Khvalko}, A. and {King}, J. and {Kirkby}, D. and {Kulkarni}, A.~M. and
+ {Kumar}, A. and {Lee}, A. and {Lenz}, D. and {Littlefair}, S.~P. and
+ {Ma}, Z. and {Macleod}, D.~M. and {Mastropietro}, M. and {McCully}, C. and
+ {Montagnac}, S. and {Morris}, B.~M. and {Mueller}, M. and {Mumford}, S.~J. and
+ {Muna}, D. and {Murphy}, N.~A. and {Nelson}, S. and {Nguyen}, G.~H. and
+ {Ninan}, J.~P. and {N{\"o}the}, M. and {Ogaz}, S. and {Oh}, S. and
+ {Parejko}, J.~K. and {Parley}, N. and {Pascual}, S. and {Patil}, R. and
+ {Patil}, A.~A. and {Plunkett}, A.~L. and {Prochaska}, J.~X. and
+ {Rastogi}, T. and {Reddy Janga}, V. and {Sabater}, J. and {Sakurikar}, P. and
+ {Seifert}, M. and {Sherbert}, L.~E. and {Sherwood-Taylor}, H. and
+ {Shih}, A.~Y. and {Sick}, J. and {Silbiger}, M.~T. and {Singanamalla}, S. and
+ {Singer}, L.~P. and {Sladen}, P.~H. and {Sooley}, K.~A. and
+ {Sornarajah}, S. and {Streicher}, O. and {Teuben}, P. and {Thomas}, S.~W. and
+ {Tremblay}, G.~R. and {Turner}, J.~E.~H. and {Terr{\'o}n}, V. and
+ {van Kerkwijk}, M.~H. and {de la Vega}, A. and {Watkins}, L.~L. and
+ {Weaver}, B.~A. and {Whitmore}, J.~B. and {Woillez}, J. and
+ {Zabalza}, V. and {Astropy Contributors}},
+ title = "{The Astropy Project: Building an Open-science Project and Status of the v2.0 Core Package}",
+ journal = {AJ},
+archivePrefix = "arXiv",
+ eprint = {1801.02634},
+ primaryClass = "astro-ph.IM",
+ keywords = {methods: data analysis, methods: miscellaneous, methods: statistical, reference systems },
+ year = 2018,
+ month = sep,
+ volume = 156,
+ eid = {123},
+ pages = {123},
+ doi = {10.3847/1538-3881/aabc4f},
+ adsurl = {http://adsabs.harvard.edu/abs/2018AJ....156..123A},
+ adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}
+
+
+
+
+
+@ARTICLE{astropy2013,
+ author = {{Astropy Collaboration} and {Robitaille}, T.~P. and {Tollerud}, E.~J. and
+ {Greenfield}, P. and {Droettboom}, M. and {Bray}, E. and {Aldcroft}, T. and
+ {Davis}, M. and {Ginsburg}, A. and {Price-Whelan}, A.~M. and
+ {Kerzendorf}, W.~E. and {Conley}, A. and {Crighton}, N. and
+ {Barbary}, K. and {Muna}, D. and {Ferguson}, H. and {Grollier}, F. and
+ {Parikh}, M.~M. and {Nair}, P.~H. and {Unther}, H.~M. and {Deil}, C. and
+ {Woillez}, J. and {Conseil}, S. and {Kramer}, R. and {Turner}, J.~E.~H. and
+ {Singer}, L. and {Fox}, R. and {Weaver}, B.~A. and {Zabalza}, V. and
+ {Edwards}, Z.~I. and {Azalee Bostroem}, K. and {Burke}, D.~J. and
+ {Casey}, A.~R. and {Crawford}, S.~M. and {Dencheva}, N. and
+ {Ely}, J. and {Jenness}, T. and {Labrie}, K. and {Lim}, P.~L. and
+ {Pierfederici}, F. and {Pontzen}, A. and {Ptak}, A. and {Refsdal}, B. and
+ {Servillat}, M. and {Streicher}, O.},
+ title = "{Astropy: A community Python package for astronomy}",
+ journal = {A\&A},
+archivePrefix = "arXiv",
+ eprint = {1307.6212},
+ primaryClass = "astro-ph.IM",
+ keywords = {methods: data analysis, methods: miscellaneous, virtual observatory tools},
+ year = 2013,
+ month = oct,
+ volume = 558,
+ eid = {A33},
+ pages = {A33},
+ doi = {10.1051/0004-6361/201322068},
+ adsurl = {http://adsabs.harvard.edu/abs/2013A\%26A...558A..33A},
+ adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}
diff --git a/reproduce/software/bibtex/cython.tex b/reproduce/software/bibtex/cython.tex
new file mode 100644
index 0000000..110cb19
--- /dev/null
+++ b/reproduce/software/bibtex/cython.tex
@@ -0,0 +1,21 @@
+%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com>
+%%
+%% Copying and distribution of this file, with or without modification,
+%% are permitted in any medium without royalty provided the copyright
+%% notice and this notice are preserved. This file is offered as-is,
+%% without any warranty.
+
+@ARTICLE{cython2011,
+ author = {{Behnel}, S. and {Bradshaw}, R. and {Citro}, C. and {Dalcin}, L. and
+ {Seljebotn}, D.~S. and {Smith}, K.},
+ title = "{Cython: The Best of Both Worlds}",
+ journal = {CiSE},
+ year = 2011,
+ month = mar,
+ volume = 13,
+ number = 2,
+ pages = {31},
+ doi = {10.1109/MCSE.2010.118},
+ adsurl = {http://adsabs.harvard.edu/abs/2011CSE....13b..31B},
+ adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}
diff --git a/reproduce/software/bibtex/fftw.tex b/reproduce/software/bibtex/fftw.tex
new file mode 100644
index 0000000..60c24a8
--- /dev/null
+++ b/reproduce/software/bibtex/fftw.tex
@@ -0,0 +1,19 @@
+%% Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org>
+%%
+%% Copying and distribution of this file, with or without modification,
+%% are permitted in any medium without royalty provided the copyright
+%% notice and this notice are preserved. This file is offered as-is,
+%% without any warranty.
+
+@ARTICLE{fftw,
+ author = {M. {Frigo} and S. G. {Johnson}},
+ journal = {IEEE Proc.},
+ title = {The Design and Implementation of FFTW3},
+ year = {2005},
+ volume = {93},
+ number = {2},
+ pages = {216},
+ doi = {10.1109/JPROC.2004.840301},
+ ISSN = {0018-9219},
+ month = {Feb},
+}
diff --git a/reproduce/software/bibtex/gnuastro.tex b/reproduce/software/bibtex/gnuastro.tex
new file mode 100644
index 0000000..91788c9
--- /dev/null
+++ b/reproduce/software/bibtex/gnuastro.tex
@@ -0,0 +1,24 @@
+%% Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org>
+%%
+%% Copying and distribution of this file, with or without modification,
+%% are permitted in any medium without royalty provided the copyright
+%% notice and this notice are preserved. This file is offered as-is,
+%% without any warranty.
+
+@ARTICLE{gnuastro,
+ author = {{Akhlaghi}, M. and {Ichikawa}, T.},
+ title = "{Noise-based Detection and Segmentation of Nebulous Objects}",
+ journal = {ApJS},
+archivePrefix = "arXiv",
+ eprint = {1505.01664},
+ primaryClass = "astro-ph.IM",
+ keywords = {galaxies: irregular, galaxies: photometry, galaxies: structure, methods: data analysis, techniques: image processing, techniques: photometric},
+ year = 2015,
+ month = sep,
+ volume = 220,
+ eid = {1},
+ pages = {1},
+ doi = {10.1088/0067-0049/220/1/1},
+ adsurl = {http://adsabs.harvard.edu/abs/2015ApJS..220....1A},
+ adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}
diff --git a/reproduce/software/bibtex/matplotlib.tex b/reproduce/software/bibtex/matplotlib.tex
new file mode 100644
index 0000000..8578899
--- /dev/null
+++ b/reproduce/software/bibtex/matplotlib.tex
@@ -0,0 +1,22 @@
+%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com>
+%%
+%% Copying and distribution of this file, with or without modification,
+%% are permitted in any medium without royalty provided the copyright
+%% notice and this notice are preserved. This file is offered as-is,
+%% without any warranty.
+
+@Article{matplotlib2007,
+ Author = {Hunter, J. D.},
+ Title = {Matplotlib: A 2D graphics environment},
+ Journal = {CiSE},
+ Volume = {9},
+ Number = {3},
+ Pages = {90},
+ abstract = {Matplotlib is a 2D graphics package used for Python
+ for application development, interactive scripting, and
+ publication-quality image generation across user
+ interfaces and operating systems.},
+ publisher = {IEEE COMPUTER SOC},
+ doi = {10.1109/MCSE.2007.55},
+ year = 2007
+}
diff --git a/reproduce/software/bibtex/mpi4py.tex b/reproduce/software/bibtex/mpi4py.tex
new file mode 100644
index 0000000..4e0045f
--- /dev/null
+++ b/reproduce/software/bibtex/mpi4py.tex
@@ -0,0 +1,20 @@
+%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com>
+%%
+%% Copying and distribution of this file, with or without modification,
+%% are permitted in any medium without royalty provided the copyright
+%% notice and this notice are preserved. This file is offered as-is,
+%% without any warranty.
+
+@ARTICLE{mpi4py2011,
+ author = {{Dalcin}, L.~D. and {Paz}, R.~R. and {Kler}, P.~A. and {Cosimo}, A.
+ },
+ title = "{Parallel distributed computing using Python}",
+ journal = {Advances in Water Resources},
+ year = 2011,
+ month = sep,
+ volume = 34,
+ pages = {1124},
+ doi = {10.1016/j.advwatres.2011.04.013},
+ adsurl = {http://adsabs.harvard.edu/abs/2011AdWR...34.1124D},
+ adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}
diff --git a/reproduce/software/bibtex/numpy.tex b/reproduce/software/bibtex/numpy.tex
new file mode 100644
index 0000000..0d9028f
--- /dev/null
+++ b/reproduce/software/bibtex/numpy.tex
@@ -0,0 +1,23 @@
+%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com>
+%%
+%% Copying and distribution of this file, with or without modification,
+%% are permitted in any medium without royalty provided the copyright
+%% notice and this notice are preserved. This file is offered as-is,
+%% without any warranty.
+
+@ARTICLE{numpy2011,
+ author = {{van der Walt}, S. and {Colbert}, S.~C. and {Varoquaux}, G.},
+ title = "{The NumPy Array: A Structure for Efficient Numerical Computation}",
+ journal = {CiSE},
+archivePrefix = "arXiv",
+ eprint = {1102.1523},
+ primaryClass = "cs.MS",
+ year = 2011,
+ month = mar,
+ volume = 13,
+ number = 2,
+ pages = {22},
+ doi = {10.1109/MCSE.2011.37},
+ adsurl = {http://adsabs.harvard.edu/abs/2011CSE....13b..22V},
+ adsnote = {Provided by the SAO/NASA Astrophysics Data System}
+}
diff --git a/reproduce/software/bibtex/scipy.tex b/reproduce/software/bibtex/scipy.tex
new file mode 100644
index 0000000..60500d2
--- /dev/null
+++ b/reproduce/software/bibtex/scipy.tex
@@ -0,0 +1,34 @@
+%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com>
+%%
+%% Copying and distribution of this file, with or without modification,
+%% are permitted in any medium without royalty provided the copyright
+%% notice and this notice are preserved. This file is offered as-is,
+%% without any warranty.
+
+@ARTICLE{scipy2011,
+ author = {K. J. {Millman} and M. {Aivazis}},
+ journal = {CiSE},
+ title = {Python for Scientists and Engineers},
+ year = {2011},
+ volume = {13},
+ number = {2},
+ pages = {9},
+ keywords = {Special issues and sections;Computer languages;Programming;Scientific computing;Numerical models;Programming languages;Python;Scientific computing;interactive research;Python libraries;Python tools},
+ doi = {10.1109/MCSE.2011.36},
+ ISSN = {1521-9615},
+ month ={March},
+}
+
+@ARTICLE{scipy2007,
+ author = {T. E. {Oliphant}},
+ journal = {CiSE},
+ title = {Python for Scientific Computing},
+ year = {2007},
+ volume = {9},
+ number = {3},
+ pages = {10},
+ keywords = {high level languages;Python;scientific computing;steering language;scientific codes;high-level language;Scientific computing;High level languages;Libraries;Writing;Application software;Embedded software;Software standards;Standards development;Internet;Prototypes;Python;computer languages;scientific programming;scientific computing},
+ doi = {10.1109/MCSE.2007.58},
+ ISSN = {1521-9615},
+ month = {May},
+}
diff --git a/reproduce/config/gnuastro/astbuildprog.conf b/reproduce/software/config/gnuastro/astbuildprog.conf
index c15baf1..c15baf1 100644
--- a/reproduce/config/gnuastro/astbuildprog.conf
+++ b/reproduce/software/config/gnuastro/astbuildprog.conf
diff --git a/reproduce/config/gnuastro/astconvertt.conf b/reproduce/software/config/gnuastro/astconvertt.conf
index 6466455..6466455 100644
--- a/reproduce/config/gnuastro/astconvertt.conf
+++ b/reproduce/software/config/gnuastro/astconvertt.conf
diff --git a/reproduce/config/gnuastro/aststatistics.conf b/reproduce/software/config/gnuastro/aststatistics.conf
index 1d13e3f..1d13e3f 100644
--- a/reproduce/config/gnuastro/aststatistics.conf
+++ b/reproduce/software/config/gnuastro/aststatistics.conf
diff --git a/reproduce/config/gnuastro/gnuastro.conf b/reproduce/software/config/gnuastro/gnuastro.conf
index 57fcadc..57fcadc 100644
--- a/reproduce/config/gnuastro/gnuastro.conf
+++ b/reproduce/software/config/gnuastro/gnuastro.conf
diff --git a/reproduce/config/pipeline/LOCAL.mk.in b/reproduce/software/config/installation/LOCAL.mk.in
index 785bb6a..785bb6a 100644
--- a/reproduce/config/pipeline/LOCAL.mk.in
+++ b/reproduce/software/config/installation/LOCAL.mk.in
diff --git a/reproduce/config/pipeline/dependency-numpy-scipy.cfg b/reproduce/software/config/installation/numpy-scipy.cfg
index 4b7a7b0..4b7a7b0 100644
--- a/reproduce/config/pipeline/dependency-numpy-scipy.cfg
+++ b/reproduce/software/config/installation/numpy-scipy.cfg
diff --git a/reproduce/config/pipeline/texlive.conf b/reproduce/software/config/installation/texlive.conf
index 53054e1..53054e1 100644
--- a/reproduce/config/pipeline/texlive.conf
+++ b/reproduce/software/config/installation/texlive.conf
diff --git a/reproduce/config/pipeline/dependency-texlive.mk b/reproduce/software/config/installation/texlive.mk
index 0acf5ab..0acf5ab 100644
--- a/reproduce/config/pipeline/dependency-texlive.mk
+++ b/reproduce/software/config/installation/texlive.mk
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/software/config/installation/versions.mk
index c6c8043..149f9ae 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/software/config/installation/versions.mk
@@ -21,8 +21,8 @@ atlas-version = 3.10.3
bash-version = 5.0
binutils-version = 2.31.1
cfitsio-version = 3.45
-cmake-version = 3.12.4
-coreutils-version = 8.30
+cmake-version = 3.14.2
+coreutils-version = 8.31
curl-version = 7.63.0
diffutils-version = 3.7
fftw-version = 3.3.8
@@ -30,10 +30,10 @@ file-version = 5.36
findutils-version = 4.6.0.199-e3fc
flock-version = 0.2.3
freetype-version = 2.9
-gawk-version = 4.2.1
+gawk-version = 5.0.0
gcc-version = 8.3.0
ghostscript-version = 9.26
-git-version = 2.20.1
+git-version = 2.21.0
gmp-version = 6.1.2
gnuastro-version = 0.8
grep-version = 3.3
@@ -59,12 +59,12 @@ openmpi-version = 4.0.1
openssl-version = 1.1.1a
patchelf-version = 0.9
pkgconfig-version = 0.29.2
-python-version = 3.6.8
+python-version = 3.7.3
readline-version = 8.0
sed-version = 4.7
-tar-version = 1.31
+tar-version = 1.32
unzip-version = 6.0
-wget-version = 1.20.1
+wget-version = 1.20.3
which-version = 2.21
xz-version = 5.2.4
zip-version = 3.0
diff --git a/reproduce/src/make/dependencies-atlas-multiple.mk b/reproduce/software/make/atlas-multiple.mk
index fef25c7..fef25c7 100755
--- a/reproduce/src/make/dependencies-atlas-multiple.mk
+++ b/reproduce/software/make/atlas-multiple.mk
diff --git a/reproduce/src/make/dependencies-atlas-single.mk b/reproduce/software/make/atlas-single.mk
index dde2926..dde2926 100755
--- a/reproduce/src/make/dependencies-atlas-single.mk
+++ b/reproduce/software/make/atlas-single.mk
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/software/make/basic.mk
index b56d01d..cfa05d2 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/software/make/basic.mk
@@ -1,20 +1,17 @@
-# Build the VERY BASIC project dependencies before everything else assuming
+# Build the VERY BASIC project software before higher-level ones. Assuming
# minimal/generic Make and Shell.
#
# ------------------------------------------------------------------------
# !!!!! IMPORTANT NOTES !!!!!
#
# This Makefile will be run by the initial `./configure' script. It is not
-# included into the reproduction pipe after that.
+# included into the project after that.
#
-# This Makefile builds very low-level and basic tools like GNU Tar, and
-# various compression programs, GNU Bash, and GNU Make. Therefore this is
-# the only Makefile in the reproduction pipeline where you MUST NOT assume
-# that modern GNU Bash or GNU Make are used. After this Makefile, other
-# Makefiles can safely assume the fixed version of all these software.
-#
-# This Makefile is a very simplified version of `dependencies.mk' in the
-# same directory. See there for more comments.
+# This Makefile builds very low-level and basic tools like GNU Tar, GNU
+# Bash, GNU Make, GCC and etc. Therefore this is the only Makefile in the
+# project where you CANNOT assume that GNU Bash or GNU Make are used. After
+# this Makefile (where GNU Bash and GNU Make are built), other Makefiles
+# can safely assume the fixed version of all these software.
#
# ------------------------------------------------------------------------
#
@@ -35,18 +32,18 @@
# Top level environment
-include reproduce/config/pipeline/LOCAL.mk
-include reproduce/src/make/dependencies-build-rules.mk
-include reproduce/config/pipeline/dependency-versions.mk
+include reproduce/software/make/build-rules.mk
+include reproduce/software/config/installation/LOCAL.mk
+include reproduce/software/config/installation/versions.mk
lockdir = $(BDIR)/locks
-ddir = $(BDIR)/dependencies
-tdir = $(BDIR)/dependencies/tarballs
-idir = $(BDIR)/dependencies/installed
-ibdir = $(BDIR)/dependencies/installed/bin
-ildir = $(BDIR)/dependencies/installed/lib
-ibidir = $(BDIR)/dependencies/installed/version-info/bin
-ilidir = $(BDIR)/dependencies/installed/version-info/lib
+tdir = $(BDIR)/software/tarballs
+ddir = $(BDIR)/software/build-tmp
+idir = $(BDIR)/software/installed
+ibdir = $(BDIR)/software/installed/bin
+ildir = $(BDIR)/software/installed/lib
+ibidir = $(BDIR)/software/installed/version-info/bin
+ilidir = $(BDIR)/software/installed/version-info/lib
# We'll need the system's PATH for making links to low-level programs we
# won't be building ourselves.
@@ -58,11 +55,11 @@ export CCACHE_DISABLE := 1
export PATH := $(ibdir):$(PATH)
export PKG_CONFIG_PATH := $(ildir)/pkgconfig
export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig
-export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS)
-export CPPFLAGS := -I$(idir)/include $(CPPFLAGS)
export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH)
+export CPPFLAGS := -I$(idir)/include $(CPPFLAGS)
+export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS)
-# Define the programs that don't depend on any other.
+# Define the top-level programs (that don't depend on any other).
top-level-programs = low-level-links wget gcc
all: $(foreach p, $(top-level-programs), $(ibidir)/$(p))
@@ -73,10 +70,10 @@ all: $(foreach p, $(top-level-programs), $(ibidir)/$(p))
# Tarballs
# --------
#
-# Prepare tarballs. Difference with that in `dependencies.mk': `.ONESHELL'
-# is not recognized by some versions of Make (even older GNU Makes). So
-# we'll have to make sure the recipe doesn't break into multiple shell
-# calls (so we can preserve the variables).
+# Prepare tarballs. Difference with that in `high-level.mk': `.ONESHELL' is
+# not recognized by some versions of Make (even older GNU Makes). So we'll
+# have to make sure the recipe doesn't break into multiple shell calls (so
+# we can preserve the variables).
#
# Software hosted at akhlaghi.org/src: As of our latest check (November
# 2018) their major release tarballs either crash or don't build on some
@@ -93,7 +90,7 @@ all: $(foreach p, $(top-level-programs), $(ibidir)/$(p))
# However, downloading from this link is slow (because its just a link). So
# its easier to just keep a with the others.
$(lockdir): | $(BDIR); mkdir $@
-downloadwrapper = ./reproduce/src/bash/download-multi-try
+downloadwrapper = ./reproduce/analysis/bash/download-multi-try
tarballs = $(foreach t, bash-$(bash-version).tar.gz \
binutils-$(binutils-version).tar.lz \
bzip2-$(bzip2-version).tar.gz \
@@ -121,9 +118,11 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \
readline-$(readline-version).tar.gz \
sed-$(sed-version).tar.xz \
tar-$(tar-version).tar.gz \
+ unzip-$(unzip-version).tar.gz \
wget-$(wget-version).tar.lz \
which-$(which-version).tar.gz \
xz-$(xz-version).tar.gz \
+ zip-$(zip-version).tar.gz \
zlib-$(zlib-version).tar.gz \
, $(tdir)/$(t) )
$(tarballs): $(tdir)/%: | $(lockdir)
@@ -164,9 +163,15 @@ $(tarballs): $(tdir)/%: | $(lockdir)
elif [ $$n = readline ]; then w=http://ftp.gnu.org/gnu/readline; \
elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed; \
elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \
+ elif [ $$n = unzip ]; then \
+ mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//'); \
+ w=ftp://ftp.info-zip.org/pub/infozip/src/unzip$$v.tgz; \
elif [ $$n = wget ]; then w=http://ftp.gnu.org/gnu/wget; \
elif [ $$n = which ]; then w=http://ftp.gnu.org/gnu/which; \
elif [ $$n = xz ]; then w=http://tukaani.org/xz; \
+ elif [ $$n = zip ]; then \
+ mergenames=0; v=$$(echo $(zip-version) | sed -e's/\.//'); \
+ w=ftp://ftp.info-zip.org/pub/infozip/src/zip$$v.tgz; \
elif [ $$n = zlib ]; then w=http://www.zlib.net; \
else \
echo; echo; echo; \
@@ -178,7 +183,7 @@ $(tarballs): $(tdir)/%: | $(lockdir)
if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*"; \
else tarballurl=$$w; \
fi; \
- \
+ \
echo "Downloading $$tarballurl"; \
if [ -f $(ibdir)/wget ]; then \
downloader="wget --no-use-server-timestamps -O"; \
@@ -344,18 +349,51 @@ $(ibidir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
&& ln -fs libbz2.so.1.0 libbz2.so \
&& echo "Bzip2 $(bzip2-version)" > $@
+$(ibidir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz
+ v=$$(echo $(unzip-version) | sed -e's/\.//'); \
+ $(call gbuild, $<, unzip$$v, static,, \
+ -f unix/Makefile generic_gcc \
+ CFLAGS="-DBIG_MEM -DMMAP",,pwd, \
+ -f unix/Makefile \
+ BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \
+ && echo "Unzip $(unzip-version)" > $@
+
+$(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz
+ v=$$(echo $(zip-version) | sed -e's/\.//'); \
+ $(call gbuild, $<, zip$$v, static,, \
+ -f unix/Makefile generic_gcc \
+ CFLAGS="-DBIG_MEM -DMMAP",,pwd, \
+ -f unix/Makefile \
+ BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \
+ && echo "Zip $(zip-version)" > $@
+
+# Some programs (like Wget and CMake) that use zlib need it to be dynamic
+# so they use our custom build. So we won't force a static-only build.
+#
+# Note for a static-only build: Zlib's `./configure' doesn't use Autoconf's
+# configure script, it just accepts a direct `--static' option.
+$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz
+ $(call gbuild, $<, zlib-$(zlib-version)) \
+ && echo "Zlib $(zlib-version)" > $@
+
# GNU Tar: When built statically, tar gives a segmentation fault on
-# unpacking Bash. So we'll build it dynamically.
+# unpacking Bash. So we'll build it dynamically. Note that technically, zip
+# and unzip aren't dependencies of Tar, but for a clean build, we'll set
+# Tar to be the last compression-related software (the first-set of
+# software to be built).
$(ibidir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
$(ibidir)/bzip2 \
- $(ibidir)/lzip \
+ $(ibidir)/unzip \
$(ibidir)/gzip \
+ $(ibidir)/lzip \
+ $(ilidir)/zlib \
+ $(ibidir)/zip \
$(ibidir)/xz
# Since all later programs depend on Tar, the configuration will be
# stuck here, only making Tar. So its more efficient to built it on
# multiple threads (when the user's Make doesn't pass down the
# number of threads).
- $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads)) \
+ $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads) V=1) \
&& echo "GNU Tar $(tar-version)" > $@
@@ -555,22 +593,11 @@ $(ibidir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
# Downloader
# ----------
#
-# Some programs (like Wget and CMake) that use zlib need it to be dynamic
-# so they use our custom build. So we won't force a static-only build.
-#
-# Note for a static-only build: Zlib's `./configure' doesn't use Autoconf's
-# configure script, it just accepts a direct `--static' option.
-$(idir)/etc:; mkdir $@
-$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \
- $(ibidir)/bash
- $(call gbuild, $<, zlib-$(zlib-version)) \
- && echo "Zlib $(zlib-version)" > $@
-
# OpenSSL: Some programs/libraries later need dynamic linking. So we'll
# build libssl (and libcrypto) dynamically also.
#
# Until we find a nice and generic way to create an updated CA file in the
-# project, the certificates will be available in a file for this pipeline
+# project, the certificates will be available in a file for this project
# along with the other tarballs.
#
# In case you do want a static OpenSSL and libcrypto, then uncomment the
@@ -579,9 +606,10 @@ $(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \
#ifeq ($(static_build),yes)
#openssl-static = no-dso no-dynamic-engine no-shared
#endif
+$(idir)/etc:; mkdir $@
$(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \
$(tdir)/cert.pem \
- $(ilidir)/zlib | $(idir)/etc
+ $(ibidir)/bash | $(idir)/etc
# According to OpenSSL's Wiki (link bellow), it can't automatically
# detect Mac OS's structure. It will need some help. So we'll use
# the `on_mac_os' Make variable that we defined in the configure
@@ -763,13 +791,15 @@ $(ibidir)/which: $(tdir)/which-$(which-version).tar.gz \
# Binutils' linker `ld' is apparently only good for GNU/Linux systems and
# other OSs have their own. So for now we aren't actually building
# Binutils (`ld' isn't a prerequisite of GCC).
-$(ibidir)/binutils: $(tdir)/binutils-$(binutils-version).tar.lz
+$(ibidir)/binutils: $(tdir)/binutils-$(binutils-version).tar.lz \
+ $(ibidir)/bash
$(call gbuild, $<, binutils-$(binutils-version), static) \
&& echo "GNU Binutils $(binutils-version)" > $@
# `file' is not a prerequisite of GCC. However, since it is low level, it is
# set as a prerequisite of GCC to have it installed.
-$(ibidir)/file: $(tdir)/file-$(file-version).tar.gz
+$(ibidir)/file: $(tdir)/file-$(file-version).tar.gz \
+ $(ibidir)/bash
$(call gbuild, $<, file-$(file-version), static) \
&& echo "File $(file-version)" > $@
diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/software/make/build-rules.mk
index a8c8731..a8c8731 100644
--- a/reproduce/src/make/dependencies-build-rules.mk
+++ b/reproduce/software/make/build-rules.mk
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/software/make/high-level.mk
index fd9bffa..f7b1f4f 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/software/make/high-level.mk
@@ -26,23 +26,23 @@
# Top level environment
-include reproduce/config/pipeline/LOCAL.mk
-include reproduce/src/make/dependencies-build-rules.mk
-include reproduce/config/pipeline/dependency-texlive.mk
-include reproduce/config/pipeline/dependency-versions.mk
+include reproduce/software/make/build-rules.mk
+include reproduce/software/config/installation/LOCAL.mk
+include reproduce/software/config/installation/texlive.mk
+include reproduce/software/config/installation/versions.mk
lockdir = $(BDIR)/locks
-ddir = $(BDIR)/dependencies
-dtexdir = $(shell pwd)/tex/dependencies
-tdir = $(BDIR)/dependencies/tarballs
-idir = $(BDIR)/dependencies/installed
-ibdir = $(BDIR)/dependencies/installed/bin
-ildir = $(BDIR)/dependencies/installed/lib
-ibidir = $(BDIR)/dependencies/installed/version-info/bin
-ilidir = $(BDIR)/dependencies/installed/version-info/lib
-itidir = $(BDIR)/dependencies/installed/version-info/tex
-ictdir = $(BDIR)/dependencies/installed/version-info/cite
-ipydir = $(BDIR)/dependencies/installed/version-info/python
+tdir = $(BDIR)/software/tarballs
+ddir = $(BDIR)/software/build-tmp
+idir = $(BDIR)/software/installed
+ibdir = $(BDIR)/software/installed/bin
+ildir = $(BDIR)/software/installed/lib
+dtexdir = $(shell pwd)/reproduce/software/bibtex
+ibidir = $(BDIR)/software/installed/version-info/bin
+ilidir = $(BDIR)/software/installed/version-info/lib
+itidir = $(BDIR)/software/installed/version-info/tex
+ictdir = $(BDIR)/software/installed/version-info/cite
+ipydir = $(BDIR)/software/installed/version-info/python
# Define the top-level programs to build (installed in `.local/bin').
#
@@ -51,8 +51,8 @@ ipydir = $(BDIR)/dependencies/installed/version-info/python
# successfully on Mac (only static) and GNU/Linux (shared and static). But,
# since it takes a few hours to build, it is not currently a target.
top-level-libraries = # atlas
-top-level-programs = gnuastro metastore unzip zip
top-level-python = astroquery matplotlib
+top-level-programs = gnuastro metastore
all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \
$(foreach p, $(top-level-programs), $(ibidir)/$(p)) \
$(foreach p, $(top-level-python), $(ipydir)/$(p)) \
@@ -87,15 +87,21 @@ export LDFLAGS := $(rpath_command) -L$(ildir)
# We want the download to happen on a single thread. So we need to define a
# lock, and call a special script we have written for this job. These are
-# placed here because we want them both in the `dependencies.mk' and
-# `dependencies-python.mk'.
+# placed here because we want them both in the `high-level.mk' and
+# `python.mk'.
$(lockdir): | $(BDIR); mkdir $@
downloader="wget --no-use-server-timestamps -O";
-downloadwrapper = ./reproduce/src/bash/download-multi-try
+downloadwrapper = ./reproduce/analysis/bash/download-multi-try
+
+
+
# Python packages
-include reproduce/src/make/dependencies-python.mk
+include reproduce/software/make/python.mk
+
+
+
# Tarballs
@@ -127,11 +133,9 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
libgit2-$(libgit2-version).tar.gz \
metastore-$(metastore-version).tar.gz \
openmpi-$(openmpi-version).tar.gz \
- unzip-$(unzip-version).tar.gz \
openblas-$(openblas-version).tar.gz \
tiff-$(libtiff-version).tar.gz \
wcslib-$(wcslib-version).tar.bz2 \
- zip-$(zip-version).tar.gz \
, $(tdir)/$(t) )
$(tarballs): $(tdir)/%: | $(lockdir)
if [ -f $(DEPENDENCIES-DIR)/$* ]; then
@@ -156,7 +160,12 @@ $(tarballs): $(tdir)/%: | $(lockdir)
elif [ $$n = atlas ]; then
mergenames=0
w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download
- elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12
+ elif [ $$n = cmake ]; then
+ mergenames=0
+ majv=$$(echo $(cmake-version) \
+ | sed -e's/\./ /' \
+ | awk '{printf("%d.%d", $$1, $$2)}')
+ w=https://cmake.org/files/v$$majv/cmake-$(cmake-version).tar.gz
elif [ $$n = curl ]; then w=https://curl.haxx.se/download
elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw
elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype
@@ -185,13 +194,7 @@ $(tarballs): $(tdir)/%: | $(lockdir)
majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}')
w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$*
elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff
- elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src
- mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//')
- w=ftp://ftp.info-zip.org/pub/infozip/src/unzip$$v.tgz
elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib
- elif [ $$n = zip ]; then
- mergenames=0; v=$$(echo $(zip-version) | sed -e's/\.//')
- w=ftp://ftp.info-zip.org/pub/infozip/src/zip$$v.tgz
else
echo; echo; echo;
echo "'$$n' not recognized as a dependency name to download."
@@ -469,19 +472,20 @@ $(ibidir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
#
# On Mac systems, the build complains about `clang' specific
# features, so we can't use our own GCC build here.
- if [ x$(on_mac_os) = xyes ]; then \
- export CC=clang; \
- export CXX=clang++; \
- fi; \
- cd $(ddir) \
- && rm -rf cmake-$(cmake-version) \
- && tar xf $< && cd cmake-$(cmake-version) \
- && ./bootstrap --prefix=$(idir) --system-curl --system-zlib\
- --system-bzip2 --system-liblzma --no-qt-gui \
- && make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 \
- && make install \
- && cd .. \
- && rm -rf cmake-$(cmake-version) \
+ if [ x$(on_mac_os) = xyes ]; then \
+ export CC=clang; \
+ export CXX=clang++; \
+ fi; \
+ cd $(ddir) \
+ && rm -rf cmake-$(cmake-version) \
+ && tar xf $< \
+ && cd cmake-$(cmake-version) \
+ && ./bootstrap --prefix=$(idir) --system-curl --system-zlib \
+ --system-bzip2 --system-liblzma --no-qt-gui \
+ && make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 \
+ && make install \
+ && cd .. \
+ && rm -rf cmake-$(cmake-version) \
&& echo "CMake $(cmake-version)" > $@
# cURL (and its library, which is needed by several programs here) can
@@ -573,7 +577,7 @@ $(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
-e's|@GROUP[@]|'$$group'|g' \
-e's|@BINDIR[@]|$(ibdir)|g' \
-e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \
- reproduce/src/bash/git-$$f > .git/hooks/$$f
+ reproduce/software/bash/git-$$f > .git/hooks/$$f
chmod +x .git/hooks/$$f
echo "Metastore (forked) $(metastore-version)" > $@
done
@@ -609,23 +613,20 @@ endif
&& cp $(dtexdir)/gnuastro.tex $(ictdir)/ \
&& echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@
-$(ibidir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz
- v=$$(echo $(unzip-version) | sed -e's/\.//')
- $(call gbuild, $<, unzip$$v, static,, \
- -f unix/Makefile generic_gcc \
- CFLAGS="-DBIG_MEM -DMMAP",,pwd, \
- -f unix/Makefile \
- BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \
- && echo "Unzip $(unzip-version)" > $@
-$(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz
- v=$$(echo $(zip-version) | sed -e's/\.//')
- $(call gbuild, $<, zip$$v, static,, \
- -f unix/Makefile generic_gcc \
- CFLAGS="-DBIG_MEM -DMMAP",,pwd, \
- -f unix/Makefile \
- BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \
- && echo "Zip $(zip-version)" > $@
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -639,7 +640,7 @@ $(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz
# `texlive-ready-tlmgr' and use its contents to mark if we can use it or
# not.
$(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \
- reproduce/config/pipeline/texlive.conf
+ reproduce/software/config/installation/texlive.conf
# Unpack, enter the directory, and install based on the given
# configuration (prerequisite of this rule).
@@ -649,7 +650,8 @@ $(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \
tar xf $(tdir)/install-tl-unx.tar.gz
cd install-tl-*
sed -e's|@installdir[@]|$(idir)|g' \
- $$topdir/reproduce/config/pipeline/texlive.conf > texlive.conf
+ $$topdir/reproduce/software/config/installation/texlive.conf \
+ > texlive.conf
# TeX Live's installation may fail due to any reason. But TeX Live
# is optional (only necessary for building the final PDF). So we
@@ -680,7 +682,7 @@ $(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \
# To keep things modular and simple, we'll break up the installation of TeX
# Live itself (only very basic TeX and LaTeX) and the installation of its
# necessary packages into two packages.
-$(itidir)/texlive: reproduce/config/pipeline/dependency-texlive.mk \
+$(itidir)/texlive: reproduce/software/config/installation/texlive.mk \
$(itidir)/texlive-ready-tlmgr
# To work with TeX live installation, we'll need the internet.
diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/software/make/python.mk
index 837b0ad..5ec99f0 100644
--- a/reproduce/src/make/dependencies-python.mk
+++ b/reproduce/software/make/python.mk
@@ -415,7 +415,7 @@ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \
else \
export LDFLAGS="$(LDFLAGS) -shared"; \
fi; \
- conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
+ conf="$$(pwd)/reproduce/software/config/installation/numpy-scipy.cfg"; \
$(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf, \
Numpy $(numpy-version)) \
&& cp $(dtexdir)/numpy.tex $(ictdir)/ \
@@ -463,7 +463,7 @@ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \
else \
export LDFLAGS="$(LDFLAGS) -shared"; \
fi; \
- conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \
+ conf="$$(pwd)/reproduce/software/config/installation/numpy-scipy.cfg"; \
$(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf) \
&& cp $(dtexdir)/scipy.tex $(ictdir)/ \
&& echo "Scipy $(scipy-version) \citep{scipy2007,scipy2011}" > $@