diff options
Diffstat (limited to 'reproduce')
| -rwxr-xr-x | reproduce/analysis/bash/download-multi-try (renamed from reproduce/src/bash/download-multi-try) | 0 | ||||
| -rw-r--r-- | reproduce/analysis/config/INPUTS.mk (renamed from reproduce/config/pipeline/INPUTS.mk) | 0 | ||||
| -rw-r--r-- | reproduce/analysis/config/delete-me-num.mk (renamed from reproduce/config/pipeline/delete-me-num.mk) | 0 | ||||
| -rw-r--r-- | reproduce/analysis/config/pdf-build.mk (renamed from reproduce/config/pipeline/pdf-build.mk) | 0 | ||||
| -rw-r--r-- | reproduce/analysis/make/delete-me.mk (renamed from reproduce/src/make/delete-me.mk) | 2 | ||||
| -rw-r--r-- | reproduce/analysis/make/download.mk (renamed from reproduce/src/make/download.mk) | 2 | ||||
| -rw-r--r-- | reproduce/analysis/make/initialize.mk (renamed from reproduce/src/make/initialize.mk) | 26 | ||||
| -rw-r--r-- | reproduce/analysis/make/paper.mk (renamed from reproduce/src/make/paper.mk) | 39 | ||||
| -rw-r--r-- | reproduce/analysis/make/top.mk (renamed from reproduce/src/make/top.mk) | 27 | ||||
| -rw-r--r-- | reproduce/software/bash/git-post-checkout (renamed from reproduce/src/bash/git-post-checkout) | 0 | ||||
| -rw-r--r-- | reproduce/software/bash/git-pre-commit (renamed from reproduce/src/bash/git-pre-commit) | 0 | ||||
| -rw-r--r-- | reproduce/software/bibtex/astropy.tex | 97 | ||||
| -rw-r--r-- | reproduce/software/bibtex/cython.tex | 21 | ||||
| -rw-r--r-- | reproduce/software/bibtex/fftw.tex | 19 | ||||
| -rw-r--r-- | reproduce/software/bibtex/gnuastro.tex | 24 | ||||
| -rw-r--r-- | reproduce/software/bibtex/matplotlib.tex | 22 | ||||
| -rw-r--r-- | reproduce/software/bibtex/mpi4py.tex | 20 | ||||
| -rw-r--r-- | reproduce/software/bibtex/numpy.tex | 23 | ||||
| -rw-r--r-- | reproduce/software/bibtex/scipy.tex | 34 | ||||
| -rw-r--r-- | reproduce/software/config/gnuastro/astbuildprog.conf (renamed from reproduce/config/gnuastro/astbuildprog.conf) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/gnuastro/astconvertt.conf (renamed from reproduce/config/gnuastro/astconvertt.conf) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/gnuastro/aststatistics.conf (renamed from reproduce/config/gnuastro/aststatistics.conf) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/gnuastro/gnuastro.conf (renamed from reproduce/config/gnuastro/gnuastro.conf) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/installation/LOCAL.mk.in (renamed from reproduce/config/pipeline/LOCAL.mk.in) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/installation/numpy-scipy.cfg (renamed from reproduce/config/pipeline/dependency-numpy-scipy.cfg) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/installation/texlive.conf (renamed from reproduce/config/pipeline/texlive.conf) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/installation/texlive.mk (renamed from reproduce/config/pipeline/dependency-texlive.mk) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/installation/versions.mk (renamed from reproduce/config/pipeline/dependency-versions.mk) | 14 | ||||
| -rwxr-xr-x | reproduce/software/make/atlas-multiple.mk (renamed from reproduce/src/make/dependencies-atlas-multiple.mk) | 0 | ||||
| -rwxr-xr-x | reproduce/software/make/atlas-single.mk (renamed from reproduce/src/make/dependencies-atlas-single.mk) | 0 | ||||
| -rw-r--r-- | reproduce/software/make/basic.mk (renamed from reproduce/src/make/dependencies-basic.mk) | 124 | ||||
| -rw-r--r-- | reproduce/software/make/build-rules.mk (renamed from reproduce/src/make/dependencies-build-rules.mk) | 0 | ||||
| -rw-r--r-- | reproduce/software/make/high-level.mk (renamed from reproduce/src/make/dependencies.mk) | 126 | ||||
| -rw-r--r-- | reproduce/software/make/python.mk (renamed from reproduce/src/make/dependencies-python.mk) | 4 |
34 files changed, 456 insertions, 168 deletions
diff --git a/reproduce/src/bash/download-multi-try b/reproduce/analysis/bash/download-multi-try index 1fd7497..1fd7497 100755 --- a/reproduce/src/bash/download-multi-try +++ b/reproduce/analysis/bash/download-multi-try diff --git a/reproduce/config/pipeline/INPUTS.mk b/reproduce/analysis/config/INPUTS.mk index eb38295..eb38295 100644 --- a/reproduce/config/pipeline/INPUTS.mk +++ b/reproduce/analysis/config/INPUTS.mk diff --git a/reproduce/config/pipeline/delete-me-num.mk b/reproduce/analysis/config/delete-me-num.mk index 17f608c..17f608c 100644 --- a/reproduce/config/pipeline/delete-me-num.mk +++ b/reproduce/analysis/config/delete-me-num.mk diff --git a/reproduce/config/pipeline/pdf-build.mk b/reproduce/analysis/config/pdf-build.mk index 3a86ff3..3a86ff3 100644 --- a/reproduce/config/pipeline/pdf-build.mk +++ b/reproduce/analysis/config/pdf-build.mk diff --git a/reproduce/src/make/delete-me.mk b/reproduce/analysis/make/delete-me.mk index 701a316..c4cfffe 100644 --- a/reproduce/src/make/delete-me.mk +++ b/reproduce/analysis/make/delete-me.mk @@ -82,7 +82,7 @@ $(histogram): $(dddemodir)/%-hist.txt: $(indir)/%.fits | $(dddemodir) # ---------------- # # This is just as a demonstration on how to get analysic configuration -# parameters from variables defined in `reproduce/config/pipeline'. +# parameters from variables defined in `reproduce/analysis/config/'. stats = $(dddemodir)/wfpc2-stats.txt $(stats): $(dddemodir)/%-stats.txt: $(indir)/%.fits | $(dddemodir) aststatistics $< -h0 --mean --median > $@ diff --git a/reproduce/src/make/download.mk b/reproduce/analysis/make/download.mk index dfc49da..a721863 100644 --- a/reproduce/src/make/download.mk +++ b/reproduce/analysis/make/download.mk @@ -48,7 +48,7 @@ # process with a file and make sure that only one downloading event is in # progress at every moment. $(indir):; mkdir $@ -downloadwrapper = $(srcdir)/bash/download-multi-try +downloadwrapper = $(bashdir)/download-multi-try inputdatasets = $(foreach i, wfpc2, $(indir)/$(i).fits) $(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir) diff --git a/reproduce/src/make/initialize.mk b/reproduce/analysis/make/initialize.mk index cd533f2..a034494 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -31,14 +31,14 @@ # be necessary to put a lock on them. This project uses the `flock' program # to achieve this. texdir = $(BDIR)/tex -srcdir = reproduce/src lockdir = $(BDIR)/locks indir = $(BDIR)/inputs mtexdir = $(texdir)/macros -pconfdir = reproduce/config/pipeline -installdir = $(BDIR)/dependencies/installed +bashdir = reproduce/analysis/bash +pconfdir = reproduce/analysis/config +installdir = $(BDIR)/software/installed # --------- Delete for no Gnuastro --------- -gconfdir = reproduce/config/gnuastro +gconfdir = reproduce/software/config/gnuastro # ------------------------------------------ @@ -233,16 +233,14 @@ $(packagecontents): | $(texdir) cp configure COPYING for-group README.md README-hacking.md $$dir/ # Build the top-level directories. - mkdir $$dir/reproduce $$dir/tex $$dir/tex/tikz $$dir/tex/pipeline + mkdir $$dir/reproduce $$dir/tex $$dir/tex/tikz $$dir/tex/build - # Copy all the `reproduce' contents except for the `build' symbolic - # link. + # Copy all the necessary `reproduce' and `tex' contents. shopt -s extglob cp -r tex/src $$dir/tex/src cp tex/tikz/*.pdf $$dir/tex/tikz - cp -r reproduce/!(build) $$dir/reproduce - cp -r tex/pipeline/!($(packagebasename)) $$dir/tex/pipeline - cp -r tex/dependencies $$dir/tex/dependencies + cp -r reproduce/ $$dir/reproduce + cp -r tex/build/!($(packagebasename)) $$dir/tex/build # Clean up un-necessary/local files: 1) the $(texdir)/build* # directories (when building in a group structure, there will be @@ -250,9 +248,9 @@ $(packagecontents): | $(texdir) # build files and don't have any relevant/hand-written files in # them. 2) The `LOCAL.mk' and `gnuastro-local.conf' files just have # this machine's local settings and are irrelevant for anyone else. - rm -rf $$dir/tex/pipeline/build* - rm $$dir/reproduce/config/pipeline/LOCAL.mk - rm $$dir/reproduce/config/gnuastro/gnuastro-local.conf + rm -rf $$dir/tex/build/build* + rm $$dir/reproduce/software/config/installation/LOCAL.mk + rm $$dir/reproduce/software/config/gnuastro/gnuastro-local.conf # PROJECT SPECIFIC: under this comment, copy any other file for # packaging, or remove any of the copied files above to suite your @@ -338,4 +336,4 @@ $(mtexdir)/initialize.tex: | $(mtexdir) # Version of the project. @v=$$(git describe --dirty --always); - echo "\newcommand{\pipelineversion}{$$v}" > $@ + echo "\newcommand{\projectversion}{$$v}" > $@ diff --git a/reproduce/src/make/paper.mk b/reproduce/analysis/make/paper.mk index 0c42bee..5378ee3 100644 --- a/reproduce/src/make/paper.mk +++ b/reproduce/analysis/make/paper.mk @@ -23,35 +23,32 @@ # # To report the input settings and results, the final report's PDF (final # target of this project) uses macros generated from various steps of the -# project. All these macros are defined in `$(mtexdir)/pipeline.tex'. +# project. All these macros are defined through `$(mtexdir)/project.tex'. # -# `$(mtexdir)/pipeline.tex' is actually just a combination of separate -# files that keep the LaTeX macros related to each workhorse Makefile (in +# `$(mtexdir)/project.tex' is actually just a combination of separate files +# that keep the LaTeX macros related to each workhorse Makefile (in # `reproduce/src/make/*.mk'). Those individual macros are pre-requisites to -# `$(mtexdir)/pipeline.tex'. The only workhorse Makefile that doesn't need +# `$(mtexdir)/project.tex'. The only workhorse Makefile that doesn't need # to produce LaTeX macros is this Makefile (`reproduce/src/make/paper.mk'). # # This file is thus the interface between the processing scripts and the # final PDF: when we get to this point, all the processing has been # completed. # -# Note that if you don't want the final PDF and just want the -# processing and file outputs, you can remove the value of -# `pdf-build-final' in `reproduce/config/pipeline/pdf-build.mk'. -$(mtexdir)/pipeline.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s).tex) +# Note that if you don't want the final PDF and just want the processing +# and file outputs, you can remove the value of `pdf-build-final' in +# `reproduce/analysis/config/pdf-build.mk'. +$(mtexdir)/project.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s).tex) # If no PDF is requested, or if LaTeX isn't available, don't # continue to building the final PDF. Otherwise, merge all the TeX # macros into one for building the PDF. @if [ -f .local/bin/pdflatex ] && [ x"$(pdf-build-final)" != x ]; then - # First make sure the `tex/pipeline' symbolic link exists. - if [ ! -e tex/pipeline ]; then ln -s $(texdir) tex/pipeline; fi - # Put a LaTeX input command for all the necessary macro files. - rm -f $(mtexdir)/pipeline.tex + rm -f $(mtexdir)/project.tex for t in $(subst paper,,$(makesrc)); do - echo "\input{tex/pipeline/macros/$$t.tex}" >> $(mtexdir)/pipeline.tex + echo "\input{tex/build/macros/$$t.tex}" >> $(mtexdir)/project.tex done else echo @@ -64,7 +61,7 @@ $(mtexdir)/pipeline.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s echo "can check by running './.local/bin/latex --version'), _AND_" echo "make sure that the 'pdf-build-final' variable has a value." echo "'pdf-build-final' is defined in: " - echo "'reproduce/config/pipeline/pdf-build.mk'." + echo "'reproduce/analysis/config/pdf-build.mk'." echo echo "If you don't have LaTeX within the project, please re-run" echo "'./configure' when you have internet access. To speed it up," @@ -88,15 +85,15 @@ $(mtexdir)/pipeline.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s # necessary bibliography before making the final paper. So we'll first have # one run of LaTeX (similar to the `paper.pdf' recipe), then `biber'. # -# NOTE: `$(mtexdir)/pipeline.tex' is an order-only-prerequisite for +# NOTE: `$(mtexdir)/project.tex' is an order-only-prerequisite for # `paper.bbl'. This is because we need to run LaTeX in both the `paper.bbl' # recipe and the `paper.pdf' recipe. But if `tex/src/references.tex' hasn't # been modified, we don't want to re-build the bibliography, only the final # PDF. $(texbdir)/paper.bbl: tex/src/references.tex \ - | $(tikzdir) $(texbdir) $(mtexdir)/pipeline.tex - # If `$(mtexdir)/pipeline.tex' is empty, don't build PDF. - @macros=$$(cat $(mtexdir)/pipeline.tex) + | $(tikzdir) $(texbdir) $(mtexdir)/project.tex + # If `$(mtexdir)/project.tex' is empty, don't build PDF. + @macros=$$(cat $(mtexdir)/project.tex) if [ x"$$macros" != x ]; then # We'll run LaTeX first to generate the `.bcf' file (necessary @@ -121,11 +118,11 @@ $(texbdir)/paper.bbl: tex/src/references.tex \ # to run everything cleanly from there, it is necessary to add the current # directory (top project directory) to the `TEXINPUTS' environment # variable. -paper.pdf: $(mtexdir)/pipeline.tex paper.tex $(texbdir)/paper.bbl \ +paper.pdf: $(mtexdir)/project.tex paper.tex $(texbdir)/paper.bbl \ | $(tikzdir) $(texbdir) - # If `$(mtexdir)/pipeline.tex' is empty, don't build the PDF. - @macros=$$(cat $(mtexdir)/pipeline.tex) + # If `$(mtexdir)/project.tex' is empty, don't build the PDF. + @macros=$$(cat $(mtexdir)/project.tex) if [ x"$$macros" != x ]; then # Go into the top TeX build directory and make the paper. diff --git a/reproduce/src/make/top.mk b/reproduce/analysis/make/top.mk index 763dbd7..a85c530 100644 --- a/reproduce/src/make/top.mk +++ b/reproduce/analysis/make/top.mk @@ -20,7 +20,7 @@ # Load the local configuration (created after running `./configure'). -include reproduce/config/pipeline/LOCAL.mk +include reproduce/software/config/installation/LOCAL.mk @@ -63,7 +63,7 @@ include reproduce/config/pipeline/LOCAL.mk # # If you are just interested in the processing and don't want to build the # PDF, you can skip the creatation of the final PDF by removing the value -# of `pdf-build-final' in `reproduce/config/pipeline/pdf-build.mk'. +# of `pdf-build-final' in `reproduce/analysis/config/pdf-build.mk'. ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) all: paper.pdf else @@ -119,17 +119,18 @@ makesrc = initialize \ -# Include all Makefiles -# --------------------- +# Include all analysis Makefiles +# ------------------------------ # -# We have two classes of Makefiles, separated by context and their location: +# 1) All the analysis configuration-Makefiles (Makefiles that only define +# variables with no rules or order). # -# 1) First, we'll include all the configuration-Makefiles. These -# Makefiles only define variables with no rules or order. We just -# won't include `LOCAL.mk' because it has already been included -# above. +# 2) From the software configuration-Makefiles, we only include the one +# containing software versions, just incase its necessary to +# use/report outside of the acknowledgments section of the paper. # -# 2) Then, we'll import the workhorse-Makefiles which contain rules to -# actually do this project's processing. -include $(filter-out %LOCAL.mk, reproduce/config/pipeline/*.mk) -include $(foreach s,$(makesrc), reproduce/src/make/$(s).mk) +# 3) Finally, we'll import all the analysis workhorse-Makefiles which +# contain rules to actually do this project's processing. +include reproduce/analysis/config/*.mk +include reproduce/software/config/installation/versions.mk +include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk) diff --git a/reproduce/src/bash/git-post-checkout b/reproduce/software/bash/git-post-checkout index 9552f01..9552f01 100644 --- a/reproduce/src/bash/git-post-checkout +++ b/reproduce/software/bash/git-post-checkout diff --git a/reproduce/src/bash/git-pre-commit b/reproduce/software/bash/git-pre-commit index dbe0ecc..dbe0ecc 100644 --- a/reproduce/src/bash/git-pre-commit +++ b/reproduce/software/bash/git-pre-commit diff --git a/reproduce/software/bibtex/astropy.tex b/reproduce/software/bibtex/astropy.tex new file mode 100644 index 0000000..450212d --- /dev/null +++ b/reproduce/software/bibtex/astropy.tex @@ -0,0 +1,97 @@ +%% Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{astropy2018, + author = {{Astropy Collaboration} and {Price-Whelan}, A.~M. and {Sip{\H o}cz}, B.~M. and + {G{\"u}nther}, H.~M. and {Lim}, P.~L. and {Crawford}, S.~M. and + {Conseil}, S. and {Shupe}, D.~L. and {Craig}, M.~W. and {Dencheva}, N. and + {Ginsburg}, A. and {VanderPlas}, J.~T. and {Bradley}, L.~D. and + {P{\'e}rez-Su{\'a}rez}, D. and {de Val-Borro}, M. and {Aldcroft}, T.~L. and + {Cruz}, K.~L. and {Robitaille}, T.~P. and {Tollerud}, E.~J. and + {Ardelean}, C. and {Babej}, T. and {Bach}, Y.~P. and {Bachetti}, M. and + {Bakanov}, A.~V. and {Bamford}, S.~P. and {Barentsen}, G. and + {Barmby}, P. and {Baumbach}, A. and {Berry}, K.~L. and {Biscani}, F. and + {Boquien}, M. and {Bostroem}, K.~A. and {Bouma}, L.~G. and {Brammer}, G.~B. and + {Bray}, E.~M. and {Breytenbach}, H. and {Buddelmeijer}, H. and + {Burke}, D.~J. and {Calderone}, G. and {Cano Rodr{\'{\i}}guez}, J.~L. and + {Cara}, M. and {Cardoso}, J.~V.~M. and {Cheedella}, S. and {Copin}, Y. and + {Corrales}, L. and {Crichton}, D. and {D'Avella}, D. and {Deil}, C. and + {Depagne}, {\'E}. and {Dietrich}, J.~P. and {Donath}, A. and + {Droettboom}, M. and {Earl}, N. and {Erben}, T. and {Fabbro}, S. and + {Ferreira}, L.~A. and {Finethy}, T. and {Fox}, R.~T. and {Garrison}, L.~H. and + {Gibbons}, S.~L.~J. and {Goldstein}, D.~A. and {Gommers}, R. and + {Greco}, J.~P. and {Greenfield}, P. and {Groener}, A.~M. and + {Grollier}, F. and {Hagen}, A. and {Hirst}, P. and {Homeier}, D. and + {Horton}, A.~J. and {Hosseinzadeh}, G. and {Hu}, L. and {Hunkeler}, J.~S. and + {Ivezi{\'c}}, {\v Z}. and {Jain}, A. and {Jenness}, T. and {Kanarek}, G. and + {Kendrew}, S. and {Kern}, N.~S. and {Kerzendorf}, W.~E. and + {Khvalko}, A. and {King}, J. and {Kirkby}, D. and {Kulkarni}, A.~M. and + {Kumar}, A. and {Lee}, A. and {Lenz}, D. and {Littlefair}, S.~P. and + {Ma}, Z. and {Macleod}, D.~M. and {Mastropietro}, M. and {McCully}, C. and + {Montagnac}, S. and {Morris}, B.~M. and {Mueller}, M. and {Mumford}, S.~J. and + {Muna}, D. and {Murphy}, N.~A. and {Nelson}, S. and {Nguyen}, G.~H. and + {Ninan}, J.~P. and {N{\"o}the}, M. and {Ogaz}, S. and {Oh}, S. and + {Parejko}, J.~K. and {Parley}, N. and {Pascual}, S. and {Patil}, R. and + {Patil}, A.~A. and {Plunkett}, A.~L. and {Prochaska}, J.~X. and + {Rastogi}, T. and {Reddy Janga}, V. and {Sabater}, J. and {Sakurikar}, P. and + {Seifert}, M. and {Sherbert}, L.~E. and {Sherwood-Taylor}, H. and + {Shih}, A.~Y. and {Sick}, J. and {Silbiger}, M.~T. and {Singanamalla}, S. and + {Singer}, L.~P. and {Sladen}, P.~H. and {Sooley}, K.~A. and + {Sornarajah}, S. and {Streicher}, O. and {Teuben}, P. and {Thomas}, S.~W. and + {Tremblay}, G.~R. and {Turner}, J.~E.~H. and {Terr{\'o}n}, V. and + {van Kerkwijk}, M.~H. and {de la Vega}, A. and {Watkins}, L.~L. and + {Weaver}, B.~A. and {Whitmore}, J.~B. and {Woillez}, J. and + {Zabalza}, V. and {Astropy Contributors}}, + title = "{The Astropy Project: Building an Open-science Project and Status of the v2.0 Core Package}", + journal = {AJ}, +archivePrefix = "arXiv", + eprint = {1801.02634}, + primaryClass = "astro-ph.IM", + keywords = {methods: data analysis, methods: miscellaneous, methods: statistical, reference systems }, + year = 2018, + month = sep, + volume = 156, + eid = {123}, + pages = {123}, + doi = {10.3847/1538-3881/aabc4f}, + adsurl = {http://adsabs.harvard.edu/abs/2018AJ....156..123A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + + + + + +@ARTICLE{astropy2013, + author = {{Astropy Collaboration} and {Robitaille}, T.~P. and {Tollerud}, E.~J. and + {Greenfield}, P. and {Droettboom}, M. and {Bray}, E. and {Aldcroft}, T. and + {Davis}, M. and {Ginsburg}, A. and {Price-Whelan}, A.~M. and + {Kerzendorf}, W.~E. and {Conley}, A. and {Crighton}, N. and + {Barbary}, K. and {Muna}, D. and {Ferguson}, H. and {Grollier}, F. and + {Parikh}, M.~M. and {Nair}, P.~H. and {Unther}, H.~M. and {Deil}, C. and + {Woillez}, J. and {Conseil}, S. and {Kramer}, R. and {Turner}, J.~E.~H. and + {Singer}, L. and {Fox}, R. and {Weaver}, B.~A. and {Zabalza}, V. and + {Edwards}, Z.~I. and {Azalee Bostroem}, K. and {Burke}, D.~J. and + {Casey}, A.~R. and {Crawford}, S.~M. and {Dencheva}, N. and + {Ely}, J. and {Jenness}, T. and {Labrie}, K. and {Lim}, P.~L. and + {Pierfederici}, F. and {Pontzen}, A. and {Ptak}, A. and {Refsdal}, B. and + {Servillat}, M. and {Streicher}, O.}, + title = "{Astropy: A community Python package for astronomy}", + journal = {A\&A}, +archivePrefix = "arXiv", + eprint = {1307.6212}, + primaryClass = "astro-ph.IM", + keywords = {methods: data analysis, methods: miscellaneous, virtual observatory tools}, + year = 2013, + month = oct, + volume = 558, + eid = {A33}, + pages = {A33}, + doi = {10.1051/0004-6361/201322068}, + adsurl = {http://adsabs.harvard.edu/abs/2013A\%26A...558A..33A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/bibtex/cython.tex b/reproduce/software/bibtex/cython.tex new file mode 100644 index 0000000..110cb19 --- /dev/null +++ b/reproduce/software/bibtex/cython.tex @@ -0,0 +1,21 @@ +%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{cython2011, + author = {{Behnel}, S. and {Bradshaw}, R. and {Citro}, C. and {Dalcin}, L. and + {Seljebotn}, D.~S. and {Smith}, K.}, + title = "{Cython: The Best of Both Worlds}", + journal = {CiSE}, + year = 2011, + month = mar, + volume = 13, + number = 2, + pages = {31}, + doi = {10.1109/MCSE.2010.118}, + adsurl = {http://adsabs.harvard.edu/abs/2011CSE....13b..31B}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/bibtex/fftw.tex b/reproduce/software/bibtex/fftw.tex new file mode 100644 index 0000000..60c24a8 --- /dev/null +++ b/reproduce/software/bibtex/fftw.tex @@ -0,0 +1,19 @@ +%% Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{fftw, + author = {M. {Frigo} and S. G. {Johnson}}, + journal = {IEEE Proc.}, + title = {The Design and Implementation of FFTW3}, + year = {2005}, + volume = {93}, + number = {2}, + pages = {216}, + doi = {10.1109/JPROC.2004.840301}, + ISSN = {0018-9219}, + month = {Feb}, +} diff --git a/reproduce/software/bibtex/gnuastro.tex b/reproduce/software/bibtex/gnuastro.tex new file mode 100644 index 0000000..91788c9 --- /dev/null +++ b/reproduce/software/bibtex/gnuastro.tex @@ -0,0 +1,24 @@ +%% Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{gnuastro, + author = {{Akhlaghi}, M. and {Ichikawa}, T.}, + title = "{Noise-based Detection and Segmentation of Nebulous Objects}", + journal = {ApJS}, +archivePrefix = "arXiv", + eprint = {1505.01664}, + primaryClass = "astro-ph.IM", + keywords = {galaxies: irregular, galaxies: photometry, galaxies: structure, methods: data analysis, techniques: image processing, techniques: photometric}, + year = 2015, + month = sep, + volume = 220, + eid = {1}, + pages = {1}, + doi = {10.1088/0067-0049/220/1/1}, + adsurl = {http://adsabs.harvard.edu/abs/2015ApJS..220....1A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/bibtex/matplotlib.tex b/reproduce/software/bibtex/matplotlib.tex new file mode 100644 index 0000000..8578899 --- /dev/null +++ b/reproduce/software/bibtex/matplotlib.tex @@ -0,0 +1,22 @@ +%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@Article{matplotlib2007, + Author = {Hunter, J. D.}, + Title = {Matplotlib: A 2D graphics environment}, + Journal = {CiSE}, + Volume = {9}, + Number = {3}, + Pages = {90}, + abstract = {Matplotlib is a 2D graphics package used for Python + for application development, interactive scripting, and + publication-quality image generation across user + interfaces and operating systems.}, + publisher = {IEEE COMPUTER SOC}, + doi = {10.1109/MCSE.2007.55}, + year = 2007 +} diff --git a/reproduce/software/bibtex/mpi4py.tex b/reproduce/software/bibtex/mpi4py.tex new file mode 100644 index 0000000..4e0045f --- /dev/null +++ b/reproduce/software/bibtex/mpi4py.tex @@ -0,0 +1,20 @@ +%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{mpi4py2011, + author = {{Dalcin}, L.~D. and {Paz}, R.~R. and {Kler}, P.~A. and {Cosimo}, A. + }, + title = "{Parallel distributed computing using Python}", + journal = {Advances in Water Resources}, + year = 2011, + month = sep, + volume = 34, + pages = {1124}, + doi = {10.1016/j.advwatres.2011.04.013}, + adsurl = {http://adsabs.harvard.edu/abs/2011AdWR...34.1124D}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/bibtex/numpy.tex b/reproduce/software/bibtex/numpy.tex new file mode 100644 index 0000000..0d9028f --- /dev/null +++ b/reproduce/software/bibtex/numpy.tex @@ -0,0 +1,23 @@ +%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{numpy2011, + author = {{van der Walt}, S. and {Colbert}, S.~C. and {Varoquaux}, G.}, + title = "{The NumPy Array: A Structure for Efficient Numerical Computation}", + journal = {CiSE}, +archivePrefix = "arXiv", + eprint = {1102.1523}, + primaryClass = "cs.MS", + year = 2011, + month = mar, + volume = 13, + number = 2, + pages = {22}, + doi = {10.1109/MCSE.2011.37}, + adsurl = {http://adsabs.harvard.edu/abs/2011CSE....13b..22V}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/bibtex/scipy.tex b/reproduce/software/bibtex/scipy.tex new file mode 100644 index 0000000..60500d2 --- /dev/null +++ b/reproduce/software/bibtex/scipy.tex @@ -0,0 +1,34 @@ +%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{scipy2011, + author = {K. J. {Millman} and M. {Aivazis}}, + journal = {CiSE}, + title = {Python for Scientists and Engineers}, + year = {2011}, + volume = {13}, + number = {2}, + pages = {9}, + keywords = {Special issues and sections;Computer languages;Programming;Scientific computing;Numerical models;Programming languages;Python;Scientific computing;interactive research;Python libraries;Python tools}, + doi = {10.1109/MCSE.2011.36}, + ISSN = {1521-9615}, + month ={March}, +} + +@ARTICLE{scipy2007, + author = {T. E. {Oliphant}}, + journal = {CiSE}, + title = {Python for Scientific Computing}, + year = {2007}, + volume = {9}, + number = {3}, + pages = {10}, + keywords = {high level languages;Python;scientific computing;steering language;scientific codes;high-level language;Scientific computing;High level languages;Libraries;Writing;Application software;Embedded software;Software standards;Standards development;Internet;Prototypes;Python;computer languages;scientific programming;scientific computing}, + doi = {10.1109/MCSE.2007.58}, + ISSN = {1521-9615}, + month = {May}, +} diff --git a/reproduce/config/gnuastro/astbuildprog.conf b/reproduce/software/config/gnuastro/astbuildprog.conf index c15baf1..c15baf1 100644 --- a/reproduce/config/gnuastro/astbuildprog.conf +++ b/reproduce/software/config/gnuastro/astbuildprog.conf diff --git a/reproduce/config/gnuastro/astconvertt.conf b/reproduce/software/config/gnuastro/astconvertt.conf index 6466455..6466455 100644 --- a/reproduce/config/gnuastro/astconvertt.conf +++ b/reproduce/software/config/gnuastro/astconvertt.conf diff --git a/reproduce/config/gnuastro/aststatistics.conf b/reproduce/software/config/gnuastro/aststatistics.conf index 1d13e3f..1d13e3f 100644 --- a/reproduce/config/gnuastro/aststatistics.conf +++ b/reproduce/software/config/gnuastro/aststatistics.conf diff --git a/reproduce/config/gnuastro/gnuastro.conf b/reproduce/software/config/gnuastro/gnuastro.conf index 57fcadc..57fcadc 100644 --- a/reproduce/config/gnuastro/gnuastro.conf +++ b/reproduce/software/config/gnuastro/gnuastro.conf diff --git a/reproduce/config/pipeline/LOCAL.mk.in b/reproduce/software/config/installation/LOCAL.mk.in index 785bb6a..785bb6a 100644 --- a/reproduce/config/pipeline/LOCAL.mk.in +++ b/reproduce/software/config/installation/LOCAL.mk.in diff --git a/reproduce/config/pipeline/dependency-numpy-scipy.cfg b/reproduce/software/config/installation/numpy-scipy.cfg index 4b7a7b0..4b7a7b0 100644 --- a/reproduce/config/pipeline/dependency-numpy-scipy.cfg +++ b/reproduce/software/config/installation/numpy-scipy.cfg diff --git a/reproduce/config/pipeline/texlive.conf b/reproduce/software/config/installation/texlive.conf index 53054e1..53054e1 100644 --- a/reproduce/config/pipeline/texlive.conf +++ b/reproduce/software/config/installation/texlive.conf diff --git a/reproduce/config/pipeline/dependency-texlive.mk b/reproduce/software/config/installation/texlive.mk index 0acf5ab..0acf5ab 100644 --- a/reproduce/config/pipeline/dependency-texlive.mk +++ b/reproduce/software/config/installation/texlive.mk diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/software/config/installation/versions.mk index c6c8043..149f9ae 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/software/config/installation/versions.mk @@ -21,8 +21,8 @@ atlas-version = 3.10.3 bash-version = 5.0 binutils-version = 2.31.1 cfitsio-version = 3.45 -cmake-version = 3.12.4 -coreutils-version = 8.30 +cmake-version = 3.14.2 +coreutils-version = 8.31 curl-version = 7.63.0 diffutils-version = 3.7 fftw-version = 3.3.8 @@ -30,10 +30,10 @@ file-version = 5.36 findutils-version = 4.6.0.199-e3fc flock-version = 0.2.3 freetype-version = 2.9 -gawk-version = 4.2.1 +gawk-version = 5.0.0 gcc-version = 8.3.0 ghostscript-version = 9.26 -git-version = 2.20.1 +git-version = 2.21.0 gmp-version = 6.1.2 gnuastro-version = 0.8 grep-version = 3.3 @@ -59,12 +59,12 @@ openmpi-version = 4.0.1 openssl-version = 1.1.1a patchelf-version = 0.9 pkgconfig-version = 0.29.2 -python-version = 3.6.8 +python-version = 3.7.3 readline-version = 8.0 sed-version = 4.7 -tar-version = 1.31 +tar-version = 1.32 unzip-version = 6.0 -wget-version = 1.20.1 +wget-version = 1.20.3 which-version = 2.21 xz-version = 5.2.4 zip-version = 3.0 diff --git a/reproduce/src/make/dependencies-atlas-multiple.mk b/reproduce/software/make/atlas-multiple.mk index fef25c7..fef25c7 100755 --- a/reproduce/src/make/dependencies-atlas-multiple.mk +++ b/reproduce/software/make/atlas-multiple.mk diff --git a/reproduce/src/make/dependencies-atlas-single.mk b/reproduce/software/make/atlas-single.mk index dde2926..dde2926 100755 --- a/reproduce/src/make/dependencies-atlas-single.mk +++ b/reproduce/software/make/atlas-single.mk diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/software/make/basic.mk index b56d01d..cfa05d2 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/software/make/basic.mk @@ -1,20 +1,17 @@ -# Build the VERY BASIC project dependencies before everything else assuming +# Build the VERY BASIC project software before higher-level ones. Assuming # minimal/generic Make and Shell. # # ------------------------------------------------------------------------ # !!!!! IMPORTANT NOTES !!!!! # # This Makefile will be run by the initial `./configure' script. It is not -# included into the reproduction pipe after that. +# included into the project after that. # -# This Makefile builds very low-level and basic tools like GNU Tar, and -# various compression programs, GNU Bash, and GNU Make. Therefore this is -# the only Makefile in the reproduction pipeline where you MUST NOT assume -# that modern GNU Bash or GNU Make are used. After this Makefile, other -# Makefiles can safely assume the fixed version of all these software. -# -# This Makefile is a very simplified version of `dependencies.mk' in the -# same directory. See there for more comments. +# This Makefile builds very low-level and basic tools like GNU Tar, GNU +# Bash, GNU Make, GCC and etc. Therefore this is the only Makefile in the +# project where you CANNOT assume that GNU Bash or GNU Make are used. After +# this Makefile (where GNU Bash and GNU Make are built), other Makefiles +# can safely assume the fixed version of all these software. # # ------------------------------------------------------------------------ # @@ -35,18 +32,18 @@ # Top level environment -include reproduce/config/pipeline/LOCAL.mk -include reproduce/src/make/dependencies-build-rules.mk -include reproduce/config/pipeline/dependency-versions.mk +include reproduce/software/make/build-rules.mk +include reproduce/software/config/installation/LOCAL.mk +include reproduce/software/config/installation/versions.mk lockdir = $(BDIR)/locks -ddir = $(BDIR)/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib -ibidir = $(BDIR)/dependencies/installed/version-info/bin -ilidir = $(BDIR)/dependencies/installed/version-info/lib +tdir = $(BDIR)/software/tarballs +ddir = $(BDIR)/software/build-tmp +idir = $(BDIR)/software/installed +ibdir = $(BDIR)/software/installed/bin +ildir = $(BDIR)/software/installed/lib +ibidir = $(BDIR)/software/installed/version-info/bin +ilidir = $(BDIR)/software/installed/version-info/lib # We'll need the system's PATH for making links to low-level programs we # won't be building ourselves. @@ -58,11 +55,11 @@ export CCACHE_DISABLE := 1 export PATH := $(ibdir):$(PATH) export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) -export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) +export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) +export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) -# Define the programs that don't depend on any other. +# Define the top-level programs (that don't depend on any other). top-level-programs = low-level-links wget gcc all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) @@ -73,10 +70,10 @@ all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) # Tarballs # -------- # -# Prepare tarballs. Difference with that in `dependencies.mk': `.ONESHELL' -# is not recognized by some versions of Make (even older GNU Makes). So -# we'll have to make sure the recipe doesn't break into multiple shell -# calls (so we can preserve the variables). +# Prepare tarballs. Difference with that in `high-level.mk': `.ONESHELL' is +# not recognized by some versions of Make (even older GNU Makes). So we'll +# have to make sure the recipe doesn't break into multiple shell calls (so +# we can preserve the variables). # # Software hosted at akhlaghi.org/src: As of our latest check (November # 2018) their major release tarballs either crash or don't build on some @@ -93,7 +90,7 @@ all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) # However, downloading from this link is slow (because its just a link). So # its easier to just keep a with the others. $(lockdir): | $(BDIR); mkdir $@ -downloadwrapper = ./reproduce/src/bash/download-multi-try +downloadwrapper = ./reproduce/analysis/bash/download-multi-try tarballs = $(foreach t, bash-$(bash-version).tar.gz \ binutils-$(binutils-version).tar.lz \ bzip2-$(bzip2-version).tar.gz \ @@ -121,9 +118,11 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \ readline-$(readline-version).tar.gz \ sed-$(sed-version).tar.xz \ tar-$(tar-version).tar.gz \ + unzip-$(unzip-version).tar.gz \ wget-$(wget-version).tar.lz \ which-$(which-version).tar.gz \ xz-$(xz-version).tar.gz \ + zip-$(zip-version).tar.gz \ zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: | $(lockdir) @@ -164,9 +163,15 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = readline ]; then w=http://ftp.gnu.org/gnu/readline; \ elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed; \ elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \ + elif [ $$n = unzip ]; then \ + mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//'); \ + w=ftp://ftp.info-zip.org/pub/infozip/src/unzip$$v.tgz; \ elif [ $$n = wget ]; then w=http://ftp.gnu.org/gnu/wget; \ elif [ $$n = which ]; then w=http://ftp.gnu.org/gnu/which; \ elif [ $$n = xz ]; then w=http://tukaani.org/xz; \ + elif [ $$n = zip ]; then \ + mergenames=0; v=$$(echo $(zip-version) | sed -e's/\.//'); \ + w=ftp://ftp.info-zip.org/pub/infozip/src/zip$$v.tgz; \ elif [ $$n = zlib ]; then w=http://www.zlib.net; \ else \ echo; echo; echo; \ @@ -178,7 +183,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*"; \ else tarballurl=$$w; \ fi; \ - \ + \ echo "Downloading $$tarballurl"; \ if [ -f $(ibdir)/wget ]; then \ downloader="wget --no-use-server-timestamps -O"; \ @@ -344,18 +349,51 @@ $(ibidir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz && ln -fs libbz2.so.1.0 libbz2.so \ && echo "Bzip2 $(bzip2-version)" > $@ +$(ibidir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz + v=$$(echo $(unzip-version) | sed -e's/\.//'); \ + $(call gbuild, $<, unzip$$v, static,, \ + -f unix/Makefile generic_gcc \ + CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ + -f unix/Makefile \ + BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ + && echo "Unzip $(unzip-version)" > $@ + +$(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz + v=$$(echo $(zip-version) | sed -e's/\.//'); \ + $(call gbuild, $<, zip$$v, static,, \ + -f unix/Makefile generic_gcc \ + CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ + -f unix/Makefile \ + BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ + && echo "Zip $(zip-version)" > $@ + +# Some programs (like Wget and CMake) that use zlib need it to be dynamic +# so they use our custom build. So we won't force a static-only build. +# +# Note for a static-only build: Zlib's `./configure' doesn't use Autoconf's +# configure script, it just accepts a direct `--static' option. +$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz + $(call gbuild, $<, zlib-$(zlib-version)) \ + && echo "Zlib $(zlib-version)" > $@ + # GNU Tar: When built statically, tar gives a segmentation fault on -# unpacking Bash. So we'll build it dynamically. +# unpacking Bash. So we'll build it dynamically. Note that technically, zip +# and unzip aren't dependencies of Tar, but for a clean build, we'll set +# Tar to be the last compression-related software (the first-set of +# software to be built). $(ibidir)/tar: $(tdir)/tar-$(tar-version).tar.gz \ $(ibidir)/bzip2 \ - $(ibidir)/lzip \ + $(ibidir)/unzip \ $(ibidir)/gzip \ + $(ibidir)/lzip \ + $(ilidir)/zlib \ + $(ibidir)/zip \ $(ibidir)/xz # Since all later programs depend on Tar, the configuration will be # stuck here, only making Tar. So its more efficient to built it on # multiple threads (when the user's Make doesn't pass down the # number of threads). - $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads)) \ + $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads) V=1) \ && echo "GNU Tar $(tar-version)" > $@ @@ -555,22 +593,11 @@ $(ibidir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ # Downloader # ---------- # -# Some programs (like Wget and CMake) that use zlib need it to be dynamic -# so they use our custom build. So we won't force a static-only build. -# -# Note for a static-only build: Zlib's `./configure' doesn't use Autoconf's -# configure script, it just accepts a direct `--static' option. -$(idir)/etc:; mkdir $@ -$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \ - $(ibidir)/bash - $(call gbuild, $<, zlib-$(zlib-version)) \ - && echo "Zlib $(zlib-version)" > $@ - # OpenSSL: Some programs/libraries later need dynamic linking. So we'll # build libssl (and libcrypto) dynamically also. # # Until we find a nice and generic way to create an updated CA file in the -# project, the certificates will be available in a file for this pipeline +# project, the certificates will be available in a file for this project # along with the other tarballs. # # In case you do want a static OpenSSL and libcrypto, then uncomment the @@ -579,9 +606,10 @@ $(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \ #ifeq ($(static_build),yes) #openssl-static = no-dso no-dynamic-engine no-shared #endif +$(idir)/etc:; mkdir $@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ $(tdir)/cert.pem \ - $(ilidir)/zlib | $(idir)/etc + $(ibidir)/bash | $(idir)/etc # According to OpenSSL's Wiki (link bellow), it can't automatically # detect Mac OS's structure. It will need some help. So we'll use # the `on_mac_os' Make variable that we defined in the configure @@ -763,13 +791,15 @@ $(ibidir)/which: $(tdir)/which-$(which-version).tar.gz \ # Binutils' linker `ld' is apparently only good for GNU/Linux systems and # other OSs have their own. So for now we aren't actually building # Binutils (`ld' isn't a prerequisite of GCC). -$(ibidir)/binutils: $(tdir)/binutils-$(binutils-version).tar.lz +$(ibidir)/binutils: $(tdir)/binutils-$(binutils-version).tar.lz \ + $(ibidir)/bash $(call gbuild, $<, binutils-$(binutils-version), static) \ && echo "GNU Binutils $(binutils-version)" > $@ # `file' is not a prerequisite of GCC. However, since it is low level, it is # set as a prerequisite of GCC to have it installed. -$(ibidir)/file: $(tdir)/file-$(file-version).tar.gz +$(ibidir)/file: $(tdir)/file-$(file-version).tar.gz \ + $(ibidir)/bash $(call gbuild, $<, file-$(file-version), static) \ && echo "File $(file-version)" > $@ diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/software/make/build-rules.mk index a8c8731..a8c8731 100644 --- a/reproduce/src/make/dependencies-build-rules.mk +++ b/reproduce/software/make/build-rules.mk diff --git a/reproduce/src/make/dependencies.mk b/reproduce/software/make/high-level.mk index fd9bffa..f7b1f4f 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/software/make/high-level.mk @@ -26,23 +26,23 @@ # Top level environment -include reproduce/config/pipeline/LOCAL.mk -include reproduce/src/make/dependencies-build-rules.mk -include reproduce/config/pipeline/dependency-texlive.mk -include reproduce/config/pipeline/dependency-versions.mk +include reproduce/software/make/build-rules.mk +include reproduce/software/config/installation/LOCAL.mk +include reproduce/software/config/installation/texlive.mk +include reproduce/software/config/installation/versions.mk lockdir = $(BDIR)/locks -ddir = $(BDIR)/dependencies -dtexdir = $(shell pwd)/tex/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib -ibidir = $(BDIR)/dependencies/installed/version-info/bin -ilidir = $(BDIR)/dependencies/installed/version-info/lib -itidir = $(BDIR)/dependencies/installed/version-info/tex -ictdir = $(BDIR)/dependencies/installed/version-info/cite -ipydir = $(BDIR)/dependencies/installed/version-info/python +tdir = $(BDIR)/software/tarballs +ddir = $(BDIR)/software/build-tmp +idir = $(BDIR)/software/installed +ibdir = $(BDIR)/software/installed/bin +ildir = $(BDIR)/software/installed/lib +dtexdir = $(shell pwd)/reproduce/software/bibtex +ibidir = $(BDIR)/software/installed/version-info/bin +ilidir = $(BDIR)/software/installed/version-info/lib +itidir = $(BDIR)/software/installed/version-info/tex +ictdir = $(BDIR)/software/installed/version-info/cite +ipydir = $(BDIR)/software/installed/version-info/python # Define the top-level programs to build (installed in `.local/bin'). # @@ -51,8 +51,8 @@ ipydir = $(BDIR)/dependencies/installed/version-info/python # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. top-level-libraries = # atlas -top-level-programs = gnuastro metastore unzip zip top-level-python = astroquery matplotlib +top-level-programs = gnuastro metastore all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ $(foreach p, $(top-level-python), $(ipydir)/$(p)) \ @@ -87,15 +87,21 @@ export LDFLAGS := $(rpath_command) -L$(ildir) # We want the download to happen on a single thread. So we need to define a # lock, and call a special script we have written for this job. These are -# placed here because we want them both in the `dependencies.mk' and -# `dependencies-python.mk'. +# placed here because we want them both in the `high-level.mk' and +# `python.mk'. $(lockdir): | $(BDIR); mkdir $@ downloader="wget --no-use-server-timestamps -O"; -downloadwrapper = ./reproduce/src/bash/download-multi-try +downloadwrapper = ./reproduce/analysis/bash/download-multi-try + + + # Python packages -include reproduce/src/make/dependencies-python.mk +include reproduce/software/make/python.mk + + + # Tarballs @@ -127,11 +133,9 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ - unzip-$(unzip-version).tar.gz \ openblas-$(openblas-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ - zip-$(zip-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: | $(lockdir) if [ -f $(DEPENDENCIES-DIR)/$* ]; then @@ -156,7 +160,12 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = atlas ]; then mergenames=0 w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download - elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 + elif [ $$n = cmake ]; then + mergenames=0 + majv=$$(echo $(cmake-version) \ + | sed -e's/\./ /' \ + | awk '{printf("%d.%d", $$1, $$2)}') + w=https://cmake.org/files/v$$majv/cmake-$(cmake-version).tar.gz elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype @@ -185,13 +194,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff - elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src - mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') - w=ftp://ftp.info-zip.org/pub/infozip/src/unzip$$v.tgz elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib - elif [ $$n = zip ]; then - mergenames=0; v=$$(echo $(zip-version) | sed -e's/\.//') - w=ftp://ftp.info-zip.org/pub/infozip/src/zip$$v.tgz else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." @@ -469,19 +472,20 @@ $(ibidir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ # # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. - if [ x$(on_mac_os) = xyes ]; then \ - export CC=clang; \ - export CXX=clang++; \ - fi; \ - cd $(ddir) \ - && rm -rf cmake-$(cmake-version) \ - && tar xf $< && cd cmake-$(cmake-version) \ - && ./bootstrap --prefix=$(idir) --system-curl --system-zlib\ - --system-bzip2 --system-liblzma --no-qt-gui \ - && make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 \ - && make install \ - && cd .. \ - && rm -rf cmake-$(cmake-version) \ + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + export CXX=clang++; \ + fi; \ + cd $(ddir) \ + && rm -rf cmake-$(cmake-version) \ + && tar xf $< \ + && cd cmake-$(cmake-version) \ + && ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ + --system-bzip2 --system-liblzma --no-qt-gui \ + && make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 \ + && make install \ + && cd .. \ + && rm -rf cmake-$(cmake-version) \ && echo "CMake $(cmake-version)" > $@ # cURL (and its library, which is needed by several programs here) can @@ -573,7 +577,7 @@ $(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ -e's|@GROUP[@]|'$$group'|g' \ -e's|@BINDIR[@]|$(ibdir)|g' \ -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ - reproduce/src/bash/git-$$f > .git/hooks/$$f + reproduce/software/bash/git-$$f > .git/hooks/$$f chmod +x .git/hooks/$$f echo "Metastore (forked) $(metastore-version)" > $@ done @@ -609,23 +613,20 @@ endif && cp $(dtexdir)/gnuastro.tex $(ictdir)/ \ && echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ -$(ibidir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz - v=$$(echo $(unzip-version) | sed -e's/\.//') - $(call gbuild, $<, unzip$$v, static,, \ - -f unix/Makefile generic_gcc \ - CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ - -f unix/Makefile \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ - && echo "Unzip $(unzip-version)" > $@ -$(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz - v=$$(echo $(zip-version) | sed -e's/\.//') - $(call gbuild, $<, zip$$v, static,, \ - -f unix/Makefile generic_gcc \ - CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ - -f unix/Makefile \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ - && echo "Zip $(zip-version)" > $@ + + + + + + + + + + + + + @@ -639,7 +640,7 @@ $(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz # `texlive-ready-tlmgr' and use its contents to mark if we can use it or # not. $(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \ - reproduce/config/pipeline/texlive.conf + reproduce/software/config/installation/texlive.conf # Unpack, enter the directory, and install based on the given # configuration (prerequisite of this rule). @@ -649,7 +650,8 @@ $(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \ tar xf $(tdir)/install-tl-unx.tar.gz cd install-tl-* sed -e's|@installdir[@]|$(idir)|g' \ - $$topdir/reproduce/config/pipeline/texlive.conf > texlive.conf + $$topdir/reproduce/software/config/installation/texlive.conf \ + > texlive.conf # TeX Live's installation may fail due to any reason. But TeX Live # is optional (only necessary for building the final PDF). So we @@ -680,7 +682,7 @@ $(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \ # To keep things modular and simple, we'll break up the installation of TeX # Live itself (only very basic TeX and LaTeX) and the installation of its # necessary packages into two packages. -$(itidir)/texlive: reproduce/config/pipeline/dependency-texlive.mk \ +$(itidir)/texlive: reproduce/software/config/installation/texlive.mk \ $(itidir)/texlive-ready-tlmgr # To work with TeX live installation, we'll need the internet. diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/software/make/python.mk index 837b0ad..5ec99f0 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/software/make/python.mk @@ -415,7 +415,7 @@ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ else \ export LDFLAGS="$(LDFLAGS) -shared"; \ fi; \ - conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + conf="$$(pwd)/reproduce/software/config/installation/numpy-scipy.cfg"; \ $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf, \ Numpy $(numpy-version)) \ && cp $(dtexdir)/numpy.tex $(ictdir)/ \ @@ -463,7 +463,7 @@ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ else \ export LDFLAGS="$(LDFLAGS) -shared"; \ fi; \ - conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + conf="$$(pwd)/reproduce/software/config/installation/numpy-scipy.cfg"; \ $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf) \ && cp $(dtexdir)/scipy.tex $(ictdir)/ \ && echo "Scipy $(scipy-version) \citep{scipy2007,scipy2011}" > $@ |