diff options
Diffstat (limited to 'reproduce/software')
25 files changed, 3148 insertions, 0 deletions
diff --git a/reproduce/software/bash/git-post-checkout b/reproduce/software/bash/git-post-checkout new file mode 100644 index 0000000..9552f01 --- /dev/null +++ b/reproduce/software/bash/git-post-checkout @@ -0,0 +1,66 @@ +#!@BINDIR@/bash +# +# The example hook script to store the metadata information of version +# controlled files (with each commit) using the `metastore' program. +# +# Copyright (C) 2016 Przemyslaw Pawelczyk <przemoc@gmail.com> +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This script is taken from the `examples/hooks/pre-commit' file of the +# `metastore' package (installed within the project, with an MIT license +# for copyright). We have just changed the name of the `MSFILE' and also +# set special characters for the installation location of meta-store so our +# own installation is found by Git. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the +# following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +# USE OR OTHER DEALINGS IN THE SOFTWARE. + + +# File containig the metadata and metastore executable. +MSFILE=".file-metadata" +MSBIN=@BINDIR@/metastore + +# If metastore is not installed, then ignore this script (exit with a +# status of 0). +if [ ! -f $MSBIN ]; then exit 0; fi + +# Delete all temporary files +find @TOP_PROJECT_DIR@/ -name "*~" -type f -delete + +# Function to help in reporting a crash. +exit_on_fail() { + "$@" + if [ $? -ne 0 ]; then + echo "Failed to execute: $@" >&2 + exit 1 + fi +} + +# Check if the metadata file exists. +if [ ! -e "$MSFILE" ]; then + echo "\"$MSFILE\" missing" >&2 + exit 1 +fi + +# Run metastore. +exit_on_fail \ + $MSBIN -a -m -e -E -q -O @USER@ -G @GROUP@ -f "$MSFILE" + +# Return with a success code (0). +exit 0 diff --git a/reproduce/software/bash/git-pre-commit b/reproduce/software/bash/git-pre-commit new file mode 100644 index 0000000..dbe0ecc --- /dev/null +++ b/reproduce/software/bash/git-pre-commit @@ -0,0 +1,83 @@ +#!@BINDIR@/bash +# +# The example hook script to store the metadata information of version +# controlled files (with each commit) using the `metastore' program. +# +# Copyright (C) 2016 Przemyslaw Pawelczyk <przemoc@gmail.com> +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# WARNING: +# +# If the commit is aborted (e.g. by not entering any synopsis), +# then updated metastore file (.metadata by default) is not reverted, +# so its new version remains in the index. +# To undo any changes in metastore file written since HEAD commit, +# you may want to reset and checkout HEAD version of the file: +# +# git reset HEAD -- .metadata +# git checkout HEAD -- .metadata +# +# This script is taken from the `examples/hooks/pre-commit' file of the +# `metastore' package (installed within the project, with an MIT license +# for copyright). Here, the name of the `MSFILE' and also set special +# characters for the installation location of meta-store so our own +# installation is found by Git. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the +# following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +# USE OR OTHER DEALINGS IN THE SOFTWARE. + +# File containig the metadata and metastore executable. +MSFILE=".file-metadata" +MSBIN=@BINDIR@/metastore + +# If metastore is not installed, then ignore this script (exit with a +# status of 0). +if [ ! -f $MSBIN ]; then exit 0; fi + +# Function to help in reporting a crash. +exit_on_fail() { + "$@" + if [ $? -ne 0 ]; then + echo "Failed to execute: $@" >&2 + exit 1 + fi +} + +# Run metastore. +exit_on_fail \ + $MSBIN -O @USER@ -G @GROUP@ -s -f "$MSFILE" + +# If it's first metastore commit, store again to include $MSFILE in $MSFILE. +if ! git-ls-tree --name-only HEAD 2>/dev/null | grep -Fqx "$MSFILE"; then + exit_on_fail \ + $MSBIN -O @USER@ -G @GROUP@ -s -f "$MSFILE" +fi + +# Check if the metadata file exists. +if [ ! -e "$MSFILE" ]; then + echo "\"$MSFILE\" missing" >&2 + exit 1 +fi + +# Add the metadata file to the Git repository. +exit_on_fail \ + git-add "$MSFILE" + +# Return with a success code (0). +exit 0 diff --git a/reproduce/software/bibtex/astropy.tex b/reproduce/software/bibtex/astropy.tex new file mode 100644 index 0000000..450212d --- /dev/null +++ b/reproduce/software/bibtex/astropy.tex @@ -0,0 +1,97 @@ +%% Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{astropy2018, + author = {{Astropy Collaboration} and {Price-Whelan}, A.~M. and {Sip{\H o}cz}, B.~M. and + {G{\"u}nther}, H.~M. and {Lim}, P.~L. and {Crawford}, S.~M. and + {Conseil}, S. and {Shupe}, D.~L. and {Craig}, M.~W. and {Dencheva}, N. and + {Ginsburg}, A. and {VanderPlas}, J.~T. and {Bradley}, L.~D. and + {P{\'e}rez-Su{\'a}rez}, D. and {de Val-Borro}, M. and {Aldcroft}, T.~L. and + {Cruz}, K.~L. and {Robitaille}, T.~P. and {Tollerud}, E.~J. and + {Ardelean}, C. and {Babej}, T. and {Bach}, Y.~P. and {Bachetti}, M. and + {Bakanov}, A.~V. and {Bamford}, S.~P. and {Barentsen}, G. and + {Barmby}, P. and {Baumbach}, A. and {Berry}, K.~L. and {Biscani}, F. and + {Boquien}, M. and {Bostroem}, K.~A. and {Bouma}, L.~G. and {Brammer}, G.~B. and + {Bray}, E.~M. and {Breytenbach}, H. and {Buddelmeijer}, H. and + {Burke}, D.~J. and {Calderone}, G. and {Cano Rodr{\'{\i}}guez}, J.~L. and + {Cara}, M. and {Cardoso}, J.~V.~M. and {Cheedella}, S. and {Copin}, Y. and + {Corrales}, L. and {Crichton}, D. and {D'Avella}, D. and {Deil}, C. and + {Depagne}, {\'E}. and {Dietrich}, J.~P. and {Donath}, A. and + {Droettboom}, M. and {Earl}, N. and {Erben}, T. and {Fabbro}, S. and + {Ferreira}, L.~A. and {Finethy}, T. and {Fox}, R.~T. and {Garrison}, L.~H. and + {Gibbons}, S.~L.~J. and {Goldstein}, D.~A. and {Gommers}, R. and + {Greco}, J.~P. and {Greenfield}, P. and {Groener}, A.~M. and + {Grollier}, F. and {Hagen}, A. and {Hirst}, P. and {Homeier}, D. and + {Horton}, A.~J. and {Hosseinzadeh}, G. and {Hu}, L. and {Hunkeler}, J.~S. and + {Ivezi{\'c}}, {\v Z}. and {Jain}, A. and {Jenness}, T. and {Kanarek}, G. and + {Kendrew}, S. and {Kern}, N.~S. and {Kerzendorf}, W.~E. and + {Khvalko}, A. and {King}, J. and {Kirkby}, D. and {Kulkarni}, A.~M. and + {Kumar}, A. and {Lee}, A. and {Lenz}, D. and {Littlefair}, S.~P. and + {Ma}, Z. and {Macleod}, D.~M. and {Mastropietro}, M. and {McCully}, C. and + {Montagnac}, S. and {Morris}, B.~M. and {Mueller}, M. and {Mumford}, S.~J. and + {Muna}, D. and {Murphy}, N.~A. and {Nelson}, S. and {Nguyen}, G.~H. and + {Ninan}, J.~P. and {N{\"o}the}, M. and {Ogaz}, S. and {Oh}, S. and + {Parejko}, J.~K. and {Parley}, N. and {Pascual}, S. and {Patil}, R. and + {Patil}, A.~A. and {Plunkett}, A.~L. and {Prochaska}, J.~X. and + {Rastogi}, T. and {Reddy Janga}, V. and {Sabater}, J. and {Sakurikar}, P. and + {Seifert}, M. and {Sherbert}, L.~E. and {Sherwood-Taylor}, H. and + {Shih}, A.~Y. and {Sick}, J. and {Silbiger}, M.~T. and {Singanamalla}, S. and + {Singer}, L.~P. and {Sladen}, P.~H. and {Sooley}, K.~A. and + {Sornarajah}, S. and {Streicher}, O. and {Teuben}, P. and {Thomas}, S.~W. and + {Tremblay}, G.~R. and {Turner}, J.~E.~H. and {Terr{\'o}n}, V. and + {van Kerkwijk}, M.~H. and {de la Vega}, A. and {Watkins}, L.~L. and + {Weaver}, B.~A. and {Whitmore}, J.~B. and {Woillez}, J. and + {Zabalza}, V. and {Astropy Contributors}}, + title = "{The Astropy Project: Building an Open-science Project and Status of the v2.0 Core Package}", + journal = {AJ}, +archivePrefix = "arXiv", + eprint = {1801.02634}, + primaryClass = "astro-ph.IM", + keywords = {methods: data analysis, methods: miscellaneous, methods: statistical, reference systems }, + year = 2018, + month = sep, + volume = 156, + eid = {123}, + pages = {123}, + doi = {10.3847/1538-3881/aabc4f}, + adsurl = {http://adsabs.harvard.edu/abs/2018AJ....156..123A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + + + + + +@ARTICLE{astropy2013, + author = {{Astropy Collaboration} and {Robitaille}, T.~P. and {Tollerud}, E.~J. and + {Greenfield}, P. and {Droettboom}, M. and {Bray}, E. and {Aldcroft}, T. and + {Davis}, M. and {Ginsburg}, A. and {Price-Whelan}, A.~M. and + {Kerzendorf}, W.~E. and {Conley}, A. and {Crighton}, N. and + {Barbary}, K. and {Muna}, D. and {Ferguson}, H. and {Grollier}, F. and + {Parikh}, M.~M. and {Nair}, P.~H. and {Unther}, H.~M. and {Deil}, C. and + {Woillez}, J. and {Conseil}, S. and {Kramer}, R. and {Turner}, J.~E.~H. and + {Singer}, L. and {Fox}, R. and {Weaver}, B.~A. and {Zabalza}, V. and + {Edwards}, Z.~I. and {Azalee Bostroem}, K. and {Burke}, D.~J. and + {Casey}, A.~R. and {Crawford}, S.~M. and {Dencheva}, N. and + {Ely}, J. and {Jenness}, T. and {Labrie}, K. and {Lim}, P.~L. and + {Pierfederici}, F. and {Pontzen}, A. and {Ptak}, A. and {Refsdal}, B. and + {Servillat}, M. and {Streicher}, O.}, + title = "{Astropy: A community Python package for astronomy}", + journal = {A\&A}, +archivePrefix = "arXiv", + eprint = {1307.6212}, + primaryClass = "astro-ph.IM", + keywords = {methods: data analysis, methods: miscellaneous, virtual observatory tools}, + year = 2013, + month = oct, + volume = 558, + eid = {A33}, + pages = {A33}, + doi = {10.1051/0004-6361/201322068}, + adsurl = {http://adsabs.harvard.edu/abs/2013A\%26A...558A..33A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/bibtex/cython.tex b/reproduce/software/bibtex/cython.tex new file mode 100644 index 0000000..110cb19 --- /dev/null +++ b/reproduce/software/bibtex/cython.tex @@ -0,0 +1,21 @@ +%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{cython2011, + author = {{Behnel}, S. and {Bradshaw}, R. and {Citro}, C. and {Dalcin}, L. and + {Seljebotn}, D.~S. and {Smith}, K.}, + title = "{Cython: The Best of Both Worlds}", + journal = {CiSE}, + year = 2011, + month = mar, + volume = 13, + number = 2, + pages = {31}, + doi = {10.1109/MCSE.2010.118}, + adsurl = {http://adsabs.harvard.edu/abs/2011CSE....13b..31B}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/bibtex/fftw.tex b/reproduce/software/bibtex/fftw.tex new file mode 100644 index 0000000..60c24a8 --- /dev/null +++ b/reproduce/software/bibtex/fftw.tex @@ -0,0 +1,19 @@ +%% Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{fftw, + author = {M. {Frigo} and S. G. {Johnson}}, + journal = {IEEE Proc.}, + title = {The Design and Implementation of FFTW3}, + year = {2005}, + volume = {93}, + number = {2}, + pages = {216}, + doi = {10.1109/JPROC.2004.840301}, + ISSN = {0018-9219}, + month = {Feb}, +} diff --git a/reproduce/software/bibtex/gnuastro.tex b/reproduce/software/bibtex/gnuastro.tex new file mode 100644 index 0000000..91788c9 --- /dev/null +++ b/reproduce/software/bibtex/gnuastro.tex @@ -0,0 +1,24 @@ +%% Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{gnuastro, + author = {{Akhlaghi}, M. and {Ichikawa}, T.}, + title = "{Noise-based Detection and Segmentation of Nebulous Objects}", + journal = {ApJS}, +archivePrefix = "arXiv", + eprint = {1505.01664}, + primaryClass = "astro-ph.IM", + keywords = {galaxies: irregular, galaxies: photometry, galaxies: structure, methods: data analysis, techniques: image processing, techniques: photometric}, + year = 2015, + month = sep, + volume = 220, + eid = {1}, + pages = {1}, + doi = {10.1088/0067-0049/220/1/1}, + adsurl = {http://adsabs.harvard.edu/abs/2015ApJS..220....1A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/bibtex/matplotlib.tex b/reproduce/software/bibtex/matplotlib.tex new file mode 100644 index 0000000..8578899 --- /dev/null +++ b/reproduce/software/bibtex/matplotlib.tex @@ -0,0 +1,22 @@ +%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@Article{matplotlib2007, + Author = {Hunter, J. D.}, + Title = {Matplotlib: A 2D graphics environment}, + Journal = {CiSE}, + Volume = {9}, + Number = {3}, + Pages = {90}, + abstract = {Matplotlib is a 2D graphics package used for Python + for application development, interactive scripting, and + publication-quality image generation across user + interfaces and operating systems.}, + publisher = {IEEE COMPUTER SOC}, + doi = {10.1109/MCSE.2007.55}, + year = 2007 +} diff --git a/reproduce/software/bibtex/mpi4py.tex b/reproduce/software/bibtex/mpi4py.tex new file mode 100644 index 0000000..4e0045f --- /dev/null +++ b/reproduce/software/bibtex/mpi4py.tex @@ -0,0 +1,20 @@ +%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{mpi4py2011, + author = {{Dalcin}, L.~D. and {Paz}, R.~R. and {Kler}, P.~A. and {Cosimo}, A. + }, + title = "{Parallel distributed computing using Python}", + journal = {Advances in Water Resources}, + year = 2011, + month = sep, + volume = 34, + pages = {1124}, + doi = {10.1016/j.advwatres.2011.04.013}, + adsurl = {http://adsabs.harvard.edu/abs/2011AdWR...34.1124D}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/bibtex/numpy.tex b/reproduce/software/bibtex/numpy.tex new file mode 100644 index 0000000..0d9028f --- /dev/null +++ b/reproduce/software/bibtex/numpy.tex @@ -0,0 +1,23 @@ +%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{numpy2011, + author = {{van der Walt}, S. and {Colbert}, S.~C. and {Varoquaux}, G.}, + title = "{The NumPy Array: A Structure for Efficient Numerical Computation}", + journal = {CiSE}, +archivePrefix = "arXiv", + eprint = {1102.1523}, + primaryClass = "cs.MS", + year = 2011, + month = mar, + volume = 13, + number = 2, + pages = {22}, + doi = {10.1109/MCSE.2011.37}, + adsurl = {http://adsabs.harvard.edu/abs/2011CSE....13b..22V}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/bibtex/scipy.tex b/reproduce/software/bibtex/scipy.tex new file mode 100644 index 0000000..60500d2 --- /dev/null +++ b/reproduce/software/bibtex/scipy.tex @@ -0,0 +1,34 @@ +%% Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com> +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{scipy2011, + author = {K. J. {Millman} and M. {Aivazis}}, + journal = {CiSE}, + title = {Python for Scientists and Engineers}, + year = {2011}, + volume = {13}, + number = {2}, + pages = {9}, + keywords = {Special issues and sections;Computer languages;Programming;Scientific computing;Numerical models;Programming languages;Python;Scientific computing;interactive research;Python libraries;Python tools}, + doi = {10.1109/MCSE.2011.36}, + ISSN = {1521-9615}, + month ={March}, +} + +@ARTICLE{scipy2007, + author = {T. E. {Oliphant}}, + journal = {CiSE}, + title = {Python for Scientific Computing}, + year = {2007}, + volume = {9}, + number = {3}, + pages = {10}, + keywords = {high level languages;Python;scientific computing;steering language;scientific codes;high-level language;Scientific computing;High level languages;Libraries;Writing;Application software;Embedded software;Software standards;Standards development;Internet;Prototypes;Python;computer languages;scientific programming;scientific computing}, + doi = {10.1109/MCSE.2007.58}, + ISSN = {1521-9615}, + month = {May}, +} diff --git a/reproduce/software/config/gnuastro/astbuildprog.conf b/reproduce/software/config/gnuastro/astbuildprog.conf new file mode 100644 index 0000000..c15baf1 --- /dev/null +++ b/reproduce/software/config/gnuastro/astbuildprog.conf @@ -0,0 +1,16 @@ +# BuildProgram's configuration file depends on the build and is installed +# along with the rest of Gnuastro. So we'll set any specific configuration +# we want here, then import those. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice and +# this notice are preserved. This file is offered as-is, without any +# warranty. + +# To keep the program's compiled excutable file, comment this line. +deletecompiled 1 + +# Import the built configuration file +config .local/etc/astbuildprog.conf
\ No newline at end of file diff --git a/reproduce/software/config/gnuastro/astconvertt.conf b/reproduce/software/config/gnuastro/astconvertt.conf new file mode 100644 index 0000000..6466455 --- /dev/null +++ b/reproduce/software/config/gnuastro/astconvertt.conf @@ -0,0 +1,33 @@ +# Default parameters (System) for ConvertType. +# ConvertType is part of GNU Astronomy Utitlies. +# +# Use the long option name of each parameter followed by a value. The name +# and value should be separated by atleast one white-space character (for +# example ` '[space], or tab). Lines starting with `#' are ignored. +# +# For more information, please run these commands: +# +# $ astconvertt --help # Full list of options, short doc. +# $ astconvertt -P # Print all options and used values. +# $ info astconvertt # All options and input/output. +# $ info gnuastro "Configuration files" # How to use configuration files. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice and +# this notice are preserved. This file is offered as-is, without any +# warranty. + +# Input: + +# Output: + quality 100 + widthincm 10.0 + borderwidth 1 + output jpg + +# Flux: + invert 0 + +# Common options diff --git a/reproduce/software/config/gnuastro/aststatistics.conf b/reproduce/software/config/gnuastro/aststatistics.conf new file mode 100644 index 0000000..1d13e3f --- /dev/null +++ b/reproduce/software/config/gnuastro/aststatistics.conf @@ -0,0 +1,36 @@ +# Default parameters (System) for Statistics. +# Statistics is part of GNU Astronomy Utitlies. +# +# Use the long option name of each parameter followed by a value. The name +# and value should be separated by atleast one white-space character (for +# example ` '[space], or tab). Lines starting with `#' are ignored. +# +# For more information, please run these commands: +# +# $ aststatistics --help # Full list of options, short doc. +# $ aststatistics -P # Print all options and used values. +# $ info aststatistics # All options and input/output. +# $ info gnuastro "Configuration files" # How to use configuration files. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice and +# this notice are preserved. This file is offered as-is, without any +# warranty. + +# Input image: + +# Sky and its STD settings + khdu 1 + meanmedqdiff 0.005 + outliersigma 10 + outliersclip 3,0.2 + smoothwidth 3 + sclipparams 3,0.1 + +# Histogram and CFP settings + numasciibins 70 + asciiheight 10 + numbins 100 + mirrordist 1.5 diff --git a/reproduce/software/config/gnuastro/gnuastro.conf b/reproduce/software/config/gnuastro/gnuastro.conf new file mode 100644 index 0000000..57fcadc --- /dev/null +++ b/reproduce/software/config/gnuastro/gnuastro.conf @@ -0,0 +1,37 @@ +# Default values for the common options to all the programs in GNU +# Astronomy Utitlies. +# +# IMPORTANT NOTE: The `lastconfig' option is very important in a +# reproducible environment. Because we don't want any of Gnuastro's +# programs to go into an un-controlled environment (user or system wide +# configuration files). +# +# The rest of this configuration file is taken from the default Gnuastro +# configuration from its source (`bin/gnuastro.conf'). +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice and +# this notice are preserved. This file is offered as-is, without any +# warranty. + +# Local project settings (`config' has to be before `lastconfig'). + config .gnuastro/gnuastro-local.conf + lastconfig 1 + +# Input: + hdu 1 + ignorecase 1 + searchin name + +# Tessellation + tilesize 50,50 + numchannels 1,1 + remainderfrac 0.1 + workoverch 0 + interpnumngb 9 + interponlyblank 0 + +# Output: + tableformat fits-binary
\ No newline at end of file diff --git a/reproduce/software/config/installation/LOCAL.mk.in b/reproduce/software/config/installation/LOCAL.mk.in new file mode 100644 index 0000000..785bb6a --- /dev/null +++ b/reproduce/software/config/installation/LOCAL.mk.in @@ -0,0 +1,16 @@ +# Local project configuration. +# +# This is just a template for the `./configure' script to fill in. Please +# don't make any change to this file. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice and +# this notice are preserved. This file is offered as-is, without any +# warranty. +BDIR = @bdir@ +INDIR = @indir@ +DEPENDENCIES-DIR = @ddir@ +DOWNLOADER = @downloader@ +GROUP-NAME = @groupname@ diff --git a/reproduce/software/config/installation/numpy-scipy.cfg b/reproduce/software/config/installation/numpy-scipy.cfg new file mode 100644 index 0000000..4b7a7b0 --- /dev/null +++ b/reproduce/software/config/installation/numpy-scipy.cfg @@ -0,0 +1,46 @@ +# THIS IS A COPY OF NUMPY'S site.cfg.example, CUSTOMIZED FOR THIS TEMPLATE +# ------------------------------------------------------------------------ + +# This file provides configuration information about non-Python +# dependencies for numpy.distutils-using packages. Create a file like this +# called "site.cfg" next to your package's setup.py file and fill in the +# appropriate sections. Not all packages will use all sections so you +# should leave out sections that your package does not use. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice and +# this notice are preserved. This file is offered as-is, without any +# warranty. + +# IMPORTANT NOTE +# -------------- +# +# The `ALL' grouping does't apply to ATLAS!!!!! + + [ALL] + library_dirs = @LIBDIR@ + include_dirs = @INCDIR@ + + +# FFT libraries +# ------------- +# There are two FFT libraries that we can configure here: FFTW (2 and 3) and djbfft. +# Note that these libraries are not used by numpy or scipy. +# +# http://fftw.org/ +# https://cr.yp.to/djbfft.html +# +# Given only this section, numpy.distutils will try to figure out which version +# of FFTW you are using. + +# Even with this section, Numpy apparently does not use fftw. But we will +# keep it here for the future (if a solution is found). +[fftw] +libraries = fftw3 +# +# For djbfft, numpy.distutils will look for either djbfft.a or libdjbfft.a . +#[djbfft] +#include_dirs = /usr/local/djbfft/include +#library_dirs = /usr/local/djbfft/lib diff --git a/reproduce/software/config/installation/texlive.conf b/reproduce/software/config/installation/texlive.conf new file mode 100644 index 0000000..53054e1 --- /dev/null +++ b/reproduce/software/config/installation/texlive.conf @@ -0,0 +1,35 @@ +# Basic profile for build. Values to set: +# +# installdir: Install directory +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice and +# this notice are preserved. This file is offered as-is, without any +# warranty. +selected_scheme scheme-basic +TEXDIR @installdir@/texlive/2018 +TEXMFCONFIG @installdir@/texlive2018/texmf-config +TEXMFLOCAL @installdir@/texlive/texmf-local +TEXMFSYSCONFIG @installdir@/texlive/2018/texmf-config +TEXMFSYSVAR @installdir@/texlive/2018/texmf-var +TEXMFVAR @installdir@/texlive2018/texmf-var +instopt_adjustpath 0 +instopt_adjustrepo 1 +instopt_letter 0 +instopt_portable 0 +instopt_write18_restricted 1 +tlpdbopt_autobackup 1 +tlpdbopt_backupdir @installdir@/texlive/2018/backups +tlpdbopt_create_formats 1 +tlpdbopt_desktop_integration 1 +tlpdbopt_file_assocs 1 +tlpdbopt_generate_updmap 0 +tlpdbopt_install_docfiles 1 +tlpdbopt_install_srcfiles 1 +tlpdbopt_post_code 1 +tlpdbopt_sys_bin @installdir@/bin +tlpdbopt_sys_info @installdir@/share/info +tlpdbopt_sys_man @installdir@/share/man +tlpdbopt_w32_multi_user 1 diff --git a/reproduce/software/config/installation/texlive.mk b/reproduce/software/config/installation/texlive.mk new file mode 100644 index 0000000..0acf5ab --- /dev/null +++ b/reproduce/software/config/installation/texlive.mk @@ -0,0 +1,22 @@ +# Necessary packages to install in TeX Live. +# +# If any extra TeX package is necessary to build your paper, just add its +# name to this variable (you can check in `ctan.org' to find the official +# name). +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice and +# this notice are preserved. This file is offered as-is, without any +# warranty. + + +# Note on `tex' and `fancyhdr': These two packages are installed along with +# the basic installation scheme that we used to install tlmgr, they will be +# ignored in the `tlmgr install' command, but will be used later when we +# want their versions. +texlive-packages = tex fancyhdr ec newtx fontaxes xkeyval etoolbox xcolor \ + setspace caption footmisc datetime fmtcount titlesec \ + preprint ulem biblatex biber logreq pgf pgfplots fp \ + courier tex-gyre txfonts times diff --git a/reproduce/software/config/installation/versions.mk b/reproduce/software/config/installation/versions.mk new file mode 100644 index 0000000..149f9ae --- /dev/null +++ b/reproduce/software/config/installation/versions.mk @@ -0,0 +1,123 @@ +# Versions of the various dependencies +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + +# C/C++ programs and libraries. +atlas-version = 3.10.3 +bash-version = 5.0 +binutils-version = 2.31.1 +cfitsio-version = 3.45 +cmake-version = 3.14.2 +coreutils-version = 8.31 +curl-version = 7.63.0 +diffutils-version = 3.7 +fftw-version = 3.3.8 +file-version = 5.36 +findutils-version = 4.6.0.199-e3fc +flock-version = 0.2.3 +freetype-version = 2.9 +gawk-version = 5.0.0 +gcc-version = 8.3.0 +ghostscript-version = 9.26 +git-version = 2.21.0 +gmp-version = 6.1.2 +gnuastro-version = 0.8 +grep-version = 3.3 +gsl-version = 2.5 +gzip-version = 1.10 +hdf5-version = 1.10.5 +isl-version = 0.18 +libbsd-version = 0.9.1 +libffi-version = 3.2.1 +libjpeg-version = v9b +libpng-version = 1.6.36 +libtiff-version = 4.0.10 +libtool-version = 2.4.6 +lzip-version = 1.20 +m4-version = 1.4.18 +make-version = 4.2.90 +metastore-version = 1.1.2-23-fa9170b +mpfr-version = 4.0.2 +mpc-version = 1.1.0 +ncurses-version = 6.1 +openblas-version = 0.3.5 +openmpi-version = 4.0.1 +openssl-version = 1.1.1a +patchelf-version = 0.9 +pkgconfig-version = 0.29.2 +python-version = 3.7.3 +readline-version = 8.0 +sed-version = 4.7 +tar-version = 1.32 +unzip-version = 6.0 +wget-version = 1.20.3 +which-version = 2.21 +xz-version = 5.2.4 +zip-version = 3.0 +zlib-version = 1.2.11 + +# Special libraries +# ----------------- +# +# When updating the version of these libraries, please look into the build +# rule first: In one way or another, the version string becomes necessary +# during their build and must be accounted for. +bzip2-version = 1.0.6 +lapack-version = 3.8.0 +libgit2-version = 0.26.0 +wcslib-version = 6.2 + +# Python packages +# --------------- +# +# IMPORTANT: Fix url in `reproduce/src/make/dependencies.mk' +# if changing the version +asn1crypto-version = 0.24.0 +astroquery-version = 0.3.9 +astropy-version = 3.1.1 +beautifulsoup4-version = 4.7.1 +certifi-version = 2018.11.29 +cffi-version = 1.12.2 +chardet-version = 3.0.4 +cryptography-version = 2.6.1 +cycler-version = 0.10.0 +cython-version = 0.29.6 +entrypoints-version = 0.3 +h5py-version = 2.9.0 +html5lib-version = 1.0.1 +idna-version = 2.8 +jeepney-version = 0.4 +kiwisolver-version = 1.0.1 +keyring-version = 18.0.0 +matplotlib-version = 3.0.2 +mpi4py-version = 3.0.1 +numpy-version = 1.16.2 +pip-version = 19.0.2 +pycparser-version = 2.19 +pyparsing-version = 2.3.1 +pypkgconfig-version = 1.5.1 +python-dateutil-version = 2.8.0 +requests-version = 2.21.0 +scipy-version = 1.2.1 +secretstorage-version = 3.1.1 +setuptools-version = 40.8.0 +setuptools_scm-version = 3.2.0 +six-version = 1.12.0 +soupsieve-version = 1.8 +urllib3-version = 1.24.1 +virtualenv-version = 16.4.0 +webencodings-version = 0.5.1 diff --git a/reproduce/software/make/atlas-multiple.mk b/reproduce/software/make/atlas-multiple.mk new file mode 100755 index 0000000..fef25c7 --- /dev/null +++ b/reproduce/software/make/atlas-multiple.mk @@ -0,0 +1,72 @@ +# Rules to build ATLAS shared libraries in multi-threaded mode on GNU/Linux +# +# ------------------------------------------------------------------------ +# !!!!! IMPORTANT NOTES !!!!! +# +# This Makefile will be run during the initial `./configure' script. It is +# not included into the reproduction pipe after that. +# +# ------------------------------------------------------------------------ +# +# Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + +ORIGLDFLAGS := $(LDFLAGS) + +include Make.inc + +all: libatlas.so libf77blas.so libptf77blas.so libstcblas.so libptcblas.so \ + libblas.so libcblas.so liblapack.so.3.6.1 libptlapack.so.3.6.1 liblapack.so.3 + +libatlas.so: libatlas.a + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \ + --whole-archive libatlas.a --no-whole-archive -lc $(LIBS) + +libf77blas.so : libf77blas.a libatlas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libf77blas.so.3 \ + -o $@ --whole-archive libf77blas.a --no-whole-archive \ + $(F77SYSLIB) -L. -latlas + +libptf77blas.so : libptf77blas.a libatlas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libblas.so.3 \ + -o $@ --whole-archive libptf77blas.a --no-whole-archive \ + $(F77SYSLIB) -L. -latlas + +libstcblas.so : libcblas.a libatlas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libstcblas.so \ + -o $@ --whole-archive libcblas.a -L. -latlas -lblas + +libptcblas.so : libptcblas.a libatlas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libcblas.so \ + -o $@ --whole-archive libptcblas.a -L. -latlas -lblas + +libblas.so: libptf77blas.so + ln -s $< $@ + +libcblas.so: libptcblas.so + ln -s $< $@ + +liblapack.so.3.6.1 : liblapack.a libstcblas.so libf77blas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libstlapack.so.3 \ + -o $@ --whole-archive liblapack.a --no-whole-archive \ + $(F77SYSLIB) -L. -lstcblas -lf77blas + +libptlapack.so.3.6.1 : libptlapack.a libcblas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname liblapack.so.3 \ + -o $@ --whole-archive libptlapack.a --no-whole-archive \ + $(F77SYSLIB) -L. -lcblas -lblas + +liblapack.so.3: libptlapack.so.3.6.1 + ln -s $< $@ diff --git a/reproduce/software/make/atlas-single.mk b/reproduce/software/make/atlas-single.mk new file mode 100755 index 0000000..dde2926 --- /dev/null +++ b/reproduce/software/make/atlas-single.mk @@ -0,0 +1,54 @@ +# Rules to build ATLAS shared libraries for single threads on GNU/Linux +# +# ------------------------------------------------------------------------ +# !!!!! IMPORTANT NOTES !!!!! +# +# This Makefile will be run during the initial `./configure' script. It is +# not included into the reproduction pipe after that. +# +# ------------------------------------------------------------------------ +# +# Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + +ORIGLDFLAGS := $(LDFLAGS) + +include Make.inc + +all: libatlas.so libf77blas.so libcblas.so libblas.so liblapack.so.3.6.1 + +libatlas.so: libatlas.a + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \ + --whole-archive libatlas.a --no-whole-archive -lc $(LIBS) + +libf77blas.so : libf77blas.a libatlas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname libblas.so.3 \ + -o $@ --whole-archive libf77blas.a --no-whole-archive \ + $(F77SYSLIB) -L. -latlas + +libcblas.so : libcblas.a libatlas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname $@ -o $@ \ + --whole-archive libcblas.a -L. -latlas -lblas + +libblas.so: libf77blas.so + ln -s $< $@ + +liblapack.so.3.6.1 : liblapack.a libcblas.so libblas.so + ld $(ORIGLDFLAGS) $(LDFLAGS) -shared -soname liblapack.so.3 \ + -o $@ --whole-archive liblapack.a --no-whole-archive \ + $(F77SYSLIB) -L. -lcblas -lblas + +liblapack.so.3: liblapack.so.3.6.1 + ln -s $< $@ diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk new file mode 100644 index 0000000..cfa05d2 --- /dev/null +++ b/reproduce/software/make/basic.mk @@ -0,0 +1,903 @@ +# Build the VERY BASIC project software before higher-level ones. Assuming +# minimal/generic Make and Shell. +# +# ------------------------------------------------------------------------ +# !!!!! IMPORTANT NOTES !!!!! +# +# This Makefile will be run by the initial `./configure' script. It is not +# included into the project after that. +# +# This Makefile builds very low-level and basic tools like GNU Tar, GNU +# Bash, GNU Make, GCC and etc. Therefore this is the only Makefile in the +# project where you CANNOT assume that GNU Bash or GNU Make are used. After +# this Makefile (where GNU Bash and GNU Make are built), other Makefiles +# can safely assume the fixed version of all these software. +# +# ------------------------------------------------------------------------ +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + + +# Top level environment +include reproduce/software/make/build-rules.mk +include reproduce/software/config/installation/LOCAL.mk +include reproduce/software/config/installation/versions.mk + +lockdir = $(BDIR)/locks +tdir = $(BDIR)/software/tarballs +ddir = $(BDIR)/software/build-tmp +idir = $(BDIR)/software/installed +ibdir = $(BDIR)/software/installed/bin +ildir = $(BDIR)/software/installed/lib +ibidir = $(BDIR)/software/installed/version-info/bin +ilidir = $(BDIR)/software/installed/version-info/lib + +# We'll need the system's PATH for making links to low-level programs we +# won't be building ourselves. +syspath := $(PATH) + +# As we build more programs, we want to use this project's built programs +# and libraries, not the host's. +export CCACHE_DISABLE := 1 +export PATH := $(ibdir):$(PATH) +export PKG_CONFIG_PATH := $(ildir)/pkgconfig +export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) +export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) +export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) + +# Define the top-level programs (that don't depend on any other). +top-level-programs = low-level-links wget gcc +all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) + + + + + +# Tarballs +# -------- +# +# Prepare tarballs. Difference with that in `high-level.mk': `.ONESHELL' is +# not recognized by some versions of Make (even older GNU Makes). So we'll +# have to make sure the recipe doesn't break into multiple shell calls (so +# we can preserve the variables). +# +# Software hosted at akhlaghi.org/src: As of our latest check (November +# 2018) their major release tarballs either crash or don't build on some +# systems (for example Make or Gzip), or they don't exist (for example +# Bzip2). +# +# In the first case, we used their Git repo and bootstrapped them (just +# like Gnuastro) and built the most recent tarball off of that. In the case +# of Bzip2: its webpage has expired and doesn't host the data any more. It +# is available on the link below (archive.org): +# +# https://web.archive.org/web/20180624184806/http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz +# +# However, downloading from this link is slow (because its just a link). So +# its easier to just keep a with the others. +$(lockdir): | $(BDIR); mkdir $@ +downloadwrapper = ./reproduce/analysis/bash/download-multi-try +tarballs = $(foreach t, bash-$(bash-version).tar.gz \ + binutils-$(binutils-version).tar.lz \ + bzip2-$(bzip2-version).tar.gz \ + cert.pem \ + coreutils-$(coreutils-version).tar.xz \ + diffutils-$(diffutils-version).tar.xz \ + file-$(file-version).tar.gz \ + findutils-$(findutils-version).tar.lz \ + gawk-$(gawk-version).tar.lz \ + gcc-$(gcc-version).tar.xz \ + gmp-$(gmp-version).tar.lz \ + grep-$(grep-version).tar.xz \ + gzip-$(gzip-version).tar.gz \ + isl-$(isl-version).tar.bz2 \ + libtool-$(libtool-version).tar.xz \ + lzip-$(lzip-version).tar.gz \ + m4-$(m4-version).tar.gz \ + make-$(make-version).tar.lz \ + mpfr-$(mpfr-version).tar.xz \ + mpc-$(mpc-version).tar.gz \ + ncurses-$(ncurses-version).tar.gz \ + openssl-$(openssl-version).tar.gz \ + patchelf-$(patchelf-version).tar.gz \ + pkg-config-$(pkgconfig-version).tar.gz \ + readline-$(readline-version).tar.gz \ + sed-$(sed-version).tar.xz \ + tar-$(tar-version).tar.gz \ + unzip-$(unzip-version).tar.gz \ + wget-$(wget-version).tar.lz \ + which-$(which-version).tar.gz \ + xz-$(xz-version).tar.gz \ + zip-$(zip-version).tar.gz \ + zlib-$(zlib-version).tar.gz \ + , $(tdir)/$(t) ) +$(tarballs): $(tdir)/%: | $(lockdir) + if [ -f $(DEPENDENCIES-DIR)/$* ]; then \ + cp $(DEPENDENCIES-DIR)/$* $@; \ + else \ + n=$$(echo $* | sed -e's/[0-9\-]/ /g' \ + -e's/\./ /g' \ + | awk '{print $$1}' ); \ + \ + mergenames=1; \ + if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash; \ + elif [ $$n = binutils ]; then w=http://ftp.gnu.org/gnu/binutils; \ + elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = cert ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils;\ + elif [ $$n = diffutils ]; then w=http://ftp.gnu.org/gnu/diffutils;\ + elif [ $$n = file ]; then w=ftp://ftp.astron.com/pub/file; \ + elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk; \ + elif [ $$n = gcc ]; then w=http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version); \ + elif [ $$n = gmp ]; then w=https://gmplib.org/download/gmp; \ + elif [ $$n = grep ]; then w=http://ftp.gnu.org/gnu/grep; \ + elif [ $$n = gzip ]; then w=http://ftp.gnu.org/gnu/gzip; \ + elif [ $$n = isl ]; then w=ftp://gcc.gnu.org/pub/gcc/infrastructure; \ + elif [ $$n = libtool ]; then w=http://ftp.gnu.org/gnu/libtool; \ + elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \ + elif [ $$n = m ]; then \ + mergenames=0; \ + w=http://akhlaghi.org/src/m4-1.4.18-patched.tar.gz; \ + elif [ $$n = make ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = mpfr ]; then w=http://www.mpfr.org/mpfr-current;\ + elif [ $$n = mpc ]; then w=http://ftp.gnu.org/gnu/mpc; \ + elif [ $$n = ncurses ]; then w=http://ftp.gnu.org/gnu/ncurses; \ + elif [ $$n = openssl ]; then w=http://www.openssl.org/source; \ + elif [ $$n = patchelf ]; then w=http://nixos.org/releases/patchelf/patchelf-$(patchelf-version); \ + elif [ $$n = pkg ]; then w=http://pkg-config.freedesktop.org/releases; \ + elif [ $$n = readline ]; then w=http://ftp.gnu.org/gnu/readline; \ + elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed; \ + elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \ + elif [ $$n = unzip ]; then \ + mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//'); \ + w=ftp://ftp.info-zip.org/pub/infozip/src/unzip$$v.tgz; \ + elif [ $$n = wget ]; then w=http://ftp.gnu.org/gnu/wget; \ + elif [ $$n = which ]; then w=http://ftp.gnu.org/gnu/which; \ + elif [ $$n = xz ]; then w=http://tukaani.org/xz; \ + elif [ $$n = zip ]; then \ + mergenames=0; v=$$(echo $(zip-version) | sed -e's/\.//'); \ + w=ftp://ftp.info-zip.org/pub/infozip/src/zip$$v.tgz; \ + elif [ $$n = zlib ]; then w=http://www.zlib.net; \ + else \ + echo; echo; echo; \ + echo "'$$n' not a basic dependency name (for downloading)." \ + echo; echo; echo; \ + exit 1; \ + fi; \ + \ + if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*"; \ + else tarballurl=$$w; \ + fi; \ + \ + echo "Downloading $$tarballurl"; \ + if [ -f $(ibdir)/wget ]; then \ + downloader="wget --no-use-server-timestamps -O"; \ + else \ + downloader="$(DOWNLOADER)"; \ + fi; \ + \ + touch $(lockdir)/download; \ + $(downloadwrapper) "$$downloader" $(lockdir)/download \ + $$tarballurl $@; \ + fi + + + + + +# Low-level (not built) programs +# ------------------------------ +# +# For the time being, we aren't building a local C compiler, but we'll use +# any C compiler that the system already has and just make a symbolic link +# to it. +# +# ccache: ccache acts like a wrapper over the C compiler and is made to +# avoid/speed-up compiling of identical files in a system (it is commonly +# used on large servers). It actually makes `gcc' or `g++' a symbolic link +# to itself so it can control them internally. So, for our purpose here, it +# is very annoying and can cause many complications. We thus remove any +# part of PATH of that has `ccache' in it before making symbolic links to +# the programs we are not building ourselves. +makelink = origpath="$$PATH"; \ + export PATH=$$(echo $(syspath) | tr : '\n' | grep -v ccache \ + | tr '\n' :); \ + a=$$(which $(1) 2> /dev/null); \ + if [ -e $(ibdir)/$(1) ]; then rm $(ibdir)/$(1); fi; \ + if [ x$$a = x ]; then \ + if [ "x$(strip $(2))" = xmandatory ]; then \ + echo "'$(1)' is necessary for higher-level tools."; \ + echo "Please install it for the configuration to continue."; \ + exit 1; \ + fi; \ + else \ + ln -s $$a $(ibdir)/$(1); \ + fi; \ + export PATH="$$origpath" +$(ibdir) $(ildir):; mkdir $@ +$(ibidir)/low-level-links: | $(ibdir) $(ildir) + + # The Assembler + $(call makelink,as) + + # Compiler (Cmake needs the clang compiler which we aren't building + # yet in the project). + $(call makelink,clang) + $(call makelink,clang++) + + # The linker + $(call makelink,ar) + $(call makelink,ld) + $(call makelink,nm) + $(call makelink,ps) + $(call makelink,ranlib) + + # Mac OS specific + $(call makelink,sysctl) + $(call makelink,sw_vers) + $(call makelink,dsymutil) + $(call makelink,install_name_tool) + + # On Mac OS, libtool is different compared to GNU Libtool. The + # libtool we'll build in the high-level dependencies has the + # executable name `glibtool'. + $(call makelink,libtool) + + # GNU Gettext (translate messages) + $(call makelink,msgfmt) + + # Needed by TeXLive specifically. + $(call makelink,perl) + + # Necessary libraries: + # Libdl (for dynamic loading libraries at runtime) + # POSIX Threads library for multi-threaded programs. + for l in dl pthread; do \ + rm -f $(ildir)/lib$$l*; \ + if [ -f /usr/lib/lib$$l.a ]; then \ + ln -s /usr/lib/lib$$l.* $(ildir)/; \ + fi; \ + done + + # We want this to be empty (so it doesn't interefere with the other + # files in `ibidir'. + touch $@ + + + + + + + + + + +# Level 1 (MOST BASIC): Compression programs +# ------------------------------------------ +# +# The first set of programs to be built are those that we need to unpack +# the source code tarballs of each program. First, we'll build the +# necessary programs, then we'll build GNU Tar. +$(ibidir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz + $(call gbuild, $<, gzip-$(gzip-version), static, , V=1) \ + && echo "GNU Gzip $(gzip-version)" > $@ + +# GNU Lzip: For a static build, the `-static' flag should be given to +# LDFLAGS on the command-line (not from the environment). +ifeq ($(static_build),yes) +lzipconf="LDFLAGS=-static" +else +lzipconf= +endif +$(ibidir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz + $(call gbuild, $<, lzip-$(lzip-version), , $(lzipconf)) \ + && echo "Lzip $(lzip-version)" > $@ + +$(ibidir)/xz: $(tdir)/xz-$(xz-version).tar.gz + $(call gbuild, $<, xz-$(xz-version), static) \ + && echo "XZ Utils $(xz-version)" > $@ + +$(ibidir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz + # Bzip2 doesn't have a `./configure' script, and its Makefile + # doesn't build a shared library. So we can't use the `gbuild' + # function here and we need to take some extra steps (inspired + # from the "Linux from Scratch" guide for Bzip2): + # 1) The `sed' call is for relative installed symbolic links. + # 2) The special Makefile-libbz2_so builds shared libraries. + # + # NOTE: the major version number appears in the final symbolic + # link. + tdir=bzip2-$(bzip2-version); \ + if [ $(static_build) = yes ]; then \ + makecommand="make LDFLAGS=-static"; \ + makeshared="echo no-shared"; \ + else \ + makecommand="make"; \ + if [ x$(on_mac_os) = xyes ]; then \ + makeshared="echo no-shared"; \ + else \ + makeshared="make -f Makefile-libbz2_so"; \ + fi; \ + fi; \ + cd $(ddir) && rm -rf $$tdir && tar xf $< && cd $$tdir \ + && sed -e 's@\(ln -s -f \)$$(PREFIX)/bin/@\1@' Makefile \ + > Makefile.sed \ + && mv Makefile.sed Makefile \ + && $$makeshared \ + && cp -a libbz2* $(ildir)/ \ + && make clean \ + && $$makecommand \ + && make install PREFIX=$(idir) \ + && cd .. \ + && rm -rf $$tdir \ + && cd $(ildir) \ + && ln -fs libbz2.so.1.0 libbz2.so \ + && echo "Bzip2 $(bzip2-version)" > $@ + +$(ibidir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz + v=$$(echo $(unzip-version) | sed -e's/\.//'); \ + $(call gbuild, $<, unzip$$v, static,, \ + -f unix/Makefile generic_gcc \ + CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ + -f unix/Makefile \ + BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ + && echo "Unzip $(unzip-version)" > $@ + +$(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz + v=$$(echo $(zip-version) | sed -e's/\.//'); \ + $(call gbuild, $<, zip$$v, static,, \ + -f unix/Makefile generic_gcc \ + CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ + -f unix/Makefile \ + BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ + && echo "Zip $(zip-version)" > $@ + +# Some programs (like Wget and CMake) that use zlib need it to be dynamic +# so they use our custom build. So we won't force a static-only build. +# +# Note for a static-only build: Zlib's `./configure' doesn't use Autoconf's +# configure script, it just accepts a direct `--static' option. +$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz + $(call gbuild, $<, zlib-$(zlib-version)) \ + && echo "Zlib $(zlib-version)" > $@ + +# GNU Tar: When built statically, tar gives a segmentation fault on +# unpacking Bash. So we'll build it dynamically. Note that technically, zip +# and unzip aren't dependencies of Tar, but for a clean build, we'll set +# Tar to be the last compression-related software (the first-set of +# software to be built). +$(ibidir)/tar: $(tdir)/tar-$(tar-version).tar.gz \ + $(ibidir)/bzip2 \ + $(ibidir)/unzip \ + $(ibidir)/gzip \ + $(ibidir)/lzip \ + $(ilidir)/zlib \ + $(ibidir)/zip \ + $(ibidir)/xz + # Since all later programs depend on Tar, the configuration will be + # stuck here, only making Tar. So its more efficient to built it on + # multiple threads (when the user's Make doesn't pass down the + # number of threads). + $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads) V=1) \ + && echo "GNU Tar $(tar-version)" > $@ + + + + + + + + + + +# Level 2 (SECOND MOST BASIC): Bash and Make +# ------------------------------------------ +# +# GNU Make and GNU Bash are the second layer that we'll need to build the +# basic dependencies. +# +# Unfortunately Make needs dynamic linking in two instances: when loading +# objects (dynamically linked libraries), or when using the `getpwnam' +# function (for tilde expansion). The first can be disabled with +# `--disable-load', but unfortunately I don't know any way to fix the +# second. So, we'll have to build it dynamically for now. +$(ibidir)/make: $(tdir)/make-$(make-version).tar.lz \ + $(ibidir)/tar + # See Tar's comments for the `-j' option. + $(call gbuild, $<, make-$(make-version), , , -j$(numthreads)) \ + && echo "GNU Make $(make-version)" > $@ + +$(ilidir)/ncurses: $(tdir)/ncurses-$(ncurses-version).tar.gz \ + $(ibidir)/make + + # Delete the library that will be installed (so we can make sure + # the build process completed afterwards and reset the links). + rm -f $(ildir)/libncursesw* + + # Delete the (possibly existing) low-level programs that depend on + # `readline', and thus `ncurses'. Since these programs are actually + # used during the building of `ncurses', we need to delete them so + # the build process doesn't use the project's Bash and AWK, but the + # host's. + rm -f $(ibdir)/bash* $(ibdir)/awk* $(ibdir)/gawk* + + # Standard build process. + $(call gbuild, $<, ncurses-$(ncurses-version), static, \ + --with-shared --enable-rpath --without-normal \ + --without-debug --with-cxx-binding \ + --with-cxx-shared --enable-widec --enable-pc-files \ + --with-pkg-config=$(ildir)/pkgconfig ) + + # Unfortunately there are many problems with `ncurses' using + # "normal" (or 8-bit) characters. The standard way that will work + # is to build it with wide character mode as you see above in the + # configuration (or the `w' prefix you see below). Also, most + # programs (and in particular Bash and AWK), first look for other + # (mostly obsolete) libraries like tinfo, which define the same + # symbols. The links below address both situations: we need to fool + # higher-level packages to find this library even if they aren't + # explicitly mentioning its name correctly (as a value to `-l' at + # link time in their configure scripts). + # + # This part is taken from the Arch Linux build script[1], then + # extended to Mac thanks to Homebrew's script [2]. + # + # [1] https://git.archlinux.org/svntogit/packages.git/tree/trunk/PKGBUILD?h=packages/ncurses + # [2] https://github.com/Homebrew/homebrew-core/blob/master/Formula/ncurses.rb + # + # Since we can't have comments, in the connected script, here is a + # summary: + # + # 1. We find the actual suffix of the library, from the file that + # is not a symbolic link (starting with `-' in the output of + # `ls -l'). + # + # 2. We make symbolic links to all the "ncurses", "ncurses++", + # "form", "panel" and "menu" libraries to point to their + # "wide" (character) library. + # + # 3. We make symbolic links to the "tic" and "tinfo" libraries to + # point to the same `libncursesw' library. + # + # 4. Some programs link with "curses" (not "ncurses", notice the + # starting "n"), so we'll also make links for these to point + # to the `libncursesw' library. + # + # 5. A link is made to also be able to include files from the + # `ncurses' headers. + if [ x$(on_mac_os) = xyes ]; then so="dylib"; else so="so"; fi; \ + if [ -f $(ildir)/libncursesw.$$so ]; then \ + \ + sov=$$(ls -l $(ildir)/libncursesw* \ + | awk '/^-/{print $$NF}' \ + | sed -e's|'$(ildir)/libncursesw.'||'); \ + \ + cd "$(ildir)"; \ + for lib in ncurses ncurses++ form panel menu; do \ + ln -fs lib$$lib"w".$$sov lib$$lib.$$so; \ + ln -fs $(ildir)/pkgconfig/"$$lib"w.pc pkgconfig/$$lib.pc; \ + done; \ + for lib in tic tinfo; do \ + ln -fs libncursesw.$$sov lib$$lib.$$so; \ + ln -fs libncursesw.$$sov lib$$lib.$$sov; \ + ln -fs $(ildir)/pkgconfig/ncursesw.pc pkgconfig/$$lib.pc; \ + done; \ + ln -fs libncursesw.$$sov libcurses.$$so; \ + ln -fs libncursesw.$$sov libcursesw.$$sov; \ + ln -fs $(ildir)/pkgconfig/ncursesw.pc pkgconfig/curses.pc; \ + ln -fs $(ildir)/pkgconfig/ncursesw.pc pkgconfig/cursesw.pc; \ + \ + ln -fs $(idir)/include/ncursesw $(idir)/include/ncurses; \ + echo "GNU NCURSES $(ncurses-version)" > $@; \ + else \ + exit 1; \ + fi + +$(ilidir)/readline: $(tdir)/readline-$(readline-version).tar.gz \ + $(ilidir)/ncurses + $(call gbuild, $<, readline-$(readline-version), static, \ + --with-curses --disable-install-examples, \ + SHLIB_LIBS="-lncursesw" ) \ + && echo "GNU Readline $(readline-version)" > $@ + +$(ibidir)/patchelf: $(tdir)/patchelf-$(patchelf-version).tar.gz \ + $(ibidir)/make + $(call gbuild, $<, patchelf-$(patchelf-version), static) \ + && echo "PatchELF $(patchelf-version)" > $@ + + +# IMPORTANT: Even though we have enabled `rpath', Bash doesn't write the +# absolute adddress of the libraries it depends on! Therefore, if we +# configure Bash with `--with-installed-readline' (so the installed version +# of Readline, that we build below as a prerequisite or AWK, is used) and +# you run `ldd $(ibdir)/bash' on the resulting binary, it will say that it +# is linking with the system's `readline'. But if you run that same command +# within a rule in this project, you'll see that it is indeed linking with +# our own built readline. +ifeq ($(on_mac_os),yes) +needpatchelf = +else +needpatchelf = $(ibidir)/patchelf +endif +$(ibidir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ + $(ilidir)/readline \ + $(needpatchelf) + + # Delete the (possibly) existing Bash executable. + rm -f $(ibdir)/bash + + # Build Bash. Note that we aren't building Bash with + # `--with-installed-readline'. This is because (as described above) + # Bash needs the `LD_LIBRARY_PATH' set properly before it is + # run. Within a recipe, things are fine (we do set + # `LD_LIBRARY_PATH'). However, Make will also call the shell + # outside of the recipe (for example in the `foreach' Make + # function!). In such cases, our new `LD_LIBRARY_PATH' is not set. + # This will cause a crash in the shell and thus the Makefile, + # complaining that it can't find `libreadline'. Therefore, even + # though we build readline below, we won't link Bash with an + # external readline. + # + # Bash has many `--enable' features which are already enabled by + # default. As described in the manual, they are mainly useful when + # you disable them all with `--enable-minimal-config' and enable a + # subset using the `--enable' options. + if [ "x$(static_build)" = xyes ]; then stopt="--enable-static-link";\ + else stopt=""; \ + fi; \ + $(call gbuild, $<, bash-$(bash-version),, \ + --with-installed-readline=$(ildir) $$stopt ) + + # Atleast on GNU/Linux systems, Bash doesn't include RPATH by + # default. So, we have to manually include it, currently we are + # only doing this on GNU/Linux systems (using the `patchelf' + # program). + if [ "x$(needpatchelf)" != x ]; then \ + if [ -f $(ibdir)/bash ]; then \ + $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/bash; fi \ + fi + + # To be generic, some systems use the `sh' command to call the + # shell. By convention, `sh' is just a symbolic link to the + # preferred shell executable. So we'll define `$(ibdir)/sh' as a + # symbolic link to the Bash that we just built and installed. + # + # Just to be sure that the installation step above went well, + # before making the link, we'll see if the file actually exists + # there. + if [ -f $(ibdir)/bash ]; then \ + ln -fs $(ibdir)/bash $(ibdir)/sh; \ + echo "GNU Bash $(bash-version)" > $@; \ + else \ + echo "GNU Bash not built!"; exit 1; fi + + + + + +# Downloader +# ---------- +# +# OpenSSL: Some programs/libraries later need dynamic linking. So we'll +# build libssl (and libcrypto) dynamically also. +# +# Until we find a nice and generic way to create an updated CA file in the +# project, the certificates will be available in a file for this project +# along with the other tarballs. +# +# In case you do want a static OpenSSL and libcrypto, then uncomment the +# following conditional and put $(openssl-static) in the configure options. +# +#ifeq ($(static_build),yes) +#openssl-static = no-dso no-dynamic-engine no-shared +#endif +$(idir)/etc:; mkdir $@ +$(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ + $(tdir)/cert.pem \ + $(ibidir)/bash | $(idir)/etc + # According to OpenSSL's Wiki (link bellow), it can't automatically + # detect Mac OS's structure. It will need some help. So we'll use + # the `on_mac_os' Make variable that we defined in the configure + # script and help it with some extra configuration options and an + # environment variable. + # + # https://wiki.openssl.org/index.php/Compilation_and_Installation + if [ x$(on_mac_os) = xyes ]; then \ + export KERNEL_BITS=64; \ + copt="shared no-ssl2 no-ssl3 enable-ec_nistp_64_gcc_128"; \ + fi; \ + $(call gbuild, $<, openssl-$(openssl-version), , \ + zlib \ + $$copt \ + $(rpath_command) \ + --openssldir=$(idir)/etc/ssl \ + --with-zlib-lib=$(ildir) \ + --with-zlib-include=$(idir)/include, , , \ + ./config ) && \ + cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem; \ + if [ $$? = 0 ]; then \ + if [ x$(on_mac_os) = xyes ]; then \ + echo "No need to fix rpath in libssl"; \ + else \ + patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \ + fi; \ + echo "OpenSSL $(openssl-version)" > $@; \ + fi + +# GNU Wget +# +# Note that on some systems (for example GNU/Linux) Wget needs to explicity +# link with `libdl', but on others (for example Mac OS) it doesn't. We +# check this at configure time and define the `needs_ldl' variable. +# +# Also note that since Wget needs to load outside libraries dynamically, it +# gives a segmentation fault when built statically. +# +# There are many network related libraries that we are currently not +# building as part of this project. So to avoid too much dependency on the +# host system (especially a crash when these libraries are updated on the +# host), they are disabled here. +$(ibidir)/wget: $(tdir)/wget-$(wget-version).tar.lz \ + $(ibidir)/pkg-config \ + $(ilidir)/openssl + libs="-pthread"; \ + if [ x$(needs_ldl) = xyes ]; then libs="$$libs -ldl"; fi; \ + $(call gbuild, $<, wget-$(wget-version), , \ + LIBS="$$LIBS $$libs" \ + --with-libssl-prefix=$(idir) \ + --with-ssl=openssl \ + --with-openssl=yes \ + --without-metalink \ + --without-libuuid \ + --without-libpsl \ + --without-libidn \ + --disable-pcre2 \ + --disable-pcre \ + --disable-iri ) \ + && echo "GNU Wget $(wget-version)" > $@ + + + + + +# Basic command-line tools and their dependencies +# ----------------------------------------------- +# +# These are basic programs which are commonly necessary in the build +# process of the higher-level programs and libraries. Note that during the +# building of those higher-level programs (after this Makefile finishes), +# there is no access to the system's PATH. +$(ibidir)/coreutils: $(tdir)/coreutils-$(coreutils-version).tar.xz \ + $(ilidir)/openssl + # Coreutils will use the hashing features of OpenSSL's `libcrypto'. + # See Tar's comments for the `-j' option. + $(call gbuild, $<, coreutils-$(coreutils-version), static, \ + LDFLAGS="$(LDFLAGS)" CPPFLAGS="$(CPPFLAGS)" \ + --enable-rpath --disable-silent-rules --with-openssl, \ + -j$(numthreads)) \ + && echo "GNU Coreutils $(coreutils-version)" > $@ + +$(ibidir)/diffutils: $(tdir)/diffutils-$(diffutils-version).tar.xz \ + $(ibidir)/bash + $(call gbuild, $<, diffutils-$(diffutils-version), static, , V=1) \ + && echo "GNU Diffutils $(diffutils-version)" > $@ + +$(ibidir)/findutils: $(tdir)/findutils-$(findutils-version).tar.lz \ + $(ibidir)/bash + $(call gbuild, $<, findutils-$(findutils-version), static, , V=1) \ + && echo "GNU Findutils $(findutils-version)" > $@ + +$(ibidir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \ + $(ibidir)/bash \ + $(ilidir)/mpfr \ + $(ilidir)/gmp + # AWK doesn't include RPATH by default, so we'll have to manually + # include it using the `patchelf' program (which was a dependency + # of Bash). Just note that AWK produces two executables (for + # example `gawk-4.2.1' and `gawk') and a symbolic link `awk' to one + # of those executables. + $(call gbuild, $<, gawk-$(gawk-version), static, \ + --with-readline=$(idir)) \ + && if [ "x$(needpatchelf)" != x ]; then \ + if [ -f $(ibdir)/gawk ]; then \ + $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/gawk; \ + fi; \ + if [ -f $(ibdir)/gawk-$(gawk-version) ]; then \ + $(ibdir)/patchelf --set-rpath $(ildir) \ + $(ibdir)/gawk-$(gawk-version); \ + fi; \ + fi \ + && echo "GNU AWK $(gawk-version)" > $@ + +$(ilidir)/gmp: $(tdir)/gmp-$(gmp-version).tar.lz \ + $(ibidir)/bash + $(call gbuild, $<, gmp-$(gmp-version), static, , , make check) \ + && echo "GNU Multiple Precision Arithmetic Library $(gmp-version)" > $@ + +# On Mac OS, libtool does different things, so to avoid confusion, we'll +# prefix GNU's libtool executables with `glibtool'. +$(ibidir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz \ + $(ibidir)/m4 + $(call gbuild, $<, libtool-$(libtool-version), static, \ + --program-prefix=g) \ + && echo "GNU Libtool $(libtool-version)" > $@ + +$(ibidir)/grep: $(tdir)/grep-$(grep-version).tar.xz \ + $(ibidir)/bash + $(call gbuild, $<, grep-$(grep-version), static) \ + && echo "GNU Grep $(grep-version)" > $@ + +$(ibidir)/m4: $(tdir)/m4-$(m4-version).tar.gz \ + $(ibidir)/bash + $(call gbuild, $<, m4-$(m4-version), static) \ + && echo "GNU M4 $(m4-version)" > $@ + +$(ilidir)/mpfr: $(tdir)/mpfr-$(mpfr-version).tar.xz \ + $(ilidir)/gmp + $(call gbuild, $<, mpfr-$(mpfr-version), static, , , make check) \ + && echo "GNU Multiple Precision Floating-Point Reliably $(mpfr-version)" > $@ + +$(ibidir)/pkg-config: $(tdir)/pkg-config-$(pkgconfig-version).tar.gz \ + $(ibidir)/bash + # Some Mac OS systems may have a version of the GNU C Compiler + # (GCC) installed that doesn't support some necessary features of + # building Glib (as part of pkg-config). So to be safe, for Mac + # systems, we'll make sure it will use LLVM's Clang. + if [ x$(on_mac_os) = xyes ]; then export compiler="CC=clang"; \ + else export compiler=""; \ + fi; \ + $(call gbuild, $<, pkg-config-$(pkgconfig-version), static, \ + $$compiler --with-internal-glib \ + --with-pc-path=$(ildir)/pkgconfig) \ + && echo "pkg-config $(pkgconfig-version)" > $@ + +$(ibidir)/sed: $(tdir)/sed-$(sed-version).tar.xz \ + $(ibidir)/bash + $(call gbuild, $<, sed-$(sed-version), static) \ + && echo "GNU Sed $(sed-version)" > $@ + +$(ibidir)/which: $(tdir)/which-$(which-version).tar.gz \ + $(ibidir)/bash + $(call gbuild, $<, which-$(which-version), static) \ + && echo "GNU Which $(which-version)" > $@ + + + + + + + + + + +# GCC and its prerequisites +# ------------------------- +# +# Binutils' linker `ld' is apparently only good for GNU/Linux systems and +# other OSs have their own. So for now we aren't actually building +# Binutils (`ld' isn't a prerequisite of GCC). +$(ibidir)/binutils: $(tdir)/binutils-$(binutils-version).tar.lz \ + $(ibidir)/bash + $(call gbuild, $<, binutils-$(binutils-version), static) \ + && echo "GNU Binutils $(binutils-version)" > $@ + +# `file' is not a prerequisite of GCC. However, since it is low level, it is +# set as a prerequisite of GCC to have it installed. +$(ibidir)/file: $(tdir)/file-$(file-version).tar.gz \ + $(ibidir)/bash + $(call gbuild, $<, file-$(file-version), static) \ + && echo "File $(file-version)" > $@ + +$(ilidir)/isl: $(tdir)/isl-$(isl-version).tar.bz2 \ + $(ilidir)/gmp + $(call gbuild, $<, isl-$(isl-version), static) \ + && echo "GNU Integer Set Library $(isl-version)" > $@ + +$(ilidir)/mpc: $(tdir)/mpc-$(mpc-version).tar.gz \ + $(ilidir)/mpfr + $(call gbuild, $<, mpc-$(mpc-version), static, , , make check) \ + && echo "GNU Multiple Precision Complex library" > $@ + +# We are having issues with `libiberty' (part of GCC) on Mac. So for now, +# GCC won't be built there. Since almost no natural science paper's +# processing depends so strongly on the compiler used, for now, this isn't +# a bad assumption, but we are indeed searching for a solution. +# +# Based on the GCC manual, the GCC build can benefit from a GNU +# environment. So, we'll build GCC after building all the basic tools that +# are often used in a configure and build scripts of GCC components. +# +# Objective C and Objective C++ is necessary for installing `matplotlib'. +# +# We are currently having problems installing GCC on macOS, so for the time +# being, if the project is being run on a macOS, we'll just set a link. +ifeq ($(host_cc),1) +gcc-prerequisites = +else +gcc-prerequisites = $(tdir)/gcc-$(gcc-version).tar.xz \ + $(ilidir)/isl \ + $(ilidir)/mpc +endif +$(ibidir)/gcc: $(gcc-prerequisites) \ + $(ibidir)/sed \ + $(ibidir)/bash \ + $(ibidir)/file \ + $(ibidir)/gawk \ + $(ibidir)/grep \ + $(ibidir)/which \ + $(ibidir)/glibtool \ + $(ibidir)/coreutils \ + $(ibidir)/diffutils \ + $(ibidir)/findutils + + # GCC builds is own libraries in '$(idir)/lib64'. But all other + # libraries are in '$(idir)/lib'. Since this project is only for a + # single architecture, we can trick GCC into building its libraries + # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic + # link to '$(idir)/lib'. + if [ $(host_cc) = 1 ]; then \ + $(call makelink,gcc); \ + $(call makelink,g++,mandatory); \ + $(call makelink,gfortran,mandatory); \ + ccinfo=$$(gcc --version | awk 'NR==1'); \ + echo "C compiler (""$$ccinfo"")" > $@; \ + else \ + rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\ + rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \ + rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\ + \ + ln -fs $(ildir) $(idir)/lib64; \ + \ + cd $(ddir); \ + rm -rf gcc-build gcc-$(gcc-version); \ + tar xf $< \ + && mkdir $(ddir)/gcc-build \ + && cd $(ddir)/gcc-build \ + && ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ + --prefix=$(idir) \ + --with-mpc=$(idir) \ + --with-mpfr=$(idir) \ + --with-gmp=$(idir) \ + --with-isl=$(idir) \ + --with-build-time-tools=$(idir) \ + --enable-shared \ + --disable-multilib \ + --disable-multiarch \ + --enable-threads=posix \ + --with-local-prefix=$(idir) \ + --enable-languages=c,c++,fortran,objc,obj-c++ \ + --disable-libada \ + --disable-nls \ + --enable-default-pie \ + --enable-default-ssp \ + --enable-cet=auto \ + --enable-decimal-float \ + && make SHELL=$(ibdir)/bash -j$$(nproc) \ + && make SHELL=$(ibdir)/bash install \ + && cd .. \ + && rm -rf gcc-build gcc-$(gcc-version) \ + \ + && if [ "x$(on_mac_os)" != xyes ]; then \ + for f in $$(find $(idir)/libexec/gcc); do \ + if ldd $$f &> /dev/null; then \ + patchelf --set-rpath $(ildir) $$f; \ + fi; \ + done; \ + fi \ + && echo "GNU Compiler Collection (GCC) $(gcc-version)" > $@; \ + fi diff --git a/reproduce/software/make/build-rules.mk b/reproduce/software/make/build-rules.mk new file mode 100644 index 0000000..a8c8731 --- /dev/null +++ b/reproduce/software/make/build-rules.mk @@ -0,0 +1,120 @@ +# Generic configurable recipes to build packages with GNU Build system or +# CMake. This is Makefile is not intended to be run directly, it will be +# imported into `dependencies-basic.mk' and `dependencies.mk'. They should +# be activated with Make's `Call' function. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + + + + + +# IMPORTANT note +# -------------- +# +# Without using `&&', if a step fails, the process will continue. However, +# in the `if' statements, we need `;' (particularly between `]' and +# `then'). So we need to put any necessary checks at the start, then when +# we start the process, every command will be separated by an `&&'. + + + + + +# GNU Build system +# ---------------- +# +# Arguments: +# 1: Tarball full address. +# 2: Directory name after unpacking. +# 3: Set to `static' for a static build. +# 4: Extra configuration options. +# 5: Extra options/arguments to pass to Make. +# 6: Step to run between `make' and `make install': usually `make check'. +# 7: The configuration script (`configure' by default). +# 8: Arguments for `make install'. +# +# NOTE: Unfortunately the configure script of `zlib' doesn't recognize +# `SHELL'. So we'll have to remove it from the call to the configure +# script. +# +# NOTE: A program might not contain any configure script. In this case, +# we'll just pass a non-relevant function like `pwd'. So SED should be used +# to modify `confscript' or to set `configop'. +gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then \ + export LDFLAGS="$$LDFLAGS -static"; \ + fi; \ + check="$(6)"; \ + if [ x"$$check" = x ]; then check="echo Skipping-check"; fi; \ + cd $(ddir); rm -rf $(2); \ + if ! tar xf $(1); then echo; echo "Tar error"; exit 1; fi; \ + cd $(2); \ + \ + if [ x"$(strip $(7))" = x ]; then confscript=./configure; \ + else confscript="$(strip $(7))"; \ + fi; \ + \ + if [ -f $(ibdir)/bash ]; then \ + if [ -f $$confscript ]; then \ + sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ + -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ + $$confscript > $$confscript-tmp; \ + mv $$confscript-tmp $$confscript; \ + chmod +x $$confscript; \ + fi; \ + shellop="SHELL=$(ibdir)/bash"; \ + elif [ -f /bin/bash ]; then shellop="SHELL=/bin/bash"; \ + else shellop="SHELL=/bin/sh"; \ + fi; \ + \ + if [ -f $$confscript ]; then \ + if [ x"$(strip $(2))" = x"zlib-$(zlib-version)" ]; then \ + configop="--prefix=$(idir)"; \ + else configop="$$shellop --prefix=$(idir)"; \ + fi; \ + fi; \ + \ + echo; echo "Using '$$confscript' to configure:"; echo; \ + echo "$$confscript $(4) $$configop"; echo; \ + $$confscript $(4) $$configop && \ + make "$$shellop" $(5) && \ + $$check && \ + make "$$shellop" install $(8) && \ + cd .. && rm -rf $(2) + + + + +# CMake +# ----- +# +# According to the link below, in CMake `/bin/sh' is hardcoded, so there is +# no way to change it. +# +# https://stackoverflow.com/questions/21167014/how-to-set-shell-variable-in-makefiles-generated-by-cmake +cbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then \ + export LDFLAGS="$$LDFLAGS -static"; \ + opts="-DBUILD_SHARED_LIBS=OFF"; \ + fi; \ + cd $(ddir) && rm -rf $(2) && tar xf $(1) && cd $(2) && \ + rm -rf project-build && mkdir project-build && \ + cd project-build && \ + cmake .. -DCMAKE_LIBRARY_PATH=$(ildir) \ + -DCMAKE_INSTALL_PREFIX=$(idir) \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON $$opts $(4) && \ + make && make install && \ + cd ../.. && \ + rm -rf $(2) diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk new file mode 100644 index 0000000..f7b1f4f --- /dev/null +++ b/reproduce/software/make/high-level.mk @@ -0,0 +1,720 @@ +# Build the project's dependencies (programs and libraries). +# +# ------------------------------------------------------------------------ +# !!!!! IMPORTANT NOTES !!!!! +# +# This Makefile will be run by the initial `./configure' script. It is not +# included into the reproduction pipe after that. +# +# ------------------------------------------------------------------------ +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + + + +# Top level environment +include reproduce/software/make/build-rules.mk +include reproduce/software/config/installation/LOCAL.mk +include reproduce/software/config/installation/texlive.mk +include reproduce/software/config/installation/versions.mk + +lockdir = $(BDIR)/locks +tdir = $(BDIR)/software/tarballs +ddir = $(BDIR)/software/build-tmp +idir = $(BDIR)/software/installed +ibdir = $(BDIR)/software/installed/bin +ildir = $(BDIR)/software/installed/lib +dtexdir = $(shell pwd)/reproduce/software/bibtex +ibidir = $(BDIR)/software/installed/version-info/bin +ilidir = $(BDIR)/software/installed/version-info/lib +itidir = $(BDIR)/software/installed/version-info/tex +ictdir = $(BDIR)/software/installed/version-info/cite +ipydir = $(BDIR)/software/installed/version-info/python + +# Define the top-level programs to build (installed in `.local/bin'). +# +# About ATLAS: currently the template does not depend on ATLAS but many +# high level software depend on it. The current rule for ATLAS is tested +# successfully on Mac (only static) and GNU/Linux (shared and static). But, +# since it takes a few hours to build, it is not currently a target. +top-level-libraries = # atlas +top-level-python = astroquery matplotlib +top-level-programs = gnuastro metastore +all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ + $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ + $(foreach p, $(top-level-python), $(ipydir)/$(p)) \ + $(itidir)/texlive + +# Other basic environment settings: We are only including the host +# operating system's PATH environment variable (after our own!) for the +# compiler and linker. For the library binaries and headers, we are only +# using our internally built libraries. +# +# To investigate: +# +# 1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all +# the parameters defined bellow as `NAME=VALUE' statements before +# calling Bash. This will enable us to completely ignore the user's +# native environment. +# +# 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the +# user's environment. +.ONESHELL: +.SHELLFLAGS := --noprofile --norc -ec +export CCACHE_DISABLE := 1 +export PATH := $(ibdir) +export SHELL := $(ibdir)/bash +export CPPFLAGS := -I$(idir)/include +export PKG_CONFIG_PATH := $(ildir)/pkgconfig +export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +export LD_RUN_PATH := $(ildir):$(il64dir) +export LD_LIBRARY_PATH := $(ildir):$(il64dir) +export LDFLAGS := $(rpath_command) -L$(ildir) + + +# We want the download to happen on a single thread. So we need to define a +# lock, and call a special script we have written for this job. These are +# placed here because we want them both in the `high-level.mk' and +# `python.mk'. +$(lockdir): | $(BDIR); mkdir $@ +downloader="wget --no-use-server-timestamps -O"; +downloadwrapper = ./reproduce/analysis/bash/download-multi-try + + + + + +# Python packages +include reproduce/software/make/python.mk + + + + + +# Tarballs +# -------- +# +# All the necessary tarballs are defined and prepared with this rule. +# +# Note that we want the tarballs to follow the convention of NAME-VERSION +# before the `tar.XX' prefix. For those programs that don't follow this +# convention, but include the name/version in their tarball names with +# another format, we'll do the modification before the download so the +# downloaded file has our desired format. +tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ + atlas-$(atlas-version).tar.bz2 \ + cmake-$(cmake-version).tar.gz \ + curl-$(curl-version).tar.gz \ + freetype-$(freetype-version).tar.gz \ + fftw-$(fftw-version).tar.gz \ + ghostscript-$(ghostscript-version).tar.gz \ + git-$(git-version).tar.xz \ + gnuastro-$(gnuastro-version).tar.lz \ + gsl-$(gsl-version).tar.gz \ + hdf5-$(hdf5-version).tar.gz \ + install-tl-unx.tar.gz \ + jpegsrc.$(libjpeg-version).tar.gz \ + lapack-$(lapack-version).tar.gz \ + libbsd-$(libbsd-version).tar.xz \ + libpng-$(libpng-version).tar.xz \ + libgit2-$(libgit2-version).tar.gz \ + metastore-$(metastore-version).tar.gz \ + openmpi-$(openmpi-version).tar.gz \ + openblas-$(openblas-version).tar.gz \ + tiff-$(libtiff-version).tar.gz \ + wcslib-$(wcslib-version).tar.bz2 \ + , $(tdir)/$(t) ) +$(tarballs): $(tdir)/%: | $(lockdir) + if [ -f $(DEPENDENCIES-DIR)/$* ]; then + cp $(DEPENDENCIES-DIR)/$* $@ + else + # Remove all numbers, `-' and `.' from the tarball name so we can + # search more easily only with the program name. + n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ + | awk '{print $$1}' ) + + # Set the top download link of the requested tarball. + mergenames=1 + if [ $$n = cfitsio ]; then + mergenames=0 + v=$$(echo $(cfitsio-version) | sed -e's/\.//' \ + | awk '{l=length($$1); \ + printf (l==4 ? "%d\n" \ + : (l==3 ? "%d0\n" \ + : (l==2 ? "%d00\n" \ + : "%d000\n") ), $$1)}') + w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz + elif [ $$n = atlas ]; then + mergenames=0 + w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download + elif [ $$n = cmake ]; then + mergenames=0 + majv=$$(echo $(cmake-version) \ + | sed -e's/\./ /' \ + | awk '{printf("%d.%d", $$1, $$2)}') + w=https://cmake.org/files/v$$majv/cmake-$(cmake-version).tar.gz + elif [ $$n = curl ]; then w=https://curl.haxx.se/download + elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw + elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype + elif [ $$n = hdf ]; then + mergenames=0 + majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') + w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$* + elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 + elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git + elif [ $$n = gnuastro ]; then w=http://ftp.gnu.org/gnu/gnuastro + elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl + elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet + elif [ $$n = jpegsrc ]; then w=http://ijg.org/files + elif [ $$n = lapack ]; then w=http://www.netlib.org/lapack + elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases + elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng + elif [ $$n = libgit ]; then + mergenames=0 + w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz + elif [ $$n = metastore ]; then w=http://akhlaghi.org/src + elif [ $$n = openblas ]; then + mergenames=0 + w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz + elif [ $$n = openmpi ]; then + mergenames=0 + majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') + w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* + elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff + elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib + else + echo; echo; echo; + echo "'$$n' not recognized as a dependency name to download." + echo; echo; echo; + exit 1 + fi + + # Download the requested tarball. Note that some packages may not + # follow our naming convention (where the package name is merged + # with its version number). In such cases, `w' will be the full + # address, not just the top directory address. But since we are + # storing all the tarballs in one directory, we want it to have + # the same naming convention, so we'll download it to a temporary + # name, then rename that. + if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*" + else tarballurl=$$w + fi + + # Download using the script specially defined for this job. + touch $(lockdir)/download + downloader="wget --no-use-server-timestamps -O" + $(downloadwrapper) "$$downloader" $(lockdir)/download \ + $$tarballurl $@ + fi + + + + + +# Libraries +# --------- +# +# We would prefer to build static libraries, but some compilers like LLVM +# don't have static capabilities, so they'll only build dynamic/shared +# libraries. Therefore, we can't use the easy `.a' suffix for static +# libraries as targets and there are different conventions for shared +# library names. +# +# For the actual build, the same compiler that built the library will build +# the programs, so exact knowledge of the suffix is ultimately irrelevant +# for us here. So, we'll make an `$(ildir)/built' directory and make a +# simple plain text file in it with the basic library name (an no prefix) +# and create/write into it when the library is successfully built. +$(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ + $(ibidir)/curl + + # CFITSIO hard-codes the absolute address of cURL's `curl-config' + # program (which gives the necessary header and linking + # information) into the configure script. So we'll have to modify + # it manually before doing the standard build. + topdir=$(pwd); cd $(ddir); tar xf $< + customtar=cfitsio-$(cfitsio-version)-custom.tar.gz + sed cfitsio/configure \ + -e's|/usr/bin/curl-config|$(ibdir)/curl-config|g' \ + > cfitsio/configure_tmp + mv cfitsio/configure_tmp cfitsio/configure + chmod +x cfitsio/configure + tar cf $$customtar cfitsio + cd $$topdir + + # Continue the standard build on the customized tarball. + $(call gbuild, $$customtar, cfitsio, static, \ + --enable-sse2 --enable-reentrant) \ + && rm $$customtar \ + && echo "CFITSIO $(cfitsio-version)" > $@ + +$(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz + $(call gbuild, $<, gsl-$(gsl-version), static) \ + && echo "GNU Scientific Library $(gsl-version)" > $@ + +$(ilidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz + $(call gbuild, $<, fftw-$(fftw-version), static, \ + --enable-shared) \ + && cp $(dtexdir)/fftw.tex $(ictdir)/ \ + && echo "FFTW $(fftw-version) \citep{fftw}" > $@ + +# Freetype is necessary to install matplotlib +$(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ + $(ilidir)/libpng + $(call gbuild, $<, freetype-$(freetype-version), static) \ + && echo "FreeType $(freetype-version)" > $@ + +$(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ + $(ilidir)/openmpi + export CC=mpicc; \ + export FC=mpif90; \ + $(call gbuild, $<, hdf5-$(hdf5-version), static, \ + --enable-parallel \ + --enable-fortran, V=1) \ + && echo "HDF5 library $(hdf5-version)" > $@ + +$(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz + $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \ + && echo "Libbsd $(libbsd-version)" > $@ + +$(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz + $(call gbuild, $<, jpeg-9b, static) \ + && echo "Libjpeg $(libjpeg-version)" > $@ + +$(ilidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz + $(call gbuild, $<, libpng-$(libpng-version), static) \ + && echo "Libpng $(libpng-version)" > $@ + +$(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ + $(ilidir)/libjpeg + $(call gbuild, $<, tiff-$(libtiff-version), static, \ + --disable-webp --disable-zstd) \ + && echo "Libtiff $(libtiff-version)" > $@ + +$(ilidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz + $(call gbuild, $<, openmpi-$(openmpi-version), static, , V=1) \ + && echo "OpenMPI $(openmpi-version)" > $@ + +$(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ + $(tdir)/lapack-$(lapack-version).tar.gz + + # Get the operating system specific features (how to get + # CPU frequency and the library suffixes). To make the steps + # more readable, the different library version suffixes are + # named with a single character: `s' for no version in the + # name, `m' for the major version suffix, and `f' for the + # full version suffix. + # GCC in Mac OS doesn't work. To work around this issue, on Mac + # systems we force ATLAS to use `clang' instead of `gcc'. + if [ x$(on_mac_os) = xyes ]; then + s=dylib + m=3.dylib + f=3.6.1.dylib + core=$$(sysctl hw.cpufrequency | awk '{print $$2/1000000}') + clangflag="--force-clang=$(ibdir)/clang" + else + s=so + m=so.3 + f=so.3.6.1 + clangflag= + core=$$(cat /proc/cpuinfo | grep "cpu MHz" \ + | head -n 1 \ + | sed "s/.*: \([0-9.]*\).*/\1/") + fi + + # See if the shared libraries should be build for a single CPU + # thread or multiple threads. + N=$$(nproc) + srcdir=$$(pwd)/reproduce/src/make + if [ $$N = 1 ]; then + sharedmk=$$srcdir/dependencies-atlas-single.mk + else + sharedmk=$$srcdir/dependencies-atlas-multiple.mk + fi + + # The linking step here doesn't recognize the `-Wl' in the + # `rpath_command'. + export LDFLAGS=-L$(ildir) + cd $(ddir) \ + && tar xf $< \ + && cd ATLAS \ + && rm -rf build \ + && mkdir build \ + && cd build \ + && ../configure -b 64 -D c -DPentiumCPS=$$core \ + --with-netlib-lapack-tarfile=$(word 2, $^) \ + --cripple-atlas-performance \ + -Fa alg -fPIC --shared $$clangflag \ + --prefix=$(idir) \ + && make \ + && if [ "x$(on_mac_os)" != xyes ]; then \ + cd lib && make -f $$sharedmk && cd .. \ + && for l in lib/*.$$s*; do \ + patchelf --set-rpath $(ildir) $$l; done \ + && cp -d lib/*.$$s* $(ildir) \ + && ln -fs $(ildir)/libblas.$$s $(ildir)/libblas.$$m \ + && ln -fs $(ildir)/libf77blas.$$s $(ildir)/libf77blas.$$m \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$s \ + && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$m; \ + fi \ + && make install + + # We need to check the existance of `libptlapack.a', but we can't + # do this in the `&&' steps above (it will conflict). So we'll do + # the check after seeing if `libtatlas.so' is installed, then we'll + # finalize the build (delete the untarred directory). + if [ "x$(on_mac_os)" != xyes ]; then \ + [ -e lib/libptlapack.a ] && cp lib/libptlapack.a $(ildir); \ + cd $(ddir); \ + rm -rf ATLAS; \ + fi + + # We'll check the full installation with the static library (not + # currently building shared library on Mac. + if [ -f $(ildir)/libatlas.a ]; then \ + echo "ATLAS $(atlas-version)" > $@; \ + fi + +$(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + fi; \ + cd $(ddir) \ + && tar xf $< \ + && cd OpenBLAS-$(openblas-version) \ + && make \ + && make PREFIX=$(idir) install \ + && cd .. \ + && rm -rf OpenBLAS-$(openblas-version) \ + && echo "OpenBLAS $(openblas-version)" > $@ + + + + +# Libraries with special attention on Mac OS +# ------------------------------------------ +# +# Libgit2 and WCSLIB don't set their installation path, or don't do it +# properly, in their finally installed shared libraries. But since we are +# linking everything (including OpenSSL and its dependencies) dynamically, +# we need to also make a shared libraries and can't use static +# libraries. So for Mac OS systems we have to correct their addresses +# manually. +# +# For example, Libgit2 page recommends doing a static build, especially for +# Mac systems (with `-DBUILD_SHARED_LIBS=OFF'): "It’s highly recommended +# that you build libgit2 as a static library for Xcode projects. This +# simplifies distribution significantly, as the resolution of dynamic +# libraries at runtime can be extremely problematic.". This is a major +# problem we have been having so far with Mac systems: +# https://libgit2.org/docs/guides/build-and-link +$(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \ + $(ibidir)/cmake \ + $(ibidir)/curl + # Build and install the library. + $(call cbuild, $<, libgit2-$(libgit2-version), static, \ + -DUSE_SSH=OFF -DBUILD_CLAR=OFF \ + -DTHREADSAFE=ON ) + + # Correct the shared library absolute address if necessary. + if [ x$(on_mac_os) = xyes ]; then + install_name_tool -id $(ildir)/libgit2.26.dylib \ + $(ildir)/libgit2.26.dylib + fi + + # Write the target file. + echo "Libgit2 $(libgit2-version)" > $@ + +$(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ + $(ilidir)/cfitsio + # Build and install the library. + $(call gbuild, $<, wcslib-$(wcslib-version), , \ + LIBS="-pthread -lcurl -lm" \ + --with-cfitsiolib=$(ildir) \ + --with-cfitsioinc=$(idir)/include \ + --without-pgplot --disable-fortran) + + # Correct the shared library absolute address if necessary. + if [ x$(on_mac_os) = xyes ]; then + install_name_tool -id $(ildir)/libwcs.6.2.dylib \ + $(ildir)/libwcs.6.2.dylib; + fi + + # Write the target file. + echo "WCSLIB $(wcslib-version)" > $@ + + + + + +# Programs +# -------- +# +# CMake can be built with its custom `./bootstrap' script. +$(ibidir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ + $(ibidir)/curl + # After searching in `bootstrap', I couldn't find `LIBS', only + # `LDFLAGS'. So the extra libraries are being added to `LDFLAGS', + # not `LIBS'. + # + # On Mac systems, the build complains about `clang' specific + # features, so we can't use our own GCC build here. + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + export CXX=clang++; \ + fi; \ + cd $(ddir) \ + && rm -rf cmake-$(cmake-version) \ + && tar xf $< \ + && cd cmake-$(cmake-version) \ + && ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ + --system-bzip2 --system-liblzma --no-qt-gui \ + && make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 \ + && make install \ + && cd .. \ + && rm -rf cmake-$(cmake-version) \ + && echo "CMake $(cmake-version)" > $@ + +# cURL (and its library, which is needed by several programs here) can +# optionally link with many different network-related libraries on the host +# system that we are not yet building in the template. Many of these are +# not relevant to most science projects, so we are explicitly using +# `--without-XXX' or `--disable-XXX' so cURL doesn't link with them. Note +# that if it does link with them, the configuration will crash when the +# library is updated/changed by the host, and the whole purpose of this +# project is avoid dependency on the host as much as possible. +$(ibidir)/curl: $(tdir)/curl-$(curl-version).tar.gz + $(call gbuild, $<, curl-$(curl-version), , \ + LIBS="-pthread" \ + --with-zlib=$(ildir) \ + --with-ssl=$(idir) \ + --without-mesalink \ + --with-ca-fallback \ + --without-librtmp \ + --without-libidn2 \ + --without-wolfssl \ + --without-brotli \ + --without-gnutls \ + --without-cyassl \ + --without-libpsl \ + --without-axtls \ + --disable-ldaps \ + --disable-ldap \ + --without-nss, V=1) \ + && echo "cURL $(curl-version)" > $@ + +$(ibidir)/ghostscript: $(tdir)/ghostscript-$(ghostscript-version).tar.gz + $(call gbuild, $<, ghostscript-$(ghostscript-version)) \ + && echo "GPL Ghostscript $(ghostscript-version)" > $@ + +$(ibidir)/git: $(tdir)/git-$(git-version).tar.xz \ + $(ibidir)/curl + $(call gbuild, $<, git-$(git-version), static, \ + --without-tcltk --with-shell=$(ibdir)/bash, \ + V=1) \ + && echo "Git $(git-version)" > $@ + +# Metastore is used (through a Git hook) to restore the source modification +# dates of files after a Git checkout. Another Git hook saves all file +# metadata just before a commit (to allow restoration after a +# checkout). Since this project is managed in Makefiles, file modification +# dates are critical to not having to redo the whole analysis after +# checking out between branches. +# +# Note that we aren't using the standard version of Metastore, but a fork +# of it that is maintained in this repository: +# https://gitlab.com/makhlaghi/metastore-fork +# +# Libbsd is not necessary on macOS systems, because macOS is already a +# BSD-based distribution. But on GNU/Linux systems, it is necessary. +ifeq ($(on_mac_os),yes) +needlibbsd = +else +needlibbsd = $(ilidir)/libbsd +endif +$(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ + $(needlibbsd) \ + $(ibidir)/git + + # The build command below will change the current directory of this + # build, so we'll fix its value here. + current_dir=$$(pwd) + + # Metastore doesn't have any `./configure' script. So we'll just + # call `pwd' as a place-holder for the `./configure' command. + # + # File attributes are also not available on some systems, since the + # main purpose here is modification dates (and not attributes), + # we'll also set the `NO_XATTR' flag. + $(call gbuild, $<, metastore-$(metastore-version), static,, \ + NO_XATTR=1 V=1,,pwd,PREFIX=$(idir)) + + # Write the relevant hooks into this system's Git hooks, so Git + # calls metastore properly on every commit and every checkout. + # + # Note that the -O and -G options used here are currently only in a + # fork of `metastore' currently hosted at: + # https://github.com/mohammad-akhlaghi/metastore + user=$$(whoami) + group=$$(groups | awk '{print $$1}') + cd $$current_dir + if [ -f $(ibdir)/metastore ]; then + for f in pre-commit post-checkout; do + sed -e's|@USER[@]|'$$user'|g' \ + -e's|@GROUP[@]|'$$group'|g' \ + -e's|@BINDIR[@]|$(ibdir)|g' \ + -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ + reproduce/software/bash/git-$$f > .git/hooks/$$f + chmod +x .git/hooks/$$f + echo "Metastore (forked) $(metastore-version)" > $@ + done + else + echo; echo; echo; + echo "*****************" + echo "metastore couldn't be installed!" + echo + echo "Its used for preserving timestamps on Git commits." + echo "Its useful for development, not simple running of the project." + echo "So we won't stop the configuration because it wasn't built." + echo "*****************" + fi + +# The order of dependencies is based on how long they take to build (how +# large they are): Libgit2 depends on CMake which takes a VERY long time to +# build. Also, Ghostscript and GSL are relatively large packages. So when +# building in parallel, its better to have these packages start building +# early. +$(ibidir)/gnuastro: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \ + $(ilidir)/gsl \ + $(ilidir)/wcslib \ + $(ilidir)/libjpeg \ + $(ilidir)/libtiff \ + $(ilidir)/libgit2 \ + $(ibidir)/ghostscript +ifeq ($(static_build),yes) + staticopts="--enable-static=yes --enable-shared=no"; +endif + $(call gbuild, $<, gnuastro-$(gnuastro-version), static, \ + $$staticopts, -j$(numthreads), \ + make check -j$(numthreads)) \ + && cp $(dtexdir)/gnuastro.tex $(ictdir)/ \ + && echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ + + + + + + + + + + + + + + + + + + + + +# Since we want to avoid complicating the PATH, we are putting a symbolic +# link of all the TeX Live executables in $(ibdir). But symbolic links are +# hard to track for Make (as a target). Also, TeX in general is optional +# for the project (the processing is the main target, not the generation of +# the final PDF). So we'll make a simple ASCII file called +# `texlive-ready-tlmgr' and use its contents to mark if we can use it or +# not. +$(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \ + reproduce/software/config/installation/texlive.conf + + # Unpack, enter the directory, and install based on the given + # configuration (prerequisite of this rule). + @topdir=$$(pwd) + cd $(ddir) + rm -rf install-tl-* + tar xf $(tdir)/install-tl-unx.tar.gz + cd install-tl-* + sed -e's|@installdir[@]|$(idir)|g' \ + $$topdir/reproduce/software/config/installation/texlive.conf \ + > texlive.conf + + # TeX Live's installation may fail due to any reason. But TeX Live + # is optional (only necessary for building the final PDF). So we + # don't want the configure script to fail if it can't run. + if ./install-tl --profile=texlive.conf; then + + # Put a symbolic link of the TeX Live executables in `ibdir'. The + # main problem is that the year and build system (for example + # `x86_64-linux') are also in the directory names, making it hard + # to be generic. We are using wildcards here, but only in this + # Makefile, not in any other. + ln -fs $(idir)/texlive/20*/bin/*/* $(ibdir)/ + + # Register that the build was successful. + echo "TeX Live is ready." > $@ + else + echo "NOT!" > $@ + fi + + # Clean up + cd .. + rm -rf install-tl-* + + + + + +# To keep things modular and simple, we'll break up the installation of TeX +# Live itself (only very basic TeX and LaTeX) and the installation of its +# necessary packages into two packages. +$(itidir)/texlive: reproduce/software/config/installation/texlive.mk \ + $(itidir)/texlive-ready-tlmgr + + # To work with TeX live installation, we'll need the internet. + @res=$$(cat $(itidir)/texlive-ready-tlmgr) + if [ x"$$res" = x"NOT!" ]; then + echo "" > $@ + else + # Install all the extra necessary packages. If LaTeX complains + # about not finding a command/file/what-ever/XXXXXX, simply run + # the following command to find which package its in, then add it + # to the `texlive-packages' variable of the first prerequisite. + # + # ./.local/bin/tlmgr info XXXXXX + # + # We are putting a notice, because if there is no internet, + # `tlmgr' just hangs waiting. + tlmgr install $(texlive-packages) + + # Make a symbolic link of all the TeX Live executables in the bin + # directory so we don't have to modify `PATH'. + ln -fs $(idir)/texlive/20*/bin/*/* $(ibdir)/ + + # Get all the necessary versions. + texlive=$$(pdflatex --version | awk 'NR==1' | sed 's/.*(\(.*\))/\1/' \ + | awk '{print $$NF}'); + + # Package names and versions. + tlmgr info $(texlive-packages) --only-installed | awk \ + '$$1=="package:" {version=0; \ + if($$NF=="tex-gyre") name="texgyre"; \ + else name=$$NF} \ + $$1=="cat-version:" {version=$$NF} \ + $$1=="cat-date:" {if(version==0) version=$$2; \ + printf("%s %s\n", name, version)}' >> $@ + fi diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk new file mode 100644 index 0000000..5ec99f0 --- /dev/null +++ b/reproduce/software/make/python.mk @@ -0,0 +1,506 @@ +# Build the project's Python dependencies. +# +# ------------------------------------------------------------------------ +# !!!!! IMPORTANT NOTES !!!!! +# +# This Makefile will be run by the initial `./configure' script. It is not +# included into the reproduction pipe after that. +# +# ------------------------------------------------------------------------ +# +# Copyright (C) 2019 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + + + + + +# Python enviroment +# ----------------- +# +# The main Python environment variable is `PYTHONPATH'. However, so far we +# have found several other Python-related environment variables on some +# systems which might interfere. To be safe, we are removing all their +# values. +export PYTHONPATH := $(installdir)/lib/python/site-packages +export PYTHONPATH2 := $(PYTHONPATH) +export PYTHONPATH3 := $(PYTHONPATH) +export _LMFILES_ := +export LOADEDMODULES := +export MPI_PYTHON_SITEARCH := +export MPI_PYTHON2_SITEARCH := +export MPI_PYTHON3_SITEARCH := + + + + + +# Tarballs +# -------- +# +# All the necessary tarballs are defined and prepared with this rule. +# +# Note that we want the tarballs to follow the convention of NAME-VERSION +# before the `tar.XX' prefix. For those programs that don't follow this +# convention, but include the name/version in their tarball names with +# another format, we'll do the modification before the download so the +# downloaded file has our desired format. +pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ + astroquery-$(astroquery-version).tar.gz \ + astropy-$(astropy-version).tar.gz \ + beautifulsoup4-$(beautifulsoup4-version).tar.gz \ + certifi-$(certifi-version).tar.gz \ + cffi-$(cffi-version).tar.gz \ + chardet-$(chardet-version).tar.gz \ + cryptography-$(cryptography-version).tar.gz \ + cycler-$(cycler-version).tar.gz \ + cython-$(cython-version).tar.gz \ + entrypoints-$(entrypoints-version).tar.gz \ + h5py-$(h5py-version).tar.gz \ + html5lib-$(html5lib-version).tar.gz \ + idna-$(idna-version).tar.gz \ + jeepney-$(jeepney-version).tar.gz \ + kiwisolver-$(kiwisolver-version).tar.gz \ + keyring-$(keyring-version).tar.gz \ + libffi-$(libffi-version).tar.gz \ + matplotlib-$(matplotlib-version).tar.gz \ + mpi4py-$(mpi4py-version).tar.gz \ + numpy-$(numpy-version).zip \ + pkgconfig-$(pypkgconfig-version).tar.gz \ + pip-$(pip-version).tar.gz \ + pycparser-$(pycparser-version).tar.gz \ + python-$(python-version).tar.gz \ + python-dateutil-$(python-dateutil-version).tar.gz \ + pyparsing-$(pyparsing-version).tar.gz \ + requests-$(requests-version).tar.gz \ + scipy-$(scipy-version).tar.gz \ + secretstorage-$(secretstorage-version).tar.gz \ + setuptools-$(setuptools-version).zip \ + setuptools_scm-$(setuptools_scm-version).tar.gz \ + six-$(six-version).tar.gz \ + soupsieve-$(soupsieve-version).tar.gz \ + urllib3-$(urllib3-version).tar.gz \ + webencodings-$(webencodings-version).tar.gz \ + virtualenv-$(virtualenv-version).tar.gz \ + , $(tdir)/$(t) ) +pytopurl=https://files.pythonhosted.org/packages +$(pytarballs): $(tdir)/%: + if [ -f $(DEPENDENCIES-DIR)/$* ]; then + cp $(DEPENDENCIES-DIR)/$* $@ + else + + # Convenience variable + # -------------------- + # + # `n' is just for convenience and to avoid having to repeat the + # package tarball name in the conditional to find its URL. + # + # For some packages (for example `python-dateutil', or those with + # a number or dash in their name), we need special consideration + # because the tokenization above will produce `python' as the + # first string. + if [ $* = python-dateutil-$(python-dateutil-version).tar.gz ]; then + n=dateutil + elif [ $* = h5py-$(h5py-version).tar.gz ]; then + n=h5py + + # elif [ $* = strange-tarball5name-version.tar.gz ]; then + # n=strange5-name + else + # Remove all numbers, `-' and `.' from the tarball name so we can + # search more easily only with the program name. + n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ + | awk '{print $$1}') + fi + + # Set the top download link of the requested tarball. The ones + # that have non-standard filenames (differing from our archived + # tarball names) are treated first, then the standard ones. + mergenames=1 + if [ $$n = cython ]; then + mergenames=0 + hash=36/da/fcb979fc8cb486a67a013d6aefefbb95a3e19e67e49dff8a35e014046c5e + h=$(pytopurl)/$$hash/Cython-$(cython-version).tar.gz + elif [ $$n = python ]; then + mergenames=0 + h=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz + elif [ $$n = libffi ]; then + mergenames=0 + h=ftp://sourceware.org/pub/libffi/libffi-$(libffi-version).tar.gz + elif [ $$n = secretstorage ]; then + mergenames=0 + hash=a6/89/df343dbc2957a317127e7ff2983230dc5336273be34f2e1911519d85aeb5 + h=$(pytopurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz + elif [ $$n = asn ]; then h=fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4 + elif [ $$n = astroquery ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396 + elif [ $$n = astropy ]; then h=eb/f7/1251bf6881861f24239efe0c24cbcfc4191ccdbb69ac3e9bb740d0c23352 + elif [ $$n = beautifulsoup ]; then h=80/f2/f6aca7f1b209bb9a7ef069d68813b091c8c3620642b568dac4eb0e507748 + elif [ $$n = certifi ]; then h=55/54/3ce77783acba5979ce16674fc98b1920d00b01d337cfaaf5db22543505ed + elif [ $$n = cffi ]; then h=64/7c/27367b38e6cc3e1f49f193deb761fe75cda9f95da37b67b422e62281fcac + elif [ $$n = chardet ]; then h=fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d + elif [ $$n = cryptography ]; then h=07/ca/bc827c5e55918ad223d59d299fff92f3563476c3b00d0a9157d9c0217449 + elif [ $$n = cycler ]; then h=c2/4b/137dea450d6e1e3d474e1d873cd1d4f7d3beed7e0dc973b06e8e10d32488 + elif [ $$n = entrypoints ]; then h=b4/ef/063484f1f9ba3081e920ec9972c96664e2edb9fdc3d8669b0e3b8fc0ad7c + elif [ $$n = h5py ]; then h=43/27/a6e7dcb8ae20a4dbf3725321058923fec262b6f7835179d78ccc8d98deec + elif [ $$n = html ]; then h=85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f + elif [ $$n = idna ]; then h=ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7 + elif [ $$n = jeepney ]; then h=16/1d/74adf3b164a8d19a60d0fcf706a751ffa2a1eaa8e5bbb1b6705c92a05263 + elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2 + elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864 + elif [ $$n = matplotlib ]; then h=89/0c/653aec68e9cfb775c4fbae8f71011206e5e7fe4d60fcf01ea1a9d3bc957f + elif [ $$n = mpi ]; then h=55/a2/c827b196070e161357b49287fa46d69f25641930fd5f854722319d431843 + elif [ $$n = numpy ]; then h=cf/8d/6345b4f32b37945fedc1e027e83970005fc9c699068d2f566b82826515f2 + elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af + elif [ $$n = pkgconfig ]; then h=6e/a9/ff67ef67217dfdf2aca847685fe789f82b931a6957a3deac861297585db6 + elif [ $$n = pycparser ]; then h=68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a + elif [ $$n = pyparsing ]; then h=b9/b8/6b32b3e84014148dcd60dd05795e35c2e7f4b72f918616c61fdce83d27fc + elif [ $$n = dateutil ]; then h=ad/99/5b2e99737edeb28c71bcbec5b5dda19d0d9ef3ca3e92e3e925e7c0bb364c + elif [ $$n = requests ]; then h=52/2c/514e4ac25da2b08ca5a464c50463682126385c4272c18193876e91f4bc38 + elif [ $$n = scipy ]; then h=a9/b4/5598a706697d1e2929eaf7fe68898ef4bea76e4950b9efbe1ef396b8813a + elif [ $$n = secretstorage ]; then h=a6/89/df343dbc2957a317127e7ff2983230dc5336273be34f2e1911519d85aeb5 + elif [ $$n = setuptools ]; then h=c2/f7/c7b501b783e5a74cf1768bc174ee4fb0a8a6ee5af6afa92274ff964703e0 + elif [ $$n = setuptools_scm ]; then h=54/85/514ba3ca2a022bddd68819f187ae826986051d130ec5b972076e4f58a9f3 + elif [ $$n = six ]; then h=dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca + elif [ $$n = soupsieve ]; then h=0c/52/e9088bb9b96e2d39fc3b33fcda5b4fde9d71473536ac660a1ca9a0958a2f + elif [ $$n = urllib ]; then h=b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1 + elif [ $$n = virtualenv ]; then h=51/aa/c395a6e6eaaedfa5a04723b6446a1df783b16cca6fec66e671cede514688 + elif [ $$n = webencodings ]; then h=0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47 +# elif [ $$n = strange5-name ]; then h=XXXXX + else + echo; echo; echo; + echo "'$$n' not recognized as a dependency name to download." + echo; echo; echo; + exit 1 + fi + + # Download the requested tarball. Note that some packages may not + # follow our naming convention (where the package name is merged + # with its version number). In such cases, `w' will be the full + # address, not just the top directory address. But since we are + # storing all the tarballs in one directory, we want it to have + # the same naming convention, so we'll download it to a temporary + # name, then rename that. + if [ $$mergenames = 1 ]; then tarballurl=$(pytopurl)/$$h/"$*" + else tarballurl=$$h + fi + + # Download using the script specially defined for this job. + touch $(lockdir)/download + downloader="wget --no-use-server-timestamps -O" + $(downloadwrapper) "$$downloader" $(lockdir)/download \ + $$tarballurl $@ + fi + + + + + +# Necessary programs and libraries +# -------------------------------- +# +# While this Makefile is for Python programs, in some cases, we need +# certain programs (like Python itself), or libraries for the modules. +$(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz + $(call gbuild, $<, libffi-$(libffi-version)) \ + echo "Libffi $(libffi-version)" > $@ + +$(ibidir)/python3: $(tdir)/python-$(python-version).tar.gz \ + $(ilidir)/libffi + # On Mac systems, the build complains about `clang' specific + # features, so we can't use our own GCC build here. + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + export CXX=clang++; \ + fi; \ + $(call gbuild, $<, Python-$(python-version),, \ + --without-ensurepip \ + --with-system-ffi \ + --enable-shared) \ + && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ + {printf "%d.%d\n", $$1, $$2}') \ + && ln -s $(ildir)/python$$v $(ildir)/python \ + && rm -rf $(ipydir) \ + && mkdir $(ipydir) \ + && echo "Python $(python-version)" > $@ + + + + + +# Non-PiP Python module installation +# ---------------------------------- +# +# To build Python packages with direct access to a `setup.py' (if no direct +# access to `setup.py' is needed, pip can be used). +# Arguments of this function are the numbers +# 1) Unpack command +# 2) Package name +# 3) Unpacked directory name after unpacking the tarball +# 4) site.cfg file (optional) +# 5) Official software name.(for paper). +pybuild = cd $(ddir); rm -rf $(3); \ + if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ + cd $(3); \ + if [ "x$(strip $(4))" != x ]; then \ + sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ + -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ + $(4) > site.cfg; \ + fi; \ + python3 setup.py build \ + && python3 setup.py install \ + && cd .. \ + && rm -rf $(3) \ + && echo "$(5)" > $@ + + + + + +# Python modules +# --------------- +# +# All the necessary Python modules go here. +$(ipydir)/asn1crypto: $(tdir)/asn1crypto-$(asn1crypto-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, asn1crypto-$(asn1crypto-version), , \ + Asn1crypto $(asn1crypto-version)) + +$(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ + $(ipydir)/beautifulsoup4 \ + $(ipydir)/html5lib \ + $(ipydir)/requests \ + $(ipydir)/astropy \ + $(ipydir)/keyring \ + $(ipydir)/numpy + $(call pybuild, tar xf, $<, astroquery-$(astroquery-version), ,\ + Astroquery $(astroquery-version)) + +$(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ + $(ipydir)/h5py \ + $(ipydir)/numpy \ + $(ipydir)/scipy + $(call pybuild, tar xf, $<, astropy-$(astropy-version)) \ + && cp $(dtexdir)/astropy.tex $(ictdir)/ \ + && echo "Astropy $(astropy-version) \citep{astropy2013,astropy2018}" > $@ + +$(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \ + $(ipydir)/soupsieve + $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version), ,\ + BeautifulSoup $(beautifulsoup4-version)) + +$(ipydir)/certifi: $(tdir)/certifi-$(certifi-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, certifi-$(certifi-version), ,\ + Certifi $(certifi-version)) + +$(ipydir)/cffi: $(tdir)/cffi-$(cffi-version).tar.gz \ + $(ilidir)/libffi \ + $(ipydir)/pycparser + $(call pybuild, tar xf, $<, cffi-$(cffi-version), ,\ + cffi $(cffi-version)) + +$(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, chardet-$(chardet-version), ,\ + Chardet $(chardet-version)) + +$(ipydir)/cryptography: $(tdir)/cryptography-$(cryptography-version).tar.gz \ + $(ipydir)/asn1crypto \ + $(ipydir)/cffi + $(call pybuild, tar xf, $<, cryptography-$(cryptography-version), ,\ + Cryptography $(cryptography-version)) + +$(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \ + $(ipydir)/six + $(call pybuild, tar xf, $<, cycler-$(cycler-version), ,\ + Cycler $(cycler-version)) + +$(ipydir)/cython: $(tdir)/cython-$(cython-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, Cython-$(cython-version)) \ + && cp $(dtexdir)/cython.tex $(ictdir)/ \ + && echo "Cython $(cython-version) \citep{cython2011}" > $@ + +$(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version), ,\ + EntryPoints $(entrypoints-version)) + +$(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \ + $(ilidir)/hdf5 \ + $(ipydir)/cython \ + $(ipydir)/pypkgconfig \ + $(ipydir)/setuptools + #$(ipydir)/mpi4py # AFTER its problem is fixed. + #export HDF5_MPI=ON; # AFTER its problem is fixed. + export HDF5_DIR=$(ildir); \ + $(call pybuild, tar xf, $<, h5py-$(h5py-version), ,\ + h5py $(h5py-version)) + +$(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \ + $(ipydir)/six \ + $(ipydir)/webencodings + $(call pybuild, tar xf, $<, html5lib-$(html5lib-version), ,\ + HTML5lib $(html5lib-version)) + +$(ipydir)/idna: $(tdir)/idna-$(idna-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, idna-$(idna-version), ,\ + idna $(idna-version)) + +$(ipydir)/jeepney: $(tdir)/jeepney-$(jeepney-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, jeepney-$(jeepney-version), ,\ + Jeepney $(jeepney-version)) + +$(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \ + $(ipydir)/entrypoints \ + $(ipydir)/secretstorage \ + $(ipydir)/setuptools_scm + $(call pybuild, tar xf, $<, keyring-$(keyring-version), ,\ + Keyring $(keyring-version)) + +$(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version), ,\ + Kiwisolver $(kiwisolver-version)) + +$(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ + $(ipydir)/cycler \ + $(ilidir)/freetype \ + $(ipydir)/kiwisolver \ + $(ipydir)/numpy \ + $(ipydir)/pyparsing \ + $(ipydir)/python-dateutil + $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) \ + && cp $(dtexdir)/matplotlib.tex $(ictdir)/ \ + && echo "Matplotlib $(matplotlib-version) \citep{matplotlib2007}" > $@ + +# Currently mpi4py doesn't build because of some conflict with OpenMPI: +# +# In file included from src/mpi4py.MPI.c:591, +# from src/MPI.c:4: +# src/mpi4py.MPI.c: In function '__pyx_f_6mpi4py_3MPI_del_Datatype': +# src/mpi4py.MPI.c:15094:36: error: expected expression before '_Static_assert' +# __pyx_t_1 = (((__pyx_v_ob[0]) == MPI_UB) != 0); +# +# But atleast on my system it fails. +$(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \ + $(ipydir)/setuptools \ + $(ilidir)/openmpi + $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version)) \ + && cp $(dtexdir)/mpi4py.tex $(ictdir)/ \ + && echo "mpi4py $(mpi4py-version) \citep{mpi4py2011}" > $@ + +$(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ + $(ipydir)/setuptools \ + $(ilidir)/openblas \ + $(ilidir)/fftw \ + $(ibidir)/unzip + if [ x$(on_mac_os) = xyes ]; then \ + export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ + else \ + export LDFLAGS="$(LDFLAGS) -shared"; \ + fi; \ + conf="$$(pwd)/reproduce/software/config/installation/numpy-scipy.cfg"; \ + $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf, \ + Numpy $(numpy-version)) \ + && cp $(dtexdir)/numpy.tex $(ictdir)/ \ + && echo "Numpy $(numpy-version) \citep{numpy2011}" > $@ + +$(ibidir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, pip-$(pip-version), ,\ + PiP $(pip-version)) + +$(ipydir)/pypkgconfig: $(tdir)/pkgconfig-$(pypkgconfig-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, pkgconfig-$(pypkgconfig-version), , + pkgconfig $(pypkgconfig-version)) + +$(ipydir)/pycparser: $(tdir)/pycparser-$(pycparser-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, pycparser-$(pycparser-version), ,\ + pycparser $(pycparser-version)) + +$(ipydir)/pyparsing: $(tdir)/pyparsing-$(pyparsing-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version), ,\ + PyParsing $(pyparsing-version)) + +$(ipydir)/python-dateutil: $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz \ + $(ipydir)/setuptools_scm \ + $(ipydir)/six + $(call pybuild, tar xf, $<, python-dateutil-$(python-dateutil-version), ,\ + python-dateutil $(python-dateutil-version)) + +$(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ + $(ipydir)/certifi \ + $(ipydir)/chardet \ + $(ipydir)/idna \ + $(ipydir)/numpy \ + $(ipydir)/urllib3 + $(call pybuild, tar xf, $<, requests-$(requests-version), ,\ + Requests $(requests-version)) + +$(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ + $(ipydir)/numpy + if [ x$(on_mac_os) = xyes ]; then \ + export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ + else \ + export LDFLAGS="$(LDFLAGS) -shared"; \ + fi; \ + conf="$$(pwd)/reproduce/software/config/installation/numpy-scipy.cfg"; \ + $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf) \ + && cp $(dtexdir)/scipy.tex $(ictdir)/ \ + && echo "Scipy $(scipy-version) \citep{scipy2007,scipy2011}" > $@ + +$(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \ + $(ipydir)/cryptography \ + $(ipydir)/jeepney + $(call pybuild, tar xf, $<, SecretStorage-$(secretstorage-version), ,\ + SecretStorage $(secretstorage-version)) + +$(ipydir)/setuptools: $(tdir)/setuptools-$(setuptools-version).zip \ + $(ibidir)/python3 \ + $(ibidir)/unzip + $(call pybuild, unzip, $<, setuptools-$(setuptools-version), ,\ + Setuptools $(setuptools-version)) + +$(ipydir)/setuptools_scm: $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version), ,\ + Setuptools-scm $(setuptools_scm-version)) + +$(ipydir)/six: $(tdir)/six-$(six-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, six-$(six-version), ,\ + Six $(six-version)) + +$(ipydir)/soupsieve: $(tdir)/soupsieve-$(soupsieve-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version), ,\ + SoupSieve $(soupsieve-version)) + +$(ipydir)/urllib3: $(tdir)/urllib3-$(urllib3-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, urllib3-$(urllib3-version), ,\ + Urllib3 $(urllib3-version)) + +$(ipydir)/webencodings: $(tdir)/webencodings-$(webencodings-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, webencodings-$(webencodings-version), ,\ + Webencodings $(webencodings-version)) |