diff options
| -rw-r--r-- | .file-metadata | bin | 9121 -> 9226 bytes | |||
| -rwxr-xr-x | project | 99 | ||||
| -rw-r--r-- | reproduce/analysis/make/initialize.mk | 9 | ||||
| -rw-r--r-- | reproduce/analysis/make/paper.mk | 11 | ||||
| -rw-r--r-- | reproduce/software/config/checksums.conf | 12 | ||||
| -rwxr-xr-x | reproduce/software/config/software_acknowledge_context.sh | 112 | ||||
| -rw-r--r-- | reproduce/software/config/urls.conf | 159 | ||||
| -rw-r--r-- | reproduce/software/config/versions.conf | 93 | ||||
| -rw-r--r--[-rwxr-xr-x] | reproduce/software/make/atlas-multiple.mk | 0 | ||||
| -rw-r--r--[-rwxr-xr-x] | reproduce/software/make/atlas-single.mk | 0 | ||||
| -rw-r--r-- | reproduce/software/make/basic.mk | 1496 | ||||
| -rw-r--r-- | reproduce/software/make/build-rules.mk | 139 | ||||
| -rw-r--r-- | reproduce/software/make/high-level.mk | 1575 | ||||
| -rw-r--r-- | reproduce/software/make/python.mk | 929 | ||||
| -rwxr-xr-x | reproduce/software/shell/configure.sh | 139 | ||||
| -rwxr-xr-x | reproduce/software/shell/pre-make-build.sh | 81 | ||||
| -rwxr-xr-x | reproduce/software/shell/run-parts.in | 71 |
17 files changed, 2674 insertions, 2251 deletions
diff --git a/.file-metadata b/.file-metadata Binary files differindex 3a4122f..9c02981 100644 --- a/.file-metadata +++ b/.file-metadata @@ -41,6 +41,7 @@ make_targets= software_dir= clean_texdir=0 prepare_redo=0 +all_highlevel=0 existing_conf=0 scriptname="./project" minmapsize=10000000000 @@ -77,6 +78,7 @@ operation is defined by the (mandatory) second argument: configure - Configure project for this machine (e.g., build software). make - Run the project (do analysis and build outputs). + shell - Execute the project's shell for interactive testing. RECOMMENDATION: If this is the first time you are configuring this template, please don't use the options and let the script explain each @@ -92,6 +94,9 @@ Project 'make' special features. uploaded to servers like 'arXiv.org'. ./project make dist-lzip Similar to 'dist', but compress to '.tar.lz'. ./project make dist-zip Similar to 'dist', but compress to '.zip'. + ./project make dist-software Build a .tar.gz tarball containing all + software source tarballs necessary for the + project. With the options below you can modify the default behavior. Configure options: @@ -105,6 +110,7 @@ Configure options: --clean-texdir Remove possibly existing build-time subdirectories under the project's 'tex/' directory (can happen when source is from arXiv for example). + --all-highlevel Build all high-level software (for development). Configure and Make options: -g, --group=STR Build and run with write permissions for a group. @@ -156,6 +162,7 @@ do # Main operation. configure) func_operation_set $1; shift;; make) func_operation_set $1; shift;; + shell) func_operation_set $1; shift;; # Configure options: @@ -179,6 +186,8 @@ do --check-config=*) on_off_option_error --check-config;; --clean-texdir) clean_texdir=1; shift;; --clean-texdir=*) on_off_option_error --clean-texdir;; + --all-highlevel) all_highlevel=1; shift;; + --all-highlevel=*) on_off_option_error --all-highlevel;; # Configure and Make options: -g|--group) group="$2"; check_v group "$group"; shift;shift;; @@ -271,6 +280,35 @@ fi +# Error when configuration isn't run +configuration_necessary() { + cat <<EOF + +The project is either (1) not configured on this system, or (2) the +configuration wasn't successful. + +(1) If it hasn't been configured at all, use the command below to configure +it (set a build directory and let it build its necessary software in it). + + $ ./project configure + +(2) If it has been configured, but the configuration failed in a step, you +can re-configure it using your previous settings with the command +below. All successful steps will be skipped, allowing a fast completion. + + $ ./project configure -e + +If there was a problem, please let us know by filling this online form: + http://savannah.nongnu.org/support/?func=additem&group=reproduce + +EOF + exit 1 +} + + + + + # Run operations in controlled environment # ---------------------------------------- controlled_env() { @@ -321,11 +359,12 @@ case $operation in # user to have to worry about any other file that needs an # executable flag. # - # Basically, all the files (shell scripts) in the two - # `reproduce/*/bash' should need executable flags, so we are giving - # them executable flags by default. If any other file in your project - # needs such flags, add them here. - chmod +x reproduce/software/shell/* reproduce/analysis/bash/* + # Basically, all the project shell scripts need executable flags so + # to make sure they have them, we are activating the executable + # flags by default here every time './project configure' is run. If + # any other file in your project needs such flags, add them here. + chmod +x reproduce/software/shell/* reproduce/software/config/*.sh \ + reproduce/analysis/bash/* # If the user requested, clean the TeX directory from the extra # (to-be-built) directories that may already be there (and will not @@ -343,6 +382,7 @@ case $operation in export minmapsize=$minmapsize export software_dir=$software_dir export existing_conf=$existing_conf + export all_highlevel=$all_highlevel export reproducible_paper_group_name=$group # Run the configuration script @@ -368,33 +408,13 @@ case $operation in - # Run the project. + # Batch execution of the project. make) # Make sure the configure script has been completed properly # (`configuration-done.txt' exists). if ! [ -f .build/software/configuration-done.txt ]; then - cat <<EOF - -The project is either (1) not configured on this system, or (2) the -configuration wasn't successful. - -(1) If it hasn't been configured at all, use the command below to configure -it (set a build directory and let it build its necessary software in it). - - $ ./project configure - -(2) If it has been configured, but the configuration failed in a step, you -can re-configure it using your previous settings with the command -below. All successful steps will be skipped, allowing a fast completion. - - $ ./project configure -e - -If there was a problem, please let us know by filling this online form: - http://savannah.nongnu.org/support/?func=additem&group=reproduce - -EOF - exit 1 + configuration_necessary fi # Run data preparation phase (optionally build Makefiles with @@ -425,7 +445,32 @@ EOF ;; + shell) + # Make sure the configure script has been completed properly + # (`configuration-done.txt' exists). + if ! [ -f .build/software/configuration-done.txt ]; then + configuration_necessary + fi + + # Run the project's own shell without inheriting any environment + # from the host. + bdir=`.local/bin/realpath .build` + instdir=$bdir/software/installed + .local/bin/env -i \ + HOME=$bdir \ + CCACHE_DISABLE=1 \ + PATH=$instdir/bin \ + LDFLAGS=-L$instdir/lib \ + SHELL=$instdir/bin/bash \ + CPPFLAGS=-I$instdir/include \ + LD_LIBRARY_PATH=$instdir/lib \ + OMPI_MCA_plm_rsh_agent=/bin/false \ + PYTHONPATH=$instdir/lib/python/site-packages \ + PYTHONPATH3=$instdir/lib/python/site-packages \ + PS1="[\[\033[32m\](maneage)\[\033[00m\] \u@\h \W]$ " \ + $instdir/bin/bash + ;; # Operation not specified. diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 489f9e3..d01cda0 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -243,6 +243,7 @@ clean: clean-mmap rm -rf $(BDIR)/tex/macros/!(dependencies.tex|dependencies-bib.tex) rm -rf $(BDIR)/!(software|tex) $(BDIR)/tex/!(macros|$(texbtopdir)) rm -rf $(BDIR)/tex/build/!(tikz) $(BDIR)/tex/build/tikz/* + rm -rf $(BDIR)/software/preparation-done.mk distclean: clean # Without cleaning the Git hooks, we won't be able to easily @@ -385,19 +386,15 @@ dist-zip: $(project-package-contents) # Package the software tarballs. dist-software: curdir=$$(pwd) + dirname=software-$(project-commit-hash) cd $(BDIR) - if [ -d .git ]; then - dirname="software-$$(git describe --dirty --always --long)" - else - dirname="software-NOGIT"; - fi mkdir $$dirname cp -L software/tarballs/* $$dirname/ tar -cf $$dirname.tar $$dirname gzip -f --best $$dirname.tar rm -rf $$dirname cd $$curdir - mv $(BDIR)/$$dir.tar.gz ./ + mv $(BDIR)/$$dirname.tar.gz ./ diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk index cc43117..6d69d49 100644 --- a/reproduce/analysis/make/paper.mk +++ b/reproduce/analysis/make/paper.mk @@ -112,8 +112,16 @@ $(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ # for `biber') and then run `biber' to generate the `.bbl' file. export TEXINPUTS=$$p: cd $(texbdir); + + # Note: the option '-shell-escape' is "normally disallowed for + # security reasons" according to the `info pdflatex' manual, but + # is enabled here in order to allow the use of PGFPlots. If you + # do not use PGFPlots, then you should remove the `-shell-escape' + # option for better security. See + # https://savannah.nongnu.org/task/?15694 for details. latex -shell-escape -halt-on-error $$p/paper.tex bibtex paper + # Hack: tidy up eprint+doi style that didn't work in .bst file. # TODO (better): read Part 4 of # http://mirrors.ctan.org/info/bibtex/tamethebeast/ttb_en.pdf @@ -150,6 +158,9 @@ paper.pdf: $(mtexdir)/project.tex paper.tex $(texbdir)/paper.bbl p=$$(pwd) export TEXINPUTS=$$p: cd $(texbdir) + + # See above for a warning and brief discussion on the the + # pdflatex option `-shell-escape'. latex -shell-escape -halt-on-error $$p/paper.tex # Convert the DVI to PostScript, and the PostScript to PDF. The diff --git a/reproduce/software/config/checksums.conf b/reproduce/software/config/checksums.conf index e23df9f..2da488b 100644 --- a/reproduce/software/config/checksums.conf +++ b/reproduce/software/config/checksums.conf @@ -38,7 +38,7 @@ libiconv-checksum = 365dac0b34b4255a0066e8033a8b3db4bdb94b9b57a9dca17ebf2d779139 libtool-checksum = a6eef35f3cbccf2c9e2667f44a476ebc80ab888725eb768e91a3a6c33b8c931afc46eb23efaee76c8696d3e4eed74ab1c71157bcb924f38ee912c8a90a6521a4 libunistring-checksum = 01dcab6e05ea4c33572bf96cc0558bcffbfc0e62fc86410cef06c1597a0073d5750525fe2dee4fdb39c9bd704557fcbab864f9645958108a2e07950bc539fe54 libxml2-checksum = cb7784ba4e72e942614e12e4f83f4ceb275f3d738b30e3b5c1f25edf8e9fa6789e854685974eed95b362049dbf6c8e7357e0327d64c681ed390534ac154e6810 -lzip-checksum = e3331bbf0166541332182a9a28c2e08f522735ef668a06dfa26055251d5142a09227d97e6ae50b38c0b8805479a307a9e55c81b120d5befad2fde44676261843 +lzip-checksum = e2e229899002072322a5bf7fb7ef37ff7cc2d0ded7d6525679ac29cec5c1534de89f76bc01e15c3d76584099957d596b9e5b32d3f9cbfc6bc8d6370fc67d7d96 m4-checksum = a92cad4441b3fd7c033837389ca3499494523d364a5fda043d92c517051510f1758b3b837f0477f42d2258a179ab79a4993e5d1694ef2673db6d96d1faff84fe make-checksum = ddf0fdcb9ee1b182ef294c5da70c1275288c99bef60e63a25c0abed2ddd44aba1770be4aab1db8cac81e5f624576f2127c5d825a1824e1c7a49df4f16445526b metastore-checksum = b2a5fdde9de5ddc1e6c368d5da1b2e97e4fdbaa138a7be281ccb40a81dd4a9bb1849d36b2d5d3f01205079bace60441f82a7002097ff3a7037340a35b0f1574a @@ -80,7 +80,7 @@ atlas-checksum = bf17306f09f2aa973cb776e2c9eacfb2409ad4d95d19802e1c4e0597d0a099f autoconf-checksum = c25e834251bfc2befe822614caf1c80d7e1314a83e7173304abc235fd15a958b8db9fbc801e8ad98328dfd6d9dbc425bfbbefec500fa268992ae7bbf4fa5bc35 automake-checksum = 47b0120a59e3e020529a6ce750297d7de1156fd2be38db5d101e50120f11b40c28741ecd5eacf2790a9e25386713dcf7717339cfa5d7943d0dbf47c417383448 bison-checksum = 00d2b37187b93100ec4b220ce2752d12ccf68f9d0d39b380d375d36dd8a22aa1d6e60156918f95e4493f9531c5d42d8fad38fd807307b491c1ca7ca4177823d9 -boost-checksum = 4378e20f18db9f186ac0f38ad98c522526e0fe48a8deb968c41325e3f69e733be3298505e91ea713e79a9d9e741305fa879d289f74e6276e0ec7286f03be87fc +boost-checksum = 2844dddd3357e76909f2f3d008f686949f8b54aed29e1c650e18becd0b50701a273bb3754f1a4e56c0c056693b27c354c9ba54ddbec92599e192c2f6736fe3be cairo-checksum = 9eb27c4cf01c0b8b56f2e15e651f6d4e52c99d0005875546405b64f1132aed12fbf84727273f493d84056a13105e065009d89e94a8bfaf2be2649e232b82377f cdsclient-checksum = 2d7abf0079189b9dd19cb8919061445fd19ea9f7dfd54e8ceee26b743218cf62ab00eba0147abe82d9294223927f04b4cc3328620dfc9184a7049f8d515b29e4 cfitsio-checksum = c0502699e266928dd25abe57730dc4b357ccc9023789fe745324ae01aa688516aceaf37321ee578f0430111d9718f0fec0dc5b54c07f935529560f00b32ce1e3 @@ -88,7 +88,7 @@ cmake-checksum = f09440681e0c414f5ca669f3aeba6666d09e0642f30a2e12c3199e7fb3da95a eigen-checksum = 34cf600914cce719d61511577ef9cd26fbdcb7a6fad1d0ab8396f98b887fac6a5577d3967e84a8f56225cc50de38f3b91f34f447d14312028383e32b34ea1972 expat-checksum = 514ff2ef3c93af0b1715b7a08732db33c13a113c4c72422716a22ee26c09235deed71ec55510cee24c33bcd6b2347602bd71ce70a432d5583fb63765ff9e0e09 fftw-checksum = ab918b742a7c7dcb56390a0a0014f517a6dff9a2e4b4591060deeb2c652bf3c6868aa74559a422a276b853289b4b701bdcbd3d4d8c08943acf29167a7be81a38 -flex-checksum = e9785f3d620a204b7d20222888917dc065c2036cae28667065bf7862dfa1b25235095a12fd04efdbd09bfd17d3452e6b9ef953a8c1137862ff671c97132a082e +flex-checksum = b4ef58d4a1d66b213e2f59df06959decf46d26b253cdc3f51cd26e2e2b505461ef23dafa974dd2005b1f0cafa5a83fe9258baf78004b2fdae6dfc299bc17bfd1 freetype-checksum = 02f0f4211f9cee5b5e46ebe61190482fca5b41bc26be06fcf0d2d717e9fb119229308398c420eeea476fa2511ca2d52948f1a3242efad30ca82ed0b07cd50e3a gdb-checksum = b4161df5adfce3eb51b3b3fc6072b2ef9f612f5e0f95b25bc57382812854836e9d1b24807c68cc8987e37e90112c6d75ffc1de3ea67a9f92858b4ce1af85c479 ghostscript-checksum = acee64fae78771bffa19b0b2bfaba3c345b420f93ceb4fc9df5fb705f785c8ed720fde2aef53546fac6aca2f7366c64c68a6e373a71999a42dc71aadc9aa782f @@ -104,15 +104,17 @@ libffi-checksum = 980ca30a8d76f963fca722432b1fe5af77d7a4e4d2eac5144fbc5374d4c596 libgit2-checksum = 0879c162e2e1dc00eadfbda22cd1f9d3a95b4ec2b653c108983f37c2f695140882de4d50d7cbc04ced247125a4e9fe6df16130e1267891aecdb2411d920db5c6 libjpeg-checksum = 74ea5af3545657d4ac03f8f7933913112cc2d982f0e379d0e5647f1acac21931468e53806297c30ebe180c7bcf84919a0ac20a4195afb03db03060d57904ef6c libnsl-checksum = a3c8f674357674b7ed4b26c05adde607f39be8d6dc9ff715448e1fcc5fc23d11fbb4ce85a6e493b79bdb0bb450dc3ffb1fb480715779f738d7bc016fae91621d +libpaper-checksum = 3bf6ebb0af89931d2f72ea4a09a7fa958b2facda5f238983ec7bac39652e08614b33f0de3af74a03457b2a4203eee4950bf18a4b726e79aa64093ace6a1fb0bc libpng-checksum = 59e8c1059013497ae616a14c3abbe239322d3873c6ded0912403fc62fb260561768230b6ab997e2cccc3b868c09f539fd13635616b9fa0dd6279a3f63ec7e074 libtiff-checksum = d213e5db09fd56b8977b187c5a756f60d6e3e998be172550c2892dbdb4b2a8e8c750202bc863fe27d0d1c577ab9de1710d15e9f6ed665aadbfd857525a81eea8 -libtirpc-checksum = 392f391f9fc1bd68d81dc44e4058831a64b32790b5c8c37338b0ab416fad2ae4d16389e632596734dba09780347918cc65c6f134e0c1afd09e81ec250785ed23 +libtirpc-checksum = bcb6b5c062c1301aa1246ec93ae0a5c1d221b8421126d020863517cb814b43ed038fb6c0c2faf4e68ff133b69abefe4f4d42bfc870671da6c27ca941a30b155a missfits-checksum = 32727f5eb30573a1cedacb8900e2536867e4815059eee32e64e3db65be9291b8a91b9f45b2c9f3cf6fc2a8cc448012ea3d502bdd9dee516008e17d5086aee795 netpbm-checksum = 064720f8a9d0a502488e1af4daecdbf3936910996507ca6f311073a0ad842346692a148eb1ddf7b717f7b108f60500246cb4b83f4d3665f5fc285a84ae1d63d6 -openblas-checksum = 91b3074eb922453bf843158b4281cde65db9e8bbdd7590e75e9e6cdcb486157f7973f2936f327bb3eb4f1702ce0ba51ae6729d8d4baf2d986c50771e8f696df0 +openblas-checksum = 64a5f983b2f6e02cdb6e0f14433498cc5daa1ccfb49246f7a2dcd38f9982fa608f2abea069fe0e35012af8c1441c43d1f6418eaccd40795f5002fed1c36ce05d openmpi-checksum = 760716974cb6b25ad820184622e1ee7926bc6fda87db6b574f76792bc1ca99522e52195866c14b7cb2df5a4981efdaf9f71d2c5533cc0e8e45c2c4b3b74cbacc openssh-checksum = e280fa2d56f550efd37c5d2477670326261aa8b94d991f9eb17aad90e0c6c9c939efa90fe87d33260d0f709485cb05c379f0fd1bd44fc0d5190298b6398c9982 patch-checksum = 75d4e1544484da12185418cd4a1571994398140a91ac606fa08dd067004187dad77d1413f0eb3319b3fe4df076714615c98b29df06af052bb65960fa8b0c86bf +pcre-checksum = abac4c4f9df9e61d7d7761a9c50843882611752e1df0842a54318f358c28f5953025eba2d78997d21ee690756b56cc9f1c04a5ed591dd60654cc78ba16d9ecfb pixman-checksum = 1b0205dbe9d9185c68813ce577a889f3c83e83fbd9955c3a72d411c3b476e6be93fc246b5b6ef4ee17e2bb8eb6fb5559e01dff7feb6a6c4c6314f980e960d690 python-checksum = c25a72ad792f7c1b4c2f79faebbe9608d04b04b2fe58ab804cb4732cdaa75ea93d175f5e52b38e91cb6ae0559ea6b645d802c8b6a869584e8bb9b5018367ce3d R-checksum = 17513e9f4dd27c61c11f7aa45227aeeeefb375bf5d4e193b471724f379a1b2da33e127cbe91aa175cbbbb048b892047e2f610280585c8159242a6c94790b07f9 diff --git a/reproduce/software/config/software_acknowledge_context.sh b/reproduce/software/config/software_acknowledge_context.sh new file mode 100755 index 0000000..b0ede85 --- /dev/null +++ b/reproduce/software/config/software_acknowledge_context.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env sh + +# Surrounding text for software acknowledgement and citation. The list of +# names, versions and citations of the software are written by an automatic +# script. However, through this file, users have the option to specify the +# text surrounding those lists. +# +# We recommend to leave these values untouched at first, after building +# your PDF, you can see how they surround the list of software you used in +# your project to make a smoothly readable English text. Afterwards, please +# feel free to modify them as you wish. +# +# Copyright (C) 2020 Boud Roukema <boud@cosmo.torun.pl> +# Copyright (C) 2020 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This script is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This script is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# <http://www.gnu.org/licenses/>. + +# WARNING: +# In contrast to most configure files in maneage, this configure file is +# technically a shell script, to be called by 'configure.sh', so the +# variables defined here must follow 'shell' syntax, not 'make' syntax. + +# These variables will be exported to the 'configure.sh' script - so +# descriptive, unique names should be used, to reduce the chance of +# conflicts between identically named variables. + +# COMMENT: +# Sentences have a tendancy to become very long. To avoid making the +# variable values too long (and thus making this file hard to read), one +# method is to break the sentence into smaller components and build the +# full sentence gradually, similar to what we have done with 'str' in the +# examples below. Each time, the value of 'str' is re-written by appending +# its previous value with the rest of the sentence. + +# COMMENT: +# As of 2020-06-10, the general issue of how to best cite software within +# maneage, especially a full list of software in the Acknowledgments +# section, remains wide open. See +# https://savannah.nongnu.org/task/index.php?15318 for some technical +# aspects of software citation. + + + + + +# Add your definitions of the LaTeX text here. +# +# To override a default but generate no text, use a string of non-zero +# length such as "{}" that will have only minor effects in LaTeX. + +# An introduction to the software acknowledgement. +thank_software_introduce= + +# The text to be used before listing C/C++ programs and libraries. +thank_progs_libs= + +# The text to be used before listing Python modules. +thank_python= + +# The text to be used before listing LaTeX packages. +thank_latex= + +# Concluding remarks. +thank_software_conclude= + + + + + +# Defaults +# -------- +# +# These are the default values which are only written into the variable if +# you don't specify a value above +if [ "x$thank_software_introduce" = "x" ]; then + thank_software_introduce="" +fi + +if [ "x$thank_progs_libs" = "x" ]; then + str="This research was done with the following free software" + str="$str programs and libraries:" + thank_progs_libs="$str" +fi + +if [ "x$thank_python" = "x" ]; then + thank_python="Within Python, the following modules were used:" +fi + +if [ "x$thank_latex" = "x" ]; then + str="The \LaTeX{} source of the paper was compiled to make the" + str="$str PDF using the following packages:" + thank_latex="$str" +fi + +if [ "x${thank_software_conclude}" = "x" ]; then + str="We are very grateful to all their creators for freely " + str="$str providing this necessary infrastructure. This research " + str="$str (and many other projects) would not be possible without " + str="$str them." + thank_software_conclude="$str" +fi diff --git a/reproduce/software/config/urls.conf b/reproduce/software/config/urls.conf new file mode 100644 index 0000000..dc5a9d8 --- /dev/null +++ b/reproduce/software/config/urls.conf @@ -0,0 +1,159 @@ +# Optional URLs of software to use for downloading their source +# +# If un-commented the URLs of this file will be used. Otherwise default +# servers (primarily on 'zenodo.org') will be checked for the tarball. +# +# Copyright (C) 2020 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2020 Raul Infante-Sainz <infantesainz@gmail.com> +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice and +# this notice are preserved. This file is offered as-is, without any +# warranty. + + + + + +# Basic/low-level programs and libraires (installed in any case) +# -------------------------------------------------------------- +#bash-url = http://akhlaghi.org/maneage-software +#binutils-url = http://ftp.gnu.org/gnu/binutils +#bzip2-url = http://akhlaghi.org/maneage-software +#cert-url = http://akhlaghi.org/maneage-software +#coreutils-url = http://ftp.gnu.org/gnu/coreutils +#curl-url = https://curl.haxx.se/download +#dash-url = http://akhlaghi.org/maneage-software +#diffutils-url = http://ftp.gnu.org/gnu/diffutils +#file-url = ftp://ftp.astron.com/pub/file +#findutils-url = http://ftp.gnu.org/gnu/findutils +#flock-url = http://akhlaghi.org/src +#gawk-url = http://ftp.gnu.org/gnu/gawk +#gcc-url = http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version) +#gettext-url = https://ftp.gnu.org/gnu/gettext +#git-url = http://mirrors.edge.kernel.org/pub/software/scm/git +#gmp-url = https://gmplib.org/download/gmp +#grep-url = http://ftp.gnu.org/gnu/grep +#gzip-url = http://akhlaghi.org/src +#isl-url = ftp://gcc.gnu.org/pub/gcc/infrastructure +#libbsd-url = http://libbsd.freedesktop.org/releases +#libiconv-url = https://ftp.gnu.org/pub/gnu/libiconv +#libtool-url = http://ftp.gnu.org/gnu/libtool +#libunistring-url = http://ftp.gnu.org/gnu/libunistring +#libxml2-url = ftp://xmlsoft.org/libxml2 +#lzip-url = http://akhlaghi.org/src +#m4-url = http://akhlaghi.org/maneage-software +#make-url = http://akhlaghi.org/src +#metastore-url = http://akhlaghi.org/maneage-software +#mpc-url = http://ftp.gnu.org/gnu/mpc +#mpfr-url = http://www.mpfr.org/mpfr-current +#ncurses-url = http://ftp.gnu.org/gnu/ncurses +#openssl-url = http://www.openssl.org/source +#patchelf-url = http://nixos.org/releases/patchelf/patchelf-$(patchelf-version) +#perl-url = $(shell echo https://www.cpan.org/src/$$(echo $(perl-version) | sed -e's/\./ /g' | awk '{printf("%d.0", $$1)}')) +#pkgconfig-url = http://pkg-config.freedesktop.org/releases +#readline-url = http://ftp.gnu.org/gnu/readline +#sed-url = http://ftp.gnu.org/gnu/sed +#tar-url = http://ftp.gnu.org/gnu/tar +#texinfo-url = http://ftp.gnu.org/gnu/texinfo +#unzip-url = $(shell echo ftp://ftp.info-zip.org/pub/infozip/src/unzip$$(echo $(unzip-version) | sed -e's/\.//').tgz) +#wget-url = http://ftp.gnu.org/gnu/wget +#which-url = http://ftp.gnu.org/gnu/which +#xz-url = http://tukaani.org/xz +#zip-url = $(shell echo ftp://ftp.info-zip.org/pub/infozip/src/zip$$(echo $(zip-version) | sed -e's/\.//').tgz) +#zlib-url = http://www.zlib.net + + + + + +# Optional/high-level programs and libraries +# ------------------------------------------ +# +# These are programs and libraries that are optional, The ones in +# `reproduce/software/config/TARGETS.conf' will be built as +# part of a project. To specify a software there, just remove the +# `-url' suffix from the list below. +#apachelog4cxx-url = http://akhlaghi.org/maneage-software +#apr-url = https://www-us.apache.org/dist/apr +#apr-util-url = https://www-us.apache.org/dist/apr +#astrometrynet-url = http://astrometry.net/downloads +#atlas-url = https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download +#autoconf-url = http://akhlaghi.org/maneage-software +#automake-url = http://ftp.gnu.org/gnu/automake +#bison-url = http://ftp.gnu.org/gnu/bison +#boost-url = $(shell vstr=$$(echo $(boost-version) | sed -e's/\./_/g'); echo https://dl.bintray.com/boostorg/release/$(boost-version)/source) +#cairo-url = https://www.cairographics.org/releases +#cdsclient-url = http://cdsarc.u-strasbg.fr/ftp/pub/sw +#cfitsio-url = https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c +#cmake-url = $(shell majv=$$(echo $(cmake-version) | sed -e's/\./ /' | awk '{printf("%d.%d", $$1, $$2)}'); echo https://cmake.org/files/v$$majv/cmake-$(cmake-version).tar.gz) +#eigen-url = http://bitbucket.org/eigen/eigen/get/$(eigen-version).tar.gz +#expat-url = $(shell vstr=$$(echo $(expat-version) | sed -e's/\./_/g'); echo https://github.com/libexpat/libexpat/releases/download/R_$$vstr/expat-$(expat-version).tar.lz) +#fftw-url = ftp://ftp.fftw.org/pub/fftw +#flex-url = https://github.com/westes/flex/files/981163 +#freetype-url = https://download.savannah.gnu.org/releases/freetype +#gdb-url = http://ftp.gnu.org/gnu/gdb +#ghostscript-url = $(shell v=$$(echo $(ghostscript-version) | sed -e's/\.//'); echo https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs$$v) +#gnuastro-url = http://ftp.gnu.org/gnu/gnuastro +#gsl-url = http://ftp.gnu.org/gnu/gsl +#hdf5-url = $(shell majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}'); echo https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src) +#healpix-url = http://akhlaghi.org/maneage-software +#help2man-url = http://ftp.gnu.org/gnu/help2man +#imagemagick-url = http://akhlaghi.org/maneage-software +#imfit-url = http://www.mpe.mpg.de/~erwin/resources/imfit +#lapack-url = http://www.netlib.org/lapack +#libffi-url = ftp://sourceware.org/pub/libffi +#libgit2-url = https://github.com/libgit2/libgit2/archive +#libjpeg-url = http://ijg.org/files +#libnsl-url = http://akhlaghi.org/maneage-software +#ibpaper-url=http://ftp.debian.org/debian/pool/main/libp/libpaper/ +#libpng-url = https://download.sourceforge.net/libpng +#libtiff-url = https://download.osgeo.org/libtiff +#libtirpc-url = https://downloads.sourceforge.net/libtirpc +#minizip-url = +#missfits-url = https://www.astromatic.net/download/missfits +#netpbm-url = http://akhlaghi.org/maneage-software +#openblas-url = https://github.com/xianyi/OpenBLAS/archive +#openmpi-url = $(shell majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}'); echo https://download.open-mpi.org/release/open-mpi/v$$majorver) +#openssh-url = https://artfiles.org/openbsd/OpenSSH/portable +#patch-url = http://ftp.gnu.org/gnu/patch +#pixman-url = https://www.cairographics.org/releases +#python-url = https://www.python.org/ftp/python/$(python-version) +#R-url = $(shell majver=$$(echo $(R-version) | sed -e's/\./ /g' | awk '{print $$1}'); echo https://cran.r-project.org/src/base/R-$$majver) +#rpcsvc-proto-url = https://github.com/thkukuk/rpcsvc-proto/releases/download +#scamp-url = http://akhlaghi.org/maneage-software +#scons-url = https://sourceforge.net/projects/scons/files/scons/$(scons-version)/scons-$(scons-version).tar.gz/download +#sextractor-url = http://akhlaghi.org/maneage-software +#swarp-url = https://www.astromatic.net/download/swarp +#swig-url = https://sourceforge.net/projects/swig/files/swig/swig-$(swig-version) +#tides-url = http://akhlaghi.org/maneage-software +#valgrind-url = https://sourceware.org/pub/valgrind +#wcslib-url = ftp://ftp.atnf.csiro.au/pub/software/wcslib +#xlsxio-url = https://github.com/brechtsanders/xlsxio/archive +#yaml-url = pyyaml.org/download/libyaml + + + + + +# Python packages +# --------------- +# +# Similar to optional programs and libraries above. +# +# IMPORTANT: Python packages are usually distributed on servers like +# 'https://pypi.org'. You can search for your package here and add the link +# to the tarball here for your desired software. The links in PyPi don't +# contain the names of the software, but only have hashes. So we aren't +# including them by default here. But adding them is trivial for a certain +# version. For example if you search "Numpy" and click on the "Download +# files" menu, you can download the source '.zip' file. For version 1.18.5, +# (with full filename of 'numpy-1.18.5.zip') the link will be like this. +# +# numpy-url = https://files.pythonhosted.org/packages/01/1b/d3ddcabd5817be02df0e6ee20d64f77ff6d0d97f83b77f65e98c8a651981/numpy-1.18.5.zip +# +# Hence, if the version changes, the checksum, and its two top host +# directories also change. This is why we aren't putting any URL here by +# default, if you need to add one, simply follow the numpy example above. +# But be sure to send us a link to the tarball so we upload it to Maneage's +# backup servers. diff --git a/reproduce/software/config/versions.conf b/reproduce/software/config/versions.conf index 1a4172a..b395e0f 100644 --- a/reproduce/software/config/versions.conf +++ b/reproduce/software/config/versions.conf @@ -14,6 +14,8 @@ # Basic/low-level programs and libraires (installed in any case) # -------------------------------------------------------------- +# +# CLASS:BASIC (important identifier, don't modify this line) bash-version = 5.0.11 binutils-version = 2.32 coreutils-version = 8.31 @@ -22,6 +24,7 @@ dash-version = 0.5.10.2 diffutils-version = 3.7 file-version = 5.36 findutils-version = 4.7.0 +flock-version = 0.2.3 gawk-version = 5.0.1 gcc-version = 9.2.0 gettext-version = 0.20.2 @@ -35,8 +38,8 @@ libiconv-version = 1.16 libtool-version = 2.4.6 libunistring-version = 0.9.10 libxml2-version = 2.9.9 -lzip-version = 1.20 -m4-version = 1.4.18 +lzip-version = 1.22-rc2 +m4-version = 1.4.18-patched make-version = 4.3 metastore-version = 1.1.2-23-fa9170b mpc-version = 1.1.0 @@ -56,6 +59,14 @@ xz-version = 5.2.4 zip-version = 3.0 zlib-version = 1.2.11 +# Version-dependent build +# ----------------------- +# +# When updating the version of these programs/libraries, please look into +# the build rule first: In one way or another, the version string becomes +# necessary during their build and must be accounted for. +bzip2-version = 1.0.6 + @@ -67,6 +78,8 @@ zlib-version = 1.2.11 # `reproduce/software/config/TARGETS.conf' will be built as # part of a project. To specify a software there, just remove the # `-version' suffix from the list below. +# +# CLASS:HIGHLEVEL (important identifier, don't modify this line.) apachelog4cxx-version = 0.10.0-603-014954db apr-version = 1.7.0 apr-util-version = 1.6.1 @@ -75,7 +88,7 @@ atlas-version = 3.10.3 autoconf-version = 2.69.200-babc automake-version = 1.16.1 bison-version = 3.4.2 -boost-version = 1.71.0 +boost-version = 1.73.0 cairo-version = 1.16.0 cdsclient-version = 3.84 cfitsio-version = 3.47 @@ -83,8 +96,7 @@ cmake-version = 3.17.0 eigen-version = 3.3.7 expat-version = 2.2.9 fftw-version = 3.3.8 -flex-version = 2.6.4 -flock-version = 0.2.3 +flex-version = 2.6.4-91-ga631f5d freetype-version = 2.9 gdb-version = 8.3 ghostscript-version = 9.50 @@ -98,14 +110,16 @@ imfit-version = 1.6.1 libffi-version = 3.2.1 libjpeg-version = v9b libnsl-version = 1.2.0-4a062cf +libpaper-version = 1.1.28 libpng-version = 1.6.37 libtiff-version = 4.0.10 -libtirpc-version = 1.1.4 +libtirpc-version = 1.2.6 missfits-version = 2.8.0 -openblas-version = 0.3.5 +openblas-version = 0.3.10 openmpi-version = 4.0.1 openssh-version = 8.0p1 patch-version = 2.7.6 +pcre-version = 8.44 pixman-version = 0.38.0 python-version = 3.7.4 R-version = 3.6.2 @@ -119,6 +133,29 @@ tides-version = 2.0 xlsxio-version = 0.2.21 yaml-version = 0.2.2 +# Version-dependent build +# ----------------------- +lapack-version = 3.8.0 +libgit2-version = 0.28.2 +wcslib-version = 6.4 + +# Netpbm's questions in the configure steps maybe change with different or +# new versions. +netpbm-version = 10.86.99 + +# Minizip is installed with the same `zlib' tarball, and they have the same +# version. +minizip-version = $(zlib-version) + +# From version 1.2 OpenSSL may not need a manual addition, as described in +# its comments and `https://savannah.nongnu.org/bugs/?58263'. If it doesn't +# cause problems, put it back in the list of "Basic/low-level" tools. +openssl-version = 1.1.1a + +# Version 3.15.0 needs two patches, please check if they are necessary on +# any future release. +valgrind-version = 3.15.0 + @@ -131,6 +168,8 @@ yaml-version = 0.2.2 # IMPORTANT: If you intend to change the version of any of the Python # modules/libraries below, please fix the hash strings of the respective # URL in `reproduce/software/make/python.mk'. +# +# CLASS:PYTHON (important identifier, don't modify this line.) asn1crypto-version = 0.24.0 asteval-version = 0.9.16 astropy-version = 4.0 @@ -151,7 +190,6 @@ flake8-version = 3.7.8 future-version = 0.18.1 galsim-version = 2.2.1 h5py-version = 2.9.0 -healpy-version = installed-with-healpix html5lib-version = 1.0.1 idna-version = 2.8 jeepney-version = 0.4 @@ -184,42 +222,9 @@ soupsieve-version = 1.8 sympy-version = 1.4 uncertainties-version = 3.1.2 urllib3-version = 1.24.1 -virtualenv-version = 16.4.0 webencodings-version = 0.5.1 - - - - -# Special programs and libraries -# ------------------------------ -# -# When updating the version of these libraries, please look into the build -# rule first: In one way or another, the version string becomes necessary -# during their build and must be accounted for. +# Special Python modules: # -# Special notes: -# - `netpbm' questions in the configure steps maybe change with different -# or new versions. - -# Basic/low-level -bzip2-version = 1.0.6 - -# Optional/high-level -lapack-version = 3.8.0 -libgit2-version = 0.28.2 -netpbm-version = 10.86.99 -wcslib-version = 6.4 - -# Minizip is installed with the same `zlib' tarball, and they have the same -# version. -minizip-version = $(zlib-version) - -# From version 1.2 OpenSSL may not need a manual addition, as described in -# its comments and `https://savannah.nongnu.org/bugs/?58263'. If it doesn't -# cause problems, put it back in the list of "Basic/low-level" tools. -openssl-version = 1.1.1a - -# Version 3.15.0 needs two patches, please check if they are necessary on -# any future release. -valgrind-version = 3.15.0 +# Healpy: When any Python module is requested, healpix will also build its +# Python module Healpy. diff --git a/reproduce/software/make/atlas-multiple.mk b/reproduce/software/make/atlas-multiple.mk index 9bd26b1..9bd26b1 100755..100644 --- a/reproduce/software/make/atlas-multiple.mk +++ b/reproduce/software/make/atlas-multiple.mk diff --git a/reproduce/software/make/atlas-single.mk b/reproduce/software/make/atlas-single.mk index 29373e0..29373e0 100755..100644 --- a/reproduce/software/make/atlas-single.mk +++ b/reproduce/software/make/atlas-single.mk diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 82bc42d..94bb5d4 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -5,13 +5,19 @@ # !!!!! IMPORTANT NOTES !!!!! # # This Makefile will be run by the initial `./project configure' script. It -# is not included into the project after that. +# is not included into the project afterwards. # -# This Makefile builds very low-level and basic tools like GNU Tar, GNU -# Bash, GNU Make, GCC and etc. Therefore this is the only Makefile in the -# project where you CANNOT assume that GNU Bash or GNU Make are used. After -# this Makefile (where GNU Bash and GNU Make are built), other Makefiles -# can safely assume the fixed version of all these software. +# This Makefile builds low-level and basic tools that are necessary in any +# project like like GNU Tar, GNU Bash, GNU Make, GCC and etc. But before +# control reaches here, the 'configure.sh' script has already built the +# extremely low-level tools: Lzip (a compressing program), GNU Make (to be +# able to run this Makefile with a fixed version), Dash (a minimalist +# POSIX-compatible shell) and Flock (to allow locking files, and +# serializing when necessary: downloading during the software building +# phase). Thanks to GNU Make and Dash, we can assume a fixed structure in +# this Makefile. However, the 'PATH's in this Makefile still include the +# host's paths because we will be using the hosts tools (gradually +# decreasing) to build our own tools. # # ------------------------------------------------------------------------ # @@ -31,13 +37,17 @@ # You should have received a copy of the GNU General Public License # along with this Makefile. If not, see <http://www.gnu.org/licenses/>. - # Top level environment include reproduce/software/config/LOCAL.conf include reproduce/software/make/build-rules.mk include reproduce/software/config/versions.conf include reproduce/software/config/checksums.conf +# The optional URLs of software. Note that these may need the software +# version, so it is important that they be loaded after 'versions.conf'. +include reproduce/software/config/urls.conf + +# Basic directories lockdir = $(BDIR)/locks tdir = $(BDIR)/software/tarballs ddir = $(BDIR)/software/build-tmp @@ -46,31 +56,34 @@ ibdir = $(BDIR)/software/installed/bin ildir = $(BDIR)/software/installed/lib ibidir = $(BDIR)/software/installed/version-info/proglib -# We'll need the system's PATH for making links to low-level programs we -# won't be building ourselves. -syspath := $(PATH) +# Ultimate Makefile target. +targets-proglib = low-level-links gcc-$(gcc-version) +all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) +# Define the shell environment +# ---------------------------- +# +# We build GNU Bash here in 'basic.mk'. So here we must must assume DASH +# shell that was built before calling this Makefile: +# http://gondor.apana.org.au/~herbert/dash. DASH is a minimalist POSIX +# shell, so it doesn't have startup options like '--noprofile --norc'. But +# from its manual, to load startup files, Dash actually requires that it be +# called with a '-' before it (for example '-dash'), so it shouldn't be +# loading any startup files if it was interpretted properly. +# # As we build more programs, we want to use this project's built programs -# and libraries, not the host's. +# and libraries, not the host's, so in all PATH-related environments, our +# own build-directory comes first. .ONESHELL: +.SHELLFLAGS := -e -c export CCACHE_DISABLE := 1 +export SHELL := $(ibdir)/dash export PATH := $(ibdir):$(PATH) export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) -# Note that we build GNU Bash here in 'basic.mk'. So we can't assume Bash -# in this Makefile and use the DASH shell that was before calling this -# Makefile: http://gondor.apana.org.au/~herbert/dash. Dash is a minimalist -# POSIX shell, so it doesn't have startup options like '--noprofile -# --norc'. But from its manual, to load startup files, Dash actually -# requires that it be called with a '-' before it (for example '-dash'), so -# it shouldn't be loading any startup files if it was interpretted -# properly. -.SHELLFLAGS := -e -c -export SHELL := $(ibdir)/dash - # This is the "basic" tools where we are relying on the host operating # system, but are slowly populating our basic software envirnoment. To run # (system or template) programs, `LD_LIBRARY_PATH' is necessary, so here, @@ -86,202 +99,29 @@ export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH):$(ildir) \ # causes crashs (see bug #56682). So we'll just give it no value at all. export DYLD_LIBRARY_PATH := -# Recipe startup script. -export PROJECT_STATUS := configure_basic -export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh - -# Define the top-level basic programs (that don't depend on any other). -top-level-programs = low-level-links gcc -all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) - -# Servers to use as backup, later this should go in a file that is not -# under version control (the actual server that the tarbal comes from is -# irrelevant). -backupservers := $(shell awk '!/^#/{printf "%s ", $$1}' \ - reproduce/software/config/servers-backup.conf) - - - - - - -# Tarballs -# -------- -# -# Prepare tarballs. Difference with that in `high-level.mk': `.ONESHELL' is -# not recognized by some versions of Make (even older GNU Makes). So we'll -# have to make sure the recipe doesn't break into multiple shell calls (so -# we can preserve the variables). -# -# Software with main webpage at our backup repository -# (http://akhlaghi.org/maneage-software): As of our latest check their -# major release tarballs either crash or don't build on some systems (for -# example Make or Gzip), or they don't exist (for example Bzip2). So we are -# building them from their Git history (which builds properly) or host them -# directly. +# Servers to use as backup. Maneage already has some fixed servers that can +# be used to download software tarballs. They are in a configuation +# file. But we give precedence to the "user" backup servers. # -# In the first case, we used their Git repo and bootstrapped them (just -# like Gnuastro) and built the most recent tarball off of that. In the case -# of Bzip2: its webpage has expired and doesn't host the data any more. It -# is available on the link below (archive.org): +# One important "user" server (which the user doesn't actually give, but is +# found at configuration time in 'configure.sh') is Zenodo (see the +# description in 'configure.sh' for more on why this depends on +# configuration time). # -# https://web.archive.org/web/20180624184806/http://www.bzip.org/1.0.6/bzip2-1.0.6.tar.gz -# -# However, downloading from this link is slow (because its just a link). So -# its easier to just keep a with the others. -$(lockdir): | $(BDIR); mkdir $@ +# Afer putting everything together, we use the first server as the +# reference for all software if their '-url' variable isn't defined (in +# 'reproduce/software/config/urls.conf'). downloadwrapper = ./reproduce/analysis/bash/download-multi-try -tarballs = $(foreach t, bash-$(bash-version).tar.lz \ - binutils-$(binutils-version).tar.lz \ - bzip2-$(bzip2-version).tar.gz \ - cert.pem \ - coreutils-$(coreutils-version).tar.xz \ - curl-$(curl-version).tar.gz \ - diffutils-$(diffutils-version).tar.xz \ - file-$(file-version).tar.gz \ - findutils-$(findutils-version).tar.xz \ - gawk-$(gawk-version).tar.lz \ - gcc-$(gcc-version).tar.xz \ - gettext-$(gettext-version).tar.xz \ - git-$(git-version).tar.xz \ - gmp-$(gmp-version).tar.lz \ - grep-$(grep-version).tar.xz \ - gzip-$(gzip-version).tar.lz \ - isl-$(isl-version).tar.bz2 \ - libbsd-$(libbsd-version).tar.xz \ - libiconv-$(libiconv-version).tar.gz \ - libtool-$(libtool-version).tar.xz \ - libunistring-$(libunistring-version).tar.xz \ - libxml2-$(libxml2-version).tar.gz \ - m4-$(m4-version).tar.gz \ - metastore-$(metastore-version).tar.gz \ - mpfr-$(mpfr-version).tar.xz \ - mpc-$(mpc-version).tar.gz \ - ncurses-$(ncurses-version).tar.gz \ - openssl-$(openssl-version).tar.gz \ - patchelf-$(patchelf-version).tar.gz \ - perl-$(perl-version).tar.gz \ - pkg-config-$(pkgconfig-version).tar.gz \ - readline-$(readline-version).tar.gz \ - sed-$(sed-version).tar.xz \ - tar-$(tar-version).tar.gz \ - texinfo-$(texinfo-version).tar.xz \ - unzip-$(unzip-version).tar.gz \ - wget-$(wget-version).tar.lz \ - which-$(which-version).tar.gz \ - xz-$(xz-version).tar.gz \ - zip-$(zip-version).tar.gz \ - zlib-$(zlib-version).tar.gz \ - , $(tdir)/$(t) ) -$(tarballs): $(tdir)/%: | $(lockdir) - - # Remove the version numbers and suffix from the tarball name so we - # can search more easily only with the program name. This requires - # the first character of the version to be a digit: packages such - # as `foo' and `foo-3' will not be distinguished, but `foo' and - # `foo2' will be distinguished. - @n=$$(echo $* | sed -e's/-[0-9]/ /' -e's/\./ /g' \ - | awk '{print $$1}' ) - - mergenames=1; \ - if [ $$n = bash ]; then c=$(bash-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = binutils ]; then c=$(binutils-checksum); w=http://ftp.gnu.org/gnu/binutils - elif [ $$n = bzip2 ]; then c=$(bzip2-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = cert ]; then c=$(cert-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = coreutils ]; then c=$(coreutils-checksum); w=http://ftp.gnu.org/gnu/coreutils - elif [ $$n = curl ]; then c=$(curl-checksum); w=https://curl.haxx.se/download - elif [ $$n = diffutils ]; then c=$(diffutils-checksum); w=http://ftp.gnu.org/gnu/diffutils - elif [ $$n = file ]; then c=$(file-checksum); w=ftp://ftp.astron.com/pub/file - elif [ $$n = findutils ]; then c=$(findutils-checksum); w=http://ftp.gnu.org/gnu/findutils - elif [ $$n = gawk ]; then c=$(gawk-checksum); w=http://ftp.gnu.org/gnu/gawk - elif [ $$n = gcc ]; then c=$(gcc-checksum); w=http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version) - elif [ $$n = gettext ]; then c=$(gettext-checksum); w=https://ftp.gnu.org/gnu/gettext - elif [ $$n = git ]; then c=$(git-checksum); w=http://mirrors.edge.kernel.org/pub/software/scm/git - elif [ $$n = gmp ]; then c=$(gmp-checksum); w=https://gmplib.org/download/gmp - elif [ $$n = grep ]; then c=$(grep-checksum); w=http://ftp.gnu.org/gnu/grep - elif [ $$n = gzip ]; then c=$(gzip-checksum); w=http://akhlaghi.org/src - elif [ $$n = isl ]; then c=$(isl-checksum); w=ftp://gcc.gnu.org/pub/gcc/infrastructure - elif [ $$n = libbsd ]; then c=$(libbsd-checksum); w=http://libbsd.freedesktop.org/releases - elif [ $$n = libiconv ]; then c=$(libiconv-checksum); w=https://ftp.gnu.org/pub/gnu/libiconv - elif [ $$n = libtool ]; then c=$(libtool-checksum); w=http://ftp.gnu.org/gnu/libtool - elif [ $$n = libunistring ]; then c=$(libunistring-checksum); w=http://ftp.gnu.org/gnu/libunistring - elif [ $$n = libxml2 ]; then c=$(libxml2-checksum); w=ftp://xmlsoft.org/libxml2 - elif [ $$n = m4 ]; then - mergenames=0 - c=$(m4-checksum) - w=http://akhlaghi.org/maneage-software/m4-1.4.18-patched.tar.gz - elif [ $$n = metastore ]; then c=$(metastore-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = mpc ]; then c=$(mpc-checksum); w=http://ftp.gnu.org/gnu/mpc - elif [ $$n = mpfr ]; then c=$(mpfr-checksum); w=http://www.mpfr.org/mpfr-current - elif [ $$n = ncurses ]; then c=$(ncurses-checksum); w=http://ftp.gnu.org/gnu/ncurses - elif [ $$n = openssl ]; then c=$(openssl-checksum); w=http://www.openssl.org/source - elif [ $$n = patchelf ]; then c=$(patchelf-checksum); w=http://nixos.org/releases/patchelf/patchelf-$(patchelf-version) - elif [ $$n = perl ]; then - c=$(perl-checksum) - v=$$(echo $(perl-version) | sed -e's/\./ /g' | awk '{printf("%d.0", $$1)}') - w=https://www.cpan.org/src/$$v - elif [ $$n = pkg-config ]; then c=$(pkgconfig-checksum); w=http://pkg-config.freedesktop.org/releases - elif [ $$n = readline ]; then c=$(readline-checksum); w=http://ftp.gnu.org/gnu/readline - elif [ $$n = sed ]; then c=$(sed-checksum); w=http://ftp.gnu.org/gnu/sed - elif [ $$n = tar ]; then c=$(tar-checksum); w=http://ftp.gnu.org/gnu/tar - elif [ $$n = texinfo ]; then c=$(texinfo-checksum); w=http://ftp.gnu.org/gnu/texinfo - elif [ $$n = unzip ]; then - c=$(unzip-checksum) - mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') - w=ftp://ftp.info-zip.org/pub/infozip/src/unzip$$v.tgz - elif [ $$n = wget ]; then c=$(wget-checksum); w=http://ftp.gnu.org/gnu/wget - elif [ $$n = which ]; then c=$(which-checksum); w=http://ftp.gnu.org/gnu/which - elif [ $$n = xz ]; then c=$(xz-checksum); w=http://tukaani.org/xz - elif [ $$n = zip ]; then - c=$(zip-checksum) - mergenames=0; v=$$(echo $(zip-version) | sed -e's/\.//') - w=ftp://ftp.info-zip.org/pub/infozip/src/zip$$v.tgz - elif [ $$n = zlib ]; then c=$(zlib-checksum); w=http://www.zlib.net - else - echo; echo; echo - echo "'$$n' not recognized as a software tarball name to download." - echo; echo; echo - exit 1 - fi +maneage_backup_urls := $(shell awk '!/^#/{printf "%s ", $$1}' \ + reproduce/software/config/servers-backup.conf) +backupservers_all = $(user_backup_urls) $(maneage_backup_urls) +topbackupserver = $(word 1, $(backupservers_all)) +backupservers = $(filter-out $(topbackupserver),$(backupservers_all)) + - # Download the raw tarball, using an '.unchecked' suffix to specify - # that it is not yet fully checked and usable. But first, since the - # download may be interrupted in a previous build and an incomplete - # '.unchecked' file may remain, we'll remove any possibly existing - # uncheked file. - rm -f "$@.unchecked" - if [ -f $(DEPENDENCIES-DIR)/$* ]; then - cp $(DEPENDENCIES-DIR)/$* "$@.unchecked" - else - if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*" - else tarballurl=$$w - fi - echo "Downloading $$tarballurl" - if [ -f $(ibdir)/wget ]; then - downloader="wget --no-use-server-timestamps -O" - else - downloader="$(DOWNLOADER)" - fi - touch $(lockdir)/download - $(downloadwrapper) "$$downloader" $(lockdir)/download \ - $$tarballurl "$@.unchecked" "$(backupservers)" - fi - # Make sure the file's Checksum is correct. - if type sha512sum > /dev/null 2>/dev/null; then - checksum=$$(sha512sum "$@.unchecked" | awk '{print $$1}') - if [ x"$$checksum" = x"$$c" ]; then - mv "$@.unchecked" "$@" - else - echo "ERROR: Non-matching checksum for '$*'." - echo "Checksum should be: $$c" - echo "Checksum is: $$checksum" - exit 1 - fi - else mv "$@.unchecked" "$@" - fi @@ -301,6 +141,10 @@ $(tarballs): $(tdir)/%: | $(lockdir) # purpose here, it is very annoying and can cause many complications. We # thus remove any part of PATH of that has `ccache' in it before making # symbolic links to the programs we are not building ourselves. +# +# We'll need the system's PATH for making links to low-level programs we +# won't be building ourselves. +syspath := $(PATH) makelink = origpath="$$PATH"; \ export PATH=$$(echo $(syspath) \ | tr : '\n' \ @@ -322,7 +166,8 @@ makelink = origpath="$$PATH"; \ export PATH="$$origpath" $(ibdir) $(ildir):; mkdir $@ -$(ibidir)/low-level-links: $(ibidir)/coreutils | $(ibdir) $(ildir) +$(ibidir)/low-level-links: $(ibidir)/coreutils-$(coreutils-version) \ + | $(ibdir) $(ildir) # Not-installed (but necessary in some cases) compilers. # Clang is necessary for CMake. @@ -331,6 +176,7 @@ $(ibidir)/low-level-links: $(ibidir)/coreutils | $(ibdir) $(ildir) # Mac OS specific $(call makelink,mig) + $(call makelink,xcrun) $(call makelink,sysctl) $(call makelink,sw_vers) $(call makelink,dsymutil) @@ -374,17 +220,25 @@ $(ibidir)/low-level-links: $(ibidir)/coreutils | $(ibdir) $(ildir) # tarballs of the other compression programs. Once all the compression # programs/libraries are complete, we build our own GNU Tar and continue # with other software. -$(ibidir)/gzip: $(tdir)/gzip-$(gzip-version).tar.lz - $(call gbuild, gzip-$(gzip-version), static, , V=1) \ - && echo "GNU Gzip $(gzip-version)" > $@ +$(lockdir): | $(BDIR); mkdir $@ +$(ibidir)/gzip-$(gzip-version): | $(ibdir) $(ildir) $(lockdir) + tarball=gzip-$(gzip-version).tar.lz + $(call import-source, $(gzip-url), $(gzip-checksum)) + $(call gbuild, gzip-$(gzip-version), static, , V=1) + echo "GNU Gzip $(gzip-version)" > $@ + +$(ibidir)/xz-$(xz-version): $(ibidir)/gzip-$(gzip-version) + tarball=xz-$(xz-version).tar.gz + $(call import-source, $(xz-url), $(xz-checksum)) + $(call gbuild, xz-$(xz-version), static) + echo "XZ Utils $(xz-version)" > $@ -$(ibidir)/xz: $(ibidir)/gzip \ - $(tdir)/xz-$(xz-version).tar.gz - $(call gbuild, xz-$(xz-version), static) \ - && echo "XZ Utils $(xz-version)" > $@ +$(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) + + # Download the tarball. + tarball=bzip2-$(bzip2-version).tar.gz + $(call import-source, $(bzip2-url), $(bzip2-checksum)) -$(ibidir)/bzip2: $(ibidir)/gzip \ - $(tdir)/bzip2-$(bzip2-version).tar.gz # Bzip2 doesn't have a `./configure' script, and its Makefile # doesn't build a shared library. So we can't use the `gbuild' # function here and we need to take some extra steps (inspired @@ -406,71 +260,77 @@ $(ibidir)/bzip2: $(ibidir)/gzip \ makeshared="make -f Makefile-libbz2_so" fi fi - cd $(ddir) && rm -rf $$tdir \ - && tar xf $(word 1,$(filter $(tdir)/%,$^)) \ - && cd $$tdir \ - && sed -e 's@\(ln -s -f \)$$(PREFIX)/bin/@\1@' Makefile \ - > Makefile.sed \ - && mv Makefile.sed Makefile \ - && $$makeshared CC=cc \ - && cp -a libbz2* $(ildir)/ \ - && make clean \ - && $$makecommand CC=cc \ - && make install PREFIX=$(idir) \ - && cd .. \ - && rm -rf $$tdir \ - && cd $(ildir) \ - && ln -fs libbz2.so.1.0 libbz2.so \ - && echo "Bzip2 $(bzip2-version)" > $@ - -$(ibidir)/unzip: $(ibidir)/gzip \ - $(tdir)/unzip-$(unzip-version).tar.gz + cd $(ddir) + rm -rf $$tdir + tar xf $(tdir)/$$tarball + cd $$tdir + sed -e 's@\(ln -s -f \)$$(PREFIX)/bin/@\1@' Makefile \ + > Makefile.sed + mv Makefile.sed Makefile + $$makeshared CC=cc + cp -a libbz2* $(ildir)/ + make clean + $$makecommand CC=cc + make install PREFIX=$(idir) + cd .. + rm -rf $$tdir + cd $(ildir) + ln -fs libbz2.so.1.0 libbz2.so + echo "Bzip2 $(bzip2-version)" > $@ + +$(ibidir)/unzip-$(unzip-version): $(ibidir)/gzip-$(gzip-version) + tarball=unzip-$(unzip-version).tar.gz v=$$(echo $(unzip-version) | sed -e's/\.//') + $(call import-source, $(unzip-url), $(unzip-checksum)) $(call gbuild, unzip$$v, static,, \ -f unix/Makefile generic \ CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ -f unix/Makefile generic \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ - && echo "Unzip $(unzip-version)" > $@ + BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) + echo "Unzip $(unzip-version)" > $@ -$(ibidir)/zip: $(ibidir)/gzip \ - $(tdir)/zip-$(zip-version).tar.gz +$(ibidir)/zip-$(zip-version): $(ibidir)/gzip-$(gzip-version) + tarball=zip-$(zip-version).tar.gz v=$$(echo $(zip-version) | sed -e's/\.//') + $(call import-source, $(zip-url), $(zip-checksum)) $(call gbuild, zip$$v, static,, \ -f unix/Makefile generic \ CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ -f unix/Makefile generic \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ - && echo "Zip $(zip-version)" > $@ + BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) + echo "Zip $(zip-version)" > $@ # Some programs (like Wget and CMake) that use zlib need it to be dynamic # so they use our custom build. So we won't force a static-only build. # # Note for a static-only build: Zlib's `./configure' doesn't use Autoconf's # configure script, it just accepts a direct `--static' option. -$(ibidir)/zlib: $(ibidir)/gzip \ - $(tdir)/zlib-$(zlib-version).tar.gz - $(call gbuild, zlib-$(zlib-version)) \ - && echo "Zlib $(zlib-version)" > $@ +$(ibidir)/zlib-$(zlib-version): $(ibidir)/gzip-$(gzip-version) + tarball=zlib-$(zlib-version).tar.gz + $(call import-source, $(zlib-url), $(zlib-checksum)) + $(call gbuild, zlib-$(zlib-version)) + echo "Zlib $(zlib-version)" > $@ # GNU Tar: When built statically, tar gives a segmentation fault on # unpacking Bash. So we'll build it dynamically. Note that technically, zip # and unzip aren't dependencies of Tar, but for a clean build, we'll set # Tar to be the last compression-related software (the first-set of # software to be built). -$(ibidir)/tar: $(ibidir)/xz \ - $(ibidir)/zip \ - $(ibidir)/gzip \ - $(ibidir)/zlib \ - $(ibidir)/bzip2 \ - $(ibidir)/unzip \ - $(tdir)/tar-$(tar-version).tar.gz +$(ibidir)/tar-$(tar-version): \ + $(ibidir)/xz-$(xz-version) \ + $(ibidir)/zip-$(zip-version) \ + $(ibidir)/gzip-$(gzip-version) \ + $(ibidir)/zlib-$(zlib-version) \ + $(ibidir)/bzip2-$(bzip2-version) \ + $(ibidir)/unzip-$(unzip-version) # Since all later programs depend on Tar, the configuration will be # stuck here, only making Tar. So its more efficient to built it on # multiple threads (when the user's Make doesn't pass down the # number of threads). - $(call gbuild, tar-$(tar-version), , , -j$(numthreads) V=1) \ - && echo "GNU Tar $(tar-version)" > $@ + tarball=tar-$(tar-version).tar.gz + $(call import-source, $(tar-url), $(tar-checksum)) + $(call gbuild, tar-$(tar-version), , , -j$(numthreads) V=1) + echo "GNU Tar $(tar-version)" > $@ @@ -481,8 +341,34 @@ $(ibidir)/tar: $(ibidir)/xz \ -# Level 2 (SECOND MOST BASIC): Bash and Make -# ------------------------------------------ +# Level 2 (necessary for linking) +# +# Patchelf is necessary for some software on GNU/Linux systems, its job is +# to manually insert RPATH into the dynamically-linked executable. Since +# all the other software depend on Pathelf, to avoid manually repeating as +# a prerequisite (and forgetting in others causing bugs), we'll put it as a +# dependancy of 'tar'. +$(ibidir)/patchelf-$(patchelf-version): $(ibidir)/tar-$(tar-version) + tarball=patchelf-$(patchelf-version).tar.gz + $(call import-source, $(patchelf-url), $(patchelf-checksum)) + if [ x$(on_mac_os) = xyes ]; then + echo "" > $@ + else + $(call gbuild, patchelf-$(patchelf-version)) + echo "PatchELF $(patchelf-version)" > $@ + fi + + + + + + + + + + +# Level 3 (THIRD MOST BASIC): Bash +# -------------------------------- # # GNU Make and GNU Bash are the second layer that we'll need to build the # basic dependencies. @@ -492,8 +378,9 @@ $(ibidir)/tar: $(ibidir)/xz \ # function (for tilde expansion). The first can be disabled with # `--disable-load', but unfortunately I don't know any way to fix the # second. So, we'll have to build it dynamically for now. -$(ibidir)/ncurses: $(ibidir)/tar \ - $(tdir)/ncurses-$(ncurses-version).tar.gz +$(ibidir)/ncurses-$(ncurses-version): $(ibidir)/patchelf-$(patchelf-version) + tarball=ncurses-$(ncurses-version).tar.gz + $(call import-source, $(ncurses-url), $(ncurses-checksum)) # Delete the library that will be installed (so we can make sure # the build process completed afterwards and reset the links). @@ -578,17 +465,14 @@ $(ibidir)/ncurses: $(ibidir)/tar \ exit 1 fi -$(ibidir)/readline: $(ibidir)/ncurses \ - $(tdir)/readline-$(readline-version).tar.gz +$(ibidir)/readline-$(readline-version): \ + $(ibidir)/ncurses-$(ncurses-version) + tarball=readline-$(readline-version).tar.gz + $(call import-source, $(readline-url), $(readline-checksum)) $(call gbuild, readline-$(readline-version), static, \ --with-curses --disable-install-examples, \ - SHLIB_LIBS="-lncursesw" -j$(numthreads)) \ - && echo "GNU Readline $(readline-version)" > $@ - -$(ibidir)/patchelf: $(ibidir)/tar \ - $(tdir)/patchelf-$(patchelf-version).tar.gz - $(call gbuild, patchelf-$(patchelf-version)) \ - && echo "PatchELF $(patchelf-version)" > $@ + SHLIB_LIBS="-lncursesw" -j$(numthreads)) + echo "GNU Readline $(readline-version)" > $@ # IMPORTANT: Even though we have enabled `rpath', Bash doesn't write the @@ -620,16 +504,13 @@ $(ibidir)/patchelf: $(ibidir)/tar \ # $ tar cf bash-5.0.$number.tar bash-5.0.$number # $ lzip --best bash-5.0.$number.tar # $ rm -rf bash50-* bash-5.0.$number bash-5.0.tar.gz +$(ibidir)/bash-$(bash-version): \ + $(ibidir)/gettext-$(gettext-version) \ + $(ibidir)/readline-$(readline-version) -ifeq ($(on_mac_os),yes) -needpatchelf = -else -needpatchelf = $(ibidir)/patchelf -endif -$(ibidir)/bash: $(needpatchelf) \ - $(ibidir)/gettext \ - $(ibidir)/readline \ - $(tdir)/bash-$(bash-version).tar.lz + # Download the tarball. + tarball=bash-$(bash-version).tar.lz + $(call import-source, $(bash-url), $(bash-checksum)) # Delete the (possibly) existing Bash executable in the project, # let it use the default shell of the host. @@ -655,9 +536,8 @@ $(ibidir)/bash: $(needpatchelf) \ # default. So, we have to manually include it, currently we are # only doing this on GNU/Linux systems (using the `patchelf' # program). - if [ "x$(needpatchelf)" != x ]; then - if [ -f $(ibdir)/bash ]; then - $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/bash; fi + if [ -f $(ibdir)/patchelf ]; then + $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/bash; fi # To be generic, some systems use the `sh' command to call the @@ -668,11 +548,8 @@ $(ibidir)/bash: $(needpatchelf) \ # Just to be sure that the installation step above went well, # before making the link, we'll see if the file actually exists # there. - if [ -f $(ibdir)/bash ]; then - ln -fs $(ibdir)/bash $(ibdir)/sh - echo "GNU Bash $(bash-version)" > $@ - else - echo "GNU Bash not built!"; exit 1; fi + ln -fs $(ibdir)/bash $(ibdir)/sh + echo "GNU Bash $(bash-version)" > $@ @@ -688,46 +565,45 @@ perl-conflddlflags = else perl-conflddlflags = -Dlddlflags="-shared $$LDFLAGS" endif -$(ibidir)/perl: $(ibidir)/tar \ - $(tdir)/perl-$(perl-version).tar.gz +$(ibidir)/perl-$(perl-version): $(ibidir)/patchelf-$(patchelf-version) + tarball=perl-$(perl-version).tar.gz + $(call import-source, $(perl-url), $(perl-checksum)) major_version=$$(echo $(perl-version) \ | sed -e's/\./ /g' \ | awk '{printf("%d", $$1)}') base_version=$$(echo $(perl-version) \ | sed -e's/\./ /g' \ | awk '{printf("%d.%d", $$1, $$2)}') - cd $(ddir) \ - && rm -rf perl-$(perl-version) \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)); then \ - echo; echo "Tar error"; exit 1; \ - fi \ - && cd perl-$(perl-version) \ - && ./Configure -des \ - -Dusethreads \ - -Duseshrplib \ - -Dprefix=$(idir) \ - -Dvendorprefix=$(idir) \ - -Dprivlib=$(idir)/share/perl$$major_version/core_perl \ - -Darchlib=$(idir)/lib/perl$$major_version/$$base_version/core_perl \ - -Dsitelib=$(idir)/share/perl$$major_version/site_perl \ - -Dsitearch=$(idir)/lib/perl$$major_version/$$basever/site_perl \ - -Dvendorlib=$(idir)/share/perl$$major_version/vendor_perl \ - -Dvendorarch=$(idir)/lib/perl$$major_version/$$base_version/vendor_perl \ - -Dscriptdir=$(idir)/bin/core_perl \ - -Dsitescript=$(idir)/bin/site_perl \ - -Dvendorscript=$(idir)/bin/vendor_perl \ - -Dinc_version_list=none \ - -Dman1ext=1perl \ - -Dman3ext=3perl \ - -Dcccdlflags='-fPIC' \ - $(perl-conflddlflags) \ - -Dldflags="$$LDFLAGS" \ - && make -j$(numthreads) \ - && make install \ - && cd .. \ - && rm -rf perl-$(perl-version) \ - && cd $$topdir \ - && echo "Perl $(perl-version)" > $@ + cd $(ddir) + rm -rf perl-$(perl-version) + tar xf $(tdir)/$$tarball + cd perl-$(perl-version) + ./Configure -des \ + -Dusethreads \ + -Duseshrplib \ + -Dprefix=$(idir) \ + -Dvendorprefix=$(idir) \ + -Dprivlib=$(idir)/share/perl$$major_version/core_perl \ + -Darchlib=$(idir)/lib/perl$$major_version/$$base_version/core_perl \ + -Dsitelib=$(idir)/share/perl$$major_version/site_perl \ + -Dsitearch=$(idir)/lib/perl$$major_version/$$basever/site_perl \ + -Dvendorlib=$(idir)/share/perl$$major_version/vendor_perl \ + -Dvendorarch=$(idir)/lib/perl$$major_version/$$base_version/vendor_perl \ + -Dscriptdir=$(idir)/bin/core_perl \ + -Dsitescript=$(idir)/bin/site_perl \ + -Dvendorscript=$(idir)/bin/vendor_perl \ + -Dinc_version_list=none \ + -Dman1ext=1perl \ + -Dman3ext=3perl \ + -Dcccdlflags='-fPIC' \ + $(perl-conflddlflags) \ + -Dldflags="$$LDFLAGS" + make -j$(numthreads) + make install + cd .. + rm -rf perl-$(perl-version) + cd $$topdir + echo "Perl $(perl-version)" > $@ @@ -754,61 +630,70 @@ $(ibidir)/perl: $(ibidir)/tar \ # an error). # # Coreutils uses Perl to create man pages! -$(ibidir)/coreutils: $(ibidir)/bash \ - $(ibidir)/perl \ - $(ibidir)/openssl \ - $(tdir)/coreutils-$(coreutils-version).tar.xz - cd $(ddir) \ - && rm -rf coreutils-$(coreutils-version) \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)); then \ - echo; echo "Tar error"; exit 1; \ - fi \ - && cd coreutils-$(coreutils-version) \ - && sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ - configure > configure-tmp \ - && mv configure-tmp configure \ - && chmod +x configure \ - && ./configure --prefix=$(idir) SHELL=$(ibdir)/bash \ - LDFLAGS="$(LDFLAGS)" CPPFLAGS="$(CPPFLAGS)" \ - --disable-silent-rules --with-openssl=yes \ - && make SHELL=$(ibdir)/bash -j$(numthreads) \ - && make SHELL=$(ibdir)/bash install \ - && if [ x$(on_mac_os) != xyes ]; then \ - make SHELL=$(ibdir)/bash install DESTDIR=junkinst; \ - instprogs=$$(ls junkinst/$(ibdir)); \ - for f in $$instprogs; do \ - $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/$$f; \ - done; \ - echo "PatchELF applied to all programs."; \ - fi \ - && cd .. \ - && rm -rf coreutils-$(coreutils-version) \ - && echo "GNU Coreutils $(coreutils-version)" > $@ +$(ibidir)/coreutils-$(coreutils-version): \ + $(ibidir)/bash-$(bash-version) \ + $(ibidir)/perl-$(perl-version) \ + $(ibidir)/openssl-$(openssl-version) + + # Import, unpack and enter the source directory. + tarball=coreutils-$(coreutils-version).tar.xz + $(call import-source, $(coreutils-url), $(coreutils-checksum)) + cd $(ddir) + rm -rf coreutils-$(coreutils-version) + tar xf $(tdir)/$$tarball + cd coreutils-$(coreutils-version) + + # Set the configure script to use our shell, note that we can't + # assume GNU SED here yet (it installs after Coreutils). + sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ + -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ + configure > configure-tmp + mv configure-tmp configure + chmod +x configure + + # Configure, build and install Coreutils. + ./configure --prefix=$(idir) SHELL=$(ibdir)/bash \ + LDFLAGS="$(LDFLAGS)" CPPFLAGS="$(CPPFLAGS)" \ + --disable-silent-rules --with-openssl=yes + make SHELL=$(ibdir)/bash -j$(numthreads) + make SHELL=$(ibdir)/bash install + + # Fix RPATH if necessary. + if [ -f $(ibdir)/patchelf ]; then + make SHELL=$(ibdir)/bash install DESTDIR=junkinst + instprogs=$$(ls junkinst/$(ibdir)) + for f in $$instprogs; do + $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/$$f + done + echo "PatchELF applied to all programs." + fi + + # Come back up to the unpacking directory, delete the source + # directory and write the final target. + cd .. + rm -rf coreutils-$(coreutils-version) + echo "GNU Coreutils $(coreutils-version)" > $@ # OpenSSL # -# Some programs/libraries later need dynamic linking. So we'll build libssl -# (and libcrypto) dynamically also. -# # Until we find a nice and generic way to create an updated CA file in the # project, the certificates will be available in a file for this project # along with the other tarballs. -# -# In case you do want a static OpenSSL and libcrypto, then uncomment the -# following conditional and put $(openssl-static) in the configure options. -# -#ifeq ($(static_build),yes) -#openssl-static = no-dso no-dynamic-engine no-shared -#endif $(idir)/etc:; mkdir $@ -# Note: cert.pm has to be AFTER the tarball, otherwise the build script -# will try to unpack cert.pm and crash (it unpacks the first dependency -# under `tdir'). -$(ibidir)/openssl: $(ibidir)/perl \ - $(tdir)/openssl-$(openssl-version).tar.gz \ - $(tdir)/cert.pem \ - | $(idir)/etc +$(idir)/etc/ssl: | $(idir)/etc; mkdir $@ +$(ibidir)/openssl-$(openssl-version): $(ibidir)/perl-$(perl-version) \ + | $(idir)/etc/ssl + + # First download the certificates and copy them into the + # installation directory. + tarball=cert.pem + $(call import-source, $(cert-url), $(cert-checksum)) + cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem + + # Now download the OpenSSL tarball. + tarball=openssl-$(openssl-version).tar.gz + $(call import-source, $(openssl-url), $(openssl-checksum)) + # According to OpenSSL's Wiki (link bellow), it can't automatically # detect Mac OS's structure. It will need some help. So we'll use # the `on_mac_os' Make variable that we defined in the configure @@ -816,18 +701,10 @@ $(ibidir)/openssl: $(ibidir)/perl \ # environment variable. # # https://wiki.openssl.org/index.php/Compilation_and_Installation - # - # Bug 58263 (https://savannah.nongnu.org/bugs/?58263): In OpenSSL - # Version 1.1.1a (also checked in 1.1.1g), `openssl/ec.h' fails to - # include `openssl/openconf.h' on some OSs. The SED hack below - # inserts a hardwired element of `openssl/openconf.h' that is - # needed to include sections of code `f` that are deprecated in - # 1.2.0, but not yet in 1.1.1. This problem may be solved in - # version 1.2.x, so please check again in that bug. - if [ x$(on_mac_os) = xyes ]; then \ - export KERNEL_BITS=64; \ - copt="shared no-ssl2 no-ssl3 enable-ec_nistp_64_gcc_128"; \ - fi; \ + if [ x$(on_mac_os) = xyes ]; then + export KERNEL_BITS=64 + copt="shared no-ssl2 no-ssl3 enable-ec_nistp_64_gcc_128" + fi $(call gbuild, openssl-$(openssl-version), , \ zlib \ $$copt \ @@ -835,19 +712,26 @@ $(ibidir)/openssl: $(ibidir)/perl \ --openssldir=$(idir)/etc/ssl \ --with-zlib-lib=$(ildir) \ --with-zlib-include=$(idir)/include, \ - -j$(numthreads), , ./config ) \ - && mv -v $(idir)/include/openssl/ec.h $(idir)/include/openssl/ec.h.orig \ - && sed -e 's,\(# include .openssl/opensslconf\.h.\),\1\n#ifndef DEPRECATEDIN_1_2_0\n#define DEPRECATEDIN_1_2_0(f) f;\n#endif\n,' \ - $(idir)/include/openssl/ec.h.orig > $(idir)/include/openssl/ec.h \ - && cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem \ - && if [ $$? = 0 ]; then \ - if [ x$(on_mac_os) = xyes ]; then \ - echo "No need to fix rpath in libssl"; \ - else \ - patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \ - fi; \ - echo "OpenSSL $(openssl-version)" > $@; \ - fi + -j$(numthreads), , ./config ) + + # Manually insert RPATH inside the OpenSSL library. + if [ -f $(ibdir)/patchelf ]; then + patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \ + fi + + # Bug 58263 (https://savannah.nongnu.org/bugs/?58263): In OpenSSL + # Version 1.1.1a (also checked in 1.1.1g), `openssl/ec.h' fails to + # include `openssl/openconf.h' on some OSs. The SED hack below + # inserts a hardwired element of `openssl/openconf.h' that is + # needed to include sections of code `f` that are deprecated in + # 1.2.0, but not yet in 1.1.1. This problem may be solved in + # version 1.2.x, so please check again in that bug. + mv -v $(idir)/include/openssl/ec.h $(idir)/include/openssl/ec.h.orig + sed -e 's,\(# include .openssl/opensslconf\.h.\),\1\n#ifndef DEPRECATEDIN_1_2_0\n#define DEPRECATEDIN_1_2_0(f) f;\n#endif\n,' \ + $(idir)/include/openssl/ec.h.orig > $(idir)/include/openssl/ec.h + + # Build the final target. + echo "OpenSSL $(openssl-version)" > $@ @@ -864,8 +748,11 @@ $(ibidir)/openssl: $(ibidir)/perl \ # them. Note that if it does link with them, the configuration will crash # when the library is updated/changed by the host, and the whole purpose of # this project is avoid dependency on the host as much as possible. -$(ibidir)/curl: $(ibidir)/coreutils \ - $(tdir)/curl-$(curl-version).tar.gz +$(ibidir)/curl-$(curl-version): $(ibidir)/coreutils-$(coreutils-version) + + tarball=curl-$(curl-version).tar.gz + $(call import-source, $(curl-url), $(curl-checksum)) + $(call gbuild, curl-$(curl-version), , \ LIBS="-pthread" \ --with-zlib=$(ildir) \ @@ -882,11 +769,12 @@ $(ibidir)/curl: $(ibidir)/coreutils \ --without-axtls \ --disable-ldaps \ --disable-ldap \ - --without-nss, V=1) \ - && if [ "x$(needpatchelf)" != x ]; then \ - $(ibdir)/patchelf --set-rpath $(ildir) $(ildir)/libcurl.so; \ - fi \ - && echo "cURL $(curl-version)" > $@ + --without-nss, V=1) + + if [ -f $(ibdir)/patchelf ]; then + $(ibdir)/patchelf --set-rpath $(ildir) $(ildir)/libcurl.so + fi + echo "cURL $(curl-version)" > $@ # GNU Wget @@ -902,13 +790,18 @@ $(ibidir)/curl: $(ibidir)/coreutils \ # building as part of this project. So to avoid too much dependency on the # host system (especially a crash when these libraries are updated on the # host), they are disabled here. -$(ibidir)/wget: $(ibidir)/libiconv \ - $(ibidir)/coreutils \ - $(tdir)/wget-$(wget-version).tar.lz +$(ibidir)/wget-$(wget-version): \ + $(ibidir)/libiconv-$(libiconv-version) \ + $(ibidir)/coreutils-$(coreutils-version) + + # Download the tarball. + tarball=wget-$(wget-version).tar.lz + $(call import-source, $(wget-url), $(wget-checksum)) + # We need to explicitly disable `libiconv', because of the # `pkg-config' and `libiconv' problem. - libs="-pthread"; \ - if [ x$(needs_ldl) = xyes ]; then libs="$$libs -ldl"; fi; \ + libs="-pthread" + if [ x$(needs_ldl) = xyes ]; then libs="$$libs -ldl"; fi $(call gbuild, wget-$(wget-version), , \ LIBS="$$LIBS $$libs" \ --with-libssl-prefix=$(idir) \ @@ -921,8 +814,8 @@ $(ibidir)/wget: $(ibidir)/libiconv \ --without-libidn \ --disable-pcre2 \ --disable-pcre \ - --disable-iri, V=1) \ - && echo "GNU Wget $(wget-version)" > $@ + --disable-iri, V=1 -j$(numthreads)) + echo "GNU Wget $(wget-version)" > $@ @@ -937,138 +830,172 @@ $(ibidir)/wget: $(ibidir)/libiconv \ # process of the higher-level programs and libraries. Note that during the # building of those higher-level programs (after this Makefile finishes), # there is no access to the system's PATH. -$(ibidir)/diffutils: $(ibidir)/coreutils \ - $(tdir)/diffutils-$(diffutils-version).tar.xz - $(call gbuild, diffutils-$(diffutils-version), static,,V=1) \ - && echo "GNU Diffutils $(diffutils-version)" > $@ - -$(ibidir)/file: $(ibidir)/coreutils \ - $(tdir)/file-$(file-version).tar.gz - $(call gbuild, file-$(file-version), static,,V=1) \ - && echo "File $(file-version)" > $@ - -$(ibidir)/findutils: $(ibidir)/coreutils \ - $(tdir)/findutils-$(findutils-version).tar.xz - $(call gbuild, findutils-$(findutils-version), static,,V=1) \ - && echo "GNU Findutils $(findutils-version)" > $@ - -$(ibidir)/gawk: $(ibidir)/gmp \ - $(ibidir)/mpfr \ - $(ibidir)/coreutils \ - $(tdir)/gawk-$(gawk-version).tar.lz +$(ibidir)/diffutils-$(diffutils-version): \ + $(ibidir)/coreutils-$(coreutils-version) + tarball=diffutils-$(diffutils-version).tar.xz + $(call import-source, $(diffutils-url), $(diffutils-checksum)) + $(call gbuild, diffutils-$(diffutils-version), static,,V=1) + echo "GNU Diffutils $(diffutils-version)" > $@ + +$(ibidir)/file-$(file-version): $(ibidir)/coreutils-$(coreutils-version) + tarball=file-$(file-version).tar.gz + $(call import-source, $(file-url), $(file-checksum)) + $(call gbuild, file-$(file-version), static,,V=1) + echo "File $(file-version)" > $@ + +$(ibidir)/findutils-$(findutils-version): \ + $(ibidir)/coreutils-$(coreutils-version) + tarball=findutils-$(findutils-version).tar.xz + $(call import-source, $(findutils-url), $(findutils-checksum)) + $(call gbuild, findutils-$(findutils-version), static,,V=1) + echo "GNU Findutils $(findutils-version)" > $@ + +$(ibidir)/gawk-$(gawk-version): \ + $(ibidir)/gmp-$(gmp-version) \ + $(ibidir)/mpfr-$(mpfr-version) \ + $(ibidir)/coreutils-$(coreutils-version) + + # Download the tarball. + tarball=gawk-$(gawk-version).tar.lz + $(call import-source, $(gawk-url), $(gawk-checksum)) + # AWK doesn't include RPATH by default, so we'll have to manually # include it using the `patchelf' program (which was a dependency # of Bash). Just note that AWK produces two executables (for # example `gawk-4.2.1' and `gawk') and a symbolic link `awk' to one # of those executables. $(call gbuild, gawk-$(gawk-version), static, \ - --with-readline=$(idir)) \ - && if [ "x$(needpatchelf)" != x ]; then \ - if [ -f $(ibdir)/gawk ]; then \ - $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/gawk; \ - fi; \ - if [ -f $(ibdir)/gawk-$(gawk-version) ]; then \ - $(ibdir)/patchelf --set-rpath $(ildir) \ - $(ibdir)/gawk-$(gawk-version); \ - fi; \ - fi \ - && echo "GNU AWK $(gawk-version)" > $@ - -$(ibidir)/libiconv: $(ibidir)/pkg-config \ - $(tdir)/libiconv-$(libiconv-version).tar.gz - $(call gbuild, libiconv-$(libiconv-version), static) \ - && echo "GNU libiconv $(libiconv-version)" > $@ - -$(ibidir)/libunistring: $(ibidir)/libiconv \ - $(tdir)/libunistring-$(libunistring-version).tar.xz + --with-readline=$(idir)) + + # Correct the RPATH on systems that have installed patchelf. + if [ -f $(ibdir)/patchelf ]; then + if [ -f $(ibdir)/gawk ]; then + $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/gawk + fi + if [ -f $(ibdir)/gawk-$(gawk-version) ]; then + $(ibdir)/patchelf --set-rpath $(ildir) \ + $(ibdir)/gawk-$(gawk-version); + fi + fi + + # Build final target. + echo "GNU AWK $(gawk-version)" > $@ + +$(ibidir)/libiconv-$(libiconv-version): \ + $(ibidir)/pkg-config-$(pkgconfig-version) + tarball=libiconv-$(libiconv-version).tar.gz + $(call import-source, $(libiconv-url), $(libiconv-checksum)) + $(call gbuild, libiconv-$(libiconv-version), static) + echo "GNU libiconv $(libiconv-version)" > $@ + +$(ibidir)/libunistring-$(libunistring-version): \ + $(ibidir)/libiconv-$(libiconv-version) + tarball=libunistring-$(libunistring-version).tar.xz + $(call import-source, $(libunistring-url), $(libunistring-checksum)) $(call gbuild, libunistring-$(libunistring-version), static,, \ - -j$(numthreads)) \ - && echo "GNU libunistring $(libunistring-version)" > $@ + -j$(numthreads)) + echo "GNU libunistring $(libunistring-version)" > $@ -$(ibidir)/libxml2: $(ibidir)/tar \ - $(tdir)/libxml2-$(libxml2-version).tar.gz +$(ibidir)/libxml2-$(libxml2-version): $(ibidir)/patchelf-$(patchelf-version) # The libxml2 tarball also contains Python bindings which are built # and installed to a system directory by default. If you don't need # the Python bindings, the easiest solution is to compile without # Python support: `./configure --without-python'. If you really need # the Python bindings, use `--with-python-install-dir=DIR' instead. + tarball=libxml2-$(libxml2-version).tar.gz + $(call import-source, $(libxml2-url), $(libxml2-checksum)) $(call gbuild, libxml2-$(libxml2-version), static, \ - --without-python) \ - && echo "Libxml2 $(libxml2-version)" > $@ - -$(ibidir)/gettext: $(ibidir)/m4 \ - $(ibidir)/libxml2 \ - $(ibidir)/ncurses \ - $(ibidir)/libiconv \ - $(ibidir)/libunistring \ - $(tdir)/gettext-$(gettext-version).tar.xz - $(call gbuild, gettext-$(gettext-version), static, V=1) \ - && echo "GNU gettext $(gettext-version)" > $@ - -$(ibidir)/git: $(ibidir)/curl \ - $(ibidir)/gettext \ - $(ibidir)/libiconv \ - $(tdir)/git-$(git-version).tar.xz - if [ x$(on_mac_os) = xyes ]; then \ - export LDFLAGS="$$LDFLAGS -lcharset"; \ - fi; \ + --without-python, V=1) + echo "Libxml2 $(libxml2-version)" > $@ + +$(ibidir)/gettext-$(gettext-version): \ + $(ibidir)/m4-$(m4-version) \ + $(ibidir)/libxml2-$(libxml2-version) \ + $(ibidir)/ncurses-$(ncurses-version) \ + $(ibidir)/libiconv-$(libiconv-version) \ + $(ibidir)/libunistring-$(libunistring-version) + tarball=gettext-$(gettext-version).tar.xz + $(call import-source, $(gettext-url), $(gettext-checksum)) + $(call gbuild, gettext-$(gettext-version), static,, \ + V=1 -j$(numthreads)) + echo "GNU gettext $(gettext-version)" > $@ + +$(ibidir)/git-$(git-version): \ + $(ibidir)/curl-$(curl-version) \ + $(ibidir)/gettext-$(gettext-version) \ + $(ibidir)/libiconv-$(libiconv-version) + tarball=git-$(git-version).tar.xz + if [ x$(on_mac_os) = xyes ]; then + export LDFLAGS="$$LDFLAGS -lcharset" + fi + $(call import-source, $(git-url), $(git-checksum)) $(call gbuild, git-$(git-version), static, \ --without-tcltk --with-shell=$(ibdir)/bash \ - --with-iconv=$(idir), V=1) \ - && echo "Git $(git-version)" > $@ - -$(ibidir)/gmp: $(ibidir)/m4 \ - $(ibidir)/coreutils \ - $(tdir)/gmp-$(gmp-version).tar.lz + --with-iconv=$(idir), V=1 -j$(numthreads)) + echo "Git $(git-version)" > $@ + +$(ibidir)/gmp-$(gmp-version): \ + $(ibidir)/m4-$(m4-version) \ + $(ibidir)/coreutils-$(coreutils-version) + tarball=gmp-$(gmp-version).tar.lz + $(call import-source, $(gmp-url), $(gmp-checksum)) $(call gbuild, gmp-$(gmp-version), static, \ - --enable-cxx --enable-fat, ,make check) \ - && echo "GNU Multiple Precision Arithmetic Library $(gmp-version)" > $@ + --enable-cxx --enable-fat, \ + -j$(numthreads) ,make check) + echo "GNU Multiple Precision Arithmetic Library $(gmp-version)" > $@ # On Mac OS, libtool does different things, so to avoid confusion, we'll # prefix GNU's libtool executables with `glibtool'. -$(ibidir)/glibtool: $(ibidir)/m4 \ - $(tdir)/libtool-$(libtool-version).tar.xz +$(ibidir)/libtool-$(libtool-version): $(ibidir)/m4-$(m4-version) + tarball=libtool-$(libtool-version).tar.xz + $(call import-source, $(libtool-url), $(libtool-checksum)) $(call gbuild, libtool-$(libtool-version), static, \ - --program-prefix=g, V=1) \ - && ln -s $(ibdir)/glibtoolize $(ibdir)/libtoolize \ - && echo "GNU Libtool $(libtool-version)" > $@ - -$(ibidir)/grep: $(ibidir)/coreutils \ - $(tdir)/grep-$(grep-version).tar.xz - $(call gbuild, grep-$(grep-version), static,,V=1) \ - && echo "GNU Grep $(grep-version)" > $@ - -$(ibidir)/libbsd: $(ibidir)/coreutils \ - $(tdir)/libbsd-$(libbsd-version).tar.xz - $(call gbuild, libbsd-$(libbsd-version), static,,V=1) \ - && echo "Libbsd $(libbsd-version)" > $@ + --program-prefix=g, V=1 -j$(numthreads)) + ln -s $(ibdir)/glibtoolize $(ibdir)/libtoolize + echo "GNU Libtool $(libtool-version)" > $@ + +$(ibidir)/grep-$(grep-version): $(ibidir)/coreutils-$(coreutils-version) + tarball=grep-$(grep-version).tar.xz + $(call import-source, $(grep-url), $(grep-checksum)) + $(call gbuild, grep-$(grep-version), static,,V=1) + echo "GNU Grep $(grep-version)" > $@ + +$(ibidir)/libbsd-$(libbsd-version): $(ibidir)/coreutils-$(coreutils-version) + tarball=libbsd-$(libbsd-version).tar.xz + $(call import-source, $(libbsd-url), $(libbsd-checksum)) + if [ x$(on_mac_os) = xyes ]; then + echo "" > $@ + else + $(call gbuild, libbsd-$(libbsd-version), static,,V=1) + echo "Libbsd $(libbsd-version)" > $@ + fi # We need to apply a patch to the M4 source to be used properly on macOS. # The patch [1] was inspired by Homebrew's build instructions [1]. # # [1] https://raw.githubusercontent.com/macports/macports-ports/edf0ee1e2cf/devel/m4/files/secure_snprintf.patch # [2] https://github.com/Homebrew/homebrew-core/blob/master/Formula/m4.rb -$(ibidir)/m4: $(ibidir)/texinfo \ - $(tdir)/m4-$(m4-version).tar.gz - cd $(ddir); \ - unpackdir=m4-$(m4-version); \ - rm -rf $$unpackdir \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)); then \ - echo; echo "Tar error"; exit 1; \ - fi \ - && cd $$unpackdir \ - && if [ x$(on_mac_os) = xyes ]; then \ - sed 's|if !(((__GLIBC__ > 2|if !defined(__APPLE__) \&\& !(((__GLIBC__ > 2|' \ - lib/vasnprintf.c > lib/vasnprintf_edited.c; \ - mv lib/vasnprintf_edited.c lib/vasnprintf.c; \ - fi \ - && ./configure --prefix=$(idir) LDFLAGS="$(LDFLAGS)" \ - CPPFLAGS="$(CPPFLAGS)" \ - && make V=1 -j$(numthreads) \ - && make V=1 install \ - && cd .. \ - && rm -rf $$unpackdir \ - && echo "GNU M4 $(m4-version)" > $@ +$(ibidir)/m4-$(m4-version): $(ibidir)/texinfo-$(texinfo-version) + tarball=m4-$(m4-version).tar.gz + $(call import-source, $(m4-url), $(m4-checksum)) + cd $(ddir) + unpackdir=m4-$(m4-version) + rm -rf $$unpackdir + tar xf $(tdir)/$$tarball + mv m4-* $$unpackdir + cd $$unpackdir + if [ x$(on_mac_os) = xyes ]; then + sed 's|if !(((__GLIBC__ > 2|if !defined(__APPLE__) \&\& !(((__GLIBC__ > 2|' \ + lib/vasnprintf.c > lib/vasnprintf_edited.c + mv lib/vasnprintf_edited.c lib/vasnprintf.c + fi + ./configure --prefix=$(idir) LDFLAGS="$(LDFLAGS)" \ + CPPFLAGS="$(CPPFLAGS)" + make V=1 -j$(numthreads) + make V=1 install + cd .. + rm -rf $$unpackdir + echo "GNU M4 $(m4-version)" > $@ # Metastore is used (through a Git hook) to restore the source modification # dates of files after a Git checkout. Another Git hook saves all file @@ -1087,17 +1014,16 @@ $(ibidir)/m4: $(ibidir)/texinfo \ # # Libbsd is not necessary on macOS systems, because macOS is already a # BSD-based distribution. But on GNU/Linux systems, it is necessary. -ifeq ($(on_mac_os),yes) -needlibbsd = -else -needlibbsd = $(ibidir)/libbsd -endif -$(ibidir)/metastore: $(needlibbsd) \ - $(ibidir)/sed \ - $(ibidir)/git \ - $(ibidir)/gawk \ - $(ibidir)/coreutils \ - $(tdir)/metastore-$(metastore-version).tar.gz +$(ibidir)/metastore-$(metastore-version): \ + $(ibidir)/sed-$(sed-version) \ + $(ibidir)/git-$(git-version) \ + $(ibidir)/gawk-$(gawk-version) \ + $(ibidir)/libbsd-$(libbsd-version) \ + $(ibidir)/coreutils-$(coreutils-version) + + # Download the tarball. + tarball=metastore-$(metastore-version).tar.gz + $(call import-source, $(metastore-url), $(metastore-checksum)) # Metastore doesn't have any `./configure' script. So we'll just # call `pwd' as a place-holder for the `./configure' command. @@ -1114,54 +1040,61 @@ $(ibidir)/metastore: $(needlibbsd) \ # are currently only available in a fork of `metastore' hosted at: # https://github.com/mohammad-akhlaghi/metastore # - # We want to inform the user if Metastore isn't built, so we don't - # continue the call to `gbuild' with an `&&'. - # # Checking for presence of `.git'. When the project source is # downloaded from a non-Git source (for example from arXiv), there # is no `.git' directory to work with. So until we find a better # solution, avoid the step to to add the Git hooks. current_dir=$$(pwd); \ $(call gbuild, metastore-$(metastore-version), static,, \ - NO_XATTR=1 V=1,,pwd,PREFIX=$(idir)); \ - if [ -f $(ibdir)/metastore ]; then \ - if [ "x$(needpatchelf)" != x ]; then \ - $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/metastore; \ - fi; \ - if [ -d .git ]; then \ - user=$$(whoami); \ - group=$$(groups | awk '{print $$1}'); \ - cd $$current_dir; \ - for f in pre-commit post-checkout; do \ + NO_XATTR=1 V=1,,pwd,PREFIX=$(idir)) + + # Correct RPATH when necessary. + if [ -f $(ibdir)/patchelf ]; then + $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/metastore + fi + + # If this project is being built in a directory version controlled + # by Git, copy the hooks into the Git configuation. + if [ -f $(ibdir)/metastore ]; then + if [ -d .git ]; then + user=$$(whoami) + group=$$(groups | awk '{print $$1}') + cd $$current_dir + for f in pre-commit post-checkout; do sed -e's|@USER[@]|'$$user'|g' \ -e's|@GROUP[@]|'$$group'|g' \ -e's|@BINDIR[@]|$(ibdir)|g' \ -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ - reproduce/software/shell/git-$$f > .git/hooks/$$f \ - && chmod +x .git/hooks/$$f; \ - done; \ - fi \ - && echo "Metastore (forked) $(metastore-version)" > $@; \ - else \ - echo; echo; echo; \ - echo "*****************"; \ - echo "metastore couldn't be installed!"; \ - echo; \ - echo "Its used for preserving timestamps on Git commits."; \ - echo "Its useful for development, not simple running of "; \ - echo "the project. So we won't stop the configuration "; \ - echo "because it wasn't built."; \ - echo "*****************"; \ + reproduce/software/shell/git-$$f > .git/hooks/$$f + chmod +x .git/hooks/$$f + done + fi + echo "Metastore (forked) $(metastore-version)" > $@ + else + echo; echo; echo + echo "*****************" + echo "metastore couldn't be installed!" + echo + echo "Its used for preserving timestamps on Git commits." + echo "Its useful for development, not simple running of " + echo "the project. So we won't stop the configuration " + echo "because it wasn't built." + echo "*****************" + echo "" > $@ fi +$(ibidir)/mpfr-$(mpfr-version): $(ibidir)/gmp-$(gmp-version) + tarball=mpfr-$(mpfr-version).tar.xz + $(call import-source, $(mpfr-url), $(mpfr-checksum)) + $(call gbuild, mpfr-$(mpfr-version), static, , , make check) + echo "GNU Multiple Precision Floating-Point Reliably $(mpfr-version)" > $@ -$(ibidir)/mpfr: $(ibidir)/gmp \ - $(tdir)/mpfr-$(mpfr-version).tar.xz - $(call gbuild, mpfr-$(mpfr-version), static, , , make check) \ - && echo "GNU Multiple Precision Floating-Point Reliably $(mpfr-version)" > $@ +$(ibidir)/pkg-config-$(pkgconfig-version): $(ibidir)/patchelf-$(patchelf-version) + + # Download the tarball. + tarball=pkg-config-$(pkgconfig-version).tar.gz + $(call import-source, $(pkgconfig-url), $(pkgconfig-checksum)) -$(ibidir)/pkg-config: $(ibidir)/tar \ - $(tdir)/pkg-config-$(pkgconfig-version).tar.gz # An existing `libiconv' can cause a conflict with `pkg-config', # this is why `libiconv' depends on `pkg-config'. On a clean build, # `pkg-config' is built first. But when we don't have a clean build @@ -1174,32 +1107,36 @@ $(ibidir)/pkg-config: $(ibidir)/tar \ # (GCC) installed that doesn't support some necessary features of # building Glib (as part of pkg-config). So to be safe, for Mac # systems, we'll make sure it will use LLVM's Clang. - if [ x$(on_mac_os) = xyes ]; then export compiler="CC=clang"; \ - else export compiler=""; \ - fi; \ + if [ x$(on_mac_os) = xyes ]; then export compiler="CC=clang" + else export compiler="" + fi $(call gbuild, pkg-config-$(pkgconfig-version), static, \ $$compiler --with-internal-glib \ - --with-pc-path=$(ildir)/pkgconfig, V=1) \ - && echo "pkg-config $(pkgconfig-version)" > $@ - -$(ibidir)/sed: $(ibidir)/coreutils \ - $(tdir)/sed-$(sed-version).tar.xz - $(call gbuild, sed-$(sed-version), static,,V=1) \ - && echo "GNU Sed $(sed-version)" > $@ - -$(ibidir)/texinfo: $(ibidir)/perl \ - $(tdir)/texinfo-$(texinfo-version).tar.xz - $(call gbuild, texinfo-$(texinfo-version), static) \ - && if [ "x$(needpatchelf)" != x ]; then \ - $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/info; \ - $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/install-info; \ - fi \ - && echo "GNU Texinfo $(texinfo-version)" > $@ + --with-pc-path=$(ildir)/pkgconfig, V=1) + echo "pkg-config $(pkgconfig-version)" > $@ + +$(ibidir)/sed-$(sed-version): $(ibidir)/coreutils-$(coreutils-version) + tarball=sed-$(sed-version).tar.xz + $(call import-source, $(sed-url), $(sed-checksum)) + $(call gbuild, sed-$(sed-version), static,,V=1) + echo "GNU Sed $(sed-version)" > $@ + +$(ibidir)/texinfo-$(texinfo-version): \ + $(ibidir)/perl-$(perl-version) + tarball=texinfo-$(texinfo-version).tar.xz + $(call import-source, $(texinfo-url), $(texinfo-checksum)) + $(call gbuild, texinfo-$(texinfo-version), static) + if [ -f $(ibdir)/patchelf ]; then + $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/info + $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/install-info + fi + echo "GNU Texinfo $(texinfo-version)" > $@ -$(ibidir)/which: $(ibidir)/coreutils \ - $(tdir)/which-$(which-version).tar.gz - $(call gbuild, which-$(which-version), static) \ - && echo "GNU Which $(which-version)" > $@ +$(ibidir)/which-$(which-version): $(ibidir)/coreutils-$(coreutils-version) + tarball=/which-$(which-version).tar.gz + $(call import-source, $(which-url), $(which-checksum)) + $(call gbuild, which-$(which-version), static) + echo "GNU Which $(which-version)" > $@ @@ -1212,80 +1149,86 @@ $(ibidir)/which: $(ibidir)/coreutils \ # GCC and its prerequisites # ------------------------- +$(ibidir)/isl-$(isl-version): $(ibidir)/gmp-$(gmp-version) + tarball=isl-$(isl-version).tar.bz2 + $(call import-source, $(isl-url), $(isl-checksum)) + if [ $(host_cc) = 1 ]; then + echo "" > $@ + else + $(call gbuild, isl-$(isl-version), static, , \ + V=1 -j$(numthreads)) + echo "GNU Integer Set Library $(isl-version)" > $@ + fi -$(ibidir)/isl: $(ibidir)/gmp \ - $(tdir)/isl-$(isl-version).tar.bz2 - $(call gbuild, isl-$(isl-version), static, , V=1) \ - && echo "GNU Integer Set Library $(isl-version)" > $@ - -$(ibidir)/mpc: $(ibidir)/mpfr \ - $(tdir)/mpc-$(mpc-version).tar.gz - $(call gbuild, mpc-$(mpc-version), static, , , make check) \ - && echo "GNU Multiple Precision Complex library" > $@ - -# Binutils' assembler (`as') and linker (`ld') will conflict with other -# compilers. So until then, on Mac systems we'll use the host opertating -# system's Binutils equivalents by just making links. - -ifeq ($(host_cc),1) -gcc-prerequisites = -else -gcc-prerequisites = $(ibidir)/isl \ - $(ibidir)/mpc -endif - -ifeq ($(on_mac_os),yes) -binutils-tarball = -else -binutils-tarball = $(tdir)/binutils-$(binutils-version).tar.lz -endif +$(ibidir)/mpc-$(mpc-version): $(ibidir)/mpfr-$(mpfr-version) + tarball=mpc-$(mpc-version).tar.gz + $(call import-source, $(mpc-url), $(mpc-checksum)) + if [ $(host_cc) = 1 ]; then + echo "" > $@ + else + $(call gbuild, mpc-$(mpc-version), static, , \ + -j$(numthreads), make check) + echo "GNU Multiple Precision Complex library" > $@ + fi # The installation of Binutils can cause problems during the build of other # programs (http://savannah.nongnu.org/bugs/?56294). Therefore, we'll set # all other basic programs as Binutils prerequisite and GCC (the final # basic target) ultimately just depends on Binutils. -# -# The `ld' linker of Binutils needs several `*crt*.o' file to run. On some -# systems these object files aren't installed in standard places. We -# defined `LIBRARY_PATH' and that fixed the problem for many -# systems. However, some software (for example ImageMagick) over-write -# `LIBRARY_PATH', therefore there is no other way than to put a link to -# these necessary files in our local build directory. IMPORTANT NOTE: -# later, when we build the GNU C Library in the project, we should remove -# this step. -$(ibidir)/binutils: $(ibidir)/sed \ - $(ibidir)/wget \ - $(ibidir)/grep \ - $(ibidir)/file \ - $(ibidir)/gawk \ - $(ibidir)/which \ - $(ibidir)/glibtool \ - $(binutils-tarball) \ - $(ibidir)/metastore \ - $(ibidir)/findutils \ - $(ibidir)/diffutils \ - $(ibidir)/coreutils \ - $(gcc-prerequisites) - - if [ x$(on_mac_os) = xyes ]; then \ - $(call makelink,as); \ - $(call makelink,ar); \ - $(call makelink,ld); \ - $(call makelink,nm); \ - $(call makelink,ps); \ - $(call makelink,ranlib); \ - echo "" > $@; \ - else \ +$(ibidir)/binutils-$(binutils-version): \ + $(ibidir)/sed-$(sed-version) \ + $(ibidir)/isl-$(isl-version) \ + $(ibidir)/mpc-$(mpc-version) \ + $(ibidir)/wget-$(wget-version) \ + $(ibidir)/grep-$(grep-version) \ + $(ibidir)/file-$(file-version) \ + $(ibidir)/gawk-$(gawk-version) \ + $(ibidir)/which-$(which-version) \ + $(ibidir)/libtool-$(libtool-version) \ + $(ibidir)/metastore-$(metastore-version) \ + $(ibidir)/findutils-$(findutils-version) \ + $(ibidir)/diffutils-$(diffutils-version) \ + $(ibidir)/coreutils-$(coreutils-version) + + # Download the tarball. + tarball=binutils-$(binutils-version).tar.lz + $(call import-source, $(binutils-url), $(binutils-checksum)) + + # Binutils' assembler (`as') and linker (`ld') will conflict with + # other compilers. So if we don't build our own compiler, we'll use + # the host opertating system's equivalents by just making links. + if [ x$(on_mac_os) = xyes ]; then + $(call makelink,as) + $(call makelink,ar) + $(call makelink,ld) + $(call makelink,nm) + $(call makelink,ps) + $(call makelink,ranlib) + echo "" > $@ + else + + # Build binutils with the standard 'gbuild' function. $(call gbuild, binutils-$(binutils-version), static, \ --with-lib-path=$(sys_library_path), \ - -j$(numthreads) ) \ - && if ! [ x"$(sys_library_path)" = x ]; then \ - for f in $(sys_library_path)/*crt*.o; do \ - b=$$($(ibdir)/basename $$f); \ - ln -s $$f $(ildir)/$$b; \ - done; \ - fi \ - && echo "GNU Binutils $(binutils-version)" > $@; \ + -j$(numthreads) ) + + # The `ld' linker of Binutils needs several `*crt*.o' file to + # run. On some systems these object files aren't installed in + # standard places. We defined `LIBRARY_PATH' and that fixed the + # problem for many systems. However, some software (for example + # ImageMagick) over-write `LIBRARY_PATH', therefore there is no + # other way than to put a link to these necessary files in our + # local build directory. IMPORTANT NOTE: later, when we build the + # GNU C Library in the project, we should remove this step. + if ! [ x"$(sys_library_path)" = x ]; then + for f in $(sys_library_path)/*crt*.o; do + b=$$($(ibdir)/basename $$f) + ln -s $$f $(ildir)/$$b + done + fi + + # Write the final target. + echo "GNU Binutils $(binutils-version)" > $@ fi # We are having issues with `libiberty' (part of GCC) on Mac. So for now, @@ -1301,117 +1244,162 @@ $(ibidir)/binutils: $(ibidir)/sed \ # # We are currently having problems installing GCC on macOS, so for the time # being, if the project is being run on a macOS, we'll just set a link. -ifeq ($(host_cc),1) -gcc-tarball = -else -gcc-tarball = $(tdir)/gcc-$(gcc-version).tar.xz -endif -$(ibidir)/gcc: $(gcc-tarball) \ - $(ibidir)/binutils +$(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) + + # Function to let the users know what to do if build fails. + error_message() { + echo; echo + echo "_________________________________________________" + echo "!!!!!!!! Warning from Maneage !!!!!!!!" + echo + echo "Unfortunately building of GCC failed on this system!" + echo "Can you please copy the last ~500 lines above and post it" + echo "as a bug here (as an attached file):" + echo " https://sv.nongnu.org/support/?func=additem&group=reproduce" + echo + echo "In the meantime, please re-configure Maneage with '--host-cc'" + echo "like below so it uses your own C compiler for building the" + echo "high-level software ('-e' is to use the existing configuration):" + echo + echo " ./project configure -e --host-cc" + echo + echo "__________ SEE NOTE FROM MANEAGE ABOVE __________" + echo; exit 1 + } + + # Download the tarball. + tarball=gcc-$(gcc-version).tar.xz + $(call import-source, $(gcc-url), $(gcc-checksum)) # GCC builds is own libraries in '$(idir)/lib64'. But all other # libraries are in '$(idir)/lib'. Since this project is only for a # single architecture, we can trick GCC into building its libraries # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic # link to '$(idir)/lib'. - if [ $(host_cc) = 1 ]; then \ - $(call makelink,cc); \ - $(call makelink,cc,,gcc); \ - $(call makelink,c++,,g++); \ - $(call makelink,gfortran); \ - $(call makelink,strip,mandatory); \ - ccinfo=$$(gcc --version | awk 'NR==1'); \ - echo "C compiler (""$$ccinfo"")" > $@; \ - else \ - current_dir=$$(pwd); \ - rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\ - rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \ - rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\ - \ - ln -fs $(ildir) $(idir)/lib64; \ - \ + if [ $(host_cc) = 1 ]; then + + # Make sure we don't have any of the program we want to link to + # in the '.local/bin' directory. + rm -f $(ibdir)/cc + rm -f $(ibdir)/c++ + rm -f $(ibdir)/gcc + rm -f $(ibdir)/cpp + rm -f $(ibdir)/strip; + rm -f $(ibdir)/gfortran; + + # Put links to the host's tools in '.local/bin'. + $(call makelink,cc) + $(call makelink,cpp) + $(call makelink,cc,,gcc) + $(call makelink,gfortran) + $(call makelink,c++,,g++) + $(call makelink,strip,mandatory) + ccinfo=$$(gcc --version | awk 'NR==1') + echo "C compiler (""$$ccinfo"")" > $@ + + else + + # We are building GCC, so to avoid any previous build in + # '.local/bin', we'll delete all the files that GCC builds from + # there. + current_dir=$$(pwd) + rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov* + rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc* + rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64* + + # We don't want '.local/lib' and '.local/lib64' to be separate. + ln -fs $(ildir) $(idir)/lib64 + + # By default we'll build GCC in the RAM to avoid building so many + # files and possibly harming the hard-drive or SSD. But if the + # RAM doesn't have enough space, we can't use it. in_ram=$$(df $(ddir) \ | awk 'NR==2{print ($$4>10000000) ? "yes" : "no"}'); \ - if [ $$in_ram = "yes" ]; then odir=$(ddir); \ - else \ - odir=$(BDIR)/software/build-tmp-gcc; \ - if [ -d $$odir ]; then rm -rf $$odir; fi; \ - mkdir $$odir; \ - fi; \ - cd $$odir; \ - rm -rf gcc-$(gcc-version); \ - tar xf $(word 1,$(filter $(tdir)/%,$^)); \ - if [ $$odir != $(ddir) ]; then \ - ln -s $$odir/gcc-$(gcc-version) $(ddir)/gcc-$(gcc-version); \ - fi; \ - if cd gcc-$(gcc-version) \ - && mkdir build \ - && cd build \ - && ../configure SHELL=$(ibdir)/bash \ - --prefix=$(idir) \ - --with-mpc=$(idir) \ - --with-gmp=$(idir) \ - --with-isl=$(idir) \ - --with-mpfr=$(idir) \ - --with-local-prefix=$(idir) \ - --with-build-time-tools=$(idir) \ - --enable-lto \ - --enable-shared \ - --enable-cet=auto \ - --enable-default-pie \ - --enable-default-ssp \ - --enable-decimal-float \ - --enable-threads=posix \ - --enable-languages=c,c++,fortran,objc,obj-c++ \ - --disable-nls \ - --disable-libada \ - --disable-multilib \ - --disable-multiarch \ - && make SHELL=$(ibdir)/bash -j$(numthreads) \ - && make SHELL=$(ibdir)/bash install \ - && cd ../.. \ - && tempname=$$odir/gcc-$(gcc-version)/build/rpath-temp-copy \ - && if [ "x$(on_mac_os)" != xyes ]; then \ - for f in $$(find $(idir)/libexec/gcc) $(ildir)/libstdc++*; do \ - isdynamic=$$(file $$f | grep "dynamically linked"); \ - if [ x"$$isdynamic" != x ]; then \ - cp $$f $$tempname; \ - patchelf --set-rpath $(ildir) $$tempname; \ - echo "$$f: added rpath"; \ - islibcpp=$$(echo $$f | grep "libstdc++"); \ - if [ x"$$islibcpp" != x ]; then \ - patchelf --add-needed $(ildir)/libiconv.so $$tempname; \ - echo "$$f: linked with libiconv"; \ - fi; \ - mv $$tempname $$f; \ - fi; \ - done; \ - fi \ - && rm -rf gcc-$(gcc-version) \ - && cd $$current_dir \ - && if [ "$$odir" != "$(ddir)" ]; then \ - rm -rf $$odir; \ - rm $(ddir)/gcc-$(gcc-version); \ - fi \ - && ln -sf $(ibdir)/gcc $(ibdir)/cc; \ - then \ - echo "GNU Compiler Collection (GCC) $(gcc-version)" > $@; \ - else echo; echo; echo; \ - echo "_________________________________________________"; \ - echo "!!!!!!!! Warning from Maneage !!!!!!!!"; \ - echo; \ - echo "Unfortunately building of GCC failed on this system!"; \ - echo "Can you please copy the last ~500 lines above and post it"; \ - echo "as a bug here (as an attached file):"; \ - echo " https://sv.nongnu.org/support/?func=additem&group=reproduce"; \ - echo; \ - echo "In the meantime, please re-configure Maneage with '--host-cc'"; \ - echo "like below so it uses your own C compiler for building the"; \ - echo "high-level software ('-e' is to use the existing configuration):"; \ - echo; \ - echo " ./project configure -e --host-cc"; \ - echo; \ - echo "__________ SEE NOTE FROM MANEAGE ABOVE __________"; \ - echo; exit 1; \ - fi; \ + if [ $$in_ram = "yes" ]; then odir=$(ddir) + else + odir=$(BDIR)/software/build-tmp-gcc + if [ -d $$odir ]; then rm -rf $$odir; fi + mkdir $$odir + fi + + # Go into the proper directory, unpack GCC and prepare the + # 'build' directory inside it for all the built files. + cd $$odir + rm -rf gcc-$(gcc-version) + tar xf $(tdir)/$$tarball + if [ $$odir != $(ddir) ]; then + ln -s $$odir/gcc-$(gcc-version) $(ddir)/gcc-$(gcc-version) + fi + cd gcc-$(gcc-version) + mkdir build + cd build + + # Configure, build and install GCC, if any of three steps fails, + # the error message will be printed. + if ! ../configure SHELL=$(ibdir)/bash \ + --prefix=$(idir) \ + --with-mpc=$(idir) \ + --with-gmp=$(idir) \ + --with-isl=$(idir) \ + --with-mpfr=$(idir) \ + --with-local-prefix=$(idir) \ + --with-build-time-tools=$(idir) \ + --enable-lto \ + --enable-shared \ + --enable-cet=auto \ + --enable-default-pie \ + --enable-default-ssp \ + --enable-decimal-float \ + --enable-threads=posix \ + --enable-languages=c,c++,fortran,objc,obj-c++ \ + --disable-nls \ + --disable-libada \ + --disable-multilib \ + --disable-multiarch; then error_message; fi + if ! make SHELL=$(ibdir)/bash -j$(numthreads); then error_message; fi + if ! make SHELL=$(ibdir)/bash install; then error_message; fi + + # We need to manually fix the RPATH inside GCC's libraries, the + # programs built by GCC already have RPATH. + tempname=$$odir/gcc-$(gcc-version)/build/rpath-temp-copy + if [ -f $(ibdir)/patchelf ]; then + + # Go over all the installed GCC libraries (its executables are + # fine!). + for f in $$(find $(idir)/libexec/gcc -type f) $(ildir)/libstdc++*; do + + # Make sure this is a static library, copy it to a temporary + # name (to avoid any possible usage of the file while it is + # being corrected), and add RPATH inside of it and put the + # corrected file back in its place. In the case of the + # standard C++ library, we also need to manually insert a + # linking to libiconv. + if file $$f | grep -q "dynamically linked"; then + cp $$f $$tempname + patchelf --set-rpath $(ildir) $$tempname + echo "$$f: added rpath" + if echo $$f | grep -q "libstdc++"; then + patchelf --add-needed $(ildir)/libiconv.so $$tempname + echo "$$f: linked with libiconv" + fi + mv $$tempname $$f + fi + done + fi + + # Come back up to the un-packing directory and delete the GCC + # source directory. + cd ../.. + rm -rf gcc-$(gcc-version) + cd $$current_dir + if [ "$$odir" != "$(ddir)" ]; then + rm -rf $$odir; + rm $(ddir)/gcc-$(gcc-version); + fi + + # Set 'cc' to point to 'gcc'. + ln -sf $(ibdir)/gcc $(ibdir)/cc + + # Write the final target. + echo "GNU Compiler Collection (GCC) $(gcc-version)" > $@ fi diff --git a/reproduce/software/make/build-rules.mk b/reproduce/software/make/build-rules.mk index 260ded8..7471e5b 100644 --- a/reproduce/software/make/build-rules.mk +++ b/reproduce/software/make/build-rules.mk @@ -1,7 +1,7 @@ # Generic configurable recipes to build packages with GNU Build system or # CMake. This is Makefile is not intended to be run directly, it will be -# imported into `basic.mk' and `high-level.mk'. They should be activated -# with Make's `Call' function. +# imported into 'basic.mk' and 'high-level.mk'. They should be activated +# with Make's 'Call' function. # # Copyright (C) 2018-2020 Mohammad Akhlaghi <mohammad@akhlaghi.org> # @@ -22,31 +22,84 @@ -# IMPORTANT note -# -------------- +# Import/download project's source +# -------------------------------- # -# Without using `&&', if a step fails, the process will continue. However, -# in the `if' statements, we need `;' (particularly between `]' and -# `then'). So we need to put any necessary checks at the start, then when -# we start the process, every command will be separated by an `&&'. +# Copy/Download the raw tarball into an '.unchecked' suffix. Then calculate +# its checksum and if it is correct, remove the extra suffix. +# +# Arguments: +# 1: The optional URL to use for this tarball. +# 2: The expeced checksum of the tarball. +# +# Necessary shell variables +# 'tarball': This is the name of the actual tarball file without a +# directory. +import-source = final=$(tdir)/$$tarball; \ + url=$(strip $(1)); \ + exp_checksum="$(strip $(2))"; \ + if [ -f $$final ]; then \ + echo "$(tdir)/$$tarball: already present in project."; \ + else \ + unchecked="$$final.unchecked"; \ + rm -f "$$unchecked"; \ + if [ -f $(DEPENDENCIES-DIR)/$$tarball ]; then \ + cp $(DEPENDENCIES-DIR)/$$tarball "$$unchecked"; \ + else \ + if [ x"$$url" = x ]; then \ + bservers="$(backupservers)"; \ + tarballurl=$(topbackupserver)/$$tarball; \ + else \ + bservers="$(backupservers_all)"; \ + tarballurl=$$url/$$tarball; \ + fi; \ + if [ -f $(ibdir)/wget ]; then \ + downloader="wget --no-use-server-timestamps -O"; \ + else \ + downloader="$(DOWNLOADER)"; \ + fi; \ + touch $(lockdir)/download; \ + $(downloadwrapper) "$$downloader" $(lockdir)/download \ + $$tarballurl "$$unchecked" "$$bservers"; \ + fi; \ + if [ x"$$exp_checksum" = x"NO-CHECK-SUM" ]; then \ + mv "$$unchecked" "$$final"; \ + else \ + if type sha512sum > /dev/null 2>/dev/null; then \ + checksum=$$(sha512sum "$$unchecked" | awk '{print $$1}'); \ + if [ x"$$checksum" = x"$$exp_checksum" ]; then \ + mv "$$unchecked" "$$final"; \ + else \ + echo "ERROR: Non-matching checksum for '$$tarball'."; \ + echo "Checksum should be: $$exp_checksum"; \ + echo "Checksum is: $$checksum"; \ + exit 1; \ + fi; \ + else mv "$$unchecked" "$$final"; \ + fi; \ + fi; \ + fi +# Unpack a tarball +# ---------------- +# # Unpack a tarball in the current directory. The issue is that until we # install GNU Tar within Maneage, we have to use the host's Tar # implementation and in some cases, they don't recognize '.lz'. -uncompress = csuffix=$$(echo $$tarball \ +uncompress = csuffix=$$(echo $$utarball \ | sed -e's/\./ /g' \ | awk '{print $$NF}'); \ if [ x$$csuffix = xlz ]; then \ intarrm=1; \ - intar=$$(echo $$tarball | sed -e's/.lz//'); \ - lzip -c -d $$tarball > $$intar; \ + intar=$$(echo $$utarball | sed -e's/.lz//'); \ + lzip -c -d $$utarball > $$intar; \ else \ intarrm=0; \ - intar=$$tarball; \ + intar=$$utarball; \ fi; \ if tar xf $$intar; then \ if [ x$$intarrm = x1 ]; then rm $$intar; fi; \ @@ -64,20 +117,20 @@ uncompress = csuffix=$$(echo $$tarball \ # # Arguments: # 1: Directory name after unpacking. -# 2: Set to `static' for a static build. +# 2: Set to 'static' for a static build. # 3: Extra configuration options. # 4: Extra options/arguments to pass to Make. -# 5: Step to run between `make' and `make install': usually `make check'. -# 6: The configuration script (`configure' by default). -# 7: Arguments for `make install'. +# 5: Step to run between 'make' and 'make install': usually 'make check'. +# 6: The configuration script ('configure' by default). +# 7: Arguments for 'make install'. # -# NOTE: Unfortunately the configure script of `zlib' doesn't recognize -# `SHELL'. So we'll have to remove it from the call to the configure +# NOTE: Unfortunately the configure script of 'zlib' doesn't recognize +# 'SHELL'. So we'll have to remove it from the call to the configure # script. # # NOTE: A program might not contain any configure script. In this case, -# we'll just pass a non-relevant function like `pwd'. So SED should be used -# to modify `confscript' or to set `configop'. +# we'll just pass a non-relevant function like 'pwd'. So SED should be used +# to modify 'confscript' or to set 'configop'. gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ export LDFLAGS="$$LDFLAGS -static"; \ fi; \ @@ -85,9 +138,8 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ if [ x"$$check" = x ]; then check="echo Skipping-check"; fi; \ cd $(ddir); \ rm -rf $(1); \ - if [ x"$$gbuild_tar" = x ]; then \ - tarball=$(word 1,$(filter $(tdir)/%,$^)); \ - else tarball=$$gbuild_tar; \ + if [ x"$$gbuild_tar" = x ]; then utarball=$(tdir)/$$tarball; \ + else utarball=$$gbuild_tar; \ fi; \ $(call uncompress); \ cd $(1); \ @@ -95,7 +147,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ if [ x"$(strip $(6))" = x ]; then confscript=./configure; \ else confscript="$(strip $(6))"; \ fi; \ - \ + \ if [ -f $(ibdir)/bash ]; then \ if [ -f "$$confscript" ]; then \ sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ @@ -108,7 +160,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ elif [ -f /bin/bash ]; then shellop="SHELL=/bin/bash"; \ else shellop="SHELL=/bin/sh"; \ fi; \ - \ + \ if [ -f "$$confscript" ]; then \ if [ x"$(strip $(1))" = x"zlib-$(zlib-version)" ]; then \ configop="--prefix=$(idir)"; \ @@ -118,12 +170,12 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ \ echo; echo "Using '$$confscript' to configure:"; echo; \ echo "$$confscript $(3) $$configop"; echo; \ - $$confscript $(3) $$configop \ - && make "$$shellop" $(4) \ - && $$check \ - && make "$$shellop" install $(7) \ - && cd .. \ - && rm -rf $(1) + $$confscript $(3) $$configop; \ + make "$$shellop" $(4); \ + $$check; \ + make "$$shellop" install $(7); \ + cd ..; \ + rm -rf $(1) @@ -131,7 +183,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ # CMake # ----- # -# According to the link below, in CMake `/bin/sh' is hardcoded, so there is +# According to the link below, in CMake '/bin/sh' is hardcoded, so there is # no way to change it unfortunately! # # https://stackoverflow.com/questions/21167014/how-to-set-shell-variable-in-makefiles-generated-by-cmake @@ -139,17 +191,18 @@ cbuild = if [ x$(static_build) = xyes ] && [ $(2)x = staticx ]; then \ export LDFLAGS="$$LDFLAGS -static"; \ opts="-DBUILD_SHARED_LIBS=OFF"; \ fi; \ - tarball=$(word 1,$(filter $(tdir)/%,$^)); \ cd $(ddir); \ rm -rf $(1); \ + utarball=$(tdir)/$$tarball; \ $(call uncompress); \ - cd $(1) \ - && rm -rf project-build \ - && mkdir project-build \ - && cd project-build \ - && cmake .. -DCMAKE_LIBRARY_PATH=$(ildir) \ - -DCMAKE_INSTALL_PREFIX=$(idir) \ - -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON $$opts $(3) \ - && make && make install \ - && cd ../.. \ - && rm -rf $(1) + cd $(1); \ + rm -rf project-build; \ + mkdir project-build; \ + cd project-build; \ + cmake .. -DCMAKE_LIBRARY_PATH=$(ildir) \ + -DCMAKE_INSTALL_PREFIX=$(idir) \ + -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON $$opts $(3); \ + make; \ + make install; \ + cd ../..; \ + rm -rf $(1) diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 75fa8ac..b756cb9 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -4,7 +4,11 @@ # !!!!! IMPORTANT NOTES !!!!! # # This Makefile will be run by the initial `./project configure' script. It -# is not included into the reproduction pipe after that. +# is not included into the project afterwards. +# +# This Makefile builds the high-level (optional) software in Maneage that +# users can choose for different projects. It thus assumes that the +# low-level tools (like GNU Tar and etc) are already build by 'basic.mk'. # # ------------------------------------------------------------------------ # @@ -24,35 +28,82 @@ # You should have received a copy of the GNU General Public License # along with this Makefile. If not, see <http://www.gnu.org/licenses/>. - - -# Top level environment +# Top level environment (same as 'basic.mk') include reproduce/software/config/LOCAL.conf include reproduce/software/make/build-rules.mk -include reproduce/software/config/TARGETS.conf include reproduce/software/config/versions.conf include reproduce/software/config/checksums.conf + +# The optional URLs of software. Note that these may need the software +# version, so it is important that they be loaded after 'versions.conf'. +include reproduce/software/config/urls.conf + +# Configurations specific to this Makefile +include reproduce/software/config/TARGETS.conf include reproduce/software/config/texlive-packages.conf +# Basic directories (similar to 'basic.mk'). lockdir = $(BDIR)/locks tdir = $(BDIR)/software/tarballs ddir = $(BDIR)/software/build-tmp idir = $(BDIR)/software/installed ibdir = $(BDIR)/software/installed/bin ildir = $(BDIR)/software/installed/lib -iidir = $(BDIR)/software/installed/include -dtexdir = $(shell pwd)/reproduce/software/bibtex -patchdir= $(shell pwd)/reproduce/software/patches -itidir = $(BDIR)/software/installed/version-info/tex -ictdir = $(BDIR)/software/installed/version-info/cite -ipydir = $(BDIR)/software/installed/version-info/python ibidir = $(BDIR)/software/installed/version-info/proglib -# Set the top-level software to build. -all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ - $(foreach p, $(top-level-python), $(ipydir)/$(p)) \ +# Basic directories (specific to this Makefile). +iidir = $(BDIR)/software/installed/include +shsrcdir = $(shell pwd)/reproduce/software/shell +dtexdir = $(shell pwd)/reproduce/software/bibtex +patchdir = $(shell pwd)/reproduce/software/patches +itidir = $(BDIR)/software/installed/version-info/tex +ictdir = $(BDIR)/software/installed/version-info/cite +ipydir = $(BDIR)/software/installed/version-info/python + +# Targets to build. +ifeq ($(strip $(all_highlevel)),1) + + # Set it to build all programs. Pay attention to special software: + # + # Versions as variables (for example minizip): they have the same as the + # version as others and the version number is actually a variable. So + # we'll need to filter it out, then add it in the end: minizip (has same + # version as zlib) + # + # Packages that are installed in the same recipe as others. These need to + # be totally removed when testing all the builds (they will be built + # anyway). lapack (installed with ATLAS) + targets-proglib := $(filter-out minizip-% lapack-%, \ + $(shell awk '/^# CLASS:PYTHON/{good=0} \ + good==1 && !/^#/ && $$1 ~ /-version$$/ { \ + printf("%s %s ", $$1, $$3)} \ + /^# CLASS:HIGHLEVEL/{good=1}' \ + reproduce/software/config/versions.conf \ + | sed 's/version //g')) \ + minizip-$(minizip-version) + + # List all existing Python packages. + targets-python := $(shell \ + awk '/^# CLASS:PYTHON/{good=1} \ + good==1 && !/^#/ && $$1 ~ /-version$$/ {printf("%s %s ",$$1,$$3)}' \ + reproduce/software/config/versions.conf | sed 's/version //g') +else + + # Append the version of each software to its name. We are using a Make + # feature where a variable name is defined with another variable. + targets-python := $(foreach p,$(top-level-python),$(p)-$($(p)-version)) + targets-proglib := $(foreach p,$(top-level-programs),$(p)-$($(p)-version)) + +endif + +# Ultimate Makefile target. +all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) \ + $(foreach p, $(targets-python), $(ipydir)/$(p)) \ $(itidir)/texlive +# Define the shell environment +# ---------------------------- +# # Other basic environment settings: We are only including the host # operating system's PATH environment variable (after our own!) for the # compiler and linker. For the library binaries and headers, we are only @@ -67,18 +118,28 @@ all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ # # 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the # user's environment. +# +# Shell settings similar to 'basic.mk': .ONESHELL: -.SHELLFLAGS := --noprofile --norc -ec -export CCACHE_DISABLE := 1 export PATH := $(ibdir) +export CCACHE_DISABLE := 1 +export SHELL := $(ibdir)/bash +export CPPFLAGS := -I$(idir)/include +.SHELLFLAGS := --noprofile --norc -ec +export PKG_CONFIG_PATH := $(ildir)/pkgconfig +export LDFLAGS := $(rpath_command) -L$(ildir) +export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig + +# Settings specific to this Makefile. export CC := $(ibdir)/gcc export CXX := $(ibdir)/g++ -export SHELL := $(ibdir)/bash export F77 := $(ibdir)/gfortran export LD_RUN_PATH := $(ildir):$(il64dir) -export PKG_CONFIG_PATH := $(ildir)/pkgconfig export LD_LIBRARY_PATH := $(ildir):$(il64dir) -export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig + +# Recipe startup script, see `reproduce/software/shell/bashrc.sh'. +export PROJECT_STATUS := configure_highlevel +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh # Until we build our own C library, without this, our GCC won't be able to # compile anything! Note that on most systems (in particular @@ -97,282 +158,45 @@ export DYLD_LIBRARY_PATH := # for `ld'. export LIBRARY_PATH := $(sys_library_path) -# Recipe startup script, see `reproduce/software/shell/bashrc.sh'. -export PROJECT_STATUS := configure_highlevel -export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh - -# Servers to use as backup, later this should go in a file that is not -# under version control (the actual server that the tarbal comes from is -# irrelevant). -backupservers := $(shell awk '!/^#/{printf "%s ", $$1}' \ - reproduce/software/config/servers-backup.conf) - # Building flags: # # C++ flags: when we build GCC, the C++ standard library needs to link with # libiconv. So it is necessary to generically include `-liconv' for all C++ # builds. -export CPPFLAGS := -I$(idir)/include -export LDFLAGS := $(rpath_command) -L$(ildir) ifeq ($(host_cc),0) export CXXFLAGS := -liconv endif - - - - -# We want the download to happen on a single thread. So we need to define a -# lock, and call a special script we have written for this job. These are -# placed here because we want them both in the `high-level.mk' and -# `python.mk'. -$(lockdir): | $(BDIR); mkdir $@ -downloader="wget --no-use-server-timestamps -O"; +# Servers to use as backup. Maneage already has some fixed servers that can +# be used to download software tarballs. They are in a configuation +# file. But we give precedence to the "user" backup servers. +# +# One important "user" server (which the user doesn't actually give, but is +# found at configuration time in 'configure.sh') is Zenodo (see the +# description in 'configure.sh' for more on why this depends on +# configuration time). +# +# Afer putting everything together, we use the first server as the +# reference for all software if their '-url' variable isn't defined (in +# 'reproduce/software/config/urls.conf'). downloadwrapper = ./reproduce/analysis/bash/download-multi-try +maneage_backup_urls := $(shell awk '!/^#/{printf "%s ", $$1}' \ + reproduce/software/config/servers-backup.conf) +backupservers_all = $(user_backup_urls) $(maneage_backup_urls) +topbackupserver = $(word 1, $(backupservers_all)) +backupservers = $(filter-out $(topbackupserver),$(backupservers_all)) -# Mini-environment software +# Import rules to build Python packages, include reproduce/software/make/python.mk -# Tarballs -# -------- -# -# All the necessary tarballs are defined and prepared with this rule. -# -# Note that we want the tarballs to follow the convention of NAME-VERSION -# before the `tar.XX' prefix. For those programs that don't follow this -# convention, but include the name/version in their tarball names with -# another format, we'll do the modification before the download so the -# downloaded file has our desired format. -tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ - apr-$(apr-version).tar.gz \ - apr-util-$(apr-util-version).tar.gz \ - astrometry.net-$(astrometrynet-version).tar.gz \ - atlas-$(atlas-version).tar.bz2 \ - autoconf-$(autoconf-version).tar.lz \ - automake-$(automake-version).tar.gz \ - bison-$(bison-version).tar.xz \ - boost-$(boost-version).tar.gz \ - cairo-$(cairo-version).tar.xz \ - cdsclient-$(cdsclient-version).tar.gz \ - cfitsio-$(cfitsio-version).tar.gz \ - cmake-$(cmake-version).tar.gz \ - eigen-$(eigen-version).tar.gz \ - expat-$(expat-version).tar.lz \ - fftw-$(fftw-version).tar.gz \ - flex-$(flex-version).tar.gz \ - freetype-$(freetype-version).tar.gz \ - gdb-$(gdb-version).tar.gz \ - ghostscript-$(ghostscript-version).tar.gz \ - gnuastro-$(gnuastro-version).tar.lz \ - gsl-$(gsl-version).tar.gz \ - hdf5-$(hdf5-version).tar.gz \ - healpix-$(healpix-version).tar.gz \ - help2man-$(help2man-version).tar.xz \ - imagemagick-$(imagemagick-version).tar.xz \ - imfit-$(imfit-version).tar.gz \ - install-tl-unx.tar.gz \ - jpegsrc.$(libjpeg-version).tar.gz \ - lapack-$(lapack-version).tar.gz \ - libgit2-$(libgit2-version).tar.gz \ - libnsl-$(libnsl-version).tar.gz \ - libpng-$(libpng-version).tar.xz \ - libtirpc-$(libtirpc-version).tar.bz2 \ - missfits-$(missfits-version).tar.gz \ - netpbm-$(netpbm-version).tar.gz \ - openblas-$(openblas-version).tar.gz \ - openmpi-$(openmpi-version).tar.gz \ - openssh-$(openssh-version).tar.gz \ - patch-$(patch-version).tar.gz \ - pixman-$(pixman-version).tar.gz \ - R-$(R-version).tar.gz \ - scamp-$(scamp-version).tar.lz \ - scons-$(scons-version).tar.gz \ - sextractor-$(sextractor-version).tar.lz \ - swarp-$(swarp-version).tar.gz \ - swig-$(swig-version).tar.gz \ - rpcsvc-proto-$(rpcsvc-proto-version).tar.xz \ - tides-$(tides-version).tar.gz \ - tiff-$(libtiff-version).tar.gz \ - valgrind-$(valgrind-version).tar.bz2 \ - wcslib-$(wcslib-version).tar.bz2 \ - xlsxio-$(xlsxio-version).tar.gz \ - yaml-$(yaml-version).tar.gz \ - zlib-$(zlib-version).tar.gz \ - , $(tdir)/$(t) ) -$(tarballs): $(tdir)/%: | $(lockdir) - - # Remove the version numbers and suffix from the tarball name so we - # can search more easily only with the program name. This requires - # the first character of the version to be a digit: packages such - # as `foo' and `foo-3' will not be distinguished, but `foo' and - # `foo2' will be distinguished. - @n=$$(echo $* | sed -e's/-[0-9]/ /' -e's/\./ /g' \ - | awk '{print $$1}' ) - - # Set the top download link of the requested tarball. - mergenames=1 - if [ $$n = apachelog4cxx ]; then c=$(apachelog4cxx-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = apr ]; then c=$(apr-checksum); w=https://www-us.apache.org/dist/apr - elif [ $$n = apr-util ]; then c=$(apr-util-checksum); w=https://www-us.apache.org/dist/apr - elif [ $$n = astrometry ]; then c=$(astrometrynet-checksum); w=http://astrometry.net/downloads - elif [ $$n = atlas ]; then - mergenames=0 - c=$(atlas-checksum) - w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download - elif [ $$n = autoconf ]; then c=$(autoconf-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = automake ]; then c=$(automake-checksum); w=http://ftp.gnu.org/gnu/automake - elif [ $$n = bison ]; then c=$(bison-checksum); w=http://ftp.gnu.org/gnu/bison - elif [ $$n = boost ]; then - mergenames=0 - c=$(boost-checksum) - vstr=$$(echo $(boost-version) | sed -e's/\./_/g') - w=https://dl.bintray.com/boostorg/release/$(boost-version)/source/boost_$$vstr.tar.gz - elif [ $$n = cairo ]; then c=$(cairo-checksum); w=https://www.cairographics.org/releases - elif [ $$n = cdsclient ]; then c=$(cdsclient-checksum); w=http://cdsarc.u-strasbg.fr/ftp/pub/sw - elif [ $$n = cfitsio ]; then c=$(cfitsio-checksum); w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c - elif [ $$n = cmake ]; then - mergenames=0 - c=$(cmake-checksum) - majv=$$(echo $(cmake-version) \ - | sed -e's/\./ /' \ - | awk '{printf("%d.%d", $$1, $$2)}') - w=https://cmake.org/files/v$$majv/cmake-$(cmake-version).tar.gz - elif [ $$n = eigen ]; then - mergenames=0 - c=$(eigen-checksum); - w=http://bitbucket.org/eigen/eigen/get/$(eigen-version).tar.gz - elif [ $$n = expat ]; then - mergenames=0 - c=$(expat-checksum) - vstr=$$(echo $(expat-version) | sed -e's/\./_/g') - w=https://github.com/libexpat/libexpat/releases/download/R_$$vstr/expat-$(expat-version).tar.lz - elif [ $$n = fftw ]; then c=$(fftw-checksum); w=ftp://ftp.fftw.org/pub/fftw - elif [ $$n = flex ]; then c=$(flex-checksum); w=https://github.com/westes/flex/files/981163 - elif [ $$n = freetype ]; then c=$(freetype-checksum); w=https://download.savannah.gnu.org/releases/freetype - elif [ $$n = gdb ]; then c=$(gdb-checksum); w=http://ftp.gnu.org/gnu/gdb - elif [ $$n = ghostscript ]; then - c=$(ghostscript-checksum) - v=$$(echo $(ghostscript-version) | sed -e's/\.//') - w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs$$v - elif [ $$n = gnuastro ]; then c=$(gnuastro-checksum); w=http://ftp.gnu.org/gnu/gnuastro - elif [ $$n = gsl ]; then c=$(gsl-checksum); w=http://ftp.gnu.org/gnu/gsl - elif [ $$n = hdf5 ]; then - mergenames=0 - c=$(hdf5-checksum) - majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') - w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$* - elif [ $$n = healpix ]; then c=$(healpix-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = help2man ]; then c=$(help2man-checksum); w=http://ftp.gnu.org/gnu/help2man - elif [ $$n = imagemagick ]; then c=$(imagemagick-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = imfit ]; then - mergenames=0 - c=$(imfit-checksum) - w=http://www.mpe.mpg.de/~erwin/resources/imfit/imfit-$(imfit-version)-source.tar.gz - elif [ $$n = install-tl-unx ]; then c=NO-CHECK-SUM; w=http://mirror.ctan.org/systems/texlive/tlnet - elif [ $$n = jpegsrc ]; then c=$(libjpeg-checksum); w=http://ijg.org/files - elif [ $$n = lapack ]; then c=$(lapack-checksum); w=http://www.netlib.org/lapack - elif [ $$n = libnsl ]; then c=$(libnsl-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = libpng ]; then c=$(libpng-checksum); w=https://download.sourceforge.net/libpng - elif [ $$n = libgit2 ]; then - mergenames=0 - c=$(libgit2-checksum) - w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz - elif [ $$n = libtirpc ]; then c=$(libtirpc-checksum); w=https://downloads.sourceforge.net/libtirpc - elif [ $$n = missfits ]; then c=$(missfits-checksum); w=https://www.astromatic.net/download/missfits - elif [ $$n = netpbm ]; then c=$(netpbm-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = openblas ]; then - mergenames=0 - c=$(openblas-checksum) - w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz - elif [ $$n = openmpi ]; then - mergenames=0 - c=$(openmpi-checksum) - majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') - w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* - elif [ $$n = openssh ]; then c=$(openssh-checksum); w=https://artfiles.org/openbsd/OpenSSH/portable - elif [ $$n = patch ]; then c=$(patch-checksum); w=http://ftp.gnu.org/gnu/patch - elif [ $$n = pixman ]; then c=$(pixman-checksum); w=https://www.cairographics.org/releases - elif [ $$n = R ]; then c=$(R-checksum); - majver=$$(echo $(R-version) | sed -e's/\./ /g' | awk '{print $$1}') - w=https://cran.r-project.org/src/base/R-$$majver - elif [ $$n = rpcsvc-proto ]; then c=$(rpcsvc-proto-checksum); w=https://github.com/thkukuk/rpcsvc-proto/releases/download/v$(rpcsvc-proto-version) - elif [ $$n = scamp ]; then c=$(scamp-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = scons ]; then - mergenames=0 - c=$(scons-checksum) - w=https://sourceforge.net/projects/scons/files/scons/$(scons-version)/scons-$(scons-version).tar.gz/download - elif [ $$n = sextractor ]; then c=$(sextractor-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = swarp ]; then c=$(swarp-checksum); w=https://www.astromatic.net/download/swarp - elif [ $$n = swig ]; then c=$(swig-checksum); w=https://sourceforge.net/projects/swig/files/swig/swig-$(swig-version) - elif [ $$n = tides ]; then c=$(tides-checksum); w=http://akhlaghi.org/maneage-software - elif [ $$n = tiff ]; then c=$(libtiff-checksum); w=https://download.osgeo.org/libtiff - elif [ $$n = valgrind ]; then c=$(valgrind-checksum); w=https://sourceware.org/pub/valgrind - elif [ $$n = wcslib ]; then c=$(wcslib-checksum); w=ftp://ftp.atnf.csiro.au/pub/software/wcslib - elif [ $$n = xlsxio ]; then - mergenames=0 - c=$(xlsxio-checksum); - w=https://github.com/brechtsanders/xlsxio/archive/$(xlsxio-version).tar.gz - elif [ $$n = yaml ]; then c=$(yaml-checksum); w=pyyaml.org/download/libyaml - elif [ $$n = zlib ]; then c=$(zlib-checksum); w=https://zlib.net - else - echo; echo; echo; - echo "'$$n' not recognized as a software tarball name to download." - echo; echo; echo; - exit 1 - fi - - # Download the requested tarball. Note that some packages may not - # follow our naming convention (where the package name is merged - # with its version number). In such cases, `w' will be the full - # address, not just the top directory address. But since we are - # storing all the tarballs in one directory, we want it to have the - # same naming convention, so we'll download it to a temporary name, - # then rename that. - rm -f "$@.unchecked" - if [ -f $(DEPENDENCIES-DIR)/$* ]; then - cp $(DEPENDENCIES-DIR)/$* "$@.unchecked" - else - if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*" - else tarballurl=$$w - fi - - # Download using the script specially defined for this job. - touch $(lockdir)/download - downloader="wget --no-use-server-timestamps -O" - $(downloadwrapper) "$$downloader" $(lockdir)/download \ - $$tarballurl "$@.unchecked" "$(backupservers)" - fi - - # Make sure this is the expected tarball. Note that we now have a - # controlled `sha512sum' build (as part of GNU Coreutils). So we - # don't need to check its existance like `basic.mk'. But for LaTeX, - # we need to ignore a checksum (it downloads the binaries). - if [ x"$$c" == x"NO-CHECK-SUM" ]; then - mv "$@.unchecked" "$@" - else - checksum=$$(sha512sum "$@.unchecked" | awk '{print $$1}') - if [ x"$$checksum" = x"$$c" ]; then - mv "$@.unchecked" "$@" - else - echo "ERROR: Non-matching checksum for '$*'." - echo "Checksum should be: $$c" - echo "Checksum is: $$checksum" - exit 1 - fi - fi - - - - - @@ -409,39 +233,50 @@ $(tarballs): $(tdir)/%: | $(lockdir) # build it because it will complain about the version of libtool, so until # the version 0.11.0 of log4cxx, we'll have to run `autogen.sh' on the # unpacked source also. -$(ibidir)/apachelog4cxx: $(ibidir)/automake \ - $(tdir)/apachelog4cxx-$(apachelog4cxx-version).tar.lz - +$(ibidir)/apachelog4cxx-$(apachelog4cxx-version): \ + $(ibidir)/expat-$(expat-version) \ + $(ibidir)/apr-util-$(apr-util-version) \ + $(ibidir)/automake-$(automake-version) + tarball=apachelog4cxx-$(apachelog4cxx-version).tar.lz + $(call import-source, $(apachelog4cxx-url), $(apachelog4cxx-checksum)) pdir=apachelog4cxx-$(apachelog4cxx-version) rm -rf $(ddir)/$$pdir topdir=$(pwd) cd $(ddir) - tar xf $(word 1,$(filter $(tdir)/%,$^)) + tar xf $(tdir)/$$tarball cd $$pdir - ./autogen.sh \ - && ./configure SHELL=$(ibdir)/bash --prefix=$(idir) \ - && make -j$(numthreads) SHELL=$(ibdir)/bash \ - && make install \ - && cd .. \ - && rm -rf $$pdir \ - && cd $$topdir \ - && echo "Apache log4cxx $(apachelog4cxx-version)" > $@ - -$(ibidir)/apr: $(tdir)/apr-$(apr-version).tar.gz - $(call gbuild, apr-$(apr-version), ,--disable-static) \ - && echo "Apache Portable Runtime $(apr-version)" > $@ - -$(ibidir)/apr-util: $(ibidir)/apr \ - $(tdir)/apr-util-$(apr-util-version).tar.gz + ./autogen.sh + ./configure SHELL=$(ibdir)/bash --prefix=$(idir) + make -j$(numthreads) SHELL=$(ibdir)/bash + make install + cd .. + rm -rf $$pdir + cd $$topdir + echo "Apache log4cxx $(apachelog4cxx-version)" > $@ + +$(ibidir)/apr-$(apr-version): + tarball=apr-$(apr-version).tar.gz + $(call import-source, $(apr-url), $(apr-checksum)) + $(call gbuild, apr-$(apr-version), ,--disable-static) + echo "Apache Portable Runtime $(apr-version)" > $@ + +$(ibidir)/apr-util-$(apr-util-version): $(ibidir)/apr-$(apr-version) + tarball=apr-util-$(apr-util-version).tar.gz + $(call import-source, $(apr-util-url), $(apr-util-checksum)) $(call gbuild, apr-util-$(apr-util-version), , \ --disable-static \ --with-apr=$(idir) \ --with-openssl=$(idir) \ - --with-crypto ) \ - && echo "Apache Portable Runtime Utility $(apr-util-version)" > $@ + --with-crypto ) + echo "Apache Portable Runtime Utility $(apr-util-version)" > $@ + +$(ibidir)/atlas-$(atlas-version): -$(ibidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ - $(tdir)/lapack-$(lapack-version).tar.gz + tarball=lapack-$(lapack-version).tar.gz + $(call import-source, $(lapack-url), $(lapack-checksum)) + + tarball=atlas-$(atlas-version).tar.bz2 + $(call import-source, $(atlas-url), $(atlas-checksum)) # Get the operating system specific features (how to get # CPU frequency and the library suffixes). To make the steps @@ -470,39 +305,46 @@ $(ibidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ # See if the shared libraries should be build for a single CPU # thread or multiple threads. N=$$(nproc) - srcdir=$$(pwd)/reproduce/src/make + srcdir=$$(pwd)/reproduce/software/make if [ $$N = 1 ]; then - sharedmk=$$srcdir/dependencies-atlas-single.mk + sharedmk=$$srcdir/atlas-single.mk else - sharedmk=$$srcdir/dependencies-atlas-multiple.mk + sharedmk=$$srcdir/atlas-multiple.mk fi # The linking step here doesn't recognize the `-Wl' in the # `rpath_command'. export LDFLAGS=-L$(ildir) - cd $(ddir) \ - && tar xf $(tdir)/atlas-$(atlas-version).tar.bz2 \ - && cd ATLAS \ - && rm -rf build \ - && mkdir build \ - && cd build \ - && ../configure -b 64 -D c -DPentiumCPS=$$core \ + cd $(ddir) + tar xf $(tdir)/atlas-$(atlas-version).tar.bz2 + cd ATLAS + rm -rf build + mkdir build + cd build + ../configure -b 64 -D c -DPentiumCPS=$$core \ --with-netlib-lapack-tarfile=$(tdir)/lapack-$(lapack-version).tar.gz \ --cripple-atlas-performance \ -Fa alg -fPIC --shared $$clangflag \ - --prefix=$(idir) \ - && make \ - && if [ "x$(on_mac_os)" != xyes ]; then \ - cd lib && make -f $$sharedmk && cd .. \ - && for l in lib/*.$$s*; do \ - patchelf --set-rpath $(ildir) $$l; done \ - && cp -d lib/*.$$s* $(ildir) \ - && ln -fs $(ildir)/libblas.$$s $(ildir)/libblas.$$m \ - && ln -fs $(ildir)/libf77blas.$$s $(ildir)/libf77blas.$$m \ - && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$s \ - && ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$m; \ - fi \ - && make install + --prefix=$(idir) + + # Static build. + make + + # Currently the shared libraries have problems on macOS. + if [ "x$(on_mac_os)" != xyes ]; then + cd lib + make -f $$sharedmk + cd .. + for l in lib/*.$$s*; do patchelf --set-rpath $(ildir) $$l; done + cp -d lib/*.$$s* $(ildir) + ln -fs $(ildir)/libblas.$$s $(ildir)/libblas.$$m + ln -fs $(ildir)/libf77blas.$$s $(ildir)/libf77blas.$$m + ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$s + ln -fs $(ildir)/liblapack.$$f $(ildir)/liblapack.$$m + fi + + # Install the libraries. + make install # We need to check the existance of `libptlapack.a', but we can't # do this in the `&&' steps above (it will conflict). So we'll do @@ -521,31 +363,37 @@ $(ibidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ fi # Boost doesn't use the standard GNU Build System. -$(ibidir)/boost: $(ibidir)/openmpi \ - $(ibidir)/python \ - $(tdir)/boost-$(boost-version).tar.gz - vstr=$$(echo $(boost-version) | sed -e's/\./_/g') - rm -rf $(ddir)/boost_$$vstr - topdir=$(pwd); cd $(ddir); - tar xf $(word 1,$(filter $(tdir)/%,$^)) \ - && cd boost_$$vstr \ - && ./bootstrap.sh --prefix=$(idir) --with-libraries=all \ - --with-python=python3 \ - && echo "using mpi ;" > project-config.jam \ - && ./b2 stage threading=multi link=shared --prefix=$(idir) -j$(numthreads) \ - && ./b2 install threading=multi link=shared --prefix=$(idir) -j$(numthreads) \ - && cd $$topdir \ - && rm -rf $(ddir)/boost_$$vstr \ - && echo "Boost $(boost-version)" > $@ - -$(ibidir)/cfitsio: $(ibidir)/curl \ - $(tdir)/cfitsio-$(cfitsio-version).tar.gz +$(ibidir)/boost-$(boost-version): \ + $(ibidir)/python-$(python-version) \ + $(ibidir)/openmpi-$(openmpi-version) + tarball=boost-$(boost-version).tar.lz + $(call import-source, $(boost-url), $(boost-checksum)) + unpackdir=boost-$(boost-version) + rm -rf $(ddir)/$$unpackdir + topdir=$(pwd) + cd $(ddir) + tar xf $(tdir)/$$tarball + cd $$unpackdir + ./bootstrap.sh --prefix=$(idir) --with-libraries=all \ + --with-python=python3 + echo "using mpi ;" > project-config.jam + ./b2 stage threading=multi link=shared --prefix=$(idir) -j$(numthreads) + ./b2 install threading=multi link=shared --prefix=$(idir) -j$(numthreads) + cd $$topdir + rm -rf $(ddir)/$$unpackdir + echo "Boost $(boost-version)" > $@ + +$(ibidir)/cfitsio-$(cfitsio-version): + + # Download the tarball + tarball=cfitsio-$(cfitsio-version).tar.gz + $(call import-source, $(cfitsio-url), $(cfitsio-checksum)) # CFITSIO hard-codes '@rpath' inside the shared library on # Mac systems. So we need to change it to our library # installation path. It doesn't affect GNU/Linux, so we'll # just do it in any case to keep things clean. - topdir=$(pwd); cd $(ddir); tar xf $(word 1,$(filter $(tdir)/%,$^)) + topdir=$(pwd); cd $(ddir); tar xf $(tdir)/$$tarball customtar=cfitsio-$(cfitsio-version)-custom.tar.gz cd cfitsio-$(cfitsio-version) sed configure -e's|@rpath|$(ildir)|g' > configure_tmp @@ -558,68 +406,82 @@ $(ibidir)/cfitsio: $(ibidir)/curl \ # Continue the standard build on the customized tarball. Note that # with the installation of CFITSIO, `fpack' and `funpack' are not # installed by default. Because of that, they are added explicity. - export gbuild_tar=$$customtar + export gbuild_tar=$(ddir)/$$customtar $(call gbuild, cfitsio-$(cfitsio-version), , \ --enable-sse2 --enable-reentrant \ - --with-bzip2=$(idir), , make shared fpack funpack) \ - && rm $$customtar \ - && echo "CFITSIO $(cfitsio-version)" > $@ - -$(ibidir)/cairo: $(ibidir)/freetype \ - $(ibidir)/libpng \ - $(ibidir)/pixman \ - $(tdir)/cairo-$(cairo-version).tar.xz + --with-bzip2=$(idir), , make shared fpack funpack) + rm $$customtar + echo "CFITSIO $(cfitsio-version)" > $@ + +$(ibidir)/cairo-$(cairo-version): \ + $(ibidir)/pixman-$(pixman-version) \ + $(ibidir)/libpng-$(libpng-version) \ + $(ibidir)/freetype-$(freetype-version) + tarball=cairo-$(cairo-version).tar.xz + $(call import-source, $(cairo-url), $(cairo-checksum)) $(call gbuild, cairo-$(cairo-version), static, \ - --with-x=no, -j$(numthreads) V=1) \ - && echo "Cairo $(cairo-version)" > $@ + --with-x=yes, -j$(numthreads) V=1) + echo "Cairo $(cairo-version)" > $@ # Eigen is just headers! So it doesn't need to be compiled. Once unpacked # it has a checksum after `eigen-eigen', so we'll just use a `*' to choose # the unpacked directory. -$(ibidir)/eigen: $(tdir)/eigen-$(eigen-version).tar.gz +$(ibidir)/eigen-$(eigen-version): + tarball=eigen-$(eigen-version).tar.gz + $(call import-source, $(eigen-url), $(eigen-checksum)) rm -rf $(ddir)/eigen-eigen-* - topdir=$(pwd); cd $(ddir); tar xf $(word 1,$(filter $(tdir)/%,$^)) + topdir=$(pwd); cd $(ddir); tar xf $(tdir)/$$tarball cd eigen-eigen-* - cp -r Eigen $(iidir)/eigen3 \ - && cd $$topdir \ - && rm -rf $(ddir)/eigen-eigen-* \ - && echo "Eigen $(eigen-version)" > $@ + cp -r Eigen $(iidir)/eigen3 + cd $$topdir + rm -rf $(ddir)/eigen-eigen-* + echo "Eigen $(eigen-version)" > $@ + +$(ibidir)/expat-$(expat-version): + tarball=expat-$(expat-version).tar.lz + $(call import-source, $(expat-url), $(expat-checksum)) + $(call gbuild, expat-$(expat-version), static) + echo "Expat $(expat-version)" > $@ -$(ibidir)/expat: $(tdir)/expat-$(expat-version).tar.lz - $(call gbuild, expat-$(expat-version), static) \ - && echo "Expat $(expat-version)" > $@ +$(ibidir)/fftw-$(fftw-version): + # Prepare the source tarball. + tarball=fftw-$(fftw-version).tar.gz + $(call import-source, $(fftw-url), $(fftw-checksum)) -$(ibidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz # FFTW's single and double precission libraries must be built # independently: for the the single-precision library, we need to # add the `--enable-float' option. We will build this first, then # the default double-precision library. confop="--enable-shared --enable-threads --enable-avx --enable-sse2" $(call gbuild, fftw-$(fftw-version), static, \ - $$confop --enable-float) \ - && $(call gbuild, fftw-$(fftw-version), static, \ - $$confop) \ - && cp $(dtexdir)/fftw.tex $(ictdir)/ \ - && echo "FFTW $(fftw-version) \citep{fftw}" > $@ - -# Freetype is necessary to install matplotlib -$(ibidir)/freetype: $(ibidir)/libpng \ - $(tdir)/freetype-$(freetype-version).tar.gz - $(call gbuild, freetype-$(freetype-version), static) \ - && echo "FreeType $(freetype-version)" > $@ - -$(ibidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz - $(call gbuild, gsl-$(gsl-version), static) \ - && echo "GNU Scientific Library $(gsl-version)" > $@ - -$(ibidir)/hdf5: $(ibidir)/openmpi \ - $(tdir)/hdf5-$(hdf5-version).tar.gz - export CC=mpicc; \ - export FC=mpif90; \ + $$confop --enable-float) + $(call gbuild, fftw-$(fftw-version), static, \ + $$confop) + cp $(dtexdir)/fftw.tex $(ictdir)/ + echo "FFTW $(fftw-version) \citep{fftw}" > $@ + +$(ibidir)/freetype-$(freetype-version): $(ibidir)/libpng-$(libpng-version) + tarball=freetype-$(freetype-version).tar.gz + $(call import-source, $(freetype-url), $(freetype-checksum)) + $(call gbuild, freetype-$(freetype-version), static) + echo "FreeType $(freetype-version)" > $@ + +$(ibidir)/gsl-$(gsl-version): + tarball=gsl-$(gsl-version).tar.gz + $(call import-source, $(gsl-url), $(gsl-checksum)) + $(call gbuild, gsl-$(gsl-version), static) + echo "GNU Scientific Library $(gsl-version)" > $@ + +$(ibidir)/hdf5-$(hdf5-version): $(ibidir)/openmpi-$(openmpi-version) + export CC=mpicc + export FC=mpif90 + tarball=hdf5-$(hdf5-version).tar.gz + $(call import-source, $(hdf5-url), $(hdf5-checksum)) $(call gbuild, hdf5-$(hdf5-version), static, \ --enable-parallel \ - --enable-fortran, -j$(numthreads) V=1) \ - && echo "HDF5 library $(hdf5-version)" > $@ + --enable-fortran, \ + -j$(numthreads) V=1) + echo "HDF5 library $(hdf5-version)" > $@ # HEALPix includes the source of its C, C++, Python (and several other # languages) libraries within one tarball. We will include the Python @@ -634,13 +496,15 @@ $(ibidir)/hdf5: $(ibidir)/openmpi \ ifeq ($(strip $(top-level-python)),) healpix-python-dep = else -healpix-python-dep = $(ipydir)/matplotlib $(ipydir)/astropy +healpix-python-dep = $(ipydir)/matplotlib-$(matplotlib-version) \ + $(ipydir)/astropy-$(astropy-version) endif -$(ibidir)/healpix: $(ibidir)/cfitsio \ - $(ibidir)/autoconf \ - $(ibidir)/automake \ - $(healpix-python-dep) \ - $(tdir)/healpix-$(healpix-version).tar.gz +$(ibidir)/healpix-$(healpix-version): $(healpix-python-dep) \ + $(ibidir)/cfitsio-$(cfitsio-version) \ + $(ibidir)/autoconf-$(autoconf-version) \ + $(ibidir)/automake-$(automake-version) + tarball=healpix-$(healpix-version).tar.gz + $(call import-source, $(healpix-url), $(healpix-checksum)) if [ x"$(healpix-python-dep)" = x ]; then pycommand1="echo no-healpy-because-no-other-python" pycommand2="echo no-healpy-because-no-other-python" @@ -650,130 +514,194 @@ $(ibidir)/healpix: $(ibidir)/cfitsio \ fi rm -rf $(ddir)/Healpix_$(healpix-version) topdir=$(pwd); cd $(ddir); - tar xf $(word 1,$(filter $(tdir)/%,$^)) - && cd Healpix_$(healpix-version)/src/C/autotools/ \ - && autoreconf --install \ - && ./configure --prefix=$(idir) \ - && make V=1 -j$(numthreads) SHELL=$(ibdir)/bash \ - && make install \ - && cd ../../cxx/autotools/ \ - && autoreconf --install \ - && ./configure --prefix=$(idir) \ - && make V=1 -j$(numthreads) SHELL=$(ibdir)/bash \ - && make install \ - && cd ../../healpy \ - && $$pycommand1 \ - && $$pycommand2 \ - && cd $$topdir \ - && rm -rf $(ddir)/Healpix_$(healpix-version) \ - && cp $(dtexdir)/healpix.tex $(ictdir)/ \ - && echo "HEALPix $(healpix-version) \citep{healpix}" > $@ - -$(ibidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz - $(call gbuild, jpeg-9b, static,,V=1) \ - && echo "Libjpeg $(libjpeg-version)" > $@ - -$(ibidir)/libnsl: $(ibidir)/libtirpc \ - $(ibidir)/rpcsvc-proto \ - $(tdir)/libnsl-$(libnsl-version).tar.gz + tar xf $(tdir)/$$tarball + cd Healpix_$(healpix-version)/src/C/autotools/ + autoreconf --install + ./configure --prefix=$(idir) + make V=1 -j$(numthreads) SHELL=$(ibdir)/bash + make install + cd ../../cxx/autotools/ + autoreconf --install + ./configure --prefix=$(idir) + make V=1 -j$(numthreads) SHELL=$(ibdir)/bash + make install + cd ../../healpy + $$pycommand1 + $$pycommand2 + cd $$topdir + rm -rf $(ddir)/Healpix_$(healpix-version) + cp $(dtexdir)/healpix.tex $(ictdir)/ + echo "HEALPix $(healpix-version) \citep{healpix}" > $@ + +$(ibidir)/libjpeg-$(libjpeg-version): + tarball=jpegsrc.$(libjpeg-version).tar.gz + $(call import-source, $(libjpeg-url), $(libjpeg-checksum)) + $(call gbuild, jpeg-9b, static,,V=1) + echo "Libjpeg $(libjpeg-version)" > $@ + +$(ibidir)/libnsl-$(libnsl-version): \ + $(ibidir)/libtirpc-$(libtirpc-version) \ + $(ibidir)/rpcsvc-proto-$(rpcsvc-proto-version) + tarball=libnsl-$(libnsl-version).tar.gz + $(call import-source, $(libnsl-url), $(libnsl-checksum)) $(call gbuild, libnsl-$(libnsl-version), static, \ - --sysconfdir=$(idir)/etc) \ - && echo "Libnsl $(libnsl-version)" > $@ + --sysconfdir=$(idir)/etc) + echo "Libnsl $(libnsl-version)" > $@ + +$(ibidir)/libpaper-$(libpaper-version): \ + $(ibidir)/automake-$(automake-version) -$(ibidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz - $(call gbuild, libpng-$(libpng-version), static) \ - && echo "Libpng $(libpng-version)" > $@ + # Download the tarball. + tarball=libpaper-$(libpaper-version).tar.gz + $(call import-source, $(libpaper-url), $(libpaper-checksum)) -$(ibidir)/libtiff: $(ibidir)/libjpeg \ - $(tdir)/tiff-$(libtiff-version).tar.gz + # Unpack, build the configure system, build and install. + cd $(ddir) + tar -xf $(tdir)/$$tarball + unpackdir=libpaper-$(libpaper-version) + cd $$unpackdir + autoreconf -fi + ./configure --prefix=$(idir) --sysconfdir=$(idir)/etc \ + --disable-static + make + make install + cd .. + rm -rf $$unpackdir + + # Post-processing: according to Linux From Scratch, libpaper + # expects that packages will install files into this directory and + # 'paperconfig' is a script which will invoke 'run-parts' if + # '/etc/libpaper.d' exists + mkdir -vp $(idir)/etc/libpaper.d + sed -e's|MANEAGESHELL|$(SHELL)|' $(shsrcdir)/run-parts.in \ + > $(ibdir)/run-parts + chmod +x $(ibdir)/run-parts + echo "Libpaper $(libpaper-version)" > $@ + +$(ibidir)/libpng-$(libpng-version): + tarball=libpng-$(libpng-version).tar.xz + $(call import-source, $(libpng-url), $(libpng-checksum)) + $(call gbuild, libpng-$(libpng-version), static) + echo "Libpng $(libpng-version)" > $@ + +$(ibidir)/libtiff-$(libtiff-version): $(ibidir)/libjpeg-$(libjpeg-version) + tarball=tiff-$(libtiff-version).tar.gz + $(call import-source, $(libtiff-url), $(libtiff-checksum)) $(call gbuild, tiff-$(libtiff-version), static, \ --disable-jbig \ --disable-webp \ - --disable-zstd) \ - && echo "Libtiff $(libtiff-version)" > $@ + --disable-zstd) + echo "Libtiff $(libtiff-version)" > $@ -$(ibidir)/libtirpc: $(tdir)/libtirpc-$(libtirpc-version).tar.bz2 +$(ibidir)/libtirpc-$(libtirpc-version): + tarball=libtirpc-$(libtirpc-version).tar.bz2 + $(call import-source, $(libtirpc-url), $(libtirpc-checksum)) $(call gbuild, libtirpc-$(libtirpc-version), static, \ - --disable-gssapi, V=1) \ + --disable-gssapi, V=1) echo "libtirpc $(libtirpc-version)" > $@ -$(ibidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz - if [ x$(on_mac_os) = xyes ]; then \ - export CC=clang; \ - fi; \ - cd $(ddir) \ - && tar xf $(word 1,$(filter $(tdir)/%,$^)) \ - && cd OpenBLAS-$(openblas-version) \ - && make \ - && make PREFIX=$(idir) install \ - && cd .. \ - && rm -rf OpenBLAS-$(openblas-version) \ - && echo "OpenBLAS $(openblas-version)" > $@ - -$(ibidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz +$(ibidir)/openblas-$(openblas-version): + tarball=OpenBLAS-$(openblas-version).tar.gz + $(call import-source, $(openblas-url), $(openblas-checksum)) + if [ x$(on_mac_os) = xyes ]; then export CC=clang; fi + cd $(ddir) + tar xf $(tdir)/$$tarball + cd OpenBLAS-$(openblas-version) + make -j$(numthreads) + make PREFIX=$(idir) install + cd .. + rm -rf OpenBLAS-$(openblas-version) + echo "OpenBLAS $(openblas-version)" > $@ + +$(ibidir)/openmpi-$(openmpi-version): + tarball=openmpi-$(openmpi-version).tar.gz + $(call import-source, $(openmpi-url), $(openmpi-checksum)) $(call gbuild, openmpi-$(openmpi-version), static, , \ - -j$(numthreads) V=1) \ - && echo "Open MPI $(openmpi-version)" > $@ + -j$(numthreads) V=1) + echo "Open MPI $(openmpi-version)" > $@ # IMPORTANT NOTE: The build instructions for OpenSSH are defined here, but # it is best that it not be prerequisite of any program and thus not built # within the project because of all the security issues it may cause. Only # enable/build it in a project with caution, and if there is no other # solution (for example to disable SSH in a program that may ask for it. -$(ibidir)/openssh: $(tdir)/openssh-$(openssh-version).tar.gz +$(ibidir)/openssh-$(openssh-version): + tarball=openssh-$(openssh-version).tar.gz + $(call import-source, $(openssh-url), $(openssh-checksum)) $(call gbuild, openssh-$(openssh-version), static, \ --with-privsep-path=$(ibdir)/.ssh_privsep \ --with-privsep-user=nobody \ --with-md5-passwords \ --with-ssl-engine \ - , -j$(numthreads) V=1) \ - && echo "OpenSSH $(openssh-version)" > $@ + , -j$(numthreads) V=1) + echo "OpenSSH $(openssh-version)" > $@ -$(ibidir)/pixman: $(tdir)/pixman-$(pixman-version).tar.gz +$(ibidir)/pixman-$(pixman-version): + tarball=pixman-$(pixman-version).tar.gz + $(call import-source, $(pixman-url), $(pixman-checksum)) $(call gbuild, pixman-$(pixman-version), static, , \ - -j$(numthreads) V=1) \ - && echo "Pixman $(pixman-version)" > $@ + -j$(numthreads) V=1) + echo "Pixman $(pixman-version)" > $@ -$(ibidir)/rpcsvc-proto: $(tdir)/rpcsvc-proto-$(rpcsvc-proto-version).tar.xz - $(call gbuild, rpcsvc-proto-$(rpcsvc-proto-version), static) \ - && echo "rpcsvc $(rpcsvc-proto-version)" > $@ +$(ibidir)/rpcsvc-proto-$(rpcsvc-proto-version): + # 'libintl' is installed as part of GNU Gettext in + # 'basic.mk'. rpcsvc-proto needs to link with it on macOS. + if [ x$(on_mac_os) = xyes ]; then + export CC=clang + export CXX=clang++ + export LDFLAGS="-lintl $$LDFLAGS" + fi -$(ibidir)/tides: $(tdir)/tides-$(tides-version).tar.gz + # Download the tarball and build rpcsvc-proto. + tarball=rpcsvc-proto-$(rpcsvc-proto-version).tar.xz + $(call import-source, $(rpcsvc-proto-url), $(rpcsvc-proto-checksum)) + $(call gbuild, rpcsvc-proto-$(rpcsvc-proto-version), static) + echo "rpcsvc $(rpcsvc-proto-version)" > $@ + +$(ibidir)/tides-$(tides-version): + tarball=tides-$(tides-version).tar.gz + $(call import-source, $(tides-url), $(tides-checksum)) $(call gbuild, tides-$(tides-version), static,\ - --with-gmp=$(idir) --with-mpfr=$(idir)) \ - && cp $(dtexdir)/tides.tex $(ictdir)/ \ - && echo "TIDES $(tides-version) \citep{tides}" > $@ - -$(ibidir)/valgrind: $(ibidir)/patch \ - $(ibidir)/autoconf \ - $(ibidir)/automake \ - $(tdir)/valgrind-$(valgrind-version).tar.bz2 + --with-gmp=$(idir) --with-mpfr=$(idir)) + cp $(dtexdir)/tides.tex $(ictdir)/ + echo "TIDES $(tides-version) \citep{tides}" > $@ + +$(ibidir)/valgrind-$(valgrind-version): \ + $(ibidir)/patch-$(patch-version) \ + $(ibidir)/autoconf-$(autoconf-version) \ + $(ibidir)/automake-$(automake-version) + # Import the tarball + tarball=valgrind-$(valgrind-version).tar.bz2 + $(call import-source, $(valgrind-url), $(valgrind-checksum)) + # For valgrind-3.15.0, see # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=946329 for a # report on an MPI-related compile bug and the two patches # below. These two patches and `automake` should allow valgrind to # compile with gcc-9.2.0. - cd $(ddir) \ - && tar -x -f $(word 1,$(filter $(tdir)/%,$^)) \ - && valgrinddir=valgrind-$(valgrind-version) \ - && cd $${valgrinddir} \ - && printf "valgrindir=$${valgrinddir} ; pwd = %s .\n" $$($(ibdir)/pwd) \ - && if [ "x$(valgrind-version)" = "x3.15.0" ]; then \ - patch --verbose -p1 < $(patchdir)/valgrind-3.15.0-mpi-fix1.patch; \ - patch --verbose -p1 < $(patchdir)/valgrind-3.15.0-mpi-fix2.patch; \ - fi \ - && autoreconf \ - && ./configure --prefix=$(idir) \ - && make -j$(numthreads) \ - && if ! make check -j$(numthreads); then \ - echo; echo "Valgrind's 'make check' failed!"; echo; \ - fi \ - && make install \ - && echo "Valgrind $(valgrind-version)" > $@ + cd $(ddir) + tar -xf $(tdir)/$$tarball + valgrinddir=valgrind-$(valgrind-version) + cd $${valgrinddir} + printf "valgrindir=$${valgrinddir} ; pwd = %s .\n" $$($(ibdir)/pwd) + if [ "x$(valgrind-version)" = "x3.15.0" ]; then + patch --verbose -p1 < $(patchdir)/valgrind-3.15.0-mpi-fix1.patch + patch --verbose -p1 < $(patchdir)/valgrind-3.15.0-mpi-fix2.patch + fi + autoreconf + ./configure --prefix=$(idir) + make -j$(numthreads) + if ! make check -j$(numthreads); then + echo; echo "Valgrind's 'make check' failed!"; echo + fi + make install + echo "Valgrind $(valgrind-version)" > $@ -$(ibidir)/yaml: $(tdir)/yaml-$(yaml-version).tar.gz - $(call gbuild, yaml-$(yaml-version), static) \ - && echo "LibYAML $(yaml-version)" > $@ +$(ibidir)/yaml-$(yaml-version): + tarball=yaml-$(yaml-version).tar.gz + $(call import-source, $(yaml-url), $(yaml-checksum)) + $(call gbuild, yaml-$(yaml-version), static) + echo "LibYAML $(yaml-version)" > $@ @@ -800,20 +728,23 @@ $(ibidir)/yaml: $(tdir)/yaml-$(yaml-version).tar.gz # functions! But apparently `libgit2' has its own implementation of libiconv # that it uses if it can't find libiconv on macOS. So, to fix this problem # it is necessary to use the option `-DUSE_ICONV=OFF` in the configure step. -$(ibidir)/libgit2: $(ibidir)/curl \ - $(ibidir)/cmake \ - $(tdir)/libgit2-$(libgit2-version).tar.gz +$(ibidir)/libgit2-$(libgit2-version): $(ibidir)/cmake-$(cmake-version) + tarball=libgit2-$(libgit2-version).tar.gz + $(call import-source, $(libgit2-url), $(libgit2-checksum)) $(call cbuild, libgit2-$(libgit2-version), static, \ -DUSE_SSH=OFF -DBUILD_CLAR=OFF \ - -DTHREADSAFE=ON -DUSE_ICONV=OFF ) \ - && if [ x$(on_mac_os) = xyes ]; then \ - install_name_tool -id $(ildir)/libgit2.28.dylib \ - $(ildir)/libgit2.28.dylib; \ - fi \ - && echo "Libgit2 $(libgit2-version)" > $@ - -$(ibidir)/wcslib: $(ibidir)/cfitsio \ - $(tdir)/wcslib-$(wcslib-version).tar.bz2 + -DTHREADSAFE=ON -DUSE_ICONV=OFF ) + if [ x$(on_mac_os) = xyes ]; then + install_name_tool -id $(ildir)/libgit2.28.dylib \ + $(ildir)/libgit2.28.dylib + fi + echo "Libgit2 $(libgit2-version)" > $@ + +$(ibidir)/wcslib-$(wcslib-version): $(ibidir)/cfitsio-$(cfitsio-version) + # Import the tarball. + tarball=wcslib-$(wcslib-version).tar.bz2 + $(call import-source, $(wcslib-url), $(wcslib-checksum)) + # If Fortran isn't present, don't build WCSLIB with it. if type gfortran &> /dev/null; then fortranopt=""; else fortranopt="--disable-fortran" @@ -824,12 +755,12 @@ $(ibidir)/wcslib: $(ibidir)/cfitsio \ LIBS="-pthread -lcurl -lm" \ --with-cfitsiolib=$(ildir) \ --with-cfitsioinc=$(idir)/include \ - --without-pgplot $$fortranopt) \ - && if [ x$(on_mac_os) = xyes ]; then \ - install_name_tool -id $(ildir)/libwcs.6.4.dylib \ - $(ildir)/libwcs.6.4.dylib; \ - fi \ - && echo "WCSLIB $(wcslib-version)" > $@ + --without-pgplot $$fortranopt) + if [ x$(on_mac_os) = xyes ]; then + install_name_tool -id $(ildir)/libwcs.6.4.dylib \ + $(ildir)/libwcs.6.4.dylib + fi + echo "WCSLIB $(wcslib-version)" > $@ @@ -846,51 +777,58 @@ $(ibidir)/wcslib: $(ibidir)/cfitsio \ # Astrometry-net contains a lot of programs. We need to specify the # installation directory and the Python executable (by default it will look # for /usr/bin/python) -$(ibidir)/astrometrynet: $(ibidir)/gsl \ - $(ibidir)/swig \ - $(ipydir)/numpy \ - $(ibidir)/cairo \ - $(ibidir)/libpng \ - $(ibidir)/netpbm \ - $(ibidir)/wcslib \ - $(ibidir)/cfitsio \ - $(ibidir)/libjpeg \ - $(tdir)/astrometry.net-$(astrometrynet-version).tar.gz +$(ibidir)/astrometrynet-$(astrometrynet-version): \ + $(ibidir)/gsl-$(gsl-version) \ + $(ibidir)/swig-$(swig-version) \ + $(ipydir)/numpy-$(numpy-version) \ + $(ibidir)/cairo-$(cairo-version) \ + $(ibidir)/libpng-$(libpng-version) \ + $(ibidir)/netpbm-$(netpbm-version) \ + $(ibidir)/wcslib-$(wcslib-version) \ + $(ibidir)/cfitsio-$(cfitsio-version) \ + $(ibidir)/libjpeg-$(libjpeg-version) + + # Import the tarball + tarball=astrometry.net-$(astrometrynet-version).tar.gz + $(call import-source, $(astrometrynet-url), $(astrometrynet-checksum)) + # We are modifying the Makefile in two steps because on Mac OS # system we do not have `/proc/cpuinfo' nor `free'. Since this is # only for the `report.txt', this changes do not causes problems in # running `astrometrynet' - cd $(ddir) \ - && rm -rf astrometry.net-$(astrometrynet-version) \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)); then \ - echo; echo "Tar error"; exit 1; \ - fi \ - && cd astrometry.net-$(astrometrynet-version) \ - && sed -e 's|cat /proc/cpuinfo|echo "Ignoring CPU info"|' \ - -e 's|-free|echo "Ignoring RAM info"|' Makefile > Makefile.tmp \ - && mv Makefile.tmp Makefile \ - && make \ - && make py \ - && make extra \ - && make install INSTALL_DIR=$(idir) PYTHON_SCRIPT="$(ibdir)/python" \ - && cd .. \ - && rm -rf astrometry.net-$(astrometrynet-version) \ - && cp $(dtexdir)/astrometrynet.tex $(ictdir)/ \ - && echo "Astrometry.net $(astrometrynet-version) \citep{astrometrynet}" > $@ - -$(ibidir)/autoconf: $(tdir)/autoconf-$(autoconf-version).tar.lz - $(call gbuild, autoconf-$(autoconf-version), static, ,V=1) \ - && echo "GNU Autoconf $(autoconf-version)" > $@ - -$(ibidir)/automake: $(ibidir)/autoconf \ - $(tdir)/automake-$(automake-version).tar.gz - $(call gbuild, automake-$(automake-version), static, ,V=1) \ - && echo "GNU Automake $(automake-version)" > $@ - -$(ibidir)/bison: $(ibidir)/help2man \ - $(tdir)/bison-$(bison-version).tar.xz - $(call gbuild, bison-$(bison-version), static, ,V=1) \ - && echo "GNU Bison $(bison-version)" > $@ + cd $(ddir) + rm -rf astrometry.net-$(astrometrynet-version) + tar xf $(tdir)/$$tarball + cd astrometry.net-$(astrometrynet-version) + sed -e 's|cat /proc/cpuinfo|echo "Ignoring CPU info"|' \ + -e 's|-free|echo "Ignoring RAM info"|' Makefile > Makefile.tmp + mv Makefile.tmp Makefile + make + make py + make extra + make install INSTALL_DIR=$(idir) PYTHON_SCRIPT="$(ibdir)/python" + cd .. + rm -rf astrometry.net-$(astrometrynet-version) + cp $(dtexdir)/astrometrynet.tex $(ictdir)/ + echo "Astrometry.net $(astrometrynet-version) \citep{astrometrynet}" > $@ + +$(ibidir)/autoconf-$(autoconf-version): + tarball=autoconf-$(autoconf-version).tar.lz + $(call import-source, $(autoconf-url), $(autoconf-checksum)) + $(call gbuild, autoconf-$(autoconf-version), static, ,V=1) + echo "GNU Autoconf $(autoconf-version)" > $@ + +$(ibidir)/automake-$(automake-version): $(ibidir)/autoconf-$(autoconf-version) + tarball=automake-$(automake-version).tar.gz + $(call import-source, $(automake-url), $(automake-checksum)) + $(call gbuild, automake-$(automake-version), static, ,V=1) + echo "GNU Automake $(automake-version)" > $@ + +$(ibidir)/bison-$(bison-version): $(ibidir)/help2man-$(help2man-version) + tarball=bison-$(bison-version).tar.xz + $(call import-source, $(bison-url), $(bison-checksum)) + $(call gbuild, bison-$(bison-version), static, ,V=1 -j$(numthreads)) + echo "GNU Bison $(bison-version)" > $@ # cdsclient is a set of software written in c to interact with astronomical # database servers. It is a dependency of `scamp' to be able to download @@ -899,57 +837,69 @@ $(ibidir)/bison: $(ibidir)/help2man \ # programs are scripts and we need to touch them before installing. # Otherwise this software will be re-built each time the configure step is # invoked. -$(ibidir)/cdsclient: $(tdir)/cdsclient-$(cdsclient-version).tar.gz - cd $(ddir) \ - && tar xf $(word 1,$(filter $(tdir)/%,$^)) \ - && cd cdsclient-$(cdsclient-version) \ - && touch * \ - && ./configure --prefix=$(idir) \ - && make \ - && make install \ - && cd .. \ - && rm -rf cdsclient-$(cdsclient-version) \ - && echo "cdsclient $(cdsclient-version)" > $@ +$(ibidir)/cdsclient-$(cdsclient-version): + tarball=cdsclient-$(cdsclient-version).tar.gz + $(call import-source, $(cdsclient-url), $(cdsclient-checksum)) + cd $(ddir) + tar xf $(tdir)/$$tarball + cd cdsclient-$(cdsclient-version) + touch * + ./configure --prefix=$(idir) + make + make install + cd .. + rm -rf cdsclient-$(cdsclient-version) + echo "cdsclient $(cdsclient-version)" > $@ # CMake can be built with its custom `./bootstrap' script. -$(ibidir)/cmake: $(ibidir)/curl \ - $(tdir)/cmake-$(cmake-version).tar.gz +$(ibidir)/cmake-$(cmake-version): $(ibidir)/curl-$(curl-version) + # Import the tarball + tarball=cmake-$(cmake-version).tar.gz + $(call import-source, $(cmake-url), $(cmake-checksum)) + # After searching in `bootstrap', I couldn't find `LIBS', only # `LDFLAGS'. So the extra libraries are being added to `LDFLAGS', # not `LIBS'. # # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. - if [ x$(on_mac_os) = xyes ]; then \ - export CC=clang; \ - export CXX=clang++; \ - fi; \ - cd $(ddir) \ - && rm -rf cmake-$(cmake-version) \ - && tar xf $(word 1,$(filter $(tdir)/%,$^)) \ - && cd cmake-$(cmake-version) \ - && ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ - --system-bzip2 --system-liblzma --no-qt-gui \ - --parallel=$(numthreads) \ - && make -j$(numthreads) LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 \ - && make install \ - && cd .. \ - && rm -rf cmake-$(cmake-version) \ - && echo "CMake $(cmake-version)" > $@ - -$(ibidir)/flex: $(ibidir)/bison \ - $(tdir)/flex-$(flex-version).tar.gz - $(call gbuild, flex-$(flex-version), static, ,V=1) \ - && echo "Flex $(flex-version)" > $@ - -$(ibidir)/gdb: $(ibidir)/python \ - $(tdir)/gdb-$(gdb-version).tar.gz - $(call gbuild, gdb-$(gdb-version),,,V=1) \ - && echo "GNU Project Debugger (GDB) $(gdb-version)" > $@ - -$(ibidir)/ghostscript: $(ibidir)/libpng \ - $(ibidir)/libtiff \ - $(tdir)/ghostscript-$(ghostscript-version).tar.gz + if [ x$(on_mac_os) = xyes ]; then + export CC=clang + export CXX=clang++ + fi + cd $(ddir) + rm -rf cmake-$(cmake-version) + tar xf $(tdir)/$$tarball + cd cmake-$(cmake-version) + ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ + --system-bzip2 --system-liblzma --no-qt-gui \ + --parallel=$(numthreads) + make -j$(numthreads) LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 + make install + cd .. + rm -rf cmake-$(cmake-version) + echo "CMake $(cmake-version)" > $@ + +$(ibidir)/flex-$(flex-version): $(ibidir)/bison-$(bison-version) + tarball=flex-$(flex-version).tar.lz + $(call import-source, $(flex-url), $(flex-checksum)) + $(call gbuild, flex-$(flex-version), static, ,V=1 -j$(numthreads)) + echo "Flex $(flex-version)" > $@ + +$(ibidir)/gdb-$(gdb-version): $(ibidir)/python-$(python-version) + tarball=gdb-$(gdb-version).tar.gz + $(call import-source, $(gdb-url), $(gdb-checksum)) + $(call gbuild, gdb-$(gdb-version),,,V=1 -j$(numthreads)) + echo "GNU Project Debugger (GDB) $(gdb-version)" > $@ + +$(ibidir)/ghostscript-$(ghostscript-version): \ + $(ibidir)/libpng-$(libpng-version) \ + $(ibidir)/libtiff-$(libtiff-version) + + # Import the tarball. + tarball=ghostscript-$(ghostscript-version).tar.gz + $(call import-source, $(ghostscript-url), $(ghostscript-checksum)) + # First we need to make sure some necessary X11 libraries that we # don't yet install in this template are present on the host # system, see https://savannah.nongnu.org/task/?15481 . @@ -986,35 +936,38 @@ $(ibidir)/ghostscript: $(ibidir)/libpng \ fi # If they were present, go onto building Ghostscript. - $(call gbuild, ghostscript-$(ghostscript-version)) \ - && echo "GPL Ghostscript $(ghostscript-version)" > $@ - -$(ibidir)/gnuastro: $(ibidir)/gsl \ - $(ibidir)/wcslib \ - $(ibidir)/libjpeg \ - $(ibidir)/libtiff \ - $(ibidir)/libgit2 \ - $(ibidir)/ghostscript \ - $(tdir)/gnuastro-$(gnuastro-version).tar.lz -ifeq ($(static_build),yes) - staticopts="--enable-static=yes --enable-shared=no"; -endif - $(call gbuild, gnuastro-$(gnuastro-version), static, \ - $$staticopts, -j$(numthreads)) \ - && cp $(dtexdir)/gnuastro.tex $(ictdir)/ \ - && echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ - -$(ibidir)/help2man: $(tdir)/help2man-$(help2man-version).tar.xz - $(call gbuild, help2man-$(help2man-version), static, ,V=1) \ - && echo "Help2man $(Help2man-version)" > $@ - -$(ibidir)/imagemagick: $(ibidir)/zlib \ - $(ibidir)/libjpeg \ - $(ibidir)/libtiff \ - $(tdir)/imagemagick-$(imagemagick-version).tar.xz + $(call gbuild, ghostscript-$(ghostscript-version),,,V=1 -j$(numthreads)) + echo "GPL Ghostscript $(ghostscript-version)" > $@ + +$(ibidir)/gnuastro-$(gnuastro-version): \ + $(ibidir)/gsl-$(gsl-version) \ + $(ibidir)/wcslib-$(wcslib-version) \ + $(ibidir)/libjpeg-$(libjpeg-version) \ + $(ibidir)/libtiff-$(libtiff-version) \ + $(ibidir)/libgit2-$(libgit2-version) \ + $(ibidir)/ghostscript-$(ghostscript-version) + tarball=gnuastro-$(gnuastro-version).tar.lz + $(call import-source, $(gnuastro-url), $(gnuastro-checksum)) + $(call gbuild, gnuastro-$(gnuastro-version), static, , \ + -j$(numthreads)) + cp $(dtexdir)/gnuastro.tex $(ictdir)/ + echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ + +$(ibidir)/help2man-$(help2man-version): + tarball=help2man-$(help2man-version).tar.xz + $(call import-source, $(help2man-url), $(help2man-checksum)) + $(call gbuild, help2man-$(help2man-version), static, ,V=1) + echo "Help2man $(Help2man-version)" > $@ + +$(ibidir)/imagemagick-$(imagemagick-version): \ + $(ibidir)/zlib-$(zlib-version) \ + $(ibidir)/libjpeg-$(libjpeg-version) \ + $(ibidir)/libtiff-$(libtiff-version) + tarball=imagemagick-$(imagemagick-version).tar.xz + $(call import-source, $(imagemagick-url), $(imagemagick-checksum)) $(call gbuild, ImageMagick-$(imagemagick-version), static, \ - --without-x --disable-openmp, V=1 -j$(numthreads)) \ - && echo "ImageMagick $(imagemagick-version)" > $@ + --without-x --disable-openmp, V=1 -j$(numthreads)) + echo "ImageMagick $(imagemagick-version)" > $@ # `imfit' doesn't use the traditional `configure' and `make' to install # itself. Instead of that, it uses `scons'. As a consequence, the @@ -1025,42 +978,51 @@ $(ibidir)/imagemagick: $(ibidir)/zlib \ # by `$(idir)'. After that, each compiled program (`imfit', `imfit-mcmc' # and `makeimage') is copied into the installation directory and an `rpath' # is added. -$(ibidir)/imfit: $(ibidir)/gsl \ - $(ibidir)/fftw \ - $(ibidir)/scons \ - $(ibidir)/cfitsio \ - $(tdir)/imfit-$(imfit-version).tar.gz - cd $(ddir) \ - && unpackdir=imfit-$(imfit-version) \ - && rm -rf $$unpackdir \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)); then \ - echo; echo "Tar error"; exit 1; \ - fi \ - && cd $$unpackdir \ - && sed -i 's|/usr/local|$(idir)|g' SConstruct \ - && sed -i 's|/usr/include|$(idir)/include|g' SConstruct \ - && sed -i 's|.append(|.insert(0,|g' SConstruct \ - && scons --no-openmp --no-nlopt \ - --cc=$(ibdir)/gcc --cpp=$(ibdir)/g++ \ - --header-path=$(idir)/include --lib-path=$(idir)/lib imfit \ - && cp imfit $(ibdir) \ - && scons --no-openmp --no-nlopt\ - --cc=$(ibdir)/gcc --cpp=$(ibdir)/g++ \ - --header-path=$(idir)/include --lib-path=$(idir)/lib \ - imfit-mcmc \ - && cp imfit-mcmc $(ibdir) \ - && scons --no-openmp --no-nlopt\ - --cc=$(ibdir)/gcc --cpp=$(ibdir)/g++ \ - --header-path=$(idir)/include --lib-path=$(idir)/lib \ - makeimage \ - && cp makeimage $(ibdir) \ - && cp $(dtexdir)/imfit.tex $(ictdir)/ \ - && if [ "x$(on_mac_os)" != xyes ]; then \ - for p in imfit imfit-mcmc makeimage; do \ - patchelf --set-rpath $(ildir) $(ibdir)/$$p; \ - done; \ - fi \ - && echo "Imfit $(imfit-version) \citep{imfit2015}" > $@ +$(ibidir)/imfit-$(imfit-version): \ + $(ibidir)/gsl-$(gsl-version) \ + $(ibidir)/fftw-$(fftw-version) \ + $(ibidir)/scons-$(scons-version) \ + $(ibidir)/cfitsio-$(cfitsio-version) + tarball=imfit-$(imfit-version).tar.gz + $(call import-source, $(imfit-url), $(imfit-checksum)) + + # If the C library is in a non-standard location. + if ! [ x$(SYS_CPATH) = x ]; then + headerpath="--header-path=$(SYS_CPATH)" + fi + + # Unpack and build imfit and its accompanying programs. + cd $(ddir) + unpackdir=imfit-$(imfit-version) + rm -rf $$unpackdir + tar xf $(tdir)/$$tarball + cd $$unpackdir + sed -i 's|/usr/local|$(idir)|g' SConstruct + sed -i 's|/usr/include|$(idir)/include|g' SConstruct + sed -i 's|.append(|.insert(0,|g' SConstruct + scons --no-openmp --no-nlopt \ + --cc=$(ibdir)/gcc --cpp=$(ibdir)/g++ \ + --header-path=$(idir)/include $$headerpath \ + --lib-path=$(idir)/lib imfit + cp imfit $(ibdir) + scons --no-openmp --no-nlopt \ + --cc=$(ibdir)/gcc --cpp=$(ibdir)/g++ \ + --header-path=$(idir)/include $$headerpath \ + --lib-path=$(idir)/lib imfit-mcmc + cp imfit-mcmc $(ibdir) + scons --no-openmp --no-nlopt \ + --cc=$(ibdir)/gcc --cpp=$(ibdir)/g++ \ + --header-path=$(idir)/include $$headerpath \ + --lib-path=$(idir)/lib makeimage + cp makeimage $(ibdir) + cp $(dtexdir)/imfit.tex $(ictdir)/ + if [ -f $(ibdir)/patchelf ]; then + for p in imfit imfit-mcmc makeimage; do + patchelf --set-rpath $(ildir) $(ibdir)/$$p + done + fi + cp $(dtexdir)/imfit.tex $(ictdir)/ + echo "Imfit $(imfit-version) \citep{imfit2015}" > $@ # Minizip 1.x is actually distributed within zlib. It doesn't have its own # independent tarball. So we need a custom build, which include the GNU @@ -1071,39 +1033,39 @@ $(ibidir)/imfit: $(ibidir)/gsl \ # # About deleting the final crypt.h file after installation, see # https://bugzilla.redhat.com/show_bug.cgi?id=1424609 -$(ibidir)/minizip: $(ibidir)/automake \ - $(tdir)/zlib-$(zlib-version).tar.gz - cd $(ddir) \ - && unpackdir=minizip-$(minizip-version) \ - && rm -rf $$unpackdir \ - && mkdir $$unpackdir \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)) \ - -C$$unpackdir --strip-components=1; then \ - echo; echo "Tar error"; exit 1; \ - fi \ - && cd $$unpackdir\ - && ./configure --prefix=$(idir) \ - && make \ - && cd contrib/minizip \ - && cp Makefile Makefile.orig \ - && cp ../README.contrib readme.txt \ - && autoreconf --install \ - && ./configure --prefix=$(idir) \ - && make \ - && cd ../../ \ - && make test \ - && cd contrib/minizip \ - && make -f Makefile.orig test \ - && make install \ - && rm $(iidir)/minizip/crypt.h \ - && cd ../../.. \ - && rm -rf $$unpackdir \ - && echo "Minizip $(minizip-version)" > $@ - -$(ibidir)/missfits: $(tdir)/missfits-$(missfits-version).tar.gz - $(call gbuild, missfits-$(missfits-version), static) \ - && cp $(dtexdir)/missfits.tex $(ictdir)/ \ - && echo "MissFITS $(missfits-version) \citep{missfits}" > $@ +$(ibidir)/minizip-$(minizip-version): $(ibidir)/automake-$(automake-version) + tarball=zlib-$(zlib-version).tar.gz + $(call import-source, $(minizip-url), $(minizip-checksum)) + cd $(ddir) + unpackdir=minizip-$(minizip-version) + rm -rf $$unpackdir + mkdir $$unpackdir + tar xf $(tdir)/$$tarball -C$$unpackdir --strip-components=1 + cd $$unpackdir + ./configure --prefix=$(idir) + make + cd contrib/minizip + cp Makefile Makefile.orig + cp ../README.contrib readme.txt + autoreconf --install + ./configure --prefix=$(idir) + make + cd ../../ + make test + cd contrib/minizip + make -f Makefile.orig test + make install + rm $(iidir)/minizip/crypt.h + cd ../../.. + rm -rf $$unpackdir + echo "Minizip $(minizip-version)" > $@ + +$(ibidir)/missfits-$(missfits-version): + tarball=missfits-$(missfits-version).tar.gz + $(call import-source, $(missfits-url), $(missfits-checksum)) + $(call gbuild, missfits-$(missfits-version), static) + cp $(dtexdir)/missfits.tex $(ictdir)/ + echo "MissFITS $(missfits-version) \citep{missfits}" > $@ # Netpbm is a prerequisite of Astrometry-net, it contains a lot of programs. # This program has a crazy dialogue installation which is override using the @@ -1111,58 +1073,78 @@ $(ibidir)/missfits: $(tdir)/missfits-$(missfits-version).tar.gz # ask to the user. We give all answers with a pipe to the scripts (configure # and install). The questions are different depending on the system (tested # on GNU/Linux and Mac OS). -$(ibidir)/netpbm: $(ibidir)/unzip \ - $(ibidir)/libpng \ - $(ibidir)/libjpeg \ - $(ibidir)/libtiff \ - $(ibidir)/libxml2 \ - $(tdir)/netpbm-$(netpbm-version).tar.gz - if [ x$(on_mac_os) = xyes ]; then \ - answers='\n\n$(ildir)\n\n\n\n\n\n$(ildir)/include\n\n$(ildir)/include\n\n$(ildir)/include\nnone\n\n'; \ - else \ - answers='\n\n\n\n\n\n\n\n\n\n\n\n\nnone\n\n\n'; \ - fi; \ - cd $(ddir) \ - && unpackdir=netpbm-$(netpbm-version) \ - && rm -rf $$unpackdir \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)); then \ - echo; echo "Tar error"; exit 1; \ - fi \ - && cd $$unpackdir \ - && printf "$$answers" | ./configure \ - && make \ - && rm -rf $(ddir)/$$unpackdir/install \ - && make package pkgdir=$(ddir)/$$unpackdir/install \ - && printf "$(ddir)/$$unpackdir/install\n$(idir)\n\n\nN\n\n\n\n\nN\n\n" \ - | ./installnetpbm \ - && cd .. \ - && rm -rf $$unpackdir \ - && echo "Netpbm $(netpbm-version)" > $@ - -$(ibidir)/patch: $(tdir)/patch-$(patch-version).tar.gz - $(call gbuild, patch-$(patch-version), static, ,V=1) \ - && echo "GNU Patch $(patch-version)" > $@ - -# R programming language -$(ibidir)/R: $(ibidir)/libpng \ - $(ibidir)/libjpeg \ - $(ibidir)/libtiff \ - $(tdir)/R-$(R-version).tar.gz - export R_SHELL=$(SHELL); \ +$(ibidir)/netpbm-$(netpbm-version): \ + $(ibidir)/libpng-$(libpng-version) \ + $(ibidir)/libjpeg-$(libjpeg-version) \ + $(ibidir)/libtiff-$(libtiff-version) \ + $(ibidir)/libxml2-$(libxml2-version) + tarball=netpbm-$(netpbm-version).tar.gz + $(call import-source, $(netpbm-url), $(netpbm-checksum)) + if [ x$(on_mac_os) = xyes ]; then + answers='\n\n$(ildir)\n\n\n\n\n\n$(ildir)/include\n\n$(ildir)/include\n\n$(ildir)/include\nnone\n\n' + else + answers='\n\n\n\n\n\n\n\n\n\n\n\n\nnone\n\n\n' + fi + cd $(ddir) + unpackdir=netpbm-$(netpbm-version) + rm -rf $$unpackdir + tar xf $(tdir)/$$tarball + cd $$unpackdir + printf "$$answers" | ./configure + make + rm -rf $(ddir)/$$unpackdir/install + make package pkgdir=$(ddir)/$$unpackdir/install + printf "$(ddir)/$$unpackdir/install\n$(idir)\n\n\nN\n\n\n\n\nN\n\n" \ + | ./installnetpbm + cd .. + rm -rf $$unpackdir + echo "Netpbm $(netpbm-version)" > $@ + +$(ibidir)/patch-$(patch-version): + tarball=patch-$(patch-version).tar.gz + $(call import-source, $(patch-url), $(patch-checksum)) + $(call gbuild, patch-$(patch-version), static, ,V=1) + echo "GNU Patch $(patch-version)" > $@ + +$(ibidir)/pcre-$(pcre-version): + tarball=pcre-$(pcre-version).tar.gz + $(call import-source, $(pcre-url), $(pcre-checksum)) + $(call gbuild, pcre-$(pcre-version), static, \ + --enable-pcretest-libreadline \ + --enable-unicode-properties \ + --includedir=$(iidir)/pcre \ + --enable-pcregrep-libbz2 \ + --enable-pcregrep-libz \ + , V=1 -j$(numthreads)) + echo "Perl Compatible Regular Expressions $(pcre-version)" > $@ + +$(ibidir)/R-$(R-version): \ + $(ibidir)/pcre-$(pcre-version) \ + $(ibidir)/cairo-$(cairo-version) \ + $(ibidir)/libpng-$(libpng-version) \ + $(ibidir)/libjpeg-$(libjpeg-version) \ + $(ibidir)/libtiff-$(libtiff-version) \ + $(ibidir)/libpaper-$(libpaper-version) + tarball=R-$(R-version).tar.gz + $(call import-source, $(R-url), $(R-checksum)) + + export R_SHELL=$(SHELL) $(call gbuild, R-$(R-version), static, \ --without-x --with-readline \ - --disable-openmp) \ - && echo "R $(R-version)" > $@ + --disable-openmp, -j$(numthreads)) + echo "R $(R-version)" > $@ # SCAMP documentation says ATLAS is a mandatory prerequisite for using # SCAMP. We have ATLAS into the project but there are some problems with the # libraries that are not yet solved. However, we tried to install it with # the option --enable-openblas and it worked (same issue happened with # `sextractor'. -$(ibidir)/scamp: $(ibidir)/fftw \ - $(ibidir)/openblas \ - $(ibidir)/cdsclient \ - $(tdir)/scamp-$(scamp-version).tar.lz +$(ibidir)/scamp-$(scamp-version): \ + $(ibidir)/fftw-$(fftw-version) \ + $(ibidir)/openblas-$(openblas-version) \ + $(ibidir)/cdsclient-$(cdsclient-version) + tarball=scamp-$(scamp-version).tar.lz + $(call import-source, $(scamp-url), $(scamp-checksum)) $(call gbuild, scamp-$(scamp-version), static, \ --enable-threads \ --enable-openblas \ @@ -1170,84 +1152,90 @@ $(ibidir)/scamp: $(ibidir)/fftw \ --with-fftw-libdir=$(idir) \ --with-fftw-incdir=$(idir)/include \ --with-openblas-libdir=$(ildir) \ - --with-openblas-incdir=$(idir)/include) \ - && cp $(dtexdir)/scamp.tex $(ictdir)/ \ - && echo "SCAMP $(scamp-version) \citep{scamp}" > $@ + --with-openblas-incdir=$(idir)/include) + cp $(dtexdir)/scamp.tex $(ictdir)/ + echo "SCAMP $(scamp-version) \citep{scamp}" > $@ # Since `scons' doesn't use the traditional GNU installation with # `configure' and `make' it is installed manually using `python'. -$(ibidir)/scons: $(ibidir)/python \ - $(tdir)/scons-$(scons-version).tar.gz - cd $(ddir) \ - && unpackdir=scons-$(scons-version) \ - && rm -rf $$unpackdir \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)); then \ - echo; echo "Tar error"; exit 1; \ - fi \ - && cd $$unpackdir \ - && python setup.py install \ - && echo "SCons $(scons-version)" > $@ +$(ibidir)/scons-$(scons-version): $(ibidir)/python-$(python-version) + tarball=scons-$(scons-version).tar.gz + $(call import-source, $(scons-url), $(scons-checksum)) + cd $(ddir) + unpackdir=scons-$(scons-version) + rm -rf $$unpackdir + tar xf $(tdir)/$$tarball + cd $$unpackdir + python setup.py install + echo "SCons $(scons-version)" > $@ # Sextractor crashes complaining about not linking with some ATLAS # libraries. But we can override this issue since we have Openblas # installed, it is just necessary to explicity tell sextractor to use it in # the configuration step. -$(ibidir)/sextractor: $(ibidir)/fftw \ - $(ibidir)/openblas \ - $(tdir)/sextractor-$(sextractor-version).tar.lz +$(ibidir)/sextractor-$(sextractor-version): \ + $(ibidir)/fftw-$(fftw-version) \ + $(ibidir)/openblas-$(openblas-version) + tarball=sextractor-$(sextractor-version).tar.lz + $(call import-source, $(sextractor-url), $(sextractor-checksum)) $(call gbuild, sextractor-$(sextractor-version), static, \ --enable-threads --enable-openblas \ --with-openblas-libdir=$(ildir) \ - --with-openblas-incdir=$(idir)/include) \ - && ln -fs $(ibdir)/sex $(ibdir)/sextractor \ - && cp $(dtexdir)/sextractor.tex $(ictdir)/ \ - && echo "SExtractor $(sextractor-version) \citep{sextractor}" > $@ - -$(ibidir)/swarp: $(ibidir)/fftw \ - $(tdir)/swarp-$(swarp-version).tar.gz + --with-openblas-incdir=$(idir)/include) + ln -fs $(ibdir)/sex $(ibdir)/sextractor + cp $(dtexdir)/sextractor.tex $(ictdir)/ + echo "SExtractor $(sextractor-version) \citep{sextractor}" > $@ + +$(ibidir)/swarp-$(swarp-version): $(ibidir)/fftw-$(fftw-version) + tarball=swarp-$(swarp-version).tar.gz + $(call import-source, $(swarp-url), $(swarp-checksum)) $(call gbuild, swarp-$(swarp-version), static, \ - --enable-threads) \ - && cp $(dtexdir)/swarp.tex $(ictdir)/ \ - && echo "SWarp $(swarp-version) \citep{swarp}" > $@ + --enable-threads) + cp $(dtexdir)/swarp.tex $(ictdir)/ + echo "SWarp $(swarp-version) \citep{swarp}" > $@ -$(ibidir)/swig: $(tdir)/swig-$(swig-version).tar.gz +$(ibidir)/swig-$(swig-version): # Option --without-pcre was a suggestion once the configure step # was tried and it failed. It was not recommended but it works! # pcr is a dependency of swig - $(call gbuild, swig-$(swig-version), static, --without-pcre) \ - && echo "Swig $(swig-version)" > $@ - -$(ibidir)/xlsxio: $(ibidir)/cmake \ - $(ibidir)/expat \ - $(ibidir)/minizip \ - $(tdir)/xlsxio-$(xlsxio-version).tar.gz - if [ x$(on_mac_os) = xyes ]; then \ - export CC=clang; \ - export CXX=clang++; \ - export LDFLAGS="$$LDFLAGS -lbz2"; \ - else \ - export LDFLAGS="$$LDFLAGS -lbz2 -lbsd"; \ - fi; \ + tarball=swig-$(swig-version).tar.gz + $(call import-source, $(swig-url), $(swig-checksum)) + $(call gbuild, swig-$(swig-version), static, --without-pcre) + echo "Swig $(swig-version)" > $@ + +$(ibidir)/xlsxio-$(xlsxio-version): \ + $(ibidir)/cmake-$(cmake-version) \ + $(ibidir)/expat-$(expat-version) \ + $(ibidir)/minizip-$(minizip-version) + tarball=xlsxio-$(xlsxio-version).tar.gz + $(call import-source, $(xlsxio-url), $(xlsxio-checksum)) + if [ x$(on_mac_os) = xyes ]; then + export CC=clang + export CXX=clang++ + export LDFLAGS="$$LDFLAGS -lbz2" + else + export LDFLAGS="$$LDFLAGS -lbz2 -lbsd" + fi $(call cbuild, xlsxio-$(xlsxio-version), static, \ -DMINIZIP_DIR:PATH=$(idir) \ -DMINIZIP_LIBRARIES=$(idir) \ - -DMINIZIP_INCLUDE_DIRS=$(iidir)) \ - && echo "Correcting internal linking of XLSX I/O executables..." \ - && if [ "x$(on_mac_os)" = xyes ]; then \ - for f in $(ibdir)/xlsxio_* $(ildir)/libxlsxio_*.dylib; do \ - install_name_tool -change libxlsxio_read.dylib \ - $(ildir)/libxlsxio_read.dylib $$f; \ - install_name_tool -change libxlsxio_write.dylib \ - $(ildir)/libxlsxio_write.dylib $$f; \ - done; \ - else \ - for f in $(ibdir)/xlsxio_* $(ildir)/libxlsxio_*.so; do \ - patchelf --set-rpath $(ildir) $$f; \ - done; \ - fi \ - && echo "Deleting XLSX I/O example files..." \ - && rm $(ibdir)/example_xlsxio_* \ - && echo "XLSX I/O $(xlsxio-version)" > $@ + -DMINIZIP_INCLUDE_DIRS=$(iidir)) + echo "Correcting internal linking of XLSX I/O executables..." + if [ "x$(on_mac_os)" = xyes ]; then + for f in $(ibdir)/xlsxio_* $(ildir)/libxlsxio_*.dylib; do + install_name_tool -change libxlsxio_read.dylib \ + $(ildir)/libxlsxio_read.dylib $$f + install_name_tool -change libxlsxio_write.dylib \ + $(ildir)/libxlsxio_write.dylib $$f + done + else + for f in $(ibdir)/xlsxio_* $(ildir)/libxlsxio_*.so; do + patchelf --set-rpath $(ildir) $$f + done + fi + echo "Deleting XLSX I/O example files..." + rm $(ibdir)/example_xlsxio_* + echo "XLSX I/O $(xlsxio-version)" > $@ @@ -1272,11 +1260,11 @@ $(ibidir)/xlsxio: $(ibidir)/cmake \ # (`ftp.dante.de'), however, it is far too slow (when I tested it). The # `rit.edu' server seems to be a good alternative (given the importance of # NY on the internet infrastructure). -tlmirror=http://mirrors.rit.edu/CTAN/systems/texlive/tlnet +texlive-url=http://mirrors.rit.edu/CTAN/systems/texlive/tlnet +$(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf -# The core TeX Live system. -$(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf \ - $(tdir)/install-tl-unx.tar.gz + tarball=install-tl-unx.tar.gz + $(call import-source, $(texlive-url), NO-CHECK-SUM) # Unpack, enter the directory, and install based on the given # configuration (prerequisite of this rule). @@ -1318,7 +1306,7 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf \ # TeXLive using that. The old tarball will be preserved, but will # have an '-OLD' suffix after it. if ./install-tl --profile=texlive.conf -repository \ - $(tlmirror) 2> log.txt; then + $(texlive-url) 2> log.txt; then # Put a symbolic link of the TeX Live executables in `ibdir' to # avoid all the complexities of its sub-directories and additions @@ -1368,7 +1356,7 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf \ $$topdir/reproduce/software/config/texlive.conf \ > texlive.conf if ./install-tl --profile=texlive.conf -repository \ - $(tlmirror); then + $(texlive-url); then ln -fs $(idir)/texlive/maneage/bin/*/* $(ibdir)/ echo "TeX Live is ready." > $@ else @@ -1398,15 +1386,8 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf \ # don't currently build biber from source. So we can't choose the library # version. But we have the source and build instructions for the `nsl' # library. When we later build biber from source, we can easily use them. -# -#ifeq ($(on_mac_os),yes) -#forbiber = -#else -#forbiber = $(ibidir)/libnsl -#endif $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ - $(itidir)/texlive-ready-tlmgr \ - $(forbiber) + $(itidir)/texlive-ready-tlmgr # To work with TeX live installation, we'll need the internet. @res=$$(cat $(itidir)/texlive-ready-tlmgr) @@ -1419,7 +1400,7 @@ $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ # Before checking LaTeX packages, update tlmgr itself. tlmgr option backupdir $$backupdir - tlmgr -repository $(tlmirror) update --self + tlmgr -repository $(texlive-url) update --self # Install all the extra necessary packages. If LaTeX complains # about not finding a command/file/what-ever/XXXXXX, simply run diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index eef8279..ccfbc72 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -3,8 +3,12 @@ # ------------------------------------------------------------------------ # !!!!! IMPORTANT NOTES !!!!! # -# This Makefile will be run by the initial `./project configure' script. It -# is not included into the reproduction pipe after that. +# This Makefile will be loaded into 'high-level.mk', which is called by the +# `./project configure' script. It is not included into the project +# afterwards. +# +# This Makefile contains instructions to build all the Python-related +# software within the project. # # ------------------------------------------------------------------------ # @@ -52,274 +56,52 @@ python-major-version = $(shell echo $(python-version) | awk 'BEGIN{FS="."} \ -# Tarballs -# -------- -# -# All the necessary tarballs are defined and prepared with this rule. -# -# Note that we want the tarballs to follow the convention of NAME-VERSION -# before the `tar.XX' prefix. For those programs that don't follow this -# convention, but include the name/version in their tarball names with -# another format, we'll do the modification before the download so the -# downloaded file has our desired format. -pytarballs = $(foreach t, asn1crypto-$(asn1crypto-version).tar.gz \ - asteval-$(asteval-version).tar.gz \ - astroquery-$(astroquery-version).tar.gz \ - astropy-$(astropy-version).tar.gz \ - beautifulsoup4-$(beautifulsoup4-version).tar.gz \ - certifi-$(certifi-version).tar.gz \ - cffi-$(cffi-version).tar.gz \ - chardet-$(chardet-version).tar.gz \ - corner-$(corner-version).tar.gz \ - cryptography-$(cryptography-version).tar.gz \ - cycler-$(cycler-version).tar.gz \ - cython-$(cython-version).tar.gz \ - eigency-$(eigency-version).tar.gz \ - emcee-$(emcee-version).tar.gz \ - esutil-$(esutil-version).tar.gz \ - entrypoints-$(entrypoints-version).tar.gz \ - flake8-$(flake8-version).tar.gz \ - future-$(future-version).tar.gz \ - galsim-$(galsim-version).tar.gz \ - h5py-$(h5py-version).tar.gz \ - html5lib-$(html5lib-version).tar.gz \ - idna-$(idna-version).tar.gz \ - jeepney-$(jeepney-version).tar.gz \ - kiwisolver-$(kiwisolver-version).tar.gz \ - keyring-$(keyring-version).tar.gz \ - libffi-$(libffi-version).tar.gz \ - lmfit-$(lmfit-version).tar.gz \ - lsstdesccoord-$(lsstdesccoord-version).tar.gz \ - matplotlib-$(matplotlib-version).tar.gz \ - mpi4py-$(mpi4py-version).tar.gz \ - mpmath-$(mpmath-version).tar.gz \ - numpy-$(numpy-version).zip \ - pkgconfig-$(pypkgconfig-version).tar.gz \ - pip-$(pip-version).tar.gz \ - pexpect-$(pexpect-version).tar.gz \ - pybind11-$(pybind11-version).tar.gz \ - pycodestyle-$(pycodestyle-version).tar.gz \ - pycparser-$(pycparser-version).tar.gz \ - pyflakes-$(pyflakes-version).tar.gz \ - python-$(python-version).tar.gz \ - python-dateutil-$(python-dateutil-version).tar.gz \ - pyparsing-$(pyparsing-version).tar.gz \ - pyyaml-$(pyyaml-version).tar.gz \ - requests-$(requests-version).tar.gz \ - scipy-$(scipy-version).tar.gz \ - secretstorage-$(secretstorage-version).tar.gz \ - setuptools-$(setuptools-version).zip \ - setuptools_scm-$(setuptools_scm-version).tar.gz \ - sip_tpv-$(sip_tpv-version).tar.gz \ - six-$(six-version).tar.gz \ - soupsieve-$(soupsieve-version).tar.gz \ - sympy-$(sympy-version).tar.gz \ - uncertainties-$(uncertainties-version).tar.gz \ - urllib3-$(urllib3-version).tar.gz \ - webencodings-$(webencodings-version).tar.gz \ - virtualenv-$(virtualenv-version).tar.gz \ - , $(tdir)/$(t) ) -pytopurl=https://files.pythonhosted.org/packages -$(pytarballs): $(tdir)/%: - - # Convenience variable - # -------------------- - # - # `n' is just for convenience and to avoid having to repeat the - # package tarball name in the conditional to find its URL. - # - # For some packages (for example `python-dateutil', or those with - # a number or dash in their name), we need special consideration - # because the tokenization above will produce `python' as the - # first string. - @if [ $* = python-dateutil-$(python-dateutil-version).tar.gz ]; then - n=dateutil - - # elif [ $* = strange-tarball5name-version.tar.gz ]; then - # n=strange5-name - else - # Remove the version numbers and suffix from the tarball name so - # we can search more easily only with the program name. This - # requires the first character of the version to be a digit: - # packages such as `foo' and `foo-3' will not be distinguished, - # but `foo' and `foo2' will be distinguished. - n=$$(echo $* | sed -e's/-[0-9]/ /' -e's/\./ /g' \ - | awk '{print $$1}' ) - - fi - - # Set the top download link of the requested tarball. The ones - # that have non-standard filenames (differing from our archived - # tarball names) are treated first, then the standard ones. - mergenames=1 - if [ $$n = cython ]; then - mergenames=0 - c=$(cython-checksum) - hash=36/da/fcb979fc8cb486a67a013d6aefefbb95a3e19e67e49dff8a35e014046c5e - h=$(pytopurl)/$$hash/Cython-$(cython-version).tar.gz - elif [ $$n = galsim ]; then - mergenames=0 - c=$(galsim-checksum) - hash=8f/3b/bbc7cff7590d3624d528564f08745f071e316c67fce154ad38210833c103 - h=$(pytopurl)/$$hash/GalSim-$(galsim-version).tar.gz - elif [ $$n = lsstdesccoord ]; then - mergenames=0 - c=$(lsstdesccoord-checksum) - hash=9d/39/ad17697571c9aed36d20ed9ae0a135e3a734fb7f15a8605f92bf27c3b02c - h=$(pytopurl)/$$hash/LSSTDESC.Coord-$(lsstdesccoord-version).tar.gz - elif [ $$n = python ]; then - mergenames=0 - c=$(python-checksum) - h=https://www.python.org/ftp/python/$(python-version)/Python-$(python-version).tgz - elif [ $$n = pyyaml ]; then - mergenames=0 - c=$(pyyaml-checksum) - hash=9f/2c/9417b5c774792634834e730932745bc09a7d36754ca00acf1ccd1ac2594d - h=$(pytopurl)/$$hash/PyYAML-$(pyyaml-version).tar.gz - elif [ $$n = libffi ]; then - mergenames=0 - c=$(libffi-checksum) - h=ftp://sourceware.org/pub/libffi/libffi-$(libffi-version).tar.gz - elif [ $$n = secretstorage ]; then - mergenames=0 - c=$(secretstorage-checksum) - hash=a6/89/df343dbc2957a317127e7ff2983230dc5336273be34f2e1911519d85aeb5 - h=$(pytopurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz - elif [ $$n = asn1crypto ]; then h=fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4; c=$(asn1crypto-checksum) - elif [ $$n = asteval ]; then h=50/3f/29b7935c6dc09ee96dc347edc66c57e8ef68d595dd35b763a36a117acc8c; c=$(asteval-checksum) - elif [ $$n = astroquery ]; then h=e2/af/a3cd3b30745832a0e81f5f13327234099aaf5d03b7979ac947a888e68e91; c=$(astroquery-checksum) - elif [ $$n = astropy ]; then h=de/96/7feaca4b9be134128838395a9d924ea0b389ed4381702dcd9d11ae31789f; c=$(astropy-checksum) - elif [ $$n = beautifulsoup4 ]; then h=80/f2/f6aca7f1b209bb9a7ef069d68813b091c8c3620642b568dac4eb0e507748; c=$(beautifulsoup4-checksum) - elif [ $$n = certifi ]; then h=55/54/3ce77783acba5979ce16674fc98b1920d00b01d337cfaaf5db22543505ed; c=$(certifi-checksum) - elif [ $$n = cffi ]; then h=64/7c/27367b38e6cc3e1f49f193deb761fe75cda9f95da37b67b422e62281fcac; c=$(cffi-checksum) - elif [ $$n = chardet ]; then h=fc/bb/a5768c230f9ddb03acc9ef3f0d4a3cf93462473795d18e9535498c8f929d; c=$(chardet-checksum) - elif [ $$n = corner ]; then h=65/af/a7ba022f2d5787f51db91b5550cbe8e8c40a6eebd8f15119e743a09a9c19; c=$(corner-checksum) - elif [ $$n = cryptography ]; then h=07/ca/bc827c5e55918ad223d59d299fff92f3563476c3b00d0a9157d9c0217449; c=$(cryptography-checksum) - elif [ $$n = cycler ]; then h=c2/4b/137dea450d6e1e3d474e1d873cd1d4f7d3beed7e0dc973b06e8e10d32488; c=$(cycler-checksum) - elif [ $$n = eigency ]; then h=fb/6e/bc4359fbfb0bb0b588ec328251b0d0836bdd7c0a4c568959ea06df023e18; c=$(eigency-checksum) - elif [ $$n = emcee ]; then h=f0/c0/cd433f2aedeef9b1e5ed7d236c82564f7518fe7fe2238fa141ea9ce08e73; c=$(emcee-checksum) - elif [ $$n = entrypoints ]; then h=b4/ef/063484f1f9ba3081e920ec9972c96664e2edb9fdc3d8669b0e3b8fc0ad7c; c=$(entrypoints-checksum) - elif [ $$n = esutil ]; then h=5b/91/77e38282fd3d47b55e351544ab179eb209b309a8d2d40f8cdb6241beda00; c=$(esutil-checksum) - elif [ $$n = flake8 ]; then h=8d/a7/99222c9200af533c1ecb1120d99adbd1c033b57296ac5cb39d121db007a8; c=$(flake8-checksum) - elif [ $$n = future ]; then h=3f/bf/57733d44afd0cf67580658507bd11d3ec629612d5e0e432beb4b8f6fbb04; c=$(future-checksum) - elif [ $$n = h5py ]; then h=43/27/a6e7dcb8ae20a4dbf3725321058923fec262b6f7835179d78ccc8d98deec; c=$(h5py-checksum) - elif [ $$n = html5lib ]; then h=85/3e/cf449cf1b5004e87510b9368e7a5f1acd8831c2d6691edd3c62a0823f98f; c=$(html5lib-checksum) - elif [ $$n = idna ]; then h=ad/13/eb56951b6f7950cadb579ca166e448ba77f9d24efc03edd7e55fa57d04b7; c=$(idna-checksum) - elif [ $$n = jeepney ]; then h=16/1d/74adf3b164a8d19a60d0fcf706a751ffa2a1eaa8e5bbb1b6705c92a05263; c=$(jeepney-checksum) - elif [ $$n = keyring ]; then h=15/88/c6ce9509438bc02d54cf214923cfba814412f90c31c95028af852b19f9b2; c=$(keyring-checksum) - elif [ $$n = kiwisolver ]; then h=31/60/494fcce70d60a598c32ee00e71542e52e27c978e5f8219fae0d4ac6e2864; c=$(kiwisolver-checksum) - elif [ $$n = lmfit ]; then h=59/6e/117794cf85b7345361877e49245870490ae438f1981dea3c6af1316b30e7; c=$(lmfit-checksum) - elif [ $$n = matplotlib ]; then h=12/d1/7b12cd79c791348cb0c78ce6e7d16bd72992f13c9f1e8e43d2725a6d8adf; c=$(matplotlib-checksum) - elif [ $$n = mpi4py ]; then h=04/f5/a615603ce4ab7f40b65dba63759455e3da610d9a155d4d4cece1d8fd6706; c=$(mpi4py-checksum) - elif [ $$n = mpmath ]; then h=ca/63/3384ebb3b51af9610086b23ea976e6d27d6d97bf140a76a365bd77a3eb32; c=$(mpmath-checksum) - elif [ $$n = numpy ]; then h=ac/36/325b27ef698684c38b1fe2e546e2e7ef9cecd7037bcdb35c87efec4356af; c=$(numpy-checksum) - elif [ $$n = pexpect ]; then h=1c/b1/362a0d4235496cb42c33d1d8732b5e2c607b0129ad5fdd76f5a583b9fcb3; c=$(pexpect-checksum) - elif [ $$n = pip ]; then h=4c/4d/88bc9413da11702cbbace3ccc51350ae099bb351febae8acc85fec34f9af; c=$(pip-checksum) - elif [ $$n = pkgconfig ]; then h=6e/a9/ff67ef67217dfdf2aca847685fe789f82b931a6957a3deac861297585db6; c=$(pypkgconfig-checksum) - elif [ $$n = pybind11 ]; then h=aa/91/deb6743e79e22ab01502296570b39b8404f10cc507a6692d612a7fee8d51; c=$(pybind11-checksum) - elif [ $$n = pycodestyle ]; then h=1c/d1/41294da5915f4cae7f4b388cea6c2cd0d6cd53039788635f6875dfe8c72f; c=$(pycodestyle-checksum) - elif [ $$n = pycparser ]; then h=68/9e/49196946aee219aead1290e00d1e7fdeab8567783e83e1b9ab5585e6206a; c=$(pycparser-checksum) - elif [ $$n = pyflakes ]; then h=52/64/87303747635c2988fcaef18af54bfdec925b6ea3b80bcd28aaca5ba41c9e; c=$(pyflakes-checksum) - elif [ $$n = pyparsing ]; then h=b9/b8/6b32b3e84014148dcd60dd05795e35c2e7f4b72f918616c61fdce83d27fc; c=$(pyparsing-checksum) - elif [ $$n = dateutil ]; then h=ad/99/5b2e99737edeb28c71bcbec5b5dda19d0d9ef3ca3e92e3e925e7c0bb364c; c=$(python-dateutil-checksum) - elif [ $$n = requests ]; then h=52/2c/514e4ac25da2b08ca5a464c50463682126385c4272c18193876e91f4bc38; c=$(requests-checksum) - elif [ $$n = scipy ]; then h=ee/5b/5afcd1c46f97b3c2ac3489dbc95d6ca28eacf8e3634e51f495da68d97f0f; c=$(scipy-checksum) - elif [ $$n = secretstorage ]; then h=a6/89/df343dbc2957a317127e7ff2983230dc5336273be34f2e1911519d85aeb5; c=$(secretstorage-checksum) - elif [ $$n = setuptools ]; then h=11/0a/7f13ef5cd932a107cd4c0f3ebc9d831d9b78e1a0e8c98a098ca17b1d7d97; c=$(setuptools-checksum) - elif [ $$n = setuptools_scm ]; then h=83/44/53cad68ce686585d12222e6769682c4bdb9686808d2739671f9175e2938b; c=$(setuptools_scm-checksum) - elif [ $$n = six ]; then h=dd/bf/4138e7bfb757de47d1f4b6994648ec67a51efe58fa907c1e11e350cddfca; c=$(six-checksum) - elif [ $$n = sip_tpv ]; then h=27/93/a973aab2a3bf0c12cb385611819710921e13b090304c6bd015026cf9c502; c=$(sip_tpv-checksum) - elif [ $$n = soupsieve ]; then h=0c/52/e9088bb9b96e2d39fc3b33fcda5b4fde9d71473536ac660a1ca9a0958a2f; c=$(soupsieve-checksum) - elif [ $$n = sympy ]; then h=54/2e/6adb11fe599d4cfb7e8833753350ac51aa2c0603c226b36f9051cc9d2425; c=$(sympy-checksum) - elif [ $$n = uncertainties ]; then h=2a/c2/babbe5b16141859dd799ed31c03987100a7b6d0ca7c0ed4429c96ce60fdf; c=$(uncertainties-checksum) - elif [ $$n = urllib3 ]; then h=b1/53/37d82ab391393565f2f831b8eedbffd57db5a718216f82f1a8b4d381a1c1; c=$(urllib3-checksum) - elif [ $$n = virtualenv ]; then h=51/aa/c395a6e6eaaedfa5a04723b6446a1df783b16cca6fec66e671cede514688; c=$(virtualenv-checksum) - elif [ $$n = webencodings ]; then h=0b/02/ae6ceac1baeda530866a85075641cec12989bd8d31af6d5ab4a3e8c92f47; c=$(webencodings-checksum) -# elif [ $$n = strange5-name ]; then h=XXXXX; c=$(XXXXX-checksum) - else - echo; echo; echo; - echo "'$$n' not recognized as a dependency name to download." - echo; echo; echo; - exit 1 - fi - - # Download the requested tarball. Note that some packages may not - # follow our naming convention (where the package name is merged - # with its version number). In such cases, `w' will be the full - # address, not just the top directory address. But since we are - # storing all the tarballs in one directory, we want it to have - # the same naming convention, so we'll download it to a temporary - # name, then rename that. - rm -f "$@.unchecked" - if [ -f $(DEPENDENCIES-DIR)/$* ]; then - cp $(DEPENDENCIES-DIR)/$* "$@.unchecked" - else - if [ $$mergenames = 1 ]; then tarballurl=$(pytopurl)/$$h/"$*" - else tarballurl=$$h - fi - - # Download using the script specially defined for this job. - touch $(lockdir)/download - downloader="wget --no-use-server-timestamps -O" - $(downloadwrapper) "$$downloader" $(lockdir)/download \ - $$tarballurl "$@.unchecked" "$(backupservers)" - fi - - # Make sure this is the expected tarball. Note that we now have a - # controlled `sha512sum' build (as part of GNU Coreutils). So we - # don't need to check its existance like `basic.mk'. - checksum=$$(sha512sum "$@.unchecked" | awk '{print $$1}') - if [ x"$$checksum" = x"$$c" ]; then - mv "$@.unchecked" "$@" - else - echo "ERROR: Non-matching checksum for '$*'." - echo "Checksum should be: $$c" - echo "Checksum is: $$checksum" - exit 1 - fi - - - - - # Necessary programs and libraries # -------------------------------- # # While this Makefile is for Python programs, in some cases, we need # certain programs (like Python itself), or libraries for the modules. -$(ibidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz +$(ibidir)/libffi-$(libffi-version): + # Prepare the source. + tarball=libffi-$(libffi-version).tar.gz + $(call import-source, $(libffi-url), $(libffi-checksum)) + + # Build libffi. + $(call gbuild, libffi-$(libffi-version), , \ + CFLAGS="-DNO_JAVA_RAW_API=1") # On some Fedora systems, libffi installs in `lib64', not # `lib'. This will cause problems when building setuptools # later. To fix this problem, we'll first check if this has indeed # happened (it exists under `lib64', but not under `lib'). If so, # we'll put a copy of the installed libffi libraries in `lib'. - $(call gbuild, libffi-$(libffi-version), , \ - CFLAGS="-DNO_JAVA_RAW_API=1") \ - && if [ -f $(idir)/lib64/libffi.a ] \ - && ! [ -f $(idir)/lib/libffi.a ]; then \ - cp $(idir)/lib64/libffi* $(ildir)/; \ - fi \ - && echo "Libffi $(libffi-version)" > $@ - -$(ibidir)/python: $(ibidir)/libffi \ - $(tdir)/python-$(python-version).tar.gz + if [ -f $(idir)/lib64/libffi.a ] && ! [ -f $(idir)/lib/libffi.a ]; then + cp $(idir)/lib64/libffi* $(ildir)/ + fi + echo "Libffi $(libffi-version)" > $@ + +$(ibidir)/python-$(python-version): $(ibidir)/libffi-$(libffi-version) + + # Download the source. + tarball=python-$(python-version).tar.gz + $(call import-source, $(python-url), $(python-checksum)) + # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. - if [ x$(on_mac_os) = xyes ]; then \ - export CC=clang; \ - export CXX=clang++; \ - fi; \ + if [ x$(on_mac_os) = xyes ]; then + export CC=clang + export CXX=clang++ + fi $(call gbuild, Python-$(python-version),, \ --without-ensurepip \ --with-system-ffi \ - --enable-shared) \ - && ln -sf $(ildir)/python$(python-major-version) $(ildir)/python \ - && ln -sf $(ibdir)/python$(python-major-version) $(ibdir)/python \ - && ln -sf $(iidir)/python$(python-major-version)m $(iidir)/python$(python-major-version) \ - && rm -rf $(ipydir) \ - && mkdir $(ipydir) \ - && echo "Python $(python-version)" > $@ + --enable-shared, -j$(numthreads)) + ln -sf $(ildir)/python$(python-major-version) $(ildir)/python + ln -sf $(ibdir)/python$(python-major-version) $(ibdir)/python + ln -sf $(iidir)/python$(python-major-version)m $(iidir)/python$(python-major-version) + rm -rf $(ipydir) + mkdir $(ipydir) + echo "Python $(python-version)" > $@ @@ -343,22 +125,22 @@ $(ibidir)/python: $(ibidir)/libffi \ # pyhook_before: optional steps before running `python setup.py build' # pyhook_after: optional steps after running `python setup.py install' pybuild = cd $(ddir); rm -rf $(2); \ - if ! $(1) $(word 1,$(filter $(tdir)/%,$^)); then \ - echo; echo "Tar error"; exit 1; \ - fi; \ - cd $(2); \ - if [ "x$(strip $(3))" != x ]; then \ - sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ - -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ - $(3) > site.cfg; \ - fi; \ - if type pyhook_before &>/dev/null; then pyhook_before; fi \ - && python setup.py build \ - && python setup.py install \ - && if type pyhook_after &>/dev/null; then pyhook_after; fi \ - && cd .. \ - && rm -rf $(2) \ - && echo "$(4)" > $@ + if ! $(1) $(tdir)/$$tarball; then \ + echo; echo "Tar error"; exit 1; \ + fi; \ + cd $(2); \ + if [ "x$(strip $(3))" != x ]; then \ + sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ + -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ + $(3) > site.cfg; \ + fi; \ + if type pyhook_before &>/dev/null; then pyhook_before; fi; \ + python setup.py build; \ + python setup.py install; \ + if type pyhook_after &>/dev/null; then pyhook_after; fi; \ + cd ..; \ + rm -rf $(2); \ + echo "$(4)" > $@ @@ -368,31 +150,40 @@ pybuild = cd $(ddir); rm -rf $(2); \ # --------------- # # All the necessary Python modules go here. -$(ipydir)/asn1crypto: $(ipydir)/setuptools \ - $(tdir)/asn1crypto-$(asn1crypto-version).tar.gz +$(ipydir)/asn1crypto-$(asn1crypto-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=asn1crypto-$(asn1crypto-version).tar.gz + $(call import-source, $(asn1crypto-url), $(asn1crypto-checksum)) $(call pybuild, tar xf, asn1crypto-$(asn1crypto-version), , \ Asn1crypto $(asn1crypto-version)) -$(ipydir)/asteval: $(ipydir)/numpy \ - $(tdir)/asteval-$(asteval-version).tar.gz +$(ipydir)/asteval-$(asteval-version): $(ipydir)/numpy-$(numpy-version) + tarball=asteval-$(asteval-version).tar.gz + $(call import-source, $(asteval-url), $(asteval-checksum)) $(call pybuild, tar xf, asteval-$(asteval-version), , \ ASTEVAL $(asteval-version)) -$(ipydir)/astroquery: $(ipydir)/astropy \ - $(ipydir)/keyring \ - $(ipydir)/requests \ - $(tdir)/astroquery-$(astroquery-version).tar.gz - $(call pybuild, tar xf, astroquery-$(astroquery-version), ,\ +$(ipydir)/astroquery-$(astroquery-version): \ + $(ipydir)/astropy-$(astropy-version) \ + $(ipydir)/keyring-$(keyring-version) \ + $(ipydir)/requests-$(requests-version) + tarball=astroquery-$(astroquery-version).tar.gz + $(call import-source, $(astroquery-url), $(astroquery-checksum)) + $(call pybuild, tar xf, astroquery-$(astroquery-version), , \ Astroquery $(astroquery-version)) -$(ipydir)/astropy: $(ipydir)/h5py \ - $(ibidir)/expat \ - $(ipydir)/scipy \ - $(ipydir)/numpy \ - $(ipydir)/pyyaml \ - $(ipydir)/html5lib \ - $(ipydir)/beautifulsoup4 \ - $(tdir)/astropy-$(astropy-version).tar.gz +$(ipydir)/astropy-$(astropy-version): \ + $(ipydir)/h5py-$(h5py-version) \ + $(ibidir)/expat-$(expat-version) \ + $(ipydir)/scipy-$(scipy-version) \ + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/pyyaml-$(pyyaml-version) \ + $(ipydir)/html5lib-$(html5lib-version) \ + $(ipydir)/beautifulsoup4-$(beautifulsoup4-version) + + # Download the source. + tarball=astropy-$(astropy-version).tar.gz + $(call import-source, $(astropy-url), $(astropy-checksum)) + # Currently, when the Expat library is already built in a project # (for example as a dependency of another program), Astropy's # internal building of Expat will conflict with the project's. So @@ -404,107 +195,133 @@ $(ipydir)/astropy: $(ipydir)/h5py \ echo "[build]" >> setup.cfg echo "use_system_expat=1" >> setup.cfg } - $(call pybuild, tar xf, astropy-$(astropy-version)) \ - && cp $(dtexdir)/astropy.tex $(ictdir)/ \ - && echo "Astropy $(astropy-version) \citep{astropy2013,astropy2018}" > $@ - -$(ipydir)/beautifulsoup4: $(ipydir)/soupsieve \ - $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz - $(call pybuild, tar xf, beautifulsoup4-$(beautifulsoup4-version), ,\ + $(call pybuild, tar xf, astropy-$(astropy-version)) + cp $(dtexdir)/astropy.tex $(ictdir)/ + echo "Astropy $(astropy-version) \citep{astropy2013,astropy2018}" > $@ + +$(ipydir)/beautifulsoup4-$(beautifulsoup4-version): \ + $(ipydir)/soupsieve-$(soupsieve-version) + tarball=beautifulsoup4-$(beautifulsoup4-version).tar.gz + $(call import-source, $(beautifulsoup4-url), $(beautifulsoup4-checksum)) + $(call pybuild, tar xf, beautifulsoup4-$(beautifulsoup4-version), , \ BeautifulSoup $(beautifulsoup4-version)) -$(ipydir)/certifi: $(ipydir)/setuptools \ - $(tdir)/certifi-$(certifi-version).tar.gz - $(call pybuild, tar xf, certifi-$(certifi-version), ,\ +$(ipydir)/certifi-$(certifi-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=certifi-$(certifi-version).tar.gz + $(call import-source, $(certifi-url), $(certifi-checksum)) + $(call pybuild, tar xf, certifi-$(certifi-version), , \ Certifi $(certifi-version)) -$(ipydir)/cffi: $(ibidir)/libffi \ - $(ipydir)/pycparser \ - $(tdir)/cffi-$(cffi-version).tar.gz - $(call pybuild, tar xf, cffi-$(cffi-version), ,\ - cffi $(cffi-version)) - -$(ipydir)/chardet: $(ipydir)/setuptools \ - $(tdir)/chardet-$(chardet-version).tar.gz - $(call pybuild, tar xf, chardet-$(chardet-version), ,\ +$(ipydir)/cffi-$(cffi-version): \ + $(ibidir)/libffi-$(libffi-version) \ + $(ipydir)/pycparser-$(pycparser-version) + tarball=cffi-$(cffi-version).tar.gz + $(call import-source, $(cffi-url), $(cffi-checksum)) + $(call pybuild, tar xf, cffi-$(cffi-version), ,cffi $(cffi-version)) + +$(ipydir)/chardet-$(chardet-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=chardet-$(chardet-version).tar.gz + $(call import-source, $(chardet-url), $(chardet-checksum)) + $(call pybuild, tar xf, chardet-$(chardet-version), , \ Chardet $(chardet-version)) -$(ipydir)/corner: $(ipydir)/matplotlib \ - $(tdir)/corner-$(corner-version).tar.gz - $(call pybuild, tar xf, corner-$(corner-version), ,\ - Corner $(corner-version)) \ - && cp $(dtexdir)/corner.tex $(ictdir)/ \ - && echo "Corner $(corner-version) \citep{corner}" > $@ - -$(ipydir)/cryptography: $(ipydir)/cffi \ - $(ipydir)/asn1crypto \ - $(tdir)/cryptography-$(cryptography-version).tar.gz - $(call pybuild, tar xf, cryptography-$(cryptography-version), ,\ +$(ipydir)/corner-$(corner-version): $(ipydir)/matplotlib-$(matplotlib-version) + tarball=corner-$(corner-version).tar.gz + $(call import-source, $(corner-url), $(corner-checksum)) + $(call pybuild, tar xf, corner-$(corner-version), , \ + Corner $(corner-version)) + cp $(dtexdir)/corner.tex $(ictdir)/ + echo "Corner $(corner-version) \citep{corner}" > $@ + +$(ipydir)/cryptography-$(cryptography-version): \ + $(ipydir)/cffi-$(cffi-version) \ + $(ipydir)/asn1crypto-$(asn1crypto-version) + tarball=cryptography-$(cryptography-version).tar.gz + $(call import-source, $(cryptography-url), $(cryptography-checksum)) + $(call pybuild, tar xf, cryptography-$(cryptography-version), , \ Cryptography $(cryptography-version)) -$(ipydir)/cycler: $(ipydir)/six \ - $(tdir)/cycler-$(cycler-version).tar.gz - $(call pybuild, tar xf, cycler-$(cycler-version), ,\ +$(ipydir)/cycler-$(cycler-version): $(ipydir)/six-$(six-version) + tarball=cycler-$(cycler-version).tar.gz + $(call import-source, $(cycler-url), $(cycler-checksum)) + $(call pybuild, tar xf, cycler-$(cycler-version), , \ Cycler $(cycler-version)) -$(ipydir)/cython: $(ipydir)/setuptools \ - $(tdir)/cython-$(cython-version).tar.gz - $(call pybuild, tar xf, Cython-$(cython-version)) \ - && cp $(dtexdir)/cython.tex $(ictdir)/ \ - && echo "Cython $(cython-version) \citep{cython2011}" > $@ - -$(ipydir)/esutil: $(ipydir)/numpy \ - $(tdir)/esutil-$(esutil-version).tar.gz - $(call pybuild, tar xf, esutil-$(esutil-version), ,\ +$(ipydir)/cython-$(cython-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=cython-$(cython-version).tar.gz + $(call import-source, $(cython-url), $(cython-checksum)) + $(call pybuild, tar xf, Cython-$(cython-version)) + cp $(dtexdir)/cython.tex $(ictdir)/ + echo "Cython $(cython-version) \citep{cython2011}" > $@ + +$(ipydir)/esutil-$(esutil-version): $(ipydir)/numpy-$(numpy-version) + tarball=esutil-$(esutil-version).tar.gz + $(call import-source, $(esutil-url), $(esutil-checksum)) + $(call pybuild, tar xf, esutil-$(esutil-version), , \ esutil $(esutil-version)) -$(ipydir)/eigency: $(ibidir)/eigen \ - $(tdir)/eigency-$(eigency-version).tar.gz - $(call pybuild, tar xf, eigency-$(eigency-version), ,\ +$(ipydir)/eigency-$(eigency-version): \ + $(ipydir)/numpy-$(numpy-version) \ + $(ibidir)/eigen-$(eigen-version) \ + $(ipydir)/cython-$(cython-version) + tarball=eigency-$(eigency-version).tar.gz + $(call import-source, $(eigency-url), $(eigency-checksum)) + $(call pybuild, tar xf, eigency-$(eigency-version), , \ eigency $(eigency-version)) -$(ipydir)/emcee: $(ipydir)/numpy \ - $(ipydir)/setuptools_scm \ - $(tdir)/emcee-$(emcee-version).tar.gz - $(call pybuild, tar xf, emcee-$(emcee-version), ,\ +$(ipydir)/emcee-$(emcee-version): \ + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/setuptools_scm-$(setuptools_scm-version) + tarball=emcee-$(emcee-version).tar.gz + $(call import-source, $(emcee-url), $(emcee-checksum)) + $(call pybuild, tar xf, emcee-$(emcee-version), , \ emcee $(emcee-version)) -$(ipydir)/entrypoints: $(ipydir)/setuptools \ - $(tdir)/entrypoints-$(entrypoints-version).tar.gz - $(call pybuild, tar xf, entrypoints-$(entrypoints-version), ,\ +$(ipydir)/entrypoints-$(entrypoints-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=entrypoints-$(entrypoints-version).tar.gz + $(call import-source, $(entrypoints-url), $(entrypoints-checksum)) + $(call pybuild, tar xf, entrypoints-$(entrypoints-version), , \ EntryPoints $(entrypoints-version)) -$(ipydir)/flake8: $(ipydir)/pyflakes \ - $(ipydir)/pycodestyle \ - $(tdir)/flake8-$(flake8-version).tar.gz - $(call pybuild, tar xf, flake8-$(flake8-version), ,\ +$(ipydir)/flake8-$(flake8-version): \ + $(ipydir)/pyflakes-$(pyflakes-version) \ + $(ipydir)/pycodestyle-$(pycodestyle-version) + tarball=flake8-$(flake8-version).tar.gz + $(call import-source, $(flake8-url), $(flake8-checksum)) + $(call pybuild, tar xf, flake8-$(flake8-version), , \ Flake8 $(flake8-version)) -$(ipydir)/future: $(ipydir)/setuptools \ - $(tdir)/future-$(future-version).tar.gz - $(call pybuild, tar xf, future-$(future-version), ,\ +$(ipydir)/future-$(future-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=future-$(future-version).tar.gz + $(call import-source, $(future-url), $(future-checksum)) + $(call pybuild, tar xf, future-$(future-version), , \ Future $(future-version)) -$(ipydir)/galsim: $(ipydir)/future \ - $(ipydir)/astropy \ - $(ipydir)/eigency \ - $(ipydir)/pybind11 \ - $(ipydir)/lsstdesccoord \ - $(tdir)/galsim-$(galsim-version).tar.gz - $(call pybuild, tar xf, GalSim-$(galsim-version)) \ - && cp $(dtexdir)/galsim.tex $(ictdir)/ \ - && echo "Galsim $(galsim-version) \citep{galsim}" > $@ - -$(ipydir)/h5py: $(ipydir)/six \ - $(ibidir)/hdf5 \ - $(ipydir)/numpy \ - $(ipydir)/cython \ - $(ipydir)/mpi4py \ - $(ipydir)/pypkgconfig \ - $(tdir)/h5py-$(h5py-version).tar.gz - export HDF5_MPI=ON; \ - export HDF5_DIR=$(ildir); \ - $(call pybuild, tar xf, h5py-$(h5py-version), ,\ +$(ipydir)/galsim-$(galsim-version): \ + $(ipydir)/future-$(future-version) \ + $(ipydir)/astropy-$(astropy-version) \ + $(ipydir)/eigency-$(eigency-version) \ + $(ipydir)/pybind11-$(pybind11-version) \ + $(ipydir)/lsstdesccoord-$(lsstdesccoord-version) + tarball=galsim-$(galsim-version).tar.gz + $(call import-source, $(galsim-url), $(galsim-checksum)) + $(call pybuild, tar xf, GalSim-$(galsim-version)) + cp $(dtexdir)/galsim.tex $(ictdir)/ + echo "Galsim $(galsim-version) \citep{galsim}" > $@ + +$(ipydir)/h5py-$(h5py-version): \ + $(ipydir)/six-$(six-version) \ + $(ibidir)/hdf5-$(hdf5-version) \ + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/cython-$(cython-version) \ + $(ipydir)/mpi4py-$(mpi4py-version) \ + $(ipydir)/pypkgconfig-$(pypkgconfig-version) + export HDF5_MPI=ON + export HDF5_DIR=$(ildir) + tarball=h5py-$(h5py-version).tar.gz + $(call import-source, $(h5py-url), $(h5py-checksum)) + $(call pybuild, tar xf, h5py-$(h5py-version), , \ h5py $(h5py-version)) # `healpy' is actually installed as part of the HEALPix package. It will be @@ -514,231 +331,283 @@ $(ipydir)/h5py: $(ipydir)/six \ # and just ask for `healpy'. To avoid confusion in such cases, we'll just # set `healpy' to be dependent on `healpix' and not download any tarball # for it, or write anything in the final target. -$(ipydir)/healpy: $(ibidir)/healpix +$(ipydir)/healpy-$(healpy-version): $(ibidir)/healpix-$(healpy-version) touch $@ -$(ipydir)/html5lib: $(ipydir)/six \ - $(ipydir)/webencodings \ - $(tdir)/html5lib-$(html5lib-version).tar.gz - $(call pybuild, tar xf, html5lib-$(html5lib-version), ,\ +$(ipydir)/html5lib-$(html5lib-version): \ + $(ipydir)/six-$(six-version) \ + $(ipydir)/webencodings-$(webencodings-version) + tarball=html5lib-$(html5lib-version).tar.gz + $(call import-source, $(html5lib-url), $(html5lib-checksum)) + $(call pybuild, tar xf, html5lib-$(html5lib-version), , \ HTML5lib $(html5lib-version)) -$(ipydir)/idna: $(ipydir)/setuptools \ - $(tdir)/idna-$(idna-version).tar.gz - $(call pybuild, tar xf, idna-$(idna-version), ,\ +$(ipydir)/idna-$(idna-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=idna-$(idna-version).tar.gz + $(call import-source, $(idna-url), $(idna-checksum)) + $(call pybuild, tar xf, idna-$(idna-version), , \ idna $(idna-version)) -$(ipydir)/jeepney: $(ipydir)/setuptools \ - $(tdir)/jeepney-$(jeepney-version).tar.gz - $(call pybuild, tar xf, jeepney-$(jeepney-version), ,\ +$(ipydir)/jeepney-$(jeepney-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=jeepney-$(jeepney-version).tar.gz + $(call import-source, $(jeepney-url), $(jeepney-checksum)) + $(call pybuild, tar xf, jeepney-$(jeepney-version), , \ Jeepney $(jeepney-version)) -$(ipydir)/keyring: $(ipydir)/entrypoints \ - $(ipydir)/secretstorage \ - $(ipydir)/setuptools_scm \ - $(tdir)/keyring-$(keyring-version).tar.gz - $(call pybuild, tar xf, keyring-$(keyring-version), ,\ +$(ipydir)/keyring-$(keyring-version): \ + $(ipydir)/entrypoints-$(entrypoints-version) \ + $(ipydir)/secretstorage-$(secretstorage-version) \ + $(ipydir)/setuptools_scm-$(setuptools_scm-version) + tarball=keyring-$(keyring-version).tar.gz + $(call import-source, $(keyring-url), $(keyring-checksum)) + $(call pybuild, tar xf, keyring-$(keyring-version), , \ Keyring $(keyring-version)) -$(ipydir)/kiwisolver: $(ipydir)/setuptools \ - $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz - $(call pybuild, tar xf, kiwisolver-$(kiwisolver-version), ,\ +$(ipydir)/kiwisolver-$(kiwisolver-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=kiwisolver-$(kiwisolver-version).tar.gz + $(call import-source, $(kiwisolver-url), $(kiwisolver-checksum)) + $(call pybuild, tar xf, kiwisolver-$(kiwisolver-version), , \ Kiwisolver $(kiwisolver-version)) -$(ipydir)/lmfit: $(ipydir)/six \ - $(ipydir)/scipy \ - $(ipydir)/emcee \ - $(ipydir)/corner \ - $(ipydir)/asteval \ - $(ipydir)/matplotlib \ - $(ipydir)/uncertainties \ - $(tdir)/lmfit-$(lmfit-version).tar.gz - $(call pybuild, tar xf, lmfit-$(lmfit-version), ,\ +$(ipydir)/lmfit-$(lmfit-version): \ + $(ipydir)/six-$(six-version) \ + $(ipydir)/scipy-$(scipy-version) \ + $(ipydir)/emcee-$(emcee-version) \ + $(ipydir)/corner-$(corner-version) \ + $(ipydir)/asteval-$(asteval-version) \ + $(ipydir)/matplotlib-$(matplotlib-version) \ + $(ipydir)/uncertainties-$(uncertainties-version) + tarball=lmfit-$(lmfit-version).tar.gz + $(call import-source, $(lmfit-url), $(lmfit-checksum)) + $(call pybuild, tar xf, lmfit-$(lmfit-version), , \ LMFIT $(lmfit-version)) -$(ipydir)/lsstdesccoord: $(ipydir)/setuptools \ - $(tdir)/lsstdesccoord-$(lsstdesccoord-version).tar.gz - $(call pybuild, tar xf, LSSTDESC.Coord-$(lsstdesccoord-version), ,\ +$(ipydir)/lsstdesccoord-$(lsstdesccoord-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=lsstdesccoord-$(lsstdesccoord-version).tar.gz + $(call import-source, $(lsstdesccoord-url), $(lsstdesccoord-checksum)) + $(call pybuild, tar xf, LSSTDESC.Coord-$(lsstdesccoord-version), , \ LSSTDESC.Coord $(lsstdesccoord-version)) -$(ipydir)/matplotlib: $(ipydir)/numpy \ - $(ipydir)/cycler \ - $(itidir)/texlive \ - $(ibidir)/freetype \ - $(ipydir)/pyparsing \ - $(ipydir)/kiwisolver \ - $(ibidir)/ghostscript \ - $(ibidir)/imagemagick \ - $(ipydir)/python-dateutil \ - $(tdir)/matplotlib-$(matplotlib-version).tar.gz +$(ipydir)/matplotlib-$(matplotlib-version): \ + $(itidir)/texlive \ + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/cycler-$(cycler-version) \ + $(ibidir)/freetype-$(freetype-version) \ + $(ipydir)/pyparsing-$(pyparsing-version) \ + $(ipydir)/kiwisolver-$(kiwisolver-version) \ + $(ibidir)/ghostscript-$(ghostscript-version) \ + $(ibidir)/imagemagick-$(imagemagick-version) \ + $(ipydir)/python-dateutil-$(python-dateutil-version) + + # Download the source. + tarball=matplotlib-$(matplotlib-version).tar.gz + $(call import-source, $(matplotlib-url), $(matplotlib-checksum)) + # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. - if [ x$(on_mac_os) = xyes ]; then \ - export CC=clang; \ - export CXX=clang++; \ - fi; \ - $(call pybuild, tar xf, matplotlib-$(matplotlib-version)) \ - && cp $(dtexdir)/matplotlib.tex $(ictdir)/ \ - && echo "Matplotlib $(matplotlib-version) \citep{matplotlib2007}" > $@ - -$(ipydir)/mpi4py: $(ibidir)/openmpi \ - $(ipydir)/setuptools \ - $(tdir)/mpi4py-$(mpi4py-version).tar.gz - $(call pybuild, tar xf, mpi4py-$(mpi4py-version)) \ - && cp $(dtexdir)/mpi4py.tex $(ictdir)/ \ - && echo "mpi4py $(mpi4py-version) \citep{mpi4py2011}" > $@ - -$(ipydir)/mpmath: $(ipydir)/setuptools \ - $(tdir)/mpmath-$(mpmath-version).tar.gz - $(call pybuild, tar xf, mpmath-$(mpmath-version), ,\ + if [ x$(on_mac_os) = xyes ]; then + export CC=clang + export CXX=clang++ + fi + $(call pybuild, tar xf, matplotlib-$(matplotlib-version)) + cp $(dtexdir)/matplotlib.tex $(ictdir)/ + echo "Matplotlib $(matplotlib-version) \citep{matplotlib2007}" > $@ + +$(ipydir)/mpi4py-$(mpi4py-version): \ + $(ibidir)/openmpi-$(openmpi-version) \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=mpi4py-$(mpi4py-version).tar.gz + $(call import-source, $(mpi4py-url), $(mpi4py-checksum)) + $(call pybuild, tar xf, mpi4py-$(mpi4py-version)) + cp $(dtexdir)/mpi4py.tex $(ictdir)/ + echo "mpi4py $(mpi4py-version) \citep{mpi4py2011}" > $@ + +$(ipydir)/mpmath-$(mpmath-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=mpmath-$(mpmath-version).tar.gz + $(call import-source, $(mpmath-url), $(mpmath-checksum)) + $(call pybuild, tar xf, mpmath-$(mpmath-version), , \ mpmath $(mpmath-version)) -$(ipydir)/numpy: $(ibidir)/unzip \ - $(ibidir)/openblas \ - $(ipydir)/setuptools \ - $(tdir)/numpy-$(numpy-version).zip - if [ x$(on_mac_os) = xyes ]; then \ - export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ - else \ - export LDFLAGS="$(LDFLAGS) -shared"; \ - fi; \ - export CFLAGS="--std=c99 $$CFLAGS"; \ - conf="$$(pwd)/reproduce/software/config/numpy-scipy.cfg"; \ +$(ipydir)/numpy-$(numpy-version): \ + $(ibidir)/unzip-$(unzip-version) \ + $(ibidir)/openblas-$(openblas-version) \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=numpy-$(numpy-version).zip + $(call import-source, $(numpy-url), $(numpy-checksum)) + if [ x$(on_mac_os) = xyes ]; then + export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle" + else + export LDFLAGS="$(LDFLAGS) -shared" + fi + export CFLAGS="--std=c99 $$CFLAGS" + conf="$$(pwd)/reproduce/software/config/numpy-scipy.cfg" $(call pybuild, unzip, numpy-$(numpy-version),$$conf, \ - Numpy $(numpy-version)) \ - && cp $(dtexdir)/numpy.tex $(ictdir)/ \ - && echo "Numpy $(numpy-version) \citep{numpy2011}" > $@ - -$(ipydir)/pexpect: $(ipydir)/setuptools \ - $(tdir)/pexpect-$(pexpect-version).tar.gz - $(call pybuild, tar xf, pexpect-$(pexpect-version), ,\ + Numpy $(numpy-version)) + cp $(dtexdir)/numpy.tex $(ictdir)/ + echo "Numpy $(numpy-version) \citep{numpy2011}" > $@ + +$(ipydir)/pexpect-$(pexpect-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=pexpect-$(pexpect-version).tar.gz + $(call import-source, $(pexpect-url), $(pexpect-checksum)) + $(call pybuild, tar xf, pexpect-$(pexpect-version), , \ Pexpect $(pexpect-version)) -$(ibidir)/pip3: $(ipydir)/setuptools \ - $(tdir)/pip-$(pip-version).tar.gz - $(call pybuild, tar xf, pip-$(pip-version), ,\ +$(ipydir)/pip-$(pip-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=pip-$(pip-version).tar.gz + $(call import-source, $(pip-url), $(pip-checksum)) + $(call pybuild, tar xf, pip-$(pip-version), , \ PiP $(pip-version)) -$(ipydir)/pycodestyle: $(ipydir)/setuptools \ - $(tdir)/pycodestyle-$(pycodestyle-version).tar.gz - $(call pybuild, tar xf, pycodestyle-$(pycodestyle-version), ,\ +$(ipydir)/pycodestyle-$(pycodestyle-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=pycodestyle-$(pycodestyle-version).tar.gz + $(call import-source, $(pycodestyle-url), $(pycodestyle-checksum)) + $(call pybuild, tar xf, pycodestyle-$(pycodestyle-version), , \ pycodestyle $(pycodestyle-version)) -$(ipydir)/pybind11: $(ibidir)/eigen \ - $(ibidir)/boost \ - $(ipydir)/setuptools \ - $(tdir)/pybind11-$(pybind11-version).tar.gz +$(ipydir)/pybind11-$(pybind11-version): \ + $(ibidir)/eigen-$(eigen-version) \ + $(ibidir)/boost-$(boost-version) \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=pybind11-$(pybind11-version).tar.gz + $(call import-source, $(pybind11-url), $(pybind11-checksum)) pyhook_after() { cp -r include/pybind11 $(iidir)/python$(python-major-version)m/ } - $(call pybuild, tar xf, pybind11-$(pybind11-version), ,\ + $(call pybuild, tar xf, pybind11-$(pybind11-version), , \ pybind11 $(pybind11-version)) -$(ipydir)/pycparser: $(ipydir)/setuptools \ - $(tdir)/pycparser-$(pycparser-version).tar.gz - $(call pybuild, tar xf, pycparser-$(pycparser-version), ,\ +$(ipydir)/pycparser-$(pycparser-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=pycparser-$(pycparser-version).tar.gz + $(call import-source, $(pycparser-url), $(pycparser-checksum)) + $(call pybuild, tar xf, pycparser-$(pycparser-version), , \ pycparser $(pycparser-version)) -$(ipydir)/pyflakes: $(ipydir)/setuptools \ - $(tdir)/pyflakes-$(pyflakes-version).tar.gz - $(call pybuild, tar xf, pyflakes-$(pyflakes-version), ,\ +$(ipydir)/pyflakes-$(pyflakes-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=pyflakes-$(pyflakes-version).tar.gz + $(call import-source, $(pyflakes-url), $(pyflakes-checksum)) + $(call pybuild, tar xf, pyflakes-$(pyflakes-version), , \ pyflakes $(pyflakes-version)) -$(ipydir)/pyparsing: $(ipydir)/setuptools \ - $(tdir)/pyparsing-$(pyparsing-version).tar.gz - $(call pybuild, tar xf, pyparsing-$(pyparsing-version), ,\ +$(ipydir)/pyparsing-$(pyparsing-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=pyparsing-$(pyparsing-version).tar.gz + $(call import-source, $(pyparsing-url), $(pyparsing-checksum)) + $(call pybuild, tar xf, pyparsing-$(pyparsing-version), , \ PyParsing $(pyparsing-version)) -$(ipydir)/pypkgconfig: $(ipydir)/setuptools \ - $(tdir)/pkgconfig-$(pypkgconfig-version).tar.gz +$(ipydir)/pypkgconfig-$(pypkgconfig-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=pkgconfig-$(pypkgconfig-version).tar.gz + $(call import-source, $(pypkgconfig-url), $(pypkgconfig-checksum)) $(call pybuild, tar xf, pkgconfig-$(pypkgconfig-version), , pkgconfig $(pypkgconfig-version)) -$(ipydir)/python-dateutil: $(ipydir)/six \ - $(ipydir)/setuptools_scm \ - $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz - $(call pybuild, tar xf, python-dateutil-$(python-dateutil-version), ,\ +$(ipydir)/python-dateutil-$(python-dateutil-version): \ + $(ipydir)/six-$(six-version) \ + $(ipydir)/setuptools_scm-$(setuptools_scm-version) + tarball=python-dateutil-$(python-dateutil-version).tar.gz + $(call import-source, $(python-dateutil-url), $(python-dateutil-checksum)) + $(call pybuild, tar xf, python-dateutil-$(python-dateutil-version), , \ python-dateutil $(python-dateutil-version)) -$(ipydir)/pyyaml: $(ibidir)/yaml \ - $(ipydir)/cython \ - $(tdir)/pyyaml-$(pyyaml-version).tar.gz - $(call pybuild, tar xf, PyYAML-$(pyyaml-version), ,\ +$(ipydir)/pyyaml-$(pyyaml-version): \ + $(ibidir)/yaml-$(yaml-version) \ + $(ipydir)/cython-$(cython-version) + tarball=pyyaml-$(pyyaml-version).tar.gz + $(call import-source, $(pyyaml-url), $(pyyaml-checksum)) + $(call pybuild, tar xf, PyYAML-$(pyyaml-version), , \ PyYAML $(pyyaml-version)) -$(ipydir)/requests: $(ipydir)/idna \ - $(ipydir)/numpy \ - $(ipydir)/certifi \ - $(ipydir)/chardet \ - $(ipydir)/urllib3 \ - $(tdir)/requests-$(requests-version).tar.gz - $(call pybuild, tar xf, requests-$(requests-version), ,\ +$(ipydir)/requests-$(requests-version): $(ipydir)/idna-$(idna-version) \ + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/certifi-$(certifi-version) \ + $(ipydir)/chardet-$(chardet-version) \ + $(ipydir)/urllib3-$(urllib3-version) + tarball=requests-$(requests-version).tar.gz + $(call import-source, $(requests-url), $(requests-checksum)) + $(call pybuild, tar xf, requests-$(requests-version), , \ Requests $(requests-version)) -$(ipydir)/scipy: $(ipydir)/numpy \ - $(tdir)/scipy-$(scipy-version).tar.gz - if [ x$(on_mac_os) = xyes ]; then \ - export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ - else \ - export LDFLAGS="$(LDFLAGS) -shared"; \ - fi; \ - conf="$$(pwd)/reproduce/software/config/numpy-scipy.cfg"; \ - $(call pybuild, tar xf, scipy-$(scipy-version),$$conf) \ - && cp $(dtexdir)/scipy.tex $(ictdir)/ \ - && echo "Scipy $(scipy-version) \citep{scipy2007,scipy2011}" > $@ - -$(ipydir)/secretstorage: $(ipydir)/jeepney \ - $(ipydir)/cryptography \ - $(tdir)/secretstorage-$(secretstorage-version).tar.gz - $(call pybuild, tar xf, SecretStorage-$(secretstorage-version), ,\ +$(ipydir)/scipy-$(scipy-version): $(ipydir)/numpy-$(numpy-version) + tarball=scipy-$(scipy-version).tar.gz + $(call import-source, $(scipy-url), $(scipy-checksum)) + if [ x$(on_mac_os) = xyes ]; then + export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle" + else + export LDFLAGS="$(LDFLAGS) -shared" + fi + conf="$$(pwd)/reproduce/software/config/numpy-scipy.cfg" + $(call pybuild, tar xf, scipy-$(scipy-version),$$conf) + cp $(dtexdir)/scipy.tex $(ictdir)/ + echo "Scipy $(scipy-version) \citep{scipy2007,scipy2011}" > $@ + +$(ipydir)/secretstorage-$(secretstorage-version): \ + $(ipydir)/jeepney-$(jeepney-version) \ + $(ipydir)/cryptography-$(cryptography-version) + tarball=secretstorage-$(secretstorage-version).tar.gz + $(call import-source, $(secretstorage-url), $(secretstorage-checksum)) + $(call pybuild, tar xf, SecretStorage-$(secretstorage-version), , \ SecretStorage $(secretstorage-version)) -$(ipydir)/setuptools: $(ibidir)/unzip \ - $(ibidir)/python \ - $(tdir)/setuptools-$(setuptools-version).zip - $(call pybuild, unzip, setuptools-$(setuptools-version), ,\ +$(ipydir)/setuptools-$(setuptools-version): \ + $(ibidir)/unzip-$(unzip-version) \ + $(ibidir)/python-$(python-version) + tarball=setuptools-$(setuptools-version).zip + $(call import-source, $(setuptools-url), $(setuptools-checksum)) + $(call pybuild, unzip, setuptools-$(setuptools-version), , \ Setuptools $(setuptools-version)) -$(ipydir)/setuptools_scm: $(ipydir)/setuptools \ - $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.gz - $(call pybuild, tar xf, setuptools_scm-$(setuptools_scm-version), ,\ +$(ipydir)/setuptools_scm-$(setuptools_scm-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=setuptools_scm-$(setuptools_scm-version).tar.gz + $(call import-source, $(setuptools_scm-url), $(setuptools_scm-checksum)) + $(call pybuild, tar xf, setuptools_scm-$(setuptools_scm-version), , \ Setuptools-scm $(setuptools_scm-version)) -$(ipydir)/sip_tpv: $(ipydir)/sympy \ - $(ipydir)/astropy \ - $(tdir)/sip_tpv-$(sip_tpv-version).tar.gz - $(call pybuild, tar xf, sip_tpv-$(sip_tpv-version), ,) \ - && cp $(dtexdir)/sip_tpv.tex $(ictdir)/ \ - && echo "sip_tpv $(sip_tpv-version) \citep{sip-tpv}" > $@ +$(ipydir)/sip_tpv-$(sip_tpv-version): \ + $(ipydir)/sympy-$(sympy-version) \ + $(ipydir)/astropy-$(astropy-version) + tarball=sip_tpv-$(sip_tpv-version).tar.gz + $(call import-source, $(sip_tpv-url), $(sip_tpv-checksum)) + $(call pybuild, tar xf, sip_tpv-$(sip_tpv-version), ,) + cp $(dtexdir)/sip_tpv.tex $(ictdir)/ + echo "sip_tpv $(sip_tpv-version) \citep{sip-tpv}" > $@ -$(ipydir)/six: $(ipydir)/setuptools \ - $(tdir)/six-$(six-version).tar.gz - $(call pybuild, tar xf, six-$(six-version), ,\ +$(ipydir)/six-$(six-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=six-$(six-version).tar.gz + $(call import-source, $(six-url), $(six-checksum)) + $(call pybuild, tar xf, six-$(six-version), , \ Six $(six-version)) -$(ipydir)/soupsieve: $(ipydir)/setuptools \ - $(tdir)/soupsieve-$(soupsieve-version).tar.gz - $(call pybuild, tar xf, soupsieve-$(soupsieve-version), ,\ +$(ipydir)/soupsieve-$(soupsieve-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=soupsieve-$(soupsieve-version).tar.gz + $(call import-source, $(soupsieve-url), $(soupsieve-checksum)) + $(call pybuild, tar xf, soupsieve-$(soupsieve-version), , \ SoupSieve $(soupsieve-version)) -$(ipydir)/sympy: $(ipydir)/mpmath \ - $(tdir)/sympy-$(sympy-version).tar.gz - $(call pybuild, tar xf, sympy-$(sympy-version), ,) \ - && cp $(dtexdir)/sympy.tex $(ictdir)/ \ - && echo "SymPy $(sympy-version) \citep{sympy}" > $@ - -$(ipydir)/uncertainties: $(ipydir)/numpy \ - $(tdir)/uncertainties-$(uncertainties-version).tar.gz - $(call pybuild, tar xf, uncertainties-$(uncertainties-version), ,\ +$(ipydir)/sympy-$(sympy-version): $(ipydir)/mpmath-$(mpmath-version) + tarball=sympy-$(sympy-version).tar.gz + $(call import-source, $(sympy-url), $(sympy-checksum)) + $(call pybuild, tar xf, sympy-$(sympy-version), ,) + cp $(dtexdir)/sympy.tex $(ictdir)/ + echo "SymPy $(sympy-version) \citep{sympy}" > $@ + +$(ipydir)/uncertainties-$(uncertainties-version): $(ipydir)/numpy-$(numpy-version) + tarball=uncertainties-$(uncertainties-version).tar.gz + $(call import-source, $(uncertainties-url), $(uncertainties-checksum)) + $(call pybuild, tar xf, uncertainties-$(uncertainties-version), , \ uncertainties $(uncertainties-version)) -$(ipydir)/urllib3: $(ipydir)/setuptools \ - $(tdir)/urllib3-$(urllib3-version).tar.gz - $(call pybuild, tar xf, urllib3-$(urllib3-version), ,\ +$(ipydir)/urllib3-$(urllib3-version): $(ipydir)/setuptools-$(setuptools-version) + tarball=urllib3-$(urllib3-version).tar.gz + $(call import-source, $(urllib3-url), $(urllib3-checksum)) + $(call pybuild, tar xf, urllib3-$(urllib3-version), , \ Urllib3 $(urllib3-version)) -$(ipydir)/webencodings: $(ipydir)/setuptools \ - $(tdir)/webencodings-$(webencodings-version).tar.gz - $(call pybuild, tar xf, webencodings-$(webencodings-version), ,\ +$(ipydir)/webencodings-$(webencodings-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=webencodings-$(webencodings-version).tar.gz + $(call import-source, $(webencodings-url), $(webencodings-checksum)) + $(call pybuild, tar xf, webencodings-$(webencodings-version), , \ Webencodings $(webencodings-version)) diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index b2e3979..38dcf7f 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -153,6 +153,38 @@ check_permission () +# Check if there is enough free space available in the build directory +# -------------------------------------------------------------------- +# +# Use this function to check if there is enough free space in a +# directory. It is meant to be passed to the 'if' statement in the +# shell. So if there is enough space, it returns 0 (which translates to +# TRUE), otherwise, the funcion returns 1 (which translates to FALSE). +# +# Expects to be called with two arguments, the first is the threshold and +# the second is the desired directory. The 'df' function checks the given +# path to see where it is mounted on, and how much free space there is on +# that partition (in units of 1024 bytes). +# +# synopsis: +# $ free_space_warning <acceptable_threshold> <path-to-check> +# +# example: +# To check if there is 5MB of space available in /path/to/check +# call the command with arguments as shown below: +# $ free_space_warning 5000 /path/to/check/free/space +free_space_warning() +{ + fs_threshold=$1 + fs_destpath=$2 + return $(df $fs_destpath \ + | awk 'FNR==2 {if($4>'$fs_threshold') print 1; \ + else print 0; }') +} + + + + # Check for C/C++ compilers # ------------------------- @@ -729,14 +761,14 @@ EOF bdir=$(absolute_dir $build_dir) rm -rf $build_dir/$junkname else - echo " ** Can't write in '$build_dir'"; echo + echo " ** Can't write in '$build_dir'"; fi else if mkdir $build_dir 2> /dev/null; then instring="the newly created" bdir=$(absolute_dir $build_dir) else - echo " ** Can't create '$build_dir'"; echo + echo " ** Can't create '$build_dir'"; fi fi @@ -759,8 +791,19 @@ EOF # file permissions. if ! [ x"$bdir" = x ]; then if ! $(check_permission $bdir); then + # Unable to handle permissions well bdir= echo " ** File permissions can't be modified in this directory" + else + # Able to handle permissions, now check for 5GB free space + # in the given partition (note that the number is in units + # of 1024 bytes). If this is not the case, print a warning. + if $(free_space_warning 5000000 $bdir); then + echo " !! LESS THAN 5GB FREE SPACE IN: $bdir" + echo " !! We recommend choosing another partition." + echo " !! Build will continue in 5 seconds..." + sleep 5 + fi fi fi @@ -1113,7 +1156,7 @@ ln -s $instdir .local ln -s $texdir tex/build ln -s $tikzdir tex/tikz # --------- Delete for no Gnuastro --------- -ln -s $topdir/reproduce/software/config/gnuastro .gnuastro +ln -s $topdir/reproduce/analysis/config/gnuastro .gnuastro # ------------------------------------------ @@ -1250,6 +1293,48 @@ fi +# Find Zenodo URL for software downloading +# ---------------------------------------- +# +# All free-software source tarballs that are potentially used in Maneage +# are also archived in Zenodo with a certain concept-DOI. A concept-DOI is +# a Zenodo terminology, meaning a fixed DOI of the project (that can have +# many sub-DOIs for different versions). By default, the concept-DOI points +# to the most recently uploaded version. However, the concept-DOI itself is +# not directly usable for downloading files. The concept-DOI will just take +# us to the top webpage of the most recent version of the upload. +# +# The problem is that as more software are added (as new Zenodo versions), +# the most recent Zenodo-URL that the concept-DOI points to, also +# changes. The most reliable solution was found to be the tiny script below +# which will download the DOI-resolved webpage, and extract the Zenodo-URL +# of the most recent version from there (using the 'coreutils' tarball as +# an example, the directory part of the URL for all the other software are +# the same). +user_backup_urls="" +zenodocheck=.build/software/zenodo-check.html +if $downloader $zenodocheck https://doi.org/10.5281/zenodo.3883409; then + zenodourl=$(sed -n -e'/coreutils/p' $zenodocheck \ + | sed -n -e'/http/p' \ + | tr ' ' '\n' \ + | grep http \ + | sed -e 's/href="//' -e 's|/coreutils| |' \ + | awk 'NR==1{print $1}') +else + zenodourl="" +fi +rm -f $zenodocheck + +# Add the Zenodo URL to the user's given back software URLs. Since the user +# can specify 'user_backup_urls' (not yet implemented as an option in +# './project'), we'll give preference to their specified servers, then add +# the Zenodo URL afterwards. +user_backup_urls="$user_backup_urls $zenodourl" + + + + + # Build core tools for project # ---------------------------- # @@ -1258,7 +1343,7 @@ fi # (minimal Bash-like shell) and Flock (to lock files and enable serial # download). ./reproduce/software/shell/pre-make-build.sh \ - "$bdir" "$ddir" "$downloader" + "$bdir" "$ddir" "$downloader" "$user_backup_urls" @@ -1272,6 +1357,7 @@ fi # tools, but we have to be very portable (and use minimal features in all). echo; echo "Building necessary software (if necessary)..." .local/bin/make -k -f reproduce/software/make/basic.mk \ + user_backup_urls="$user_backup_urls" \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ static_build=$static_build \ @@ -1293,20 +1379,22 @@ echo; echo "Building necessary software (if necessary)..." # script. Bash and Make were the tools we need to run Makefiles, so we had # to build them in this script. But after this, we can rely on Makefiles. if [ $jobs = 0 ]; then - numthreads=$($instdir/bin/nproc --all) + numthreads=$(.local/bin/nproc --all) else numthreads=$jobs fi .local/bin/env -i HOME=$bdir \ - .local/bin/make -k -f reproduce/software/make/high-level.mk \ - sys_library_path=$sys_library_path \ - rpath_command=$rpath_command \ - static_build=$static_build \ - numthreads=$numthreads \ - on_mac_os=$on_mac_os \ - sys_cpath=$sys_cpath \ - host_cc=$host_cc \ - -j$numthreads + .local/bin/make -k -f reproduce/software/make/high-level.mk \ + user_backup_urls="$user_backup_urls" \ + sys_library_path=$sys_library_path \ + rpath_command=$rpath_command \ + all_highlevel=$all_highlevel \ + static_build=$static_build \ + numthreads=$numthreads \ + on_mac_os=$on_mac_os \ + sys_cpath=$sys_cpath \ + host_cc=$host_cc \ + -j$numthreads @@ -1400,26 +1488,27 @@ prepare_name_version () fi } +# Import the context/sentences for placing between the list of software +# names during their acknowledgment. +. $cdir/software_acknowledge_context.sh + # Report the different software in separate contexts (separating Python and # TeX packages from the C/C++ programs and libraries). proglibs=$(prepare_name_version $verdir/proglib/*) pymodules=$(prepare_name_version $verdir/python/*) texpkg=$(prepare_name_version $verdir/tex/texlive) -# Write them as one paragraph for LaTeX. +# Acknowledge these software packages in a LaTeX paragraph. pkgver=$mtexdir/dependencies.tex -.local/bin/echo "This research was done with the following free" > $pkgver -.local/bin/echo "software programs and libraries: $proglibs." >> $pkgver + +# Add the text to the ${pkgver} file. +.local/bin/echo "$thank_software_introduce " > $pkgver +.local/bin/echo "$thank_progs_libs $proglibs. " >> $pkgver if [ x"$pymodules" != x ]; then - .local/bin/echo "Within Python, the following modules" >> $pkgver - echo "were used: $pymodules." >> $pkgver + .local/bin/echo "$thank_python $pymodules. " >> $pkgver fi -.local/bin/echo "The \LaTeX{} source of the paper was compiled" >> $pkgver -.local/bin/echo "to make the PDF using the following packages:" >> $pkgver -.local/bin/echo "$texpkg. We are very grateful to all their" >> $pkgver -.local/bin/echo "creators for freely providing this necessary" >> $pkgver -.local/bin/echo "infrastructure. This research (and many " >> $pkgver -.local/bin/echo "others) would not be possible without them." >> $pkgver +.local/bin/echo "$thank_latex $texpkg. " >> $pkgver +.local/bin/echo "$thank_software_conclude" >> $pkgver # Prepare the BibTeX entries for the used software (if there are any). hasentry=0 diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh index e2ac789..05a4143 100755 --- a/reproduce/software/shell/pre-make-build.sh +++ b/reproduce/software/shell/pre-make-build.sh @@ -34,6 +34,7 @@ set -e bdir=$1 ddir=$2 downloader="$3" +user_backup_urls="$4" @@ -51,6 +52,7 @@ downloadwrapper=reproduce/analysis/bash/download-multi-try # Derived directories bindir=$instdir/bin +urlfile=$confdir/urls.conf versionsfile=$confdir/versions.conf checksumsfile=$confdir/checksums.conf backupfile=$confdir/servers-backup.conf @@ -65,8 +67,18 @@ export PATH="$bindir:$PATH" -# Load the backup servers -backupservers=$(awk '!/^#/{printf "%s ", $1}' $backupfile) +# Load the backup servers, but separate the first one. +backupservers="" +topbackupserver="" +maneage_backup_urls=$(awk '!/^#/{printf "%s ", $1}' $backupfile) +backupservers_all="$user_backup_urls $maneage_backup_urls" +for b in $backupservers_all; do + if [ x$topbackupserver = x ]; then + topbackupserver=$b + else + backupservers="$backupservers $b" + fi +done @@ -83,12 +95,21 @@ download_tarball() { else ucname=$tardir/$tarball.unchecked + # If the URL is empty, use the top backup server + if [ x$w = x ]; then + bservers="$backupservers" + tarballurl=$topbackupserver/$tarball + else + bservers="$backupservers_all" + tarballurl=$url/$tarball + fi + # See if it is in the input software directory. if [ -f "$ddir/$tarball" ]; then cp $ddir/$tarball $ucname else - $downloadwrapper "$downloader" nolock $url/$tarball $ucname \ - "$backupservers" + $downloadwrapper "$downloader" nolock $tarballurl $ucname \ + "$bservers" fi # Make sure this is the correct tarball. @@ -119,10 +140,14 @@ download_tarball() { -# Build the program from the tarball +# Build the program from the tarball. This function takes one argument +# which is the configure-time options. build_program() { if ! [ -f $ibidir/$progname ]; then + # Options + configoptions=$1 + # Go into the temporary building directory. cd $tmpblddir unpackdir="$progname"-"$version" @@ -140,13 +165,31 @@ build_program() { intar=$tardir/$tarball fi - # Unpack the tarball and build the program. + # Unpack the tarball and go into it. tar xf $intar if [ x$intarrm = x1 ]; then rm $intar; fi cd $unpackdir - ./configure --prefix=$instdir - make - make install + + # build the project, either with Make and either without it. + if [ x$progname = xlzip ]; then + ./configure --build --check --installdir=$instdir/bin $configoptions + else + # All others accept the configure script. + ./configure --prefix=$instdir $configoptions + + # To build GNU Make, we don't want to assume the existance of a + # Make program, so we use its 'build.sh' script and its own built + # 'make' program to install itself. + if [ x$progname = xmake ]; then + /bin/sh build.sh + ./make install + else + make + make install + fi + fi + + # Clean up the source directory cd $topdir rm -rf $tmpblddir/$unpackdir echo "$progname_tex $version" > $ibidir/$progname @@ -167,7 +210,7 @@ build_program() { # won't rely on the host's compression tools at all. progname="lzip" progname_tex="Lzip" -url=http://akhlaghi.org/src +url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) tarball=$progname-$version.tar download_tarball @@ -180,19 +223,17 @@ build_program # GNU Make # -------- # -# The job orchestrator of Maneage is GNU Make. Although it is not -# impossible to account for all the differences between various Make -# implementations, its much easier (for reading the code and -# writing/debugging it) if we can count on a special implementation. So -# before going into the complex job orchestration in building high-level -# software, we start by building GNU Make. +# The job orchestrator of Maneage is GNU Make. The +# '--disable-dependency-tracking' configure-time option is necessary so +# Make doesn't check for an existing 'make' implementation (recall that we +# aren't assuming any 'make' on the host). progname="make" progname_tex="GNU Make" -url=http://akhlaghi.org/src +url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) tarball=$progname-$version.tar.lz download_tarball -build_program +build_program --disable-dependency-tracking @@ -206,7 +247,7 @@ build_program # (which builds GNU Bash). progname="dash" progname_tex="Dash" -url=http://akhlaghi.org/src +url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) tarball=$progname-$version.tar.lz download_tarball @@ -235,7 +276,7 @@ fi # many simultaneous download commands are called. progname="flock" progname_tex="Discoteq flock" -url=http://akhlaghi.org/src +url=$(awk '/^'$progname'-url/{print $3}' $urlfile) version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) tarball=$progname-$version.tar.lz download_tarball diff --git a/reproduce/software/shell/run-parts.in b/reproduce/software/shell/run-parts.in new file mode 100755 index 0000000..9213585 --- /dev/null +++ b/reproduce/software/shell/run-parts.in @@ -0,0 +1,71 @@ +#!MANEAGESHELL +# run-parts: Runs all the scripts found in a directory. +# from Slackware, by Patrick J. Volkerding with ideas borrowed +# from the Red Hat and Debian versions of this utility. +# +# USAGE IN MANEAGE: this script is built with the 'libpaper' package. +# +# The original file was taken from Linux From Scratch: +# http://www.linuxfromscratch.org/blfs/view/svn/general/libpaper.html +# However, it didn't have a copyright statement. So one is being added +# here. +# +# Copyright (C) 2020 Authors mentioned above. +# Copyright (C) 2020 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# +# This script is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This script is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this script. If not, see <http://www.gnu.org/licenses/>. + +# keep going when something fails +set +e + +if [ $# -lt 1 ]; then + echo "Usage: run-parts <directory>" + exit 1 +fi + +if [ ! -d $1 ]; then + echo "Not a directory: $1" + echo "Usage: run-parts <directory>" + exit 1 +fi + +# There are several types of files that we would like to +# ignore automatically, as they are likely to be backups +# of other scripts: +IGNORE_SUFFIXES="~ ^ , .bak .new .rpmsave .rpmorig .rpmnew .swp" + +# Main loop: +for SCRIPT in $1/* ; do + # If this is not a regular file, skip it: + if [ ! -f $SCRIPT ]; then + continue + fi + # Determine if this file should be skipped by suffix: + SKIP=false + for SUFFIX in $IGNORE_SUFFIXES ; do + if [ ! "$(basename $SCRIPT $SUFFIX)" = "$(basename $SCRIPT)" ]; then + SKIP=true + break + fi + done + if [ "$SKIP" = "true" ]; then + continue + fi + # If we've made it this far, then run the script if it's executable: + if [ -x $SCRIPT ]; then + $SCRIPT || echo "$SCRIPT failed." + fi +done + +exit 0 |