From 22d915e020643797eab6f3f3017752cb1f6b0178 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 6 May 2020 00:45:22 +0100 Subject: Software are rebuilt automatically with change of version Until now, when you changed the version of a software in an already-built system, its tarball would be downloaded, but it wouldn't actually build. The only way would be to force the build by deleting the main target of that file (under `.local/version-info/TYPE/PROGRAM'). This was because the tarballs were an order-only prerequisite which was implemented some time ago based on some theoretical argument that if the tarball dates changes, the software should not be rebuilt (because we check the checksum). However, the problems this causes are more than those it solves: Users may forget to delete the main target of the program and mistakenly think that they are using the new version. The fact that all the numbers going into the paper also contain this number further hides this. With this commit, tarballs are no longer order-only and any time a version of a software is updated, it will be automatically built and not cause confusion and manual intervention by the users. As a result of this change, I also had to correct the way we find the tarball from the list of prerequisites. --- reproduce/software/make/python.mk | 112 +++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 56 deletions(-) (limited to 'reproduce/software/make/python.mk') diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index f378650..7537275 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -283,7 +283,7 @@ $(pytarballs): $(tdir)/%: # # While this Makefile is for Python programs, in some cases, we need # certain programs (like Python itself), or libraries for the modules. -$(ibidir)/libffi: | $(tdir)/libffi-$(libffi-version).tar.gz +$(ibidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz # On some Fedora systems, libffi installs in `lib64', not # `lib'. This will cause problems when building setuptools @@ -299,7 +299,7 @@ $(ibidir)/libffi: | $(tdir)/libffi-$(libffi-version).tar.gz && echo "Libffi $(libffi-version)" > $@ $(ibidir)/python: $(ibidir)/libffi \ - | $(tdir)/python-$(python-version).tar.gz + $(tdir)/python-$(python-version).tar.gz # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. if [ x$(on_mac_os) = xyes ]; then \ @@ -339,7 +339,7 @@ $(ibidir)/python: $(ibidir)/libffi \ # pyhook_before: optional steps before running `python setup.py build' # pyhook_after: optional steps after running `python setup.py install' pybuild = cd $(ddir); rm -rf $(2); \ - if ! $(1) $(word 1,$(filter $(tdir)/%,$|)); then \ + if ! $(1) $(word 1,$(filter $(tdir)/%,$^)); then \ echo; echo "Tar error"; exit 1; \ fi; \ cd $(2); \ @@ -365,19 +365,19 @@ pybuild = cd $(ddir); rm -rf $(2); \ # # All the necessary Python modules go here. $(ipydir)/asn1crypto: $(ipydir)/setuptools \ - | $(tdir)/asn1crypto-$(asn1crypto-version).tar.gz + $(tdir)/asn1crypto-$(asn1crypto-version).tar.gz $(call pybuild, tar xf, asn1crypto-$(asn1crypto-version), , \ Asn1crypto $(asn1crypto-version)) $(ipydir)/asteval: $(ipydir)/numpy \ - | $(tdir)/asteval-$(asteval-version).tar.gz + $(tdir)/asteval-$(asteval-version).tar.gz $(call pybuild, tar xf, asteval-$(asteval-version), , \ ASTEVAL $(asteval-version)) $(ipydir)/astroquery: $(ipydir)/astropy \ $(ipydir)/keyring \ $(ipydir)/requests \ - | $(tdir)/astroquery-$(astroquery-version).tar.gz + $(tdir)/astroquery-$(astroquery-version).tar.gz $(call pybuild, tar xf, astroquery-$(astroquery-version), ,\ Astroquery $(astroquery-version)) @@ -388,7 +388,7 @@ $(ipydir)/astropy: $(ipydir)/h5py \ $(ipydir)/pyyaml \ $(ipydir)/html5lib \ $(ipydir)/beautifulsoup4 \ - | $(tdir)/astropy-$(astropy-version).tar.gz + $(tdir)/astropy-$(astropy-version).tar.gz # Currently, when the Expat library is already built in a project # (for example as a dependency of another program), Astropy's # internal building of Expat will conflict with the project's. So @@ -405,28 +405,28 @@ $(ipydir)/astropy: $(ipydir)/h5py \ && echo "Astropy $(astropy-version) \citep{astropy2013,astropy2018}" > $@ $(ipydir)/beautifulsoup4: $(ipydir)/soupsieve \ - | $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz + $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz $(call pybuild, tar xf, beautifulsoup4-$(beautifulsoup4-version), ,\ BeautifulSoup $(beautifulsoup4-version)) $(ipydir)/certifi: $(ipydir)/setuptools \ - | $(tdir)/certifi-$(certifi-version).tar.gz + $(tdir)/certifi-$(certifi-version).tar.gz $(call pybuild, tar xf, certifi-$(certifi-version), ,\ Certifi $(certifi-version)) $(ipydir)/cffi: $(ibidir)/libffi \ $(ipydir)/pycparser \ - | $(tdir)/cffi-$(cffi-version).tar.gz + $(tdir)/cffi-$(cffi-version).tar.gz $(call pybuild, tar xf, cffi-$(cffi-version), ,\ cffi $(cffi-version)) $(ipydir)/chardet: $(ipydir)/setuptools \ - | $(tdir)/chardet-$(chardet-version).tar.gz + $(tdir)/chardet-$(chardet-version).tar.gz $(call pybuild, tar xf, chardet-$(chardet-version), ,\ Chardet $(chardet-version)) $(ipydir)/corner: $(ipydir)/matplotlib \ - | $(tdir)/corner-$(corner-version).tar.gz + $(tdir)/corner-$(corner-version).tar.gz $(call pybuild, tar xf, corner-$(corner-version), ,\ Corner $(corner-version)) \ && cp $(dtexdir)/corner.tex $(ictdir)/ \ @@ -434,50 +434,50 @@ $(ipydir)/corner: $(ipydir)/matplotlib \ $(ipydir)/cryptography: $(ipydir)/cffi \ $(ipydir)/asn1crypto \ - | $(tdir)/cryptography-$(cryptography-version).tar.gz + $(tdir)/cryptography-$(cryptography-version).tar.gz $(call pybuild, tar xf, cryptography-$(cryptography-version), ,\ Cryptography $(cryptography-version)) $(ipydir)/cycler: $(ipydir)/six \ - | $(tdir)/cycler-$(cycler-version).tar.gz + $(tdir)/cycler-$(cycler-version).tar.gz $(call pybuild, tar xf, cycler-$(cycler-version), ,\ Cycler $(cycler-version)) $(ipydir)/cython: $(ipydir)/setuptools \ - | $(tdir)/cython-$(cython-version).tar.gz + $(tdir)/cython-$(cython-version).tar.gz $(call pybuild, tar xf, Cython-$(cython-version)) \ && cp $(dtexdir)/cython.tex $(ictdir)/ \ && echo "Cython $(cython-version) \citep{cython2011}" > $@ $(ipydir)/esutil: $(ipydir)/numpy \ - | $(tdir)/esutil-$(esutil-version).tar.gz + $(tdir)/esutil-$(esutil-version).tar.gz $(call pybuild, tar xf, esutil-$(esutil-version), ,\ esutil $(esutil-version)) $(ipydir)/eigency: $(ibidir)/eigen \ - | $(tdir)/eigency-$(eigency-version).tar.gz + $(tdir)/eigency-$(eigency-version).tar.gz $(call pybuild, tar xf, eigency-$(eigency-version), ,\ eigency $(eigency-version)) $(ipydir)/emcee: $(ipydir)/numpy \ $(ipydir)/setuptools_scm \ - | $(tdir)/emcee-$(emcee-version).tar.gz + $(tdir)/emcee-$(emcee-version).tar.gz $(call pybuild, tar xf, emcee-$(emcee-version), ,\ emcee $(emcee-version)) $(ipydir)/entrypoints: $(ipydir)/setuptools \ - | $(tdir)/entrypoints-$(entrypoints-version).tar.gz + $(tdir)/entrypoints-$(entrypoints-version).tar.gz $(call pybuild, tar xf, entrypoints-$(entrypoints-version), ,\ EntryPoints $(entrypoints-version)) $(ipydir)/flake8: $(ipydir)/pyflakes \ $(ipydir)/pycodestyle \ - | $(tdir)/flake8-$(flake8-version).tar.gz + $(tdir)/flake8-$(flake8-version).tar.gz $(call pybuild, tar xf, flake8-$(flake8-version), ,\ Flake8 $(flake8-version)) $(ipydir)/future: $(ipydir)/setuptools \ - | $(tdir)/future-$(future-version).tar.gz + $(tdir)/future-$(future-version).tar.gz $(call pybuild, tar xf, future-$(future-version), ,\ Future $(future-version)) @@ -486,7 +486,7 @@ $(ipydir)/galsim: $(ipydir)/future \ $(ipydir)/eigency \ $(ipydir)/pybind11 \ $(ipydir)/lsstdesccoord \ - | $(tdir)/galsim-$(galsim-version).tar.gz + $(tdir)/galsim-$(galsim-version).tar.gz $(call pybuild, tar xf, GalSim-$(galsim-version)) \ && cp $(dtexdir)/galsim.tex $(ictdir)/ \ && echo "Galsim $(galsim-version) \citep{galsim}" > $@ @@ -497,7 +497,7 @@ $(ipydir)/h5py: $(ipydir)/six \ $(ipydir)/cython \ $(ipydir)/mpi4py \ $(ipydir)/pypkgconfig \ - | $(tdir)/h5py-$(h5py-version).tar.gz + $(tdir)/h5py-$(h5py-version).tar.gz export HDF5_MPI=ON; \ export HDF5_DIR=$(ildir); \ $(call pybuild, tar xf, h5py-$(h5py-version), ,\ @@ -515,29 +515,29 @@ $(ipydir)/healpy: $(ibidir)/healpix $(ipydir)/html5lib: $(ipydir)/six \ $(ipydir)/webencodings \ - | $(tdir)/html5lib-$(html5lib-version).tar.gz + $(tdir)/html5lib-$(html5lib-version).tar.gz $(call pybuild, tar xf, html5lib-$(html5lib-version), ,\ HTML5lib $(html5lib-version)) $(ipydir)/idna: $(ipydir)/setuptools \ - | $(tdir)/idna-$(idna-version).tar.gz + $(tdir)/idna-$(idna-version).tar.gz $(call pybuild, tar xf, idna-$(idna-version), ,\ idna $(idna-version)) $(ipydir)/jeepney: $(ipydir)/setuptools \ - | $(tdir)/jeepney-$(jeepney-version).tar.gz + $(tdir)/jeepney-$(jeepney-version).tar.gz $(call pybuild, tar xf, jeepney-$(jeepney-version), ,\ Jeepney $(jeepney-version)) $(ipydir)/keyring: $(ipydir)/entrypoints \ $(ipydir)/secretstorage \ $(ipydir)/setuptools_scm \ - | $(tdir)/keyring-$(keyring-version).tar.gz + $(tdir)/keyring-$(keyring-version).tar.gz $(call pybuild, tar xf, keyring-$(keyring-version), ,\ Keyring $(keyring-version)) $(ipydir)/kiwisolver: $(ipydir)/setuptools \ - | $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz + $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz $(call pybuild, tar xf, kiwisolver-$(kiwisolver-version), ,\ Kiwisolver $(kiwisolver-version)) @@ -548,12 +548,12 @@ $(ipydir)/lmfit: $(ipydir)/six \ $(ipydir)/asteval \ $(ipydir)/matplotlib \ $(ipydir)/uncertainties \ - | $(tdir)/lmfit-$(lmfit-version).tar.gz + $(tdir)/lmfit-$(lmfit-version).tar.gz $(call pybuild, tar xf, lmfit-$(lmfit-version), ,\ LMFIT $(lmfit-version)) $(ipydir)/lsstdesccoord: $(ipydir)/setuptools \ - | $(tdir)/lsstdesccoord-$(lsstdesccoord-version).tar.gz + $(tdir)/lsstdesccoord-$(lsstdesccoord-version).tar.gz $(call pybuild, tar xf, LSSTDESC.Coord-$(lsstdesccoord-version), ,\ LSSTDESC.Coord $(lsstdesccoord-version)) @@ -566,7 +566,7 @@ $(ipydir)/matplotlib: $(ipydir)/numpy \ $(ibidir)/ghostscript \ $(ibidir)/imagemagick \ $(ipydir)/python-dateutil \ - | $(tdir)/matplotlib-$(matplotlib-version).tar.gz + $(tdir)/matplotlib-$(matplotlib-version).tar.gz # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. if [ x$(on_mac_os) = xyes ]; then \ @@ -579,20 +579,20 @@ $(ipydir)/matplotlib: $(ipydir)/numpy \ $(ipydir)/mpi4py: $(ibidir)/openmpi \ $(ipydir)/setuptools \ - | $(tdir)/mpi4py-$(mpi4py-version).tar.gz + $(tdir)/mpi4py-$(mpi4py-version).tar.gz $(call pybuild, tar xf, mpi4py-$(mpi4py-version)) \ && cp $(dtexdir)/mpi4py.tex $(ictdir)/ \ && echo "mpi4py $(mpi4py-version) \citep{mpi4py2011}" > $@ $(ipydir)/mpmath: $(ipydir)/setuptools \ - | $(tdir)/mpmath-$(mpmath-version).tar.gz + $(tdir)/mpmath-$(mpmath-version).tar.gz $(call pybuild, tar xf, mpmath-$(mpmath-version), ,\ mpmath $(mpmath-version)) $(ipydir)/numpy: $(ibidir)/unzip \ $(ibidir)/openblas \ $(ipydir)/setuptools \ - | $(tdir)/numpy-$(numpy-version).zip + $(tdir)/numpy-$(numpy-version).zip if [ x$(on_mac_os) = xyes ]; then \ export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ else \ @@ -606,24 +606,24 @@ $(ipydir)/numpy: $(ibidir)/unzip \ && echo "Numpy $(numpy-version) \citep{numpy2011}" > $@ $(ipydir)/pexpect: $(ipydir)/setuptools \ - | $(tdir)/pexpect-$(pexpect-version).tar.gz + $(tdir)/pexpect-$(pexpect-version).tar.gz $(call pybuild, tar xf, pexpect-$(pexpect-version), ,\ Pexpect $(pexpect-version)) $(ibidir)/pip3: $(ipydir)/setuptools \ - | $(tdir)/pip-$(pip-version).tar.gz + $(tdir)/pip-$(pip-version).tar.gz $(call pybuild, tar xf, pip-$(pip-version), ,\ PiP $(pip-version)) $(ipydir)/pycodestyle: $(ipydir)/setuptools \ - | $(tdir)/pycodestyle-$(pycodestyle-version).tar.gz + $(tdir)/pycodestyle-$(pycodestyle-version).tar.gz $(call pybuild, tar xf, pycodestyle-$(pycodestyle-version), ,\ pycodestyle $(pycodestyle-version)) $(ipydir)/pybind11: $(ibidir)/eigen \ $(ibidir)/boost \ $(ipydir)/setuptools \ - | $(tdir)/pybind11-$(pybind11-version).tar.gz + $(tdir)/pybind11-$(pybind11-version).tar.gz pyhook_after() { cp -r include/pybind11 $(iidir)/python$(python-major-version)m/ } @@ -631,34 +631,34 @@ $(ipydir)/pybind11: $(ibidir)/eigen \ pybind11 $(pybind11-version)) $(ipydir)/pycparser: $(ipydir)/setuptools \ - | $(tdir)/pycparser-$(pycparser-version).tar.gz + $(tdir)/pycparser-$(pycparser-version).tar.gz $(call pybuild, tar xf, pycparser-$(pycparser-version), ,\ pycparser $(pycparser-version)) $(ipydir)/pyflakes: $(ipydir)/setuptools \ - | $(tdir)/pyflakes-$(pyflakes-version).tar.gz + $(tdir)/pyflakes-$(pyflakes-version).tar.gz $(call pybuild, tar xf, pyflakes-$(pyflakes-version), ,\ pyflakes $(pyflakes-version)) $(ipydir)/pyparsing: $(ipydir)/setuptools \ - | $(tdir)/pyparsing-$(pyparsing-version).tar.gz + $(tdir)/pyparsing-$(pyparsing-version).tar.gz $(call pybuild, tar xf, pyparsing-$(pyparsing-version), ,\ PyParsing $(pyparsing-version)) $(ipydir)/pypkgconfig: $(ipydir)/setuptools \ - | $(tdir)/pkgconfig-$(pypkgconfig-version).tar.gz + $(tdir)/pkgconfig-$(pypkgconfig-version).tar.gz $(call pybuild, tar xf, pkgconfig-$(pypkgconfig-version), , pkgconfig $(pypkgconfig-version)) $(ipydir)/python-dateutil: $(ipydir)/six \ $(ipydir)/setuptools_scm \ - | $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz + $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz $(call pybuild, tar xf, python-dateutil-$(python-dateutil-version), ,\ python-dateutil $(python-dateutil-version)) $(ipydir)/pyyaml: $(ibidir)/yaml \ $(ipydir)/cython \ - | $(tdir)/pyyaml-$(pyyaml-version).tar.gz + $(tdir)/pyyaml-$(pyyaml-version).tar.gz $(call pybuild, tar xf, PyYAML-$(pyyaml-version), ,\ PyYAML $(pyyaml-version)) @@ -667,12 +667,12 @@ $(ipydir)/requests: $(ipydir)/idna \ $(ipydir)/certifi \ $(ipydir)/chardet \ $(ipydir)/urllib3 \ - | $(tdir)/requests-$(requests-version).tar.gz + $(tdir)/requests-$(requests-version).tar.gz $(call pybuild, tar xf, requests-$(requests-version), ,\ Requests $(requests-version)) $(ipydir)/scipy: $(ipydir)/numpy \ - | $(tdir)/scipy-$(scipy-version).tar.gz + $(tdir)/scipy-$(scipy-version).tar.gz if [ x$(on_mac_os) = xyes ]; then \ export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ else \ @@ -685,56 +685,56 @@ $(ipydir)/scipy: $(ipydir)/numpy \ $(ipydir)/secretstorage: $(ipydir)/jeepney \ $(ipydir)/cryptography \ - | $(tdir)/secretstorage-$(secretstorage-version).tar.gz + $(tdir)/secretstorage-$(secretstorage-version).tar.gz $(call pybuild, tar xf, SecretStorage-$(secretstorage-version), ,\ SecretStorage $(secretstorage-version)) $(ipydir)/setuptools: $(ibidir)/unzip \ $(ibidir)/python \ - | $(tdir)/setuptools-$(setuptools-version).zip + $(tdir)/setuptools-$(setuptools-version).zip $(call pybuild, unzip, setuptools-$(setuptools-version), ,\ Setuptools $(setuptools-version)) $(ipydir)/setuptools_scm: $(ipydir)/setuptools \ - | $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.gz + $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.gz $(call pybuild, tar xf, setuptools_scm-$(setuptools_scm-version), ,\ Setuptools-scm $(setuptools_scm-version)) $(ipydir)/sip_tpv: $(ipydir)/sympy \ $(ipydir)/astropy \ - | $(tdir)/sip_tpv-$(sip_tpv-version).tar.gz + $(tdir)/sip_tpv-$(sip_tpv-version).tar.gz $(call pybuild, tar xf, sip_tpv-$(sip_tpv-version), ,) \ && cp $(dtexdir)/sip_tpv.tex $(ictdir)/ \ && echo "sip\_tpv $(sip_tpv-version) \citep{sip-tpv}" > $@ $(ipydir)/six: $(ipydir)/setuptools \ - | $(tdir)/six-$(six-version).tar.gz + $(tdir)/six-$(six-version).tar.gz $(call pybuild, tar xf, six-$(six-version), ,\ Six $(six-version)) $(ipydir)/soupsieve: $(ipydir)/setuptools \ - | $(tdir)/soupsieve-$(soupsieve-version).tar.gz + $(tdir)/soupsieve-$(soupsieve-version).tar.gz $(call pybuild, tar xf, soupsieve-$(soupsieve-version), ,\ SoupSieve $(soupsieve-version)) $(ipydir)/sympy: $(ipydir)/mpmath \ - | $(tdir)/sympy-$(sympy-version).tar.gz + $(tdir)/sympy-$(sympy-version).tar.gz $(call pybuild, tar xf, sympy-$(sympy-version), ,) \ && cp $(dtexdir)/sympy.tex $(ictdir)/ \ && echo "SymPy $(sympy-version) \citep{sympy}" > $@ $(ipydir)/uncertainties: $(ipydir)/numpy \ - | $(tdir)/uncertainties-$(uncertainties-version).tar.gz + $(tdir)/uncertainties-$(uncertainties-version).tar.gz $(call pybuild, tar xf, uncertainties-$(uncertainties-version), ,\ uncertainties $(uncertainties-version)) $(ipydir)/urllib3: $(ipydir)/setuptools \ - | $(tdir)/urllib3-$(urllib3-version).tar.gz + $(tdir)/urllib3-$(urllib3-version).tar.gz $(call pybuild, tar xf, urllib3-$(urllib3-version), ,\ Urllib3 $(urllib3-version)) $(ipydir)/webencodings: $(ipydir)/setuptools \ - | $(tdir)/webencodings-$(webencodings-version).tar.gz + $(tdir)/webencodings-$(webencodings-version).tar.gz $(call pybuild, tar xf, webencodings-$(webencodings-version), ,\ Webencodings $(webencodings-version)) -- cgit v1.2.1 From c0f9371b48962b32635d8ed4adcdb30a3728c8ef Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 6 May 2020 22:35:28 +0100 Subject: Corrected latex citation of sip_tpv program (no more \ before _) Until now there we had manually inserted a `\' before the `_' of sip_tpv program. However, we also recently added a step in the configure script to add a `\' before every `_' when writing the final LaTeX macro. This was because some C compilers (when the host's is used) have an `_' in their version that we had no control over. With this commit, the `\' is removed from `sip_tpv' in its build-rule and we let the backslash be inserted automatically. --- reproduce/software/make/python.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/software/make/python.mk') diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 7537275..8aa6ec8 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -705,7 +705,7 @@ $(ipydir)/sip_tpv: $(ipydir)/sympy \ $(tdir)/sip_tpv-$(sip_tpv-version).tar.gz $(call pybuild, tar xf, sip_tpv-$(sip_tpv-version), ,) \ && cp $(dtexdir)/sip_tpv.tex $(ictdir)/ \ - && echo "sip\_tpv $(sip_tpv-version) \citep{sip-tpv}" > $@ + && echo "sip_tpv $(sip_tpv-version) \citep{sip-tpv}" > $@ $(ipydir)/six: $(ipydir)/setuptools \ -- cgit v1.2.1 From c0a512e0631a024f18bec847125a9a03e6aa23a1 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 22 May 2020 23:35:21 +0100 Subject: Corrected copyright notices to fit GPL suggested format In time, some of the copyright license description had been mistakenly shortened to two paragraphs instead of the original three that is recommended in the GPL. With this commit, they are corrected to be exactly in the same three paragraph format suggested by GPL. The following files also didn't have a copyright notice, so one was added for them: reproduce/software/make/README.md reproduce/software/bibtex/healpix.tex reproduce/analysis/config/delete-me-num.conf reproduce/analysis/config/verify-outputs.conf --- reproduce/software/make/python.mk | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'reproduce/software/make/python.mk') diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 8aa6ec8..43499c7 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -11,15 +11,18 @@ # Copyright (C) 2019-2020 Raul Infante-Sainz # Copyright (C) 2019-2020 Mohammad Akhlaghi # -# This Makefile is part of Maneage. Maneage is free software: you can -# redistribute it and/or modify it under the terms of the GNU General -# Public License as published by the Free Software Foundation, either -# version 3 of the License, or (at your option) any later version. +# This Makefile is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # -# Maneage is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. See . +# This Makefile is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this Makefile. If not, see . -- cgit v1.2.1 From 3d8aa5953c4e0b79278ab2e27ec4e1051310d04f Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 1 Jun 2020 03:05:37 +0100 Subject: Core software build before using Make to build other software Until now, Maneage would only build Flock before building everything else using Make (calling 'basic.mk') in parallel. Flock was necessary to avoid parallel downloads during the building of software (which could cause network problems). But after recently trying Maneage on FreeBSD (which is not yet complete, see bug #58465), we noticed that the BSD implemenation of Make couldn't parse 'basic.mk' (in particular, complaining with the 'ifeq' parts) and its shell also had some peculiarities. It was thus decided to also install our own minimalist shell, Make and compressor program before calling 'basic.mk'. In this way, 'basic.mk' can now assume the same GNU Make features that high-level.mk and python.mk assume. The pre-make building of software is now organized in 'reproduce/software/shell/pre-make-build.sh'. Another nice feature of this commit is for macOS users: until now the default macOS Make had problems for parallel building of software, so 'basic.mk' was built in one thread. But now that we can build the core tools with GNU Make on macOS too, it uses all threads. Furthermore, since we now run 'basic.mk' with GNU Make, we can use '.ONESHELL' and don't have to finish every line of a long rule with a backslash to keep variables and such. Generally, the pre-make software are now organized like this: first we build Lzip before anything else: it is downloaded as a simple '.tar' file that is not compressed (only ~400kb). Once Lzip is built, the pre-make phase continues with building GNU Make, Dash (a minimalist shell) and Flock. All of their tarballs are in '.tar.lz'. Maneage then enters 'basic.mk' and the first program it builds is GNU Gzip (itself packaged as '.tar.lz'). Once Gzip is built, we build all the other compression software (all downloaded as '.tar.gz'). Afterwards, any compression standard for other software is fine because we have it. In the process, a bug related to using backup servers was found in 'reproduce/analysis/bash/download-multi-try' for calling outside of 'basic.mk' and removed Bash-specific features. As a result of that bug-fix, because we now have multiple servers for software tarballs, the backup servers now have their own configuration file in 'reproduce/software/config/servers-backup.conf'. This makes it much easier to maintain the backup server list across the multiple places that we need it. Some other minor fixes: - In building Bzip2, we need to specify 'CC' so it doesn't use 'gcc'. - In building Zip, the 'generic_gcc' Make option caused a crash on FreeBSD (which doesn't have GCC). - We are now using 'uname -s' to specify if we are on a Linux kernel or not, if not, we are still using the old 'on_mac_os' variable. - While I was trying to build on FreeBSD, I noticed some further corrections that could help. For example the 'makelink' Make-function now takes a third argument which can be a different name compared to the actual program (used for examle to make a link to '/usr/bin/cc' from 'gcc'. - Until now we didn't know if the host's Make implementation supports placing a '@' at the start of the recipe (to avoid printing the actual commands to standard output). Especially in the tarball download phase, there are many lines that are printed for each download which was really annoying. We already used '@' in 'high-level.mk' and 'python.mk' before, but now that we also know that 'basic.mk' is called with our custom GNU Make, we can use it at the start for a cleaner stdout. - Until now, WCSLIB assumed a Fortran compiler, but when the user is on a system where we can't install GCC (or has activated the '--host-cc' option), it may not be present and the project shouldn't break because of this. So with this commit, when a Fortran compiler isn't present, WCSLIB will be built with the '--disable-fortran' configuration option. This commit (task #15667) was completed with help/checks by Raul Infante-Sainz and Boud Roukema. --- reproduce/software/make/python.mk | 1 + 1 file changed, 1 insertion(+) (limited to 'reproduce/software/make/python.mk') diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 43499c7..eef8279 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -250,6 +250,7 @@ $(pytarballs): $(tdir)/%: # storing all the tarballs in one directory, we want it to have # the same naming convention, so we'll download it to a temporary # name, then rename that. + rm -f "$@.unchecked" if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* "$@.unchecked" else -- cgit v1.2.1