From 7540d7d7b3c57b2442002e46a7e879f31e503752 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 11 Apr 2019 10:16:37 +0100 Subject: Unzip set as prerequisite of Numpy and Setuptools Since we mixed the installation of Python packages with all other software, it may occur that some Python packages start to be installed before having installed `unzip'. As a consecuence, they could not be decompressed and the installation will fail. In particular, tarballs of Numpy and Setuptools are .zip files. With this commit, we fix this issue by setting `unzip' as a prerequisit of Numpy and Setuptools. --- .file-metadata | Bin 4780 -> 4780 bytes reproduce/src/make/dependencies-python.mk | 6 ++++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.file-metadata b/.file-metadata index 115bae6..d1dea6f 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 5b12757..a6634ff 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -410,7 +410,8 @@ $(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ipydir)/setuptools \ $(ilidir)/openblas \ - $(ilidir)/fftw + $(ilidir)/fftw \ + $(ibidir)/unzip if [ x$(on_mac_os) = xyes ]; then \ export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle"; \ else \ @@ -473,7 +474,8 @@ $(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \ SecretStorage $(secretstorage-version)) $(ipydir)/setuptools: $(tdir)/setuptools-$(setuptools-version).zip \ - $(ibidir)/python3 + $(ibidir)/python3 \ + $(ibidir)/unzip $(call pybuild, unzip, $<, setuptools-$(setuptools-version), ,\ Setuptools $(setuptools-version)) -- cgit v1.2.1 From 5a8974d1cabd236a1b47909f452be480d0c970bf Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 12 Apr 2019 07:30:37 +0100 Subject: Acknowledged Scipy-related packages: Cython, Matplotlib, Numpy and Scipy Until now, name and version of all Python packages were indicated in the final paper, but not the main paper of them (if it exists). With this commit, some Python packages (Cython, Matplotlib, Numpy and Scipy) are now properly acknoledged by citating the source paper. `mpi4py' is also cited although this package is not yet included into the pipeline. --- .file-metadata | Bin 4780 -> 4780 bytes reproduce/src/make/dependencies-python.mk | 23 ++++---- tex/src/references.tex | 85 ++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 11 deletions(-) diff --git a/.file-metadata b/.file-metadata index d1dea6f..a1e8ad4 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index a6634ff..221fad2 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -254,7 +254,7 @@ $(ibidir)/python3: $(tdir)/python-$(python-version).tar.gz \ # 3) Unpacked directory name after unpacking the tarball # 4) site.cfg file (optional) # 5) Official software name.(for paper). -pybuild = cd $(ddir); rm -rf $(3); \ +pybuild = cd $(ddir); rm -rf $(3); \ if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ cd $(3); \ if [ "x$(strip $(4))" != x ]; then \ @@ -335,8 +335,8 @@ $(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \ $(ipydir)/cython: $(tdir)/cython-$(cython-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, Cython-$(cython-version), ,\ - Cython $(cython-version)) + $(call pybuild, tar xf, $<, Cython-$(cython-version)) \ + && echo "Cython $(cython-version) \citep{cython2011}" > $@ $(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ $(ipydir)/setuptools @@ -389,8 +389,8 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/numpy \ $(ipydir)/pyparsing \ $(ipydir)/python-dateutil - $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version), ,\ - Matplotlib $(matplotlib-version)) + $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) \ + && echo "Matplotlib $(matplotlib-version) \citep{matplotlib2007}" > $@ # Currently mpi4py doesn't build because of some conflict with OpenMPI: # @@ -404,8 +404,8 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \ $(ipydir)/setuptools \ $(ilidir)/openmpi - $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version), ,\ - mpi4py $(mpi4py-version)) + $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version)) \ + && echo "mpi4py $(mpi4py-version) \citep{mpi4py2011}" > $@ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ipydir)/setuptools \ @@ -418,8 +418,9 @@ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ export LDFLAGS="$(LDFLAGS) -shared"; \ fi; \ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ - $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf, \ - Numpy $(numpy-version)) + $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf, \ + Numpy $(numpy-version)) \ + && echo "Numpy $(numpy-version) \citep{numpy2011}" > $@ $(ibidir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ $(ipydir)/setuptools @@ -464,8 +465,8 @@ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ export LDFLAGS="$(LDFLAGS) -shared"; \ fi; \ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ - $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf,\ - Scipy $(scipy-version)) + $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf)       \ + && echo "Scipy $(scipy-version) \citep{scipy2001}" > $@ $(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \ $(ipydir)/cryptography \ diff --git a/tex/src/references.tex b/tex/src/references.tex index c1c4084..47015bb 100644 --- a/tex/src/references.tex +++ b/tex/src/references.tex @@ -158,3 +158,88 @@ archivePrefix = "arXiv", adsurl = {http://adsabs.harvard.edu/abs/2013A\%26A...558A..33A}, adsnote = {Provided by the SAO/NASA Astrophysics Data System} } + + + + + +@ARTICLE{cython2011, + author = {{Behnel}, S. and {Bradshaw}, R. and {Citro}, C. and {Dalcin}, L. and + {Seljebotn}, D.~S. and {Smith}, K.}, + title = "{Cython: The Best of Both Worlds}", + journal = {Computing in Science and Engineering}, + year = 2011, + month = mar, + volume = 13, + number = 2, + pages = {31-39}, + doi = {10.1109/MCSE.2010.118}, + adsurl = {http://adsabs.harvard.edu/abs/2011CSE....13b..31B}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + + + + + +@ARTICLE{mpi4py2011, + author = {{Dalcin}, L.~D. and {Paz}, R.~R. and {Kler}, P.~A. and {Cosimo}, A. + }, + title = "{Parallel distributed computing using Python}", + journal = {Advances in Water Resources}, + year = 2011, + month = sep, + volume = 34, + pages = {1124-1139}, + doi = {10.1016/j.advwatres.2011.04.013}, + adsurl = {http://adsabs.harvard.edu/abs/2011AdWR...34.1124D}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + + + +@ARTICLE{numpy2011, + author = {{van der Walt}, S. and {Colbert}, S.~C. and {Varoquaux}, G.}, + title = "{The NumPy Array: A Structure for Efficient Numerical Computation}", + journal = {Computing in Science and Engineering}, +archivePrefix = "arXiv", + eprint = {1102.1523}, + primaryClass = "cs.MS", + year = 2011, + month = mar, + volume = 13, + number = 2, + pages = {22-30}, + doi = {10.1109/MCSE.2011.37}, + adsurl = {http://adsabs.harvard.edu/abs/2011CSE....13b..22V}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + + + + +@Article{matplotlib2007, + Author = {Hunter, J. D.}, + Title = {Matplotlib: A 2D graphics environment}, + Journal = {Computing In Science \& Engineering}, + Volume = {9}, + Number = {3}, + Pages = {90--95}, + abstract = {Matplotlib is a 2D graphics package used for Python + for application development, interactive scripting, and + publication-quality image generation across user + interfaces and operating systems.}, + publisher = {IEEE COMPUTER SOC}, + doi = {10.1109/MCSE.2007.55}, + year = 2007 +} + + + + +@Misc{scipy2001, + author = {Eric Jones and Travis Oliphant and Pearu Peterson and others}, + title = {{SciPy}: Open source scientific tools for {Python}}, + year = {2001--}, + url = "http://www.scipy.org/" +} -- cgit v1.2.1 From f4b0b28c3ccfe9dc49b529773eee75627253689d Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 12 Apr 2019 13:29:07 +0100 Subject: Fixed some Scipy-related packages citations Until now, the Scipy citation was only one paper and not the correct one (it was the online manual). With this commit, Scipy is properly cited using the two papers. Also some modifications in the `tex/src/references.tex' have been done (remove last page number). --- .file-metadata | Bin 4780 -> 4780 bytes reproduce/src/make/dependencies-python.mk | 2 +- tex/src/references.tex | 54 +++++++++++++++++++++++------- 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/.file-metadata b/.file-metadata index a1e8ad4..4d9046a 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index 221fad2..11c6b1f 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -466,7 +466,7 @@ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ fi; \ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf)       \ - && echo "Scipy $(scipy-version) \citep{scipy2001}" > $@ + && echo "Scipy $(scipy-version) \citep{scipy2007,scipy2011}" > $@ $(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \ $(ipydir)/cryptography \ diff --git a/tex/src/references.tex b/tex/src/references.tex index 47015bb..b4f4cb4 100644 --- a/tex/src/references.tex +++ b/tex/src/references.tex @@ -129,6 +129,7 @@ archivePrefix = "arXiv", + @ARTICLE{astropy2013, author = {{Astropy Collaboration} and {Robitaille}, T.~P. and {Tollerud}, E.~J. and {Greenfield}, P. and {Droettboom}, M. and {Bray}, E. and {Aldcroft}, T. and @@ -167,12 +168,12 @@ archivePrefix = "arXiv", author = {{Behnel}, S. and {Bradshaw}, R. and {Citro}, C. and {Dalcin}, L. and {Seljebotn}, D.~S. and {Smith}, K.}, title = "{Cython: The Best of Both Worlds}", - journal = {Computing in Science and Engineering}, + journal = {CiSE}, year = 2011, month = mar, volume = 13, number = 2, - pages = {31-39}, + pages = {31}, doi = {10.1109/MCSE.2010.118}, adsurl = {http://adsabs.harvard.edu/abs/2011CSE....13b..31B}, adsnote = {Provided by the SAO/NASA Astrophysics Data System} @@ -190,7 +191,7 @@ archivePrefix = "arXiv", year = 2011, month = sep, volume = 34, - pages = {1124-1139}, + pages = {1124}, doi = {10.1016/j.advwatres.2011.04.013}, adsurl = {http://adsabs.harvard.edu/abs/2011AdWR...34.1124D}, adsnote = {Provided by the SAO/NASA Astrophysics Data System} @@ -198,10 +199,30 @@ archivePrefix = "arXiv", + + +@ARTICLE{scipy2011, +author={K. J. {Millman} and M. {Aivazis}}, +journal={CiSE}, +title={Python for Scientists and Engineers}, +year={2011}, +volume={13}, +number={2}, +pages={9}, +keywords={Special issues and sections;Computer languages;Programming;Scientific computing;Numerical models;Programming languages;Python;Scientific computing;interactive research;Python libraries;Python tools}, +doi={10.1109/MCSE.2011.36}, +ISSN={1521-9615}, +month={March}, +} + + + + + @ARTICLE{numpy2011, author = {{van der Walt}, S. and {Colbert}, S.~C. and {Varoquaux}, G.}, title = "{The NumPy Array: A Structure for Efficient Numerical Computation}", - journal = {Computing in Science and Engineering}, + journal = {CiSE}, archivePrefix = "arXiv", eprint = {1102.1523}, primaryClass = "cs.MS", @@ -209,7 +230,7 @@ archivePrefix = "arXiv", month = mar, volume = 13, number = 2, - pages = {22-30}, + pages = {22}, doi = {10.1109/MCSE.2011.37}, adsurl = {http://adsabs.harvard.edu/abs/2011CSE....13b..22V}, adsnote = {Provided by the SAO/NASA Astrophysics Data System} @@ -218,13 +239,14 @@ archivePrefix = "arXiv", + @Article{matplotlib2007, Author = {Hunter, J. D.}, Title = {Matplotlib: A 2D graphics environment}, - Journal = {Computing In Science \& Engineering}, + Journal = {CiSE}, Volume = {9}, Number = {3}, - Pages = {90--95}, + Pages = {90}, abstract = {Matplotlib is a 2D graphics package used for Python for application development, interactive scripting, and publication-quality image generation across user @@ -237,9 +259,17 @@ archivePrefix = "arXiv", -@Misc{scipy2001, - author = {Eric Jones and Travis Oliphant and Pearu Peterson and others}, - title = {{SciPy}: Open source scientific tools for {Python}}, - year = {2001--}, - url = "http://www.scipy.org/" + +@ARTICLE{scipy2007, +author={T. E. {Oliphant}}, +journal={CiSE}, +title={Python for Scientific Computing}, +year={2007}, +volume={9}, +number={3}, +pages={10}, +keywords={high level languages;Python;scientific computing;steering language;scientific codes;high-level language;Scientific computing;High level languages;Libraries;Writing;Application software;Embedded software;Software standards;Standards development;Internet;Prototypes;Python;computer languages;scientific programming;scientific computing}, +doi={10.1109/MCSE.2007.58}, +ISSN={1521-9615}, +month={May}, } -- cgit v1.2.1 From 1442a5ebe5f9ca66f25c0f0a585c6ac4083101d8 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 12 Apr 2019 14:57:12 +0100 Subject: File is built as a dependency of GCC Until now, we did not have `file'. It was in other project, where a problem with `Astrometry-net' software, ends up with the necessity of having `file' into the pipeline. With this commit, we add `file' to the project. Since it is a low level program, it is set in `dependencies-basic.mk' as a prerequisite of GCC. --- .file-metadata | Bin 4780 -> 4780 bytes reproduce/config/pipeline/dependency-versions.mk | 1 + reproduce/src/make/dependencies-basic.mk | 13 +++++++++++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.file-metadata b/.file-metadata index 4d9046a..0607790 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 3b43aae..91d1265 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -29,6 +29,7 @@ coreutils-version = 8.30 curl-version = 7.63.0 diffutils-version = 3.7 fftw-version = 3.3.8 +file-version = 5.36 findutils-version = 4.6.0.199-e3fc flock-version = 0.2.3 freetype-version = 2.9 diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index bedbc38..9e277f7 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -67,7 +67,7 @@ export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) # Define the programs that don't depend on any other. -top-level-programs = low-level-links wget gcc +top-level-programs = low-level-links wget gcc file all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) @@ -104,6 +104,7 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \ cert.pem \ coreutils-$(coreutils-version).tar.xz \ diffutils-$(diffutils-version).tar.xz \ + file-$(file-version).tar.gz \ findutils-$(findutils-version).tar.lz \ gawk-$(gawk-version).tar.lz \ gcc-$(gcc-version).tar.xz \ @@ -144,6 +145,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = cert ]; then w=http://akhlaghi.org/src; \ elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils;\ elif [ $$n = diffutils ]; then w=http://ftp.gnu.org/gnu/diffutils;\ + elif [ $$n = file ]; then w=ftp://ftp.astron.com/pub/file; \ elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \ elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk; \ elif [ $$n = gcc ]; then w=http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version); \ @@ -769,6 +771,12 @@ $(ibidir)/binutils: $(tdir)/binutils-$(binutils-version).tar.lz $(call gbuild, $<, binutils-$(binutils-version), static) \ && echo "GNU Binutils $(binutils-version)" > $@ +# `file' is not a prerequisite of GCC. However, since it is low level, it is +# set as a prerequisite of GCC to have it installed. +$(ibidir)/file: $(tdir)/file-$(file-version).tar.gz + $(call gbuild, $<, file-$(file-version), static) \ + && echo "File $(file-version)" > $@ + $(ilidir)/isl: $(tdir)/isl-$(isl-version).tar.bz2 \ $(ilidir)/gmp $(call gbuild, $<, isl-$(isl-version), static) \ @@ -801,9 +809,10 @@ gcc-prerequisites = $(tdir)/gcc-$(gcc-version).tar.xz \ endif $(ibidir)/gcc: $(gcc-prerequisites) \ $(ibidir)/sed \ + $(ibidir)/bash \ + $(ibidir)/file \ $(ibidir)/gawk \ $(ibidir)/grep \ - $(ibidir)/bash \ $(ibidir)/which \ $(ibidir)/glibtool \ $(ibidir)/findutils \ -- cgit v1.2.1