From 6ec4881a1bd19052db633adec3bedbac0f4bc21a Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sat, 18 Jan 2020 20:47:35 +0000 Subject: TeXLive version printing updated to include revision Until now, when find the versions of the TeXLive packages, we would assume that `cat-date' is always present (because some packages don't have a version!). However, apparently an update has been made in the TeXLive Manager (`tlmgr') and `cat-date' is no longer present! As a result, none of the TeXLive packages were being printed. With this commit, it now assumes that `revision' is always present for every package, but it also attempts to read `cat-date' (for backwards compatability). When `cat-version' isn't present, it will try printing `revision' and if that is also not present, it will print the date. --- reproduce/software/make/high-level.mk | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index f46480a..735a24a 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1220,13 +1220,32 @@ $(itidir)/texlive: reproduce/software/config/installation/texlive.mk \ texlive=$$(pdflatex --version | awk 'NR==1' | sed 's/.*(\(.*\))/\1/' \ | awk '{print $$NF}'); - # Package names and versions. - rm -f $@ + # Package names and versions. Note that all TeXLive packages + # don't have a version unfortunately! So we need to also read the + # `revision' and `cat-date' elements and print them incase + # version isn't available. tlmgr info $(texlive-packages) --only-installed | awk \ - '$$1=="package:" {version=0; \ - if($$NF=="tex-gyre") name="texgyre"; \ - else name=$$NF} \ + '$$1=="package:" { \ + if(name!=0) \ + { \ + if(version=="") \ + { \ + if(revision=="") \ + { \ + if(date="") printf("%s (no version)\n", name); \ + else printf("%s %s (date)\n", name, date); \ + } \ + else + printf("%s %s (revision)\n", name, revision); \ + } \ + else \ + printf("%s %s\n", name, version); \ + } \ + name=""; version=""; revision=""; date=""; \ + if($$NF=="tex-gyre") name="texgyre"; \ + else name=$$NF \ + } \ + $$1=="cat-date:" {date=$$NF} \ $$1=="cat-version:" {version=$$NF} \ - $$1=="cat-date:" {if(version==0) version=$$2; \ - printf("%s %s\n", name, version)}' >> $@ + $$1=="revision:" {revision=$$NF}' > $@ fi -- cgit v1.2.1 From be73ea496657099603cce7e20413b9c6ee993c94 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 19 Jan 2020 21:15:37 +0000 Subject: LIBRARY_PATH is set accordingly based on the host Until now, GCC wouldn't build properly on Debian-based operating systems because `ld' needed to link with several necessary C library features like `crti.o' and `crtn.o' (this is an `ld' issue, not GCC). The solution is to add the directory containing them to `LIBRARY_PATH'. In the previous commit, I actually searched for these files, but while testing on another system, I noticed that it can be problematic (other architectures may exist). With this commit, we are actually finding the build architecture of the running GCC (which is the same as the `ld') and using that to fix a fixed directory to `LIBRARY_PATH'. --- reproduce/software/make/basic.mk | 15 +++++++++++---- reproduce/software/make/high-level.mk | 7 +++++++ 2 files changed, 18 insertions(+), 4 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index e7d00df..9eba04b 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -1174,6 +1174,7 @@ $(ibidir)/binutils: | $(ibidir)/sed \ $(ibidir)/diffutils \ $(ibidir)/coreutils \ $(gcc-prerequisites) + if [ x$(on_mac_os) = xyes ]; then \ $(call makelink,as); \ $(call makelink,ar); \ @@ -1183,7 +1184,9 @@ $(ibidir)/binutils: | $(ibidir)/sed \ $(call makelink,ranlib); \ echo "" > $@; \ else \ - $(call gbuild, binutils-$(binutils-version), static) \ + $(call gbuild, binutils-$(binutils-version), static, \ + --with-lib-path=$(sys_library_path), \ + -j$(numthreads) ) \ && echo "GNU Binutils $(binutils-version)" > $@; \ fi @@ -1257,15 +1260,19 @@ $(ibidir)/gcc: | $(ibidir)/binutils \ && make SHELL=$(ibdir)/bash -j$(numthreads) \ && make SHELL=$(ibdir)/bash install \ && cd ../.. \ - && rm -rf gcc-$(gcc-version) \ + && tempname=$(ddir)/gcc-$(gcc-version)/build/rpath-temp-copy \ && if [ "x$(on_mac_os)" != xyes ]; then \ patchelf --add-needed $(ildir)/libiconv.so $(ildir)/libstdc++.so; \ for f in $$(find $(idir)/libexec/gcc) $(ildir)/libstdc++*; do \ - if ldd $$f &> /dev/null; then \ - patchelf --set-rpath $(ildir) $$f; \ + isdynamic=$$(file $$f | grep "dynamically linked"); \ + if [ x"$$isdynamic" != x ]; then \ + cp $$f $$tempname; \ + patchelf --set-rpath $(ildir) $$tempname; \ + mv $$tempname $$f; echo "corrected"; \ fi; \ done; \ fi \ + && rm -rf gcc-$(gcc-version) \ && ln -sf $(ibdir)/gcc $(ibdir)/cc \ && echo "GNU Compiler Collection (GCC) $(gcc-version)" > $@; \ fi diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 735a24a..0afeaba 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -84,6 +84,13 @@ export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig # causes crashs (see bug #56682). So we'll just give it no value at all. export DYLD_LIBRARY_PATH := +# On Debian-based OSs, the basic C libraries are in a target-specific +# location, not in standard places. Until we merge the building of the C +# library, it is thus necessary to include this location here. On systems +# that don't need it, `sys_library_path' is just empty. This is necessary +# for `ld'. +export LIBRARY_PATH := $(sys_library_path) + # Recipe startup script, see `reproduce/software/bash/bashrc.sh'. export PROJECT_STATUS := configure_highlevel export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh -- cgit v1.2.1 From e3bbc17c051c7ccbf9e5375b85c90e5e41af2adb Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 19 Jan 2020 23:30:26 +0000 Subject: GNU Make updated to version 4.3 GNU Make 4.3 was just announced, so I have updated it here is well. This was important because until now the installable version was in alpha-mode (4.2.90), now its a stable version. --- reproduce/software/make/basic.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 9eba04b..343b3b0 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -176,7 +176,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) mergenames=0; \ c=$(m4-checksum); \ w=http://akhlaghi.org/reproduce-software/m4-1.4.18-patched.tar.gz; \ - elif [ $$n = make ]; then c=$(make-checksum); w=https://alpha.gnu.org/gnu/make; \ + elif [ $$n = make ]; then c=$(make-checksum); w=https://ftp.gnu.org/gnu/make; \ elif [ $$n = metastore ]; then c=$(metastore-checksum); w=http://akhlaghi.org/reproduce-software; \ elif [ $$n = mpc ]; then c=$(mpc-checksum); w=http://ftp.gnu.org/gnu/mpc; \ elif [ $$n = mpfr ]; then c=$(mpfr-checksum); w=http://www.mpfr.org/mpfr-current;\ -- cgit v1.2.1 From 7ac86df891798fd1f0ef4d7a40aff43ec9621854 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 20 Jan 2020 00:05:41 +0000 Subject: IMPORTANT!!! Configuration Makefiles now have a .conf suffix Until now, the configuration Makefiles (in `reproduce/software/config/installation' and `reproduce/analysis/config') had a `.mk' suffix, similar to the workhorse Makefiles. Although they are indeed Makefiles, but given their nature (to only keep configuration parameters), it is confusing (especially to early users) for them to also have a `.mk' (similar to the analysis or software building Makefiles). To address this issue, with this commit, all the configuration Makefiles (in those directories) are now given a `.conf' suffix. This is also assumed for all the files that are loaded. The configuration (software building) and running of the template have been checked with this change from scratch, but please report any error that may not have been noticed. THIS IS AN IMPORTANT CHANGE AND WILL CAUSE CRASHES OR UNEXPECTED BEHAVIORS FOR PROJECTS THAT HAVE BRANCHED FROM THIS TEMPLATE. PLEASE CORRECT THE SUFFIX OF ALL YOUR PROJECT'S CONFIGURATION MAKEFILES (IN THE DIRECTORIES ABOVE), OTHERWISE THEY AREN'T AUTOMATICALLY LOADED ANYMORE. --- reproduce/software/make/basic.mk | 6 +++--- reproduce/software/make/build-rules.mk | 4 ++-- reproduce/software/make/high-level.mk | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 343b3b0..93dcdc2 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -34,9 +34,9 @@ # Top level environment include reproduce/software/make/build-rules.mk -include reproduce/software/config/installation/LOCAL.mk -include reproduce/software/config/installation/versions.mk -include reproduce/software/config/installation/checksums.mk +include reproduce/software/config/installation/LOCAL.conf +include reproduce/software/config/installation/versions.conf +include reproduce/software/config/installation/checksums.conf lockdir = $(BDIR)/locks tdir = $(BDIR)/software/tarballs diff --git a/reproduce/software/make/build-rules.mk b/reproduce/software/make/build-rules.mk index cbde70b..f3ca7d2 100644 --- a/reproduce/software/make/build-rules.mk +++ b/reproduce/software/make/build-rules.mk @@ -1,7 +1,7 @@ # Generic configurable recipes to build packages with GNU Build system or # CMake. This is Makefile is not intended to be run directly, it will be -# imported into `dependencies-basic.mk' and `dependencies.mk'. They should -# be activated with Make's `Call' function. +# imported into `basic.mk' and `high-level.mk'. They should be activated +# with Make's `Call' function. # # Copyright (C) 2018-2020 Mohammad Akhlaghi # diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 0afeaba..bc77dab 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -28,11 +28,11 @@ # Top level environment include reproduce/software/make/build-rules.mk -include reproduce/software/config/installation/LOCAL.mk -include reproduce/software/config/installation/TARGETS.mk -include reproduce/software/config/installation/texlive.mk -include reproduce/software/config/installation/versions.mk -include reproduce/software/config/installation/checksums.mk +include reproduce/software/config/installation/LOCAL.conf +include reproduce/software/config/installation/TARGETS.conf +include reproduce/software/config/installation/versions.conf +include reproduce/software/config/installation/checksums.conf +include reproduce/software/config/installation/texlive-packages.conf lockdir = $(BDIR)/locks tdir = $(BDIR)/software/tarballs @@ -588,7 +588,7 @@ $(ibidir)/hdf5: $(ibidir)/openmpi \ # HEALPix includes the source of its C, C++, Python (and several other # languages) libraries within one tarball. We will include the Python # installation only when any other Python module is requested (in -# `TARGETS.mk'). +# `TARGETS.conf'). # # Note that the default `./configure' script is an interactive script which # is hard to automate. So we need to go into the `autotools' directory of @@ -1191,7 +1191,7 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/installation/texlive.co #else #forbiber = $(ibidir)/libnsl #endif -$(itidir)/texlive: reproduce/software/config/installation/texlive.mk \ +$(itidir)/texlive: reproduce/software/config/installation/texlive-packages.conf \ $(itidir)/texlive-ready-tlmgr \ $(forbiber) -- cgit v1.2.1 From efa5baf0fd68f4ba178a0a4f0bb59ccfbecaf7e7 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 22 Jan 2020 14:31:50 +0000 Subject: Perl is now a dependency of Coreutils, LD_LIBRARY_PATH in basic.mk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until now Perl was built after Coreutils, but I recently noticed that Coreutils actually uses Perl while creating its manpages. So it is now built before Coreutils. Also, while testing on an Amazon AWS EC2 server, we noticed that Coreutils can't build its man page for `md5sum'. The problem was found to be due to the fact that until now, we weren't actually setting LD_LIBRARY_PATH to our installed library path in `basic.mk'. Therefore, it would crash because the server had an older version of OpenSSL than the one that the template's Coreutils was built with. In the meantime (while addressing the issues above, because we only had one thread on the AWS server) I also noticed a few programs that were using a summarize compilation command (that just prints `CC xxx.c' instead of the whole command) so I fixed them by adding `V=1'. This bug was found by Idafen Santana Pérez. --- reproduce/software/make/basic.mk | 145 ++++++++++++++++++---------------- reproduce/software/make/high-level.mk | 2 +- 2 files changed, 80 insertions(+), 67 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 93dcdc2..48ba3de 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -58,7 +58,15 @@ export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) -export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH) \ + +# This is the "basic" tools where we are relying on the host operating +# system, but are slowly populating our basic software envirnoment. To run +# (system or template) programs, `LD_LIBRARY_PATH' is necessary, so here, +# we'll first tell the programs to look into any possible pre-defined +# `LD_LIBRARY_PATH', then we'll add our own newly installed libraries. We +# will also make sure that there is no "current directory" in it (by +# removing a starting or trailing `:' and any occurance of `::'. +export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH):$(ildir) \ | sed -e's/::/:/g' -e's/^://' -e's/:$$//') # RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is @@ -658,6 +666,66 @@ $(ibidir)/bash: $(ibidir)/readline \ +# The `-shared' flag will cause problems while building Perl on macOS, so +# we'll only use this configuration option when we are GNU/Linux +# systems. However, since the whole option must be used (which includes `=' +# and empty space), its easier to define the variable as a Make variable +# outside the recipe, not as a shell variable inside it. +ifeq ($(on_mac_os),yes) +perl-conflddlflags = +else +perl-conflddlflags = -Dlddlflags="-shared $$LDFLAGS" +endif +$(ibidir)/perl: | $(ibidir)/bash \ + $(tdir)/perl-$(perl-version).tar.gz + major_version=$$(echo $(perl-version) \ + | sed -e's/\./ /g' \ + | awk '{printf("%d", $$1)}'); \ + base_version=$$(echo $(perl-version) \ + | sed -e's/\./ /g' \ + | awk '{printf("%d.%d", $$1, $$2)}'); \ + cd $(ddir) \ + && rm -rf perl-$(perl-version) \ + && if ! tar xf $(word 1,$(filter $(tdir)/%,$|)); then \ + echo; echo "Tar error"; exit 1; \ + fi \ + && cd perl-$(perl-version) \ + && sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ + -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ + Configure > Configure-tmp \ + && mv -f Configure-tmp Configure \ + && chmod +x Configure \ + && ./Configure -des \ + -Dusethreads \ + -Duseshrplib \ + -Dprefix=$(idir) \ + -Dvendorprefix=$(idir) \ + -Dprivlib=$(idir)/share/perl$$major_version/core_perl \ + -Darchlib=$(idir)/lib/perl$$major_version/$$base_version/core_perl \ + -Dsitelib=$(idir)/share/perl$$major_version/site_perl \ + -Dsitearch=$(idir)/lib/perl$$major_version/$$basever/site_perl \ + -Dvendorlib=$(idir)/share/perl$$major_version/vendor_perl \ + -Dvendorarch=$(idir)/lib/perl$$major_version/$$base_version/vendor_perl \ + -Dscriptdir=$(idir)/bin/core_perl \ + -Dsitescript=$(idir)/bin/site_perl \ + -Dvendorscript=$(idir)/bin/vendor_perl \ + -Dinc_version_list=none \ + -Dman1ext=1perl \ + -Dman3ext=3perl \ + -Dcccdlflags='-fPIC' \ + $(perl-conflddlflags) \ + -Dldflags="$$LDFLAGS" \ + && make SHELL=$(ibdir)/bash -j$(numthreads) \ + && make SHELL=$(ibdir)/bash install \ + && cd .. \ + && rm -rf perl-$(perl-version) \ + && cd $$topdir \ + && echo "Perl $(perl-version)" > $@ + + + + + # Coreutils # --------- # @@ -677,8 +745,10 @@ $(ibidir)/bash: $(ibidir)/readline \ # The echo after the PatchELF loop is to avoid a crash if the last # file that PatchELF encounters is not usable (and it returns with # an error). +# +# Coreutils uses Perl to create man pages! $(ibidir)/coreutils: $(ibidir)/openssl \ - | $(ibidir)/bash \ + | $(ibidir)/perl \ $(tdir)/coreutils-$(coreutils-version).tar.xz cd $(ddir) \ && rm -rf coreutils-$(coreutils-version) \ @@ -847,17 +917,17 @@ $(ibidir)/wget: $(ibidir)/libiconv \ # there is no access to the system's PATH. $(ibidir)/diffutils: | $(ibidir)/coreutils \ $(tdir)/diffutils-$(diffutils-version).tar.xz - $(call gbuild, diffutils-$(diffutils-version), static, , V=1) \ + $(call gbuild, diffutils-$(diffutils-version), static,,V=1) \ && echo "GNU Diffutils $(diffutils-version)" > $@ $(ibidir)/file: | $(ibidir)/coreutils \ $(tdir)/file-$(file-version).tar.gz - $(call gbuild, file-$(file-version), static) \ + $(call gbuild, file-$(file-version), static,,V=1) \ && echo "File $(file-version)" > $@ $(ibidir)/findutils: | $(ibidir)/coreutils \ $(tdir)/findutils-$(findutils-version).tar.xz - $(call gbuild, findutils-$(findutils-version), static, , V=1) \ + $(call gbuild, findutils-$(findutils-version), static,,V=1) \ && echo "GNU Findutils $(findutils-version)" > $@ $(ibidir)/gawk: $(ibidir)/gmp \ @@ -910,13 +980,13 @@ $(ibidir)/gmp: | $(ibidir)/m4 \ $(ibidir)/glibtool: | $(ibidir)/m4 \ $(tdir)/libtool-$(libtool-version).tar.xz $(call gbuild, libtool-$(libtool-version), static, \ - --program-prefix=g) \ + --program-prefix=g, V=1) \ && ln -s $(ibdir)/glibtoolize $(ibdir)/libtoolize \ && echo "GNU Libtool $(libtool-version)" > $@ $(ibidir)/grep: | $(ibidir)/coreutils \ $(tdir)/grep-$(grep-version).tar.xz - $(call gbuild, grep-$(grep-version), static) \ + $(call gbuild, grep-$(grep-version), static,,V=1) \ && echo "GNU Grep $(grep-version)" > $@ $(ibidir)/libbsd: | $(ibidir)/coreutils \ @@ -927,7 +997,7 @@ $(ibidir)/libbsd: | $(ibidir)/coreutils \ $(ibidir)/m4: | $(ibidir)/coreutils \ $(ibidir)/texinfo \ $(tdir)/m4-$(m4-version).tar.gz - $(call gbuild, m4-$(m4-version), static) \ + $(call gbuild, m4-$(m4-version), static,,V=1) \ && echo "GNU M4 $(m4-version)" > $@ # Metastore is used (through a Git hook) to restore the source modification @@ -1020,63 +1090,6 @@ $(ibidir)/mpfr: $(ibidir)/gmp \ $(call gbuild, mpfr-$(mpfr-version), static, , , make check) \ && echo "GNU Multiple Precision Floating-Point Reliably $(mpfr-version)" > $@ -# The `-shared' flag will cause problems while building Perl on macOS, so -# we'll only use this configuration option when we are GNU/Linux -# systems. However, since the whole option must be used (which includes `=' -# and empty space), its easier to define the variable as a Make variable -# outside the recipe, not as a shell variable inside it. -ifeq ($(on_mac_os),yes) -perl-conflddlflags = -else -perl-conflddlflags = -Dlddlflags="-shared $$LDFLAGS" -endif -$(ibidir)/perl: | $(ibidir)/coreutils \ - $(tdir)/perl-$(perl-version).tar.gz - major_version=$$(echo $(perl-version) \ - | sed -e's/\./ /g' \ - | awk '{printf("%d", $$1)}'); \ - base_version=$$(echo $(perl-version) \ - | sed -e's/\./ /g' \ - | awk '{printf("%d.%d", $$1, $$2)}'); \ - cd $(ddir) \ - && rm -rf perl-$(perl-version) \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$|)); then \ - echo; echo "Tar error"; exit 1; \ - fi \ - && cd perl-$(perl-version) \ - && sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ - Configure > Configure-tmp \ - && mv -f Configure-tmp Configure \ - && chmod +x Configure \ - && ./Configure -des \ - -Dusethreads \ - -Duseshrplib \ - -Dprefix=$(idir) \ - -Dvendorprefix=$(idir) \ - -Dprivlib=$(idir)/share/perl$$major_version/core_perl \ - -Darchlib=$(idir)/lib/perl$$major_version/$$base_version/core_perl \ - -Dsitelib=$(idir)/share/perl$$major_version/site_perl \ - -Dsitearch=$(idir)/lib/perl$$major_version/$$basever/site_perl \ - -Dvendorlib=$(idir)/share/perl$$major_version/vendor_perl \ - -Dvendorarch=$(idir)/lib/perl$$major_version/$$base_version/vendor_perl \ - -Dscriptdir=$(idir)/bin/core_perl \ - -Dsitescript=$(idir)/bin/site_perl \ - -Dvendorscript=$(idir)/bin/vendor_perl \ - -Dinc_version_list=none \ - -Dman1ext=1perl \ - -Dman3ext=3perl \ - -Dcccdlflags='-fPIC' \ - $(perl-conflddlflags) \ - -Dldflags="$$LDFLAGS" \ - && make SHELL=$(ibdir)/bash -j$(numthreads) \ - && make SHELL=$(ibdir)/bash install \ - && cd .. \ - && rm -rf perl-$(perl-version) \ - && cd $$topdir \ - && echo "Perl $(perl-version)" > $@ - - $(ibidir)/pkg-config: | $(ibidir)/coreutils \ $(tdir)/pkg-config-$(pkgconfig-version).tar.gz # An existing `libiconv' can cause a conflict with `pkg-config', @@ -1101,7 +1114,7 @@ $(ibidir)/pkg-config: | $(ibidir)/coreutils \ $(ibidir)/sed: | $(ibidir)/coreutils \ $(tdir)/sed-$(sed-version).tar.xz - $(call gbuild, sed-$(sed-version), static) \ + $(call gbuild, sed-$(sed-version), static,,V=1) \ && echo "GNU Sed $(sed-version)" > $@ $(ibidir)/texinfo: | $(ibidir)/perl \ diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index bc77dab..0d05626 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -634,7 +634,7 @@ $(ibidir)/healpix: $(ibidir)/cfitsio \ && echo "HEALPix $(healpix-version) \citep{healpix}" > $@ $(ibidir)/libjpeg: | $(tdir)/jpegsrc.$(libjpeg-version).tar.gz - $(call gbuild, jpeg-9b, static) \ + $(call gbuild, jpeg-9b, static,,V=1) \ && echo "Libjpeg $(libjpeg-version)" > $@ $(ibidir)/libnsl: $(ibidir)/libtirpc \ -- cgit v1.2.1 From f37005b729065f0e4ff6bfa99e5410ebb210cd60 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 27 Jan 2020 15:15:40 +0000 Subject: Initial scripts compatible with Dash (minimalistic POSIX) Until now, the initial project scripts were primarily tested with GNU Bash. But Bash is not generally available on all systems (it has many features beyond POSIX). Because of this, effectively we were imposing the requirement on the user that they must have Bash installed. We recently started this with setting the shebang of `project' and `reproduce/software/bash/configure.sh' to `/bin/sh'. After doing so, Raul and Gaspar reported an error on their systems. To fix the problem, I installed Dash (a minimalist POSIX-compliant shell) on my computer and temporarily set the shebangs to `/bin/dash', ran the project configuration step and fixed all issues that came up. With this commit, it can go all the way to building GCC on my system's Dash. After this stage (when `high-level.mk' is called), there is no problem, because we have our own version of GNU Bash and that installed version is used. Probably some more issues still remain and will hopefully be found in the future. While doing this, I also noticed the following two minor issues: - The `./project configure' option `--input-dir' was not recognized because it was mistakenly checking `--inputdir'. It has been corrected. - The test C programs now use the `< Date: Fri, 31 Jan 2020 22:43:11 +0100 Subject: Configure step: compiler checks done before basic settings Until now, the project would first ask for the basic directories, then it would start testing the compiler. But that was problematic because the build directory can come from a previous setting (with `./project configure -e'). Also, it could confuse users to first ask for details, then suddently tell them that you don't have a working C library! We also need to store the CPATH variable in the `LOCAL.conf' because in some cases, the compiler won't work without it. With this commit, the compiler checking has been moved at the start of the configure script. Instead of putting the test program in the build directory, we now make a temporary hidden directory in the source directory and delete that directory as soon as the tests are done. In the process, I also noticed that the copyright year of the two hidden files weren't updated and corrected them. --- reproduce/software/make/high-level.mk | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 0d05626..02ae1a5 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -70,8 +70,8 @@ all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ .SHELLFLAGS := --noprofile --norc -ec export CCACHE_DISABLE := 1 export PATH := $(ibdir) -export CXX := $(ibdir)/g++ export CC := $(ibdir)/gcc +export CXX := $(ibdir)/g++ export SHELL := $(ibdir)/bash export F77 := $(ibdir)/gfortran export LD_RUN_PATH := $(ildir):$(il64dir) @@ -79,6 +79,11 @@ export PKG_CONFIG_PATH := $(ildir)/pkgconfig export LD_LIBRARY_PATH := $(ildir):$(il64dir) export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +# Until we build our own C library, without this, our GCC won't be able to +# compile anything! Note that on most systems (in particular +# non-Debian-based), `sys_cpath' will be empty. +export CPATH := $(sys_cpath) + # RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is # ultimately redundant. But on some systems, even having a single value # causes crashs (see bug #56682). So we'll just give it no value at all. -- cgit v1.2.1 From 35ed6cf0df743175688b49a4559793cb7f6e9d66 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sat, 1 Feb 2020 21:24:00 +0100 Subject: IMPORTANT: reproduce/software/bash renamed to reproduce/software/shell Until now the shell scripts in the software building phase were in the `reproduce/software/bash' directory. But given our recent change to a POSIX-only start, the `configure.sh' shell script (which is the main component of this directory) is no longer written with Bash. With this commit, to fix that problem, that directory's name has been changed to `reproduce/software/shell'. --- reproduce/software/make/README.md | 2 +- reproduce/software/make/basic.mk | 6 +++--- reproduce/software/make/high-level.mk | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/README.md b/reproduce/software/make/README.md index fc04dfb..d71a3d5 100644 --- a/reproduce/software/make/README.md +++ b/reproduce/software/make/README.md @@ -2,7 +2,7 @@ Software building instructions ============================== This directory contains Makefiles that are called by the high-level -`reproduce/software/bash/configure.sh` script. The main target for the +`reproduce/software/shell/configure.sh` script. The main target for the installation of each software is a simple plain text file that contains the name of the software and its version (which is put in the paper in the end). Once built, these plain-text files are all put in the proper diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 7e900b8..d08ba62 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -74,9 +74,9 @@ export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH):$(ildir) \ # causes crashs (see bug #56682). So we'll just give it no value at all. export DYLD_LIBRARY_PATH := -# Recipe startup script, see `reproduce/software/bash/bashrc.sh'. +# Recipe startup script. export PROJECT_STATUS := configure_basic -export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh # Define the top-level basic programs (that don't depend on any other). top-level-programs = low-level-links gcc @@ -1067,7 +1067,7 @@ $(ibidir)/metastore: $(needlibbsd) \ -e's|@GROUP[@]|'$$group'|g' \ -e's|@BINDIR[@]|$(ibdir)|g' \ -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ - reproduce/software/bash/git-$$f > .git/hooks/$$f \ + reproduce/software/shell/git-$$f > .git/hooks/$$f \ && chmod +x .git/hooks/$$f; \ done; \ fi \ diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 02ae1a5..53bef55 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -96,9 +96,9 @@ export DYLD_LIBRARY_PATH := # for `ld'. export LIBRARY_PATH := $(sys_library_path) -# Recipe startup script, see `reproduce/software/bash/bashrc.sh'. +# Recipe startup script, see `reproduce/software/shell/bashrc.sh'. export PROJECT_STATUS := configure_highlevel -export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh # Building flags: # -- cgit v1.2.1 From 6d68865a5d083b8892c3f4854284bb2036f1efdf Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Tue, 11 Feb 2020 00:38:11 +0100 Subject: Using backup server when original download server fails Until now, the main download script could only check one server for the given URL. However, ultimately the actual server that a file is downloaded from is irrelevant for this project: we actually check its checksum. Especially in the case of software (which are distributed over many servers), this can usually be very annoying: the servers may not properly communicate with the running system and even the 10 trials won't be enough. With this commit, the download script `reproduce/analysis/bash/download-multi-try' can take a new optional argument (a 5th argument). It assumes this argument is a space-separated list of server(s) to use as backup for the original URL. When downloading from the original URL fails, it will look into this list and try downloading the same file from each given server. --- reproduce/software/make/basic.mk | 16 ++++++++++++---- reproduce/software/make/high-level.mk | 7 ++++++- reproduce/software/make/python.mk | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index d08ba62..2cad9f9 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -82,6 +82,10 @@ export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh top-level-programs = low-level-links gcc all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) +# Servers to use as backup, later this should go in a file that is not +# under version control (the actual server that the tarbal comes from is +# irrelevant). +backupservers = http://akhlaghi.org/reproduce-software @@ -236,15 +240,19 @@ $(tarballs): $(tdir)/%: | $(lockdir) \ touch $(lockdir)/download; \ $(downloadwrapper) "$$downloader" $(lockdir)/download \ - $$tarballurl "$@.unchecked"; \ + $$tarballurl "$@.unchecked" "$(backupservers)"; \ fi; \ \ \ if type sha512sum > /dev/null 2>/dev/null; then \ checksum=$$(sha512sum "$@.unchecked" | awk '{print $$1}'); \ - echo "$*: should be '$$c', is '$$checksum'"; \ - if [ x$$checksum = x$$c ]; then mv "$@.unchecked" "$@"; \ - else echo "ERROR: Non-matching checksum for '$*'."; exit 1; \ + if [ x"$$checksum" = x"$$c" ]; then \ + mv "$@.unchecked" "$@"; \ + else \ + echo "ERROR: Non-matching checksum for '$*'."; \ + echo "Checksum should be: $$c"; \ + echo "Checksum is: $$checksum"; \ + exit 1; \ fi; \ else mv "$@.unchecked" "$@"; \ fi; diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 53bef55..8457690 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -100,6 +100,11 @@ export LIBRARY_PATH := $(sys_library_path) export PROJECT_STATUS := configure_highlevel export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh +# Servers to use as backup, later this should go in a file that is not +# under version control (the actual server that the tarbal comes from is +# irrelevant). +backupservers = http://akhlaghi.org/reproduce-software + # Building flags: # # C++ flags: when we build GCC, the C++ standard library needs to link with @@ -321,7 +326,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) touch $(lockdir)/download downloader="wget --no-use-server-timestamps -O" $(downloadwrapper) "$$downloader" $(lockdir)/download \ - $$tarballurl "$@.unchecked" + $$tarballurl "$@.unchecked" "$(backupservers)" fi # Make sure this is the expected tarball. Note that we now have a diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 8c7b6f0..2e74c16 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -259,7 +259,7 @@ $(pytarballs): $(tdir)/%: touch $(lockdir)/download downloader="wget --no-use-server-timestamps -O" $(downloadwrapper) "$$downloader" $(lockdir)/download \ - $$tarballurl "$@.unchecked" + $$tarballurl "$@.unchecked" "$(backupservers)" fi # Make sure this is the expected tarball. Note that we now have a -- cgit v1.2.1 From 5baaa500439481514ca173db650f7b45e4994b54 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Thu, 13 Feb 2020 17:42:10 +0000 Subject: Adding a link to the *crt*.o files in the local install directory Until now, we defined `LIBRARY_PATH' to fix the problem of the `ld' linker of Binutils needing several `*crt*.o' files to run. However, some software (for example ImageMagick) over-write `LIBRARY_PATH', therefore there is no other way than to put a link to these necessary files in our local build directory. With this commit, we fixed the problem by putting a link to the system's relevant files in the local library directory. This fixed the problem with ImageMagick. Later, when we build the GNU C Library in the project, we should remove this step. This bug reported by Raul Castellanos Sanchez. --- reproduce/software/make/basic.mk | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 2cad9f9..d8afccd 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -1182,6 +1182,15 @@ endif # programs (http://savannah.nongnu.org/bugs/?56294). Therefore, we'll set # all other basic programs as Binutils prerequisite and GCC (the final # basic target) ultimately just depends on Binutils. +# +# The `ld' linker of Binutils needs several `*crt*.o' file to run. On some +# systems these object files aren't installed in standard places. We +# defined `LIBRARY_PATH' and that fixed the problem for many +# systems. However, some software (for example ImageMagick) over-write +# `LIBRARY_PATH', therefore there is no other way than to put a link to +# these necessary files in our local build directory. IMPORTANT NOTE: +# later, when we build the GNU C Library in the project, we should remove +# this step. $(ibidir)/binutils: | $(ibidir)/sed \ $(ibidir)/wget \ $(ibidir)/grep \ @@ -1203,11 +1212,17 @@ $(ibidir)/binutils: | $(ibidir)/sed \ $(call makelink,nm); \ $(call makelink,ps); \ $(call makelink,ranlib); \ - echo "" > $@; \ + echo "" > $@; \ else \ $(call gbuild, binutils-$(binutils-version), static, \ --with-lib-path=$(sys_library_path), \ -j$(numthreads) ) \ + && if ! [ x"$(sys_library_path)" = x ]; then \ + for f in $(sys_library_path)/*crt*.o; do \ + b=$$($(ibdir)/basename $$f); \ + ln -s $$f $(ildir)/$$b; \ + done; \ + fi \ && echo "GNU Binutils $(binutils-version)" > $@; \ fi -- cgit v1.2.1 From 1c77635ff9cbf10adadbc11a90851f635de323af Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Thu, 13 Feb 2020 17:50:24 +0000 Subject: Corrected version of Texinfo when reporting Until now we were mistakenly reporting the version of SED instead of Texinfo. With this commit, we corrected it! This was reported by Raul Infante Sainz. --- reproduce/software/make/basic.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index d8afccd..4a290b6 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -1132,7 +1132,7 @@ $(ibidir)/texinfo: | $(ibidir)/perl \ $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/info; \ $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/install-info; \ fi \ - && echo "GNU Texinfo $(sed-version)" > $@ + && echo "GNU Texinfo $(texinfo-version)" > $@ $(ibidir)/which: | $(ibidir)/coreutils \ $(tdir)/which-$(which-version).tar.gz -- cgit v1.2.1 From dd769b64a6edcebd6a15183d5cb836698c4a25dc Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 16 Feb 2020 00:59:33 +0000 Subject: XLSX I/O installed with its two dependencies: expat and minizip XLSX I/O is a very simple and fast program and library for reading and writing `.xls' and `.xlsx' files (mainly used by Microsoft Excel) to CSV files. It has two separate executables that can be called for an Excel file and will output a CSV plain text file that can then be used within the pipeline with more standard tools. --- reproduce/software/make/high-level.mk | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 8457690..a83108e 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -163,6 +163,7 @@ tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ eigen-$(eigen-version).tar.gz \ + expat-$(expat-version).tar.lz \ fftw-$(fftw-version).tar.gz \ flex-$(flex-version).tar.gz \ freetype-$(freetype-version).tar.gz \ @@ -183,6 +184,7 @@ tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ libpng-$(libpng-version).tar.xz \ libtirpc-$(libtirpc-version).tar.bz2 \ libxml2-$(libxml2-version).tar.gz \ + minizip-$(minizip-version).tar.gz \ netpbm-$(netpbm-version).tar.gz \ openblas-$(openblas-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ @@ -198,6 +200,7 @@ tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ tides-$(tides-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ + xlsxio-$(xlsxio-version).tar.gz \ yaml-$(yaml-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: | $(lockdir) @@ -240,6 +243,11 @@ $(tarballs): $(tdir)/%: | $(lockdir) mergenames=0 c=$(eigen-checksum); w=http://bitbucket.org/eigen/eigen/get/$(eigen-version).tar.gz + elif [ $$n = expat ]; then + mergenames=0 + c=$(expat-checksum) + vstr=$$(echo $(expat-version) | sed -e's/\./_/g') + w=https://github.com/libexpat/libexpat/releases/download/R_$$vstr/expat-$(expat-version).tar.lz elif [ $$n = fftw ]; then c=$(fftw-checksum); w=ftp://ftp.fftw.org/pub/fftw elif [ $$n = flex ]; then c=$(flex-checksum); w=https://github.com/westes/flex/files/981163 elif [ $$n = freetype ]; then c=$(freetype-checksum); w=https://download.savannah.gnu.org/releases/freetype @@ -273,6 +281,10 @@ $(tarballs): $(tdir)/%: | $(lockdir) w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = libtirpc ]; then c=$(libtirpc-checksum); w=https://downloads.sourceforge.net/libtirpc elif [ $$n = libxml ]; then c=$(libxml2-checksum); w=ftp://xmlsoft.org/libxml2 + elif [ $$n = minizip ]; then + mergenames=0 + c=$(minizip-checksum); + w=https://github.com/nmoinvaz/minizip/archive/$(minizip-version).tar.gz elif [ $$n = netpbm ]; then c=$(netpbm-checksum); w=http://akhlaghi.org/reproduce-software elif [ $$n = openblas ]; then mergenames=0 @@ -300,6 +312,10 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = tides ]; then c=$(tides-checksum); w=http://akhlaghi.org/reproduce-software elif [ $$n = tiff ]; then c=$(libtiff-checksum); w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then c=$(wcslib-checksum); w=ftp://ftp.atnf.csiro.au/pub/software/wcslib + elif [ $$n = xlsxio ]; then + mergenames=0 + c=$(xlsxio-checksum); + w=https://github.com/brechtsanders/xlsxio/archive/$(xlsxio-version).tar.gz elif [ $$n = yaml ]; then c=$(yaml-checksum); w=pyyaml.org/download/libyaml else echo; echo; echo; @@ -563,6 +579,10 @@ $(ibidir)/eigen: | $(tdir)/eigen-$(eigen-version).tar.gz && rm -rf $(ddir)/eigen-eigen-* \ && echo "Eigen $(eigen-version)" > $@ +$(ibidir)/expat: | $(tdir)/expat-$(expat-version).tar.lz + $(call gbuild, expat-$(expat-version), static) \ + && echo "Expat $(expat-version)" > $@ + $(ibidir)/fftw: | $(tdir)/fftw-$(fftw-version).tar.gz # FFTW's single and double precission libraries must be built # independently: for the the single-precision library, we need to @@ -1014,6 +1034,11 @@ $(ibidir)/imfit: $(ibidir)/gsl \ fi \ && echo "Imfit $(imfit-version) \citep{imfit2015}" > $@ +$(ibidir)/minizip: $(ibidir)/cmake \ + | $(tdir)/minizip-$(minizip-version).tar.gz + $(call cbuild, minizip-$(minizip-version), static) \ + && echo "minizip $(minizip-version)" > $@ + # Netpbm is a prerequisite of Astrometry-net, it contains a lot of programs. # This program has a crazy dialogue installation which is override using the # printf statment. Each `\n' is a new question that the installation process @@ -1120,6 +1145,13 @@ $(ibidir)/swig: | $(tdir)/swig-$(swig-version).tar.gz $(call gbuild, swig-$(swig-version), static, --without-pcre) \ && echo "Swig $(swig-version)" > $@ +$(ibidir)/xlsxio: $(ibidir)/expat \ + $(ibidir)/minizip \ + | $(tdir)/xlsxio-$(xlsxio-version).tar.gz + export LDFLAGS="-lbz2 -lbsd"; \ + $(call cbuild, xlsxio-$(xlsxio-version), static) \ + && echo "XLSX I/O $(xlsxio-version)" > $@ + -- cgit v1.2.1 From 6dcfac476fdb8334fc682304e12a5db6f21b3a22 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 19 Feb 2020 03:38:51 +0000 Subject: Building of GCC now only done when /dev/shm has more than 10GB free Until now, like all software on GNU/Linux systems GCC would be built in RAM (to speed up the build slightly and also not put too much stress on the HDDs/SSDs). But some systems don't have enough RAM for building GCC and will complain and crash. With this commit, we have added a check on the amount of free space in the `build_tmp' directory (which will be `/dev/shm' on GNU/Linux systems). If the amount of free space isn't more than 10GB, then GCC won't be built there and a temporary directory will be built under the `$(BDIR)/software' directory for it. This bug was found by Zahra Sharbaf. This fixes bug #57853. --- reproduce/software/make/basic.mk | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 4a290b6..212c738 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -1261,15 +1261,27 @@ $(ibidir)/gcc: | $(ibidir)/binutils \ ccinfo=$$(gcc --version | awk 'NR==1'); \ echo "C compiler (""$$ccinfo"")" > $@; \ else \ + current_dir=$$(pwd); \ rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\ rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \ rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\ \ ln -fs $(ildir) $(idir)/lib64; \ - \ - cd $(ddir); \ + \ + in_ram=$$(df $(ddir) \ + | awk 'NR==2{print ($$4>10000000) ? "yes" : "no"}'); \ + if [ $$in_ram = "yes" ]; then odir=$(ddir); \ + else \ + odir=$(BDIR)/software/build-tmp-gcc; \ + if [ -d $$odir ]; then rm -rf $$odir; fi; \ + mkdir $$odir; \ + fi; \ + cd $$odir; \ rm -rf gcc-$(gcc-version); \ - tar xf $(word 1,$(filter $(tdir)/%,$|)) \ + tar xf $(word 1,$(filter $(tdir)/%,$|)); \ + if [ $$odir != $(ddir) ]; then \ + ln -s $$odir/gcc-$(gcc-version) $(ddir)/gcc-$(gcc-version); \ + fi \ && cd gcc-$(gcc-version) \ && mkdir build \ && cd build \ @@ -1296,7 +1308,7 @@ $(ibidir)/gcc: | $(ibidir)/binutils \ && make SHELL=$(ibdir)/bash -j$(numthreads) \ && make SHELL=$(ibdir)/bash install \ && cd ../.. \ - && tempname=$(ddir)/gcc-$(gcc-version)/build/rpath-temp-copy \ + && tempname=$$odir/gcc-$(gcc-version)/build/rpath-temp-copy \ && if [ "x$(on_mac_os)" != xyes ]; then \ patchelf --add-needed $(ildir)/libiconv.so $(ildir)/libstdc++.so; \ for f in $$(find $(idir)/libexec/gcc) $(ildir)/libstdc++*; do \ @@ -1309,6 +1321,11 @@ $(ibidir)/gcc: | $(ibidir)/binutils \ done; \ fi \ && rm -rf gcc-$(gcc-version) \ + && cd $$current_dir \ + && if [ "$$odir" != "$(ddir)" ]; then \ + rm -rf $$odir; \ + rm $(ddir)/gcc-$(gcc-version); \ + fi \ && ln -sf $(ibdir)/gcc $(ibdir)/cc \ && echo "GNU Compiler Collection (GCC) $(gcc-version)" > $@; \ fi -- cgit v1.2.1 From 2f0417995da0c21c894a003af5804b3c732a34c3 Mon Sep 17 00:00:00 2001 From: Surena Fatemi Date: Mon, 24 Feb 2020 22:35:20 +0330 Subject: MissFITS is now added to the template MissFITS is package for manipulating FITS files. I added it as my first commit to the project for educational purposes. --- reproduce/software/make/high-level.mk | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index a83108e..655eb9d 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -185,6 +185,7 @@ tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ libtirpc-$(libtirpc-version).tar.bz2 \ libxml2-$(libxml2-version).tar.gz \ minizip-$(minizip-version).tar.gz \ + missfits-$(missfits-version).tar.gz \ netpbm-$(netpbm-version).tar.gz \ openblas-$(openblas-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ @@ -285,6 +286,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) mergenames=0 c=$(minizip-checksum); w=https://github.com/nmoinvaz/minizip/archive/$(minizip-version).tar.gz + elif [ $$n = missfits ]; then c=$(missfits-checksum); w=https://www.astromatic.net/download/missfits elif [ $$n = netpbm ]; then c=$(netpbm-checksum); w=http://akhlaghi.org/reproduce-software elif [ $$n = openblas ]; then mergenames=0 @@ -1039,6 +1041,11 @@ $(ibidir)/minizip: $(ibidir)/cmake \ $(call cbuild, minizip-$(minizip-version), static) \ && echo "minizip $(minizip-version)" > $@ +$(ibidir)/missfits: | $(tdir)/missfits-$(missfits-version).tar.gz + $(call gbuild, missfits-$(missfits-version), static) \ + && cp $(dtexdir)/missfits.tex $(ictdir)/ \ + && echo "MissFITS $(missfits-version) \citep{missfits}" > $@ + # Netpbm is a prerequisite of Astrometry-net, it contains a lot of programs. # This program has a crazy dialogue installation which is override using the # printf statment. Each `\n' is a new question that the installation process -- cgit v1.2.1 From c1bf1cac9c6bde529e3a67ded0a0a11ea034f077 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 17 Mar 2020 11:28:51 +0000 Subject: Astroquery updated to version 0.4 In the last update of Astropy to version 4.0 they removed some things that the previous version of Astroquery needs. As a consequence, it is also necessary to update the Astroquery version to be a ble to run with the Astropy 4.0. With this commit, the update of Astroquery to it most recent version (0.4) has been done. --- reproduce/software/make/python.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 2e74c16..11ac543 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -185,7 +185,7 @@ $(pytarballs): $(tdir)/%: h=$(pytopurl)/$$hash/SecretStorage-$(secretstorage-version).tar.gz elif [ $$n = asn ]; then h=fc/f1/8db7daa71f414ddabfa056c4ef792e1461ff655c2ae2928a2b675bfed6b4; c=$(asn1crypto-checksum) elif [ $$n = asteval ]; then h=50/3f/29b7935c6dc09ee96dc347edc66c57e8ef68d595dd35b763a36a117acc8c; c=$(asteval-checksum) - elif [ $$n = astroquery ]; then h=61/50/a7a08f9e54d7d9d97e69433cd88231e1ad2901811c9d1ae9ac7ccaef9396; c=$(astroquery-checksum) + elif [ $$n = astroquery ]; then h=e2/af/a3cd3b30745832a0e81f5f13327234099aaf5d03b7979ac947a888e68e91; c=$(astroquery-checksum) elif [ $$n = astropy ]; then h=de/96/7feaca4b9be134128838395a9d924ea0b389ed4381702dcd9d11ae31789f; c=$(astropy-checksum) elif [ $$n = beautifulsoup ]; then h=80/f2/f6aca7f1b209bb9a7ef069d68813b091c8c3620642b568dac4eb0e507748; c=$(beautifulsoup4-checksum) elif [ $$n = certifi ]; then h=55/54/3ce77783acba5979ce16674fc98b1920d00b01d337cfaaf5db22543505ed; c=$(certifi-checksum) -- cgit v1.2.1 From 9000d98e7f273447c424b18fed8d3c84beeb6a79 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 20 Mar 2020 19:05:27 +0000 Subject: Adding PyYAML, Html5lib, and Beautifulsoup4 as prerequsites of Astropy Until this commit, PyYAML was not set as prerequisite of Astropy. This package is an optional dependency of Astropy for some particular functions. However, we have already included PyYAML into this project so it is available. With this commit, PyYAML has been set as a prerequisite of Astropy. In addition to this, Html5lib and Beautifulsoup4 have been also added as prerequsites of Astropy (and removed from Astroquery prerequisites). I noticed that both of them are optional dependencies of Astropy. --- reproduce/software/make/python.mk | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 11ac543..fef1605 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -373,12 +373,9 @@ $(ipydir)/asteval: $(ipydir)/numpy \ $(call pybuild, tar xf, asteval-$(asteval-version), , \ ASTEVAL $(asteval-version)) -$(ipydir)/astroquery: $(ipydir)/numpy \ - $(ipydir)/astropy \ +$(ipydir)/astroquery: $(ipydir)/astropy \ $(ipydir)/keyring \ - $(ipydir)/html5lib \ $(ipydir)/requests \ - $(ipydir)/beautifulsoup4 \ | $(tdir)/astroquery-$(astroquery-version).tar.gz $(call pybuild, tar xf, astroquery-$(astroquery-version), ,\ Astroquery $(astroquery-version)) @@ -386,6 +383,9 @@ $(ipydir)/astroquery: $(ipydir)/numpy \ $(ipydir)/astropy: $(ipydir)/h5py \ $(ipydir)/scipy \ $(ipydir)/numpy \ + $(ipydir)/pyyaml \ + $(ipydir)/html5lib \ + $(ipydir)/beautifulsoup4 \ | $(tdir)/astropy-$(astropy-version).tar.gz $(call pybuild, tar xf, astropy-$(astropy-version)) \ && cp $(dtexdir)/astropy.tex $(ictdir)/ \ -- cgit v1.2.1 From e909553d88dbf15f764f3f797956af75595adcf9 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 1 Apr 2020 19:42:18 +0100 Subject: Removed multiple tabs in MissFITS tarball definition With this commit, multiples tabs in the definition of MissFITS tarball have been removed. Now they are white spaces. --- reproduce/software/make/high-level.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 655eb9d..274bcdf 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -185,7 +185,7 @@ tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ libtirpc-$(libtirpc-version).tar.bz2 \ libxml2-$(libxml2-version).tar.gz \ minizip-$(minizip-version).tar.gz \ - missfits-$(missfits-version).tar.gz \ + missfits-$(missfits-version).tar.gz \ netpbm-$(netpbm-version).tar.gz \ openblas-$(openblas-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ -- cgit v1.2.1