From aee6d612073ce9be8aef4183c5ac7688ed4733e8 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 1 Jul 2020 13:19:23 +0100 Subject: Minor typo corrected in referencing Libidn Until this commit, once Libidn was installed, insted of its own name and version, the name and version of Libjpeg were saved (in the target if Libidn). This robably come from a copy/paste of the rule. With this commit, this minor bug has been corrected. I also added my name as an author of `reproduce/software/make/xorg.mk' Makefile since I added some code there. --- reproduce/software/make/high-level.mk | 2 +- reproduce/software/make/xorg.mk | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index cf104e6..6e14b2d 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -555,7 +555,7 @@ $(ibidir)/libidn-$(libidn-version): $(call import-source, $(libidn-url), $(libidn-checksum)) $(call gbuild, libidn-$(libidn-version), static, \ --disable-doc, -j$(numthreads) V=1) - echo "Libjpeg $(libjpeg-version)" > $@ + echo "Libidn $(libidn-version)" > $@ $(ibidir)/libjpeg-$(libjpeg-version): tarball=jpegsrc.$(libjpeg-version).tar.gz diff --git a/reproduce/software/make/xorg.mk b/reproduce/software/make/xorg.mk index e3b1f71..f59fb04 100644 --- a/reproduce/software/make/xorg.mk +++ b/reproduce/software/make/xorg.mk @@ -15,6 +15,7 @@ # ------------------------------------------------------------------------ # # Copyright (C) 2020 Mohammad Akhlaghi +# Copyright (C) 2020 Raul Infante-Sainz # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From deba07c43ebddc31ecdbcf520e2bbaf57e6f8027 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 1 Jul 2020 18:14:44 +0100 Subject: Properly accounting for space characters in host's PATH Until now, when reading the host's PATH environment variable we weren't accounting for directory names with a space character. This was most prominently visible in the 'low-level-links' step where we put links to some core system components into the project's build directory (mainly for prorietary systems like macOS). To address the problem, double quotations have been placed around the part that we extract 'ccache' from the PATH, and the part where we make the symbolic link. In the process the comments above 'makelink' were made more clear and 'low-level-links' now depends on 'grep' (which is the highest-level program it uses). This bug was reported by Mahdieh Navabi. --- reproduce/software/make/basic.mk | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 453eddb..2c7401e 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -130,9 +130,17 @@ backupservers = $(filter-out $(topbackupserver),$(backupservers_all)) # Low-level (not built) programs # ------------------------------ # -# For the time being, some components of the project on some systems, so we -# are simply making a symbolic link to the system's files here. We'll do -# this after building GNU Coreutils to have trustable elements. +# For the time being, some components of the project aren't being built on +# some systems (primarily on proprietary operating systems). So we are +# simply making a symbolic link to the system's programs/libraries in the +# build directory. +# +# The logical position of this rule is irrelevant in this Makefile (because +# programs being built here have full access to the system's PATH +# already). This is done for the high-level programs installed in +# 'high-level.mk', 'xorg.mk' or 'python.mk'. So this step is done after +# building our own GNU Grep (which is the highest-level program used in +# 'makelink') to have trustable elements. # # About ccache: ccache acts like a wrapper over the C compiler and is made # to avoid/speed-up compiling of identical files in a system (it is @@ -142,19 +150,19 @@ backupservers = $(filter-out $(topbackupserver),$(backupservers_all)) # thus remove any part of PATH of that has `ccache' in it before making # symbolic links to the programs we are not building ourselves. # -# We'll need the system's PATH for making links to low-level programs we -# won't be building ourselves. +# The double quotations after the starting 'export PATH' are necessary in +# case the user's PATH has space-characters in it. syspath := $(PATH) makelink = origpath="$$PATH"; \ - export PATH=$$(echo $(syspath) \ - | tr : '\n' \ - | grep -v ccache \ - | tr '\n' :); \ + export PATH="$$(echo $(syspath) \ + | tr : '\n' \ + | grep -v ccache \ + | tr '\n' :)"; \ if type $(1) > /dev/null 2> /dev/null; then \ if [ x$(3) = x ]; then \ - ln -sf $$(which $(1)) $(ibdir)/$(1); \ + ln -sf "$$(which $(1))" $(ibdir)/$(1); \ else \ - ln -sf $$(which $(1)) $(ibdir)/$(3); \ + ln -sf "$$(which $(1))" $(ibdir)/$(3); \ fi; \ else \ if [ "x$(strip $(2))" = xmandatory ]; then \ @@ -166,7 +174,7 @@ makelink = origpath="$$PATH"; \ export PATH="$$origpath" $(ibdir) $(ildir):; mkdir $@ -$(ibidir)/low-level-links: $(ibidir)/coreutils-$(coreutils-version) \ +$(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ | $(ibdir) $(ildir) # Not-installed (but necessary in some cases) compilers. -- cgit v1.2.1 From 1bc00c9e64bba6ebd4c90301cc2a22f25466f72f Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Thu, 2 Jul 2020 17:56:46 +0100 Subject: Only using clang in macOS systems that also have GCC Until now, when Maneage was built on a macOS that had both a clang and GCC, we would make links to both. But this cause many conflicts in some high-level programs (for example Numpy and etc, all the programs where we have explicity set 'export CC=clang' before the build recipe). This happens because the GCC that is built on a macOS isn't complete for some operations. To fix this problem, when we are on a macOS, we explicity set 'gcc' to point to 'clang' and 'g++' to point to 'clang++'. We also don't link to the host's C-preprocessor ('cpp') on macOS systems because this is only a GNU feature and using the GNU CPP is also known to have some basic problems. For example this was reported by Mahdieh Nabavi (which was the main trigger for this work): ld: Symbol not found: ___keymgr_global Referenced from: /Users/Mahdieh/build/software/installed/bin/cpp Expected in: /usr/lib/libSystem.B.dylib Also, to avoid linking to another link on the host tools (in the 'makelink' function of 'basic.mk'), we are now using 'realpath'. --- reproduce/software/make/basic.mk | 85 +++++++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 27 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 2c7401e..b9678d0 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -152,6 +152,12 @@ backupservers = $(filter-out $(topbackupserver),$(backupservers_all)) # # The double quotations after the starting 'export PATH' are necessary in # case the user's PATH has space-characters in it. +# +# We use 'realpath' here (part of GNU Coreutils which is already installed +# by the time we use 'makelink') to avoid linking to a link (on the +# host). 'realpath' will follow a link (and possibly other links in the +# middle) to an actual file and return its address. When the location isn't +# a link, it will just return it. syspath := $(PATH) makelink = origpath="$$PATH"; \ export PATH="$$(echo $(syspath) \ @@ -160,9 +166,9 @@ makelink = origpath="$$PATH"; \ | tr '\n' :)"; \ if type $(1) > /dev/null 2> /dev/null; then \ if [ x$(3) = x ]; then \ - ln -sf "$$(which $(1))" $(ibdir)/$(1); \ + ln -sf "$$(realpath $$(which $(1)))" $(ibdir)/$(1); \ else \ - ln -sf "$$(which $(1))" $(ibdir)/$(3); \ + ln -sf "$$(realpath $$(which $(1)))" $(ibdir)/$(3); \ fi; \ else \ if [ "x$(strip $(2))" = xmandatory ]; then \ @@ -177,11 +183,6 @@ $(ibdir) $(ildir):; mkdir $@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ | $(ibdir) $(ildir) - # Not-installed (but necessary in some cases) compilers. - # Clang is necessary for CMake. - $(call makelink,clang) - $(call makelink,clang++) - # Mac OS specific $(call makelink,mig) $(call makelink,xcrun) @@ -1278,6 +1279,16 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) tarball=gcc-$(gcc-version).tar.xz $(call import-source, $(gcc-url), $(gcc-checksum)) + # To avoid any previous build in '.local/bin' causing problems in + # this build/links of this GCC, we'll first delete all the possibly + # built/existing compilers in this project. Note that GCC also + # installs several executables like this 'x86_64-pc-linux-gnu-gcc', + # 'x86_64-pc-linux-gnu-gcc-ar' or 'x86_64-pc-linux-gnu-g++'. + rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc* + rm -f $(ibdir)/*gcc* $(ibdir)/gcov* $(ibdir)/cc $(ibdir)/c++ + rm -f $(ibdir)/*g++ $(ibdir)/cpp $(ibdir)/gfortran $(ibdir)/strip + rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64* + # GCC builds is own libraries in '$(idir)/lib64'. But all other # libraries are in '$(idir)/lib'. Since this project is only for a # single architecture, we can trick GCC into building its libraries @@ -1285,34 +1296,53 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) # link to '$(idir)/lib'. if [ $(host_cc) = 1 ]; then - # Make sure we don't have any of the program we want to link to - # in the '.local/bin' directory. - rm -f $(ibdir)/cc - rm -f $(ibdir)/c++ - rm -f $(ibdir)/gcc - rm -f $(ibdir)/cpp - rm -f $(ibdir)/strip; - rm -f $(ibdir)/gfortran; - - # Put links to the host's tools in '.local/bin'. - $(call makelink,cc) - $(call makelink,cpp) - $(call makelink,cc,,gcc) + # Put links to the host's tools in '.local/bin'. Note that some + # macOS systems have both a native clang *and* a GNU C Compiler + # (note that this is different from the "normal" macOS situation + # where 'gcc' actually points to clang, here we mean when 'gcc' + # is actually the GNU C Compiler). + # + # In such cases, the GCC isn't complete and using it will cause + # problems when building high-level tools (for example openBLAS, + # rpcsvc-proto, CMake, xlsxio, Python or Matplotlib among + # others). To avoid such situations macOSs are configured like + # this: we'll simply set 'gcc' to point to 'clang' and won't set + # 'gcc' to point to the system's 'gcc'. + # + # Also, note that LLVM's clang doesn't have a C Pre-Processor. So + # we will only put a link to the host's 'cpp' if the system is + # not macOS. On macOS systems that have a real GCC installed, + # having GNU CPP in the project build directory is known to cause + # problems with 'libX11'. $(call makelink,gfortran) - $(call makelink,c++,,g++) $(call makelink,strip,mandatory) + if [ x$(on_mac_os) = xyes ]; then + $(call makelink,clang) + $(call makelink,clang++) + $(call makelink,clang,,gcc) + $(call makelink,clang++,,g++) + else + $(call makelink,cpp) + $(call makelink,gcc) + $(call makelink,g++) + fi + + # We also want to have the two 'cc' and 'c++' in the build + # directory that point to the selected compiler. With the checks + # above, 'gcc' and 'g++' will point to the proper compiler, so + # we'll use them to define 'cc' and 'c++'. + $(call makelink,gcc,,cc) + $(call makelink,g++,,c++) + + # Get the first line of the compiler's '--version' output and put + # that into the target (so we know want compiler was used). ccinfo=$$(gcc --version | awk 'NR==1') echo "C compiler (""$$ccinfo"")" > $@ else - # We are building GCC, so to avoid any previous build in - # '.local/bin', we'll delete all the files that GCC builds from - # there. + # Mark the current directory. current_dir=$$(pwd) - rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov* - rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc* - rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64* # We don't want '.local/lib' and '.local/lib64' to be separate. ln -fs $(ildir) $(idir)/lib64 @@ -1406,6 +1436,7 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) # Set 'cc' to point to 'gcc'. ln -sf $(ibdir)/gcc $(ibdir)/cc + ln -sf $(ibdir)/g++ $(ibdir)/c++ # Write the final target. echo "GNU Compiler Collection (GCC) $(gcc-version)" > $@ -- cgit v1.2.1 From 0e4d4b357f4c2209aea8012847b1309fe8b33b13 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 5 Jul 2020 19:17:05 +0100 Subject: Configure script prefers clang for macOS systems In the previous commit (Commit 1bc00c9: Only using clang in macOS systems that also have GCC) we set the used C compiler for high-level programs to be 'clang' on macOS systems. But I forgot to do the same kind of change in the configure script (to prefer 'clang' when we are testing for a C compiler on the host). With this commit, the compiler checking phases of the configure script have been improved, so on macOS systems, we now first search for 'clang', then search for 'gcc'. While doing this, I also noticed that the 'rpath' checking command was done before we actually define 'instdir'!!! So in effect, the 'rpath' directory was being set to '/lib'! So with this commit, this test has been taken to after defining 'instdir'. --- reproduce/software/make/high-level.mk | 1 + 1 file changed, 1 insertion(+) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 6e14b2d..3ac3b49 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -52,6 +52,7 @@ ildir = $(BDIR)/software/installed/lib ibidir = $(BDIR)/software/installed/version-info/proglib # Basic directories (specific to this Makefile). +il64dir = $(BDIR)/software/installed/lib64 iidir = $(BDIR)/software/installed/include shsrcdir = $(shell pwd)/reproduce/software/shell dtexdir = $(shell pwd)/reproduce/software/bibtex -- cgit v1.2.1 From c85272705b7544edaf3dadda14581157fc18826c Mon Sep 17 00:00:00 2001 From: Marius Peper Date: Tue, 7 Jul 2020 22:58:07 +0200 Subject: Fixed typo that lead to crash when building healpy Until now, if a project needed the healpy software package, Maneage would crash with the following error message (abridged for full name in build directory). This was caused by a typo in the version of 'healpix' (the dependency of 'healpy'). make: *** No rule to make target '.../version-info/proglib/healpix-' With this commit, the typo in line 334 of 'python.mk' is fixed, so that when '$(ipydir)/healpy-$(healpy-version)' gets called it correctly searches for a rule to make '$(ibidir)/healpix-$(healpix-version)'. --- reproduce/software/make/python.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index ccfbc72..2fede35 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -331,7 +331,7 @@ $(ipydir)/h5py-$(h5py-version): \ # and just ask for `healpy'. To avoid confusion in such cases, we'll just # set `healpy' to be dependent on `healpix' and not download any tarball # for it, or write anything in the final target. -$(ipydir)/healpy-$(healpy-version): $(ibidir)/healpix-$(healpy-version) +$(ipydir)/healpy-$(healpy-version): $(ibidir)/healpix-$(healpix-version) touch $@ $(ipydir)/html5lib-$(html5lib-version): \ -- cgit v1.2.1 From cbd4a41555c7d4aecaea03a4cc4298da8320f01c Mon Sep 17 00:00:00 2001 From: Boud Roukema Date: Fri, 31 Jul 2020 19:04:08 +0200 Subject: OpenMPI build with slurm compatibility Prior to this commit, compilation of OpenMPI used the default OpenMPI choices of deciding which libraries should be used in relating to a job scheduler [1] (such as Slurm [2]). Given that the user on a multi-user cluster has to accept the sysadmin's choice of a job scheduler, the question of whether to (1) link with OpenMPI's own libraries (and increase the reproducibility of the science project) or rather (2) link with the sysadmin managed libraries (more likely to be compatible with the host's job scheduler), is an open question of which the best strategy for reproducibility needs to be debated and studied. In this commit, strategy (1) is adopted. The options '--withpmix=internal' and '--with-hwloc=internal' are added to the configure command. The working assumption is that the Maneage version of OpenMPI is likely to be modern enough to be compatible with the native job scheduler such as Slurm. Compilation without any 'pmix' option gave a fail in at least one case; it appears that an external pmix library was sought by the configure script. As of OpenMPI 4.0.1, the internal libevent library is used by default, so there appears to be no option to force it to be chosen internally. This commit also includes the option '--without-verbs'. This option removes a library related to "infiniband", "verbs", "openib" and "BTL"; this library appears to be deprecated. See [3], [4] for discussion. Please add feedback and discussion to the Maneage task about openmpi linking strategies (1) (internal) and (2) (external) at Savannah [5]. [1] https://en.wikipedia.org/wiki/Job_scheduler#Batch_queuing_for_HPC_clusters [2] https://en.wikipedia.org/wiki/Slurm_Workload_Manager - To avoid a name clash, 'slurm-wlm' is the metapackage in Debian for the client commands, the compute node daemon, and the central node daemon. An unrelated package 'slurm' also exists. [3] https://www-lb.open-mpi.org/faq/?category=openfabrics#ofa-device-error [4] https://www-lb.open-mpi.org/faq/?category=building [5] https://savannah.nongnu.org/task/index.php?15737 --- reproduce/software/make/high-level.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 3ac3b49..4052ed4 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -641,7 +641,10 @@ $(ibidir)/openblas-$(openblas-version): $(ibidir)/openmpi-$(openmpi-version): tarball=openmpi-$(openmpi-version).tar.gz $(call import-source, $(openmpi-url), $(openmpi-checksum)) - $(call gbuild, openmpi-$(openmpi-version), static, , \ + $(call gbuild, openmpi-$(openmpi-version), static, \ + --with-pmix=internal \ + --with-hwloc=internal \ + --without-verbs, \ -j$(numthreads) V=1) echo "Open MPI $(openmpi-version)" > $@ -- cgit v1.2.1 From b3b44798a3f472e5f5a6633de4a582d4c902b008 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Thu, 18 Jun 2020 18:01:33 +0100 Subject: IMPORTANT: New software versions (17 basic, 16 high-level and 7 Python) It was a long time that the Maneage software versions hadn't been updated. With this commit, the versions of all basic software were checked and 17 of that had newer versions were updated. Also, 16 high-level programs and libraries were updated as well as 7 Python modules. The full list is available below. Basic Software (affecting all projects) --------------------------------------- bash 5.0.11 -> 5.0.18 binutils 2.32 -> 2.35 coreutils 8.31 -> 8.32 curl 7.65.3 -> 7.71.1 file 5.36 -> 5.39 gawk 5.0.1 -> 5.1.0 gcc 9.2.0 -> 10.2.0 gettext 0.20.2 -> 0.21 git 2.26.2 -> 2.28.0 gmp 6.1.2 -> 6.2.0 grep 3.3 -> 3.4 libbsd 0.9.1 -> 0.10.0 ncurses 6.1 -> 6.2 perl 5.30.0 -> 5.32.0 sed 4.7 -> 4.8 texinfo 6.6 -> 6.7 xz 5.2.4 -> 5.2.5 Custom programs/libraries ------------------------- astrometrynet 0.77 -> 0.80 automake 0.16.1 -> 0.16.2 bison 3.6 -> 3.7 cfitsio 3.47 -> 3.48 cmake 3.17.0 -> 3.18.1 freetype 2.9 -> 2.10.2 gdb 8.3 -> 9.2 ghostscript 9.50 -> 9.52 gnuastro 0.11 -> 0.12 libgit2 0.28.2 -> 1.0.1 libidn 1.35 -> 1.36 openmpi 4.0.1 -> 4.0.4 R 3.6.2 -> 4.0.2 python 3.7.4 -> 3.8.5 wcslib 6.4 -> 7.3 yaml 0.2.2 -> 0.2.5 Python modules -------------- cython 0.29.6 -> 0.29.21 h5py 2.9.0 -> 2.10.0 matplotlib 3.1.1 -> 3.3.0 mpi4py 3.0.2 -> 3.0.3 numpy 1.17.2 -> 1.19.1 pybind11 2.4.3 -> 2.5.0 scipy 1.3.1 -> 1.5.2 --- reproduce/software/make/basic.mk | 29 ++++++++++-------- reproduce/software/make/build-rules.mk | 20 +++++++++---- reproduce/software/make/high-level.mk | 55 +++++++++++++++++++++++++++------- reproduce/software/make/python.mk | 5 +++- 4 files changed, 81 insertions(+), 28 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index b9678d0..0b672d8 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -183,6 +183,10 @@ $(ibdir) $(ildir):; mkdir $@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ | $(ibdir) $(ildir) + # Hardware specific + $(call makelink,lp) # For printing, necessary for R. + $(call makelink,lpr) # For printing, necessary for R. + # Mac OS specific $(call makelink,mig) $(call makelink,xcrun) @@ -922,7 +926,7 @@ $(ibidir)/gettext-$(gettext-version): \ $(ibidir)/ncurses-$(ncurses-version) \ $(ibidir)/libiconv-$(libiconv-version) \ $(ibidir)/libunistring-$(libunistring-version) - tarball=gettext-$(gettext-version).tar.xz + tarball=gettext-$(gettext-version).tar.lz $(call import-source, $(gettext-url), $(gettext-checksum)) $(call gbuild, gettext-$(gettext-version), static,, \ V=1 -j$(numthreads)) @@ -959,7 +963,7 @@ $(ibidir)/libtool-$(libtool-version): $(ibidir)/m4-$(m4-version) $(call import-source, $(libtool-url), $(libtool-checksum)) $(call gbuild, libtool-$(libtool-version), static, \ --program-prefix=g, V=1 -j$(numthreads)) - ln -s $(ibdir)/glibtoolize $(ibdir)/libtoolize + ln -sf $(ibdir)/glibtoolize $(ibdir)/libtoolize echo "GNU Libtool $(libtool-version)" > $@ $(ibidir)/grep-$(grep-version): $(ibidir)/coreutils-$(coreutils-version) @@ -1211,6 +1215,7 @@ $(ibidir)/binutils-$(binutils-version): \ $(call makelink,ld) $(call makelink,nm) $(call makelink,ps) + $(call makelink,strip) $(call makelink,ranlib) echo "" > $@ else @@ -1220,14 +1225,15 @@ $(ibidir)/binutils-$(binutils-version): \ --with-lib-path=$(sys_library_path), \ -j$(numthreads) ) - # The `ld' linker of Binutils needs several `*crt*.o' file to - # run. On some systems these object files aren't installed in - # standard places. We defined `LIBRARY_PATH' and that fixed the - # problem for many systems. However, some software (for example - # ImageMagick) over-write `LIBRARY_PATH', therefore there is no - # other way than to put a link to these necessary files in our - # local build directory. IMPORTANT NOTE: later, when we build the - # GNU C Library in the project, we should remove this step. + # The `ld' linker of Binutils needs several `*crt*.o' files from + # the host's GNU C Library to run. On some systems these object + # files aren't installed in standard places. We defined + # `LIBRARY_PATH' and that fixed the problem for many + # systems. However, some software (for example ImageMagick) + # over-write `LIBRARY_PATH', therefore there is no other way than + # to put a link to these necessary files in our local build + # directory. IMPORTANT NOTE: later, when we build the GNU C + # Library in the project, we should remove this step. if ! [ x"$(sys_library_path)" = x ]; then for f in $(sys_library_path)/*crt*.o; do b=$$($(ibdir)/basename $$f) @@ -1284,9 +1290,9 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) # built/existing compilers in this project. Note that GCC also # installs several executables like this 'x86_64-pc-linux-gnu-gcc', # 'x86_64-pc-linux-gnu-gcc-ar' or 'x86_64-pc-linux-gnu-g++'. + rm -f $(ibdir)/*g++ $(ibdir)/cpp $(ibdir)/gfortran rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc* rm -f $(ibdir)/*gcc* $(ibdir)/gcov* $(ibdir)/cc $(ibdir)/c++ - rm -f $(ibdir)/*g++ $(ibdir)/cpp $(ibdir)/gfortran $(ibdir)/strip rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64* # GCC builds is own libraries in '$(idir)/lib64'. But all other @@ -1315,7 +1321,6 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) # having GNU CPP in the project build directory is known to cause # problems with 'libX11'. $(call makelink,gfortran) - $(call makelink,strip,mandatory) if [ x$(on_mac_os) = xyes ]; then $(call makelink,clang) $(call makelink,clang++) diff --git a/reproduce/software/make/build-rules.mk b/reproduce/software/make/build-rules.mk index 7471e5b..86370ab 100644 --- a/reproduce/software/make/build-rules.mk +++ b/reproduce/software/make/build-rules.mk @@ -170,11 +170,21 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ \ echo; echo "Using '$$confscript' to configure:"; echo; \ echo "$$confscript $(3) $$configop"; echo; \ - $$confscript $(3) $$configop; \ - make "$$shellop" $(4); \ - $$check; \ - make "$$shellop" install $(7); \ - cd ..; \ + if [ x$$configure_in_different_directory = x1 ]; then \ + mkdir build; \ + cd build; \ + ../$$confscript $(3) $$configop; \ + make "$$shellop" $(4); \ + $$check; \ + make "$$shellop" install $(7); \ + cd ../..; \ + else \ + $$confscript $(3) $$configop; \ + make "$$shellop" $(4); \ + $$check; \ + make "$$shellop" install $(7); \ + cd ..; \ + fi; \ rm -rf $(1) diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 4052ed4..9f434bc 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -763,8 +763,8 @@ $(ibidir)/libgit2-$(libgit2-version): $(ibidir)/cmake-$(cmake-version) -DUSE_SSH=OFF -DBUILD_CLAR=OFF \ -DTHREADSAFE=ON -DUSE_ICONV=OFF ) if [ x$(on_mac_os) = xyes ]; then - install_name_tool -id $(ildir)/libgit2.28.dylib \ - $(ildir)/libgit2.28.dylib + install_name_tool -id $(ildir)/libgit2.1.0.dylib \ + $(ildir)/libgit2.1.0.dylib fi echo "Libgit2 $(libgit2-version)" > $@ @@ -785,8 +785,8 @@ $(ibidir)/wcslib-$(wcslib-version): $(ibidir)/cfitsio-$(cfitsio-version) --with-cfitsioinc=$(idir)/include \ --without-pgplot $$fortranopt) if [ x$(on_mac_os) = xyes ]; then - install_name_tool -id $(ildir)/libwcs.6.4.dylib \ - $(ildir)/libwcs.6.4.dylib + install_name_tool -id $(ildir)/libwcs.7.3.dylib \ + $(ildir)/libwcs.7.3.dylib fi echo "WCSLIB $(wcslib-version)" > $@ @@ -853,7 +853,7 @@ $(ibidir)/automake-$(automake-version): $(ibidir)/autoconf-$(autoconf-version) echo "GNU Automake $(automake-version)" > $@ $(ibidir)/bison-$(bison-version): $(ibidir)/help2man-$(help2man-version) - tarball=bison-$(bison-version).tar.xz + tarball=bison-$(bison-version).tar.lz $(call import-source, $(bison-url), $(bison-checksum)) $(call gbuild, bison-$(bison-version), static, ,V=1 -j$(numthreads)) echo "GNU Bison $(bison-version)" > $@ @@ -916,6 +916,7 @@ $(ibidir)/flex-$(flex-version): $(ibidir)/bison-$(bison-version) $(ibidir)/gdb-$(gdb-version): $(ibidir)/python-$(python-version) tarball=gdb-$(gdb-version).tar.gz + export configure_in_different_directory=1; $(call import-source, $(gdb-url), $(gdb-checksum)) $(call gbuild, gdb-$(gdb-version),,,V=1 -j$(numthreads)) echo "GNU Project Debugger (GDB) $(gdb-version)" > $@ @@ -1096,10 +1097,17 @@ $(ibidir)/minizip-$(minizip-version): $(ibidir)/automake-$(automake-version) rm -rf $$unpackdir echo "Minizip $(minizip-version)" > $@ +# The Astromatic software packages (including missfits, sextractor, swarp +# and others) need the '-fcommon' flag to compile properly on GCC 10 and +# after. Previous to GCC 10, it was the default, but from GCC 10, the +# default is '-fno-common'. This is known by the author (as SExtractor +# issue 12: https://github.com/astromatic/sextractor/issues/12) and will +# hopefully be fixed in the future. $(ibidir)/missfits-$(missfits-version): tarball=missfits-$(missfits-version).tar.gz $(call import-source, $(missfits-url), $(missfits-checksum)) - $(call gbuild, missfits-$(missfits-version), static) + $(call gbuild, missfits-$(missfits-version), static, \ + CFLAGS="-fcommon") cp $(dtexdir)/missfits.tex $(ictdir)/ echo "MissFITS $(missfits-version) \citep{missfits}" > $@ @@ -1163,11 +1171,25 @@ $(ibidir)/R-$(R-version): \ $(ibidir)/libpaper-$(libpaper-version) tarball=R-$(R-version).tar.gz $(call import-source, $(R-url), $(R-checksum)) + cd $(ddir) + tar xf $(tdir)/$$tarball + cd R-$(R-version) + # We need to manually remove the lines with '~autodetect~', they + # cause the configure script to crash in version 4.0.2. They are + # used in relation to Java, and we don't use Java anyway. + sed -i -e '/\~autodetect\~/ s/^/#/g' configure export R_SHELL=$(SHELL) - $(call gbuild, R-$(R-version), static, \ - --without-x --with-readline \ - --disable-openmp, -j$(numthreads)) + ./configure --prefix=$(idir) \ + --without-x \ + --with-pcre1 \ + --disable-java \ + --with-readline \ + --disable-openmp + make -j$(numthreads) + make install + cd .. + rm -rf R-$(R-version) echo "R $(R-version)" > $@ # SCAMP documentation says ATLAS is a mandatory prerequisite for using @@ -1181,7 +1203,10 @@ $(ibidir)/scamp-$(scamp-version): \ $(ibidir)/cdsclient-$(cdsclient-version) tarball=scamp-$(scamp-version).tar.lz $(call import-source, $(scamp-url), $(scamp-checksum)) + + # See comment above 'missfits' for '-fcommon'. $(call gbuild, scamp-$(scamp-version), static, \ + CFLAGS="-fcommon" \ --enable-threads \ --enable-openblas \ --enable-plplot=no \ @@ -1209,13 +1234,20 @@ $(ibidir)/scons-$(scons-version): $(ibidir)/python-$(python-version) # libraries. But we can override this issue since we have Openblas # installed, it is just necessary to explicity tell sextractor to use it in # the configuration step. +# +# The '-fcommon' is a necessary C compilation flag for GCC 10 and above. It +# is necessary for astromatic libraries, otherwise their build will crash. $(ibidir)/sextractor-$(sextractor-version): \ $(ibidir)/fftw-$(fftw-version) \ $(ibidir)/openblas-$(openblas-version) tarball=sextractor-$(sextractor-version).tar.lz $(call import-source, $(sextractor-url), $(sextractor-checksum)) + + # See comment above 'missfits' for '-fcommon'. $(call gbuild, sextractor-$(sextractor-version), static, \ - --enable-threads --enable-openblas \ + CFLAGS="-fcommon" \ + --enable-threads \ + --enable-openblas \ --with-openblas-libdir=$(ildir) \ --with-openblas-incdir=$(idir)/include) ln -fs $(ibdir)/sex $(ibdir)/sextractor @@ -1225,7 +1257,10 @@ $(ibidir)/sextractor-$(sextractor-version): \ $(ibidir)/swarp-$(swarp-version): $(ibidir)/fftw-$(fftw-version) tarball=swarp-$(swarp-version).tar.gz $(call import-source, $(swarp-url), $(swarp-checksum)) + + # See comment above 'missfits' for '-fcommon'. $(call gbuild, swarp-$(swarp-version), static, \ + CFLAGS="-fcommon" \ --enable-threads) cp $(dtexdir)/swarp.tex $(ictdir)/ echo "SWarp $(swarp-version) \citep{swarp}" > $@ diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 2fede35..d39208f 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -431,6 +431,7 @@ $(ipydir)/mpmath-$(mpmath-version): $(ipydir)/setuptools-$(setuptools-version) $(ipydir)/numpy-$(numpy-version): \ $(ibidir)/unzip-$(unzip-version) \ + $(ipydir)/cython-$(cython-version) \ $(ibidir)/openblas-$(openblas-version) \ $(ipydir)/setuptools-$(setuptools-version) tarball=numpy-$(numpy-version).zip @@ -528,7 +529,9 @@ $(ipydir)/requests-$(requests-version): $(ipydir)/idna-$(idna-version) \ $(call pybuild, tar xf, requests-$(requests-version), , \ Requests $(requests-version)) -$(ipydir)/scipy-$(scipy-version): $(ipydir)/numpy-$(numpy-version) +$(ipydir)/scipy-$(scipy-version): \ + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/pybind11-$(pybind11-version) tarball=scipy-$(scipy-version).tar.gz $(call import-source, $(scipy-url), $(scipy-checksum)) if [ x$(on_mac_os) = xyes ]; then -- cgit v1.2.1 From 09840874c3af449032a07cac56e23a78d7e3a666 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sat, 8 Aug 2020 22:48:53 +0100 Subject: Software tarballs saved as symlinks if already in filesystem Until now, if the software source tarballs already existed on the system they would be copied inside the project. However, the software source tarballs are sometimes/mostly larger than their actual product and can consume significant space (~375 MB in the core branch!). With this commit, when the software are present on the system, their symbolic link will be placed in 'BDIR/software/tarballs', not a full copy. Also, because the tarballs in software tarball directory may themselves be links, we use 'realpath' to find the final place of the actual file and link to that location. Therefore if 'realpath' can't be found (prior to installing Coreutils in Maneage), we will copy the tarballs from the given software tarball directory. After Maneage has installed Coreutils, the project's own 'realpath' will be used. Of course, if the software are downloaded, their full downloaded copy will be kept in 'BDIR/software/tarballs', nothing has changed in the downloading scenario. --- reproduce/software/make/build-rules.mk | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/build-rules.mk b/reproduce/software/make/build-rules.mk index 86370ab..d50c301 100644 --- a/reproduce/software/make/build-rules.mk +++ b/reproduce/software/make/build-rules.mk @@ -44,7 +44,12 @@ import-source = final=$(tdir)/$$tarball; \ unchecked="$$final.unchecked"; \ rm -f "$$unchecked"; \ if [ -f $(DEPENDENCIES-DIR)/$$tarball ]; then \ - cp $(DEPENDENCIES-DIR)/$$tarball "$$unchecked"; \ + if type realpath > /dev/null 2> /dev/null; then \ + ln -sf "$$(realpath $(DEPENDENCIES-DIR)/$$tarball)" \ + "$$unchecked"; \ + else \ + cp $(DEPENDENCIES-DIR)/$$tarball "$$unchecked"; \ + fi; \ else \ if [ x"$$url" = x ]; then \ bservers="$(backupservers)"; \ @@ -70,7 +75,7 @@ import-source = final=$(tdir)/$$tarball; \ if [ x"$$checksum" = x"$$exp_checksum" ]; then \ mv "$$unchecked" "$$final"; \ else \ - echo "ERROR: Non-matching checksum for '$$tarball'."; \ + echo "ERROR: Non-matching checksum: $$tarball"; \ echo "Checksum should be: $$exp_checksum"; \ echo "Checksum is: $$checksum"; \ exit 1; \ -- cgit v1.2.1