From aee6d612073ce9be8aef4183c5ac7688ed4733e8 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 1 Jul 2020 13:19:23 +0100 Subject: Minor typo corrected in referencing Libidn Until this commit, once Libidn was installed, insted of its own name and version, the name and version of Libjpeg were saved (in the target if Libidn). This robably come from a copy/paste of the rule. With this commit, this minor bug has been corrected. I also added my name as an author of `reproduce/software/make/xorg.mk' Makefile since I added some code there. --- reproduce/software/make/high-level.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index cf104e6..6e14b2d 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -555,7 +555,7 @@ $(ibidir)/libidn-$(libidn-version): $(call import-source, $(libidn-url), $(libidn-checksum)) $(call gbuild, libidn-$(libidn-version), static, \ --disable-doc, -j$(numthreads) V=1) - echo "Libjpeg $(libjpeg-version)" > $@ + echo "Libidn $(libidn-version)" > $@ $(ibidir)/libjpeg-$(libjpeg-version): tarball=jpegsrc.$(libjpeg-version).tar.gz -- cgit v1.2.1 From 0e4d4b357f4c2209aea8012847b1309fe8b33b13 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 5 Jul 2020 19:17:05 +0100 Subject: Configure script prefers clang for macOS systems In the previous commit (Commit 1bc00c9: Only using clang in macOS systems that also have GCC) we set the used C compiler for high-level programs to be 'clang' on macOS systems. But I forgot to do the same kind of change in the configure script (to prefer 'clang' when we are testing for a C compiler on the host). With this commit, the compiler checking phases of the configure script have been improved, so on macOS systems, we now first search for 'clang', then search for 'gcc'. While doing this, I also noticed that the 'rpath' checking command was done before we actually define 'instdir'!!! So in effect, the 'rpath' directory was being set to '/lib'! So with this commit, this test has been taken to after defining 'instdir'. --- reproduce/software/make/high-level.mk | 1 + 1 file changed, 1 insertion(+) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 6e14b2d..3ac3b49 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -52,6 +52,7 @@ ildir = $(BDIR)/software/installed/lib ibidir = $(BDIR)/software/installed/version-info/proglib # Basic directories (specific to this Makefile). +il64dir = $(BDIR)/software/installed/lib64 iidir = $(BDIR)/software/installed/include shsrcdir = $(shell pwd)/reproduce/software/shell dtexdir = $(shell pwd)/reproduce/software/bibtex -- cgit v1.2.1 From cbd4a41555c7d4aecaea03a4cc4298da8320f01c Mon Sep 17 00:00:00 2001 From: Boud Roukema Date: Fri, 31 Jul 2020 19:04:08 +0200 Subject: OpenMPI build with slurm compatibility Prior to this commit, compilation of OpenMPI used the default OpenMPI choices of deciding which libraries should be used in relating to a job scheduler [1] (such as Slurm [2]). Given that the user on a multi-user cluster has to accept the sysadmin's choice of a job scheduler, the question of whether to (1) link with OpenMPI's own libraries (and increase the reproducibility of the science project) or rather (2) link with the sysadmin managed libraries (more likely to be compatible with the host's job scheduler), is an open question of which the best strategy for reproducibility needs to be debated and studied. In this commit, strategy (1) is adopted. The options '--withpmix=internal' and '--with-hwloc=internal' are added to the configure command. The working assumption is that the Maneage version of OpenMPI is likely to be modern enough to be compatible with the native job scheduler such as Slurm. Compilation without any 'pmix' option gave a fail in at least one case; it appears that an external pmix library was sought by the configure script. As of OpenMPI 4.0.1, the internal libevent library is used by default, so there appears to be no option to force it to be chosen internally. This commit also includes the option '--without-verbs'. This option removes a library related to "infiniband", "verbs", "openib" and "BTL"; this library appears to be deprecated. See [3], [4] for discussion. Please add feedback and discussion to the Maneage task about openmpi linking strategies (1) (internal) and (2) (external) at Savannah [5]. [1] https://en.wikipedia.org/wiki/Job_scheduler#Batch_queuing_for_HPC_clusters [2] https://en.wikipedia.org/wiki/Slurm_Workload_Manager - To avoid a name clash, 'slurm-wlm' is the metapackage in Debian for the client commands, the compute node daemon, and the central node daemon. An unrelated package 'slurm' also exists. [3] https://www-lb.open-mpi.org/faq/?category=openfabrics#ofa-device-error [4] https://www-lb.open-mpi.org/faq/?category=building [5] https://savannah.nongnu.org/task/index.php?15737 --- reproduce/software/make/high-level.mk | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 3ac3b49..4052ed4 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -641,7 +641,10 @@ $(ibidir)/openblas-$(openblas-version): $(ibidir)/openmpi-$(openmpi-version): tarball=openmpi-$(openmpi-version).tar.gz $(call import-source, $(openmpi-url), $(openmpi-checksum)) - $(call gbuild, openmpi-$(openmpi-version), static, , \ + $(call gbuild, openmpi-$(openmpi-version), static, \ + --with-pmix=internal \ + --with-hwloc=internal \ + --without-verbs, \ -j$(numthreads) V=1) echo "Open MPI $(openmpi-version)" > $@ -- cgit v1.2.1 From b3b44798a3f472e5f5a6633de4a582d4c902b008 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Thu, 18 Jun 2020 18:01:33 +0100 Subject: IMPORTANT: New software versions (17 basic, 16 high-level and 7 Python) It was a long time that the Maneage software versions hadn't been updated. With this commit, the versions of all basic software were checked and 17 of that had newer versions were updated. Also, 16 high-level programs and libraries were updated as well as 7 Python modules. The full list is available below. Basic Software (affecting all projects) --------------------------------------- bash 5.0.11 -> 5.0.18 binutils 2.32 -> 2.35 coreutils 8.31 -> 8.32 curl 7.65.3 -> 7.71.1 file 5.36 -> 5.39 gawk 5.0.1 -> 5.1.0 gcc 9.2.0 -> 10.2.0 gettext 0.20.2 -> 0.21 git 2.26.2 -> 2.28.0 gmp 6.1.2 -> 6.2.0 grep 3.3 -> 3.4 libbsd 0.9.1 -> 0.10.0 ncurses 6.1 -> 6.2 perl 5.30.0 -> 5.32.0 sed 4.7 -> 4.8 texinfo 6.6 -> 6.7 xz 5.2.4 -> 5.2.5 Custom programs/libraries ------------------------- astrometrynet 0.77 -> 0.80 automake 0.16.1 -> 0.16.2 bison 3.6 -> 3.7 cfitsio 3.47 -> 3.48 cmake 3.17.0 -> 3.18.1 freetype 2.9 -> 2.10.2 gdb 8.3 -> 9.2 ghostscript 9.50 -> 9.52 gnuastro 0.11 -> 0.12 libgit2 0.28.2 -> 1.0.1 libidn 1.35 -> 1.36 openmpi 4.0.1 -> 4.0.4 R 3.6.2 -> 4.0.2 python 3.7.4 -> 3.8.5 wcslib 6.4 -> 7.3 yaml 0.2.2 -> 0.2.5 Python modules -------------- cython 0.29.6 -> 0.29.21 h5py 2.9.0 -> 2.10.0 matplotlib 3.1.1 -> 3.3.0 mpi4py 3.0.2 -> 3.0.3 numpy 1.17.2 -> 1.19.1 pybind11 2.4.3 -> 2.5.0 scipy 1.3.1 -> 1.5.2 --- reproduce/software/make/high-level.mk | 55 ++++++++++++++++++++++++++++------- 1 file changed, 45 insertions(+), 10 deletions(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 4052ed4..9f434bc 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -763,8 +763,8 @@ $(ibidir)/libgit2-$(libgit2-version): $(ibidir)/cmake-$(cmake-version) -DUSE_SSH=OFF -DBUILD_CLAR=OFF \ -DTHREADSAFE=ON -DUSE_ICONV=OFF ) if [ x$(on_mac_os) = xyes ]; then - install_name_tool -id $(ildir)/libgit2.28.dylib \ - $(ildir)/libgit2.28.dylib + install_name_tool -id $(ildir)/libgit2.1.0.dylib \ + $(ildir)/libgit2.1.0.dylib fi echo "Libgit2 $(libgit2-version)" > $@ @@ -785,8 +785,8 @@ $(ibidir)/wcslib-$(wcslib-version): $(ibidir)/cfitsio-$(cfitsio-version) --with-cfitsioinc=$(idir)/include \ --without-pgplot $$fortranopt) if [ x$(on_mac_os) = xyes ]; then - install_name_tool -id $(ildir)/libwcs.6.4.dylib \ - $(ildir)/libwcs.6.4.dylib + install_name_tool -id $(ildir)/libwcs.7.3.dylib \ + $(ildir)/libwcs.7.3.dylib fi echo "WCSLIB $(wcslib-version)" > $@ @@ -853,7 +853,7 @@ $(ibidir)/automake-$(automake-version): $(ibidir)/autoconf-$(autoconf-version) echo "GNU Automake $(automake-version)" > $@ $(ibidir)/bison-$(bison-version): $(ibidir)/help2man-$(help2man-version) - tarball=bison-$(bison-version).tar.xz + tarball=bison-$(bison-version).tar.lz $(call import-source, $(bison-url), $(bison-checksum)) $(call gbuild, bison-$(bison-version), static, ,V=1 -j$(numthreads)) echo "GNU Bison $(bison-version)" > $@ @@ -916,6 +916,7 @@ $(ibidir)/flex-$(flex-version): $(ibidir)/bison-$(bison-version) $(ibidir)/gdb-$(gdb-version): $(ibidir)/python-$(python-version) tarball=gdb-$(gdb-version).tar.gz + export configure_in_different_directory=1; $(call import-source, $(gdb-url), $(gdb-checksum)) $(call gbuild, gdb-$(gdb-version),,,V=1 -j$(numthreads)) echo "GNU Project Debugger (GDB) $(gdb-version)" > $@ @@ -1096,10 +1097,17 @@ $(ibidir)/minizip-$(minizip-version): $(ibidir)/automake-$(automake-version) rm -rf $$unpackdir echo "Minizip $(minizip-version)" > $@ +# The Astromatic software packages (including missfits, sextractor, swarp +# and others) need the '-fcommon' flag to compile properly on GCC 10 and +# after. Previous to GCC 10, it was the default, but from GCC 10, the +# default is '-fno-common'. This is known by the author (as SExtractor +# issue 12: https://github.com/astromatic/sextractor/issues/12) and will +# hopefully be fixed in the future. $(ibidir)/missfits-$(missfits-version): tarball=missfits-$(missfits-version).tar.gz $(call import-source, $(missfits-url), $(missfits-checksum)) - $(call gbuild, missfits-$(missfits-version), static) + $(call gbuild, missfits-$(missfits-version), static, \ + CFLAGS="-fcommon") cp $(dtexdir)/missfits.tex $(ictdir)/ echo "MissFITS $(missfits-version) \citep{missfits}" > $@ @@ -1163,11 +1171,25 @@ $(ibidir)/R-$(R-version): \ $(ibidir)/libpaper-$(libpaper-version) tarball=R-$(R-version).tar.gz $(call import-source, $(R-url), $(R-checksum)) + cd $(ddir) + tar xf $(tdir)/$$tarball + cd R-$(R-version) + # We need to manually remove the lines with '~autodetect~', they + # cause the configure script to crash in version 4.0.2. They are + # used in relation to Java, and we don't use Java anyway. + sed -i -e '/\~autodetect\~/ s/^/#/g' configure export R_SHELL=$(SHELL) - $(call gbuild, R-$(R-version), static, \ - --without-x --with-readline \ - --disable-openmp, -j$(numthreads)) + ./configure --prefix=$(idir) \ + --without-x \ + --with-pcre1 \ + --disable-java \ + --with-readline \ + --disable-openmp + make -j$(numthreads) + make install + cd .. + rm -rf R-$(R-version) echo "R $(R-version)" > $@ # SCAMP documentation says ATLAS is a mandatory prerequisite for using @@ -1181,7 +1203,10 @@ $(ibidir)/scamp-$(scamp-version): \ $(ibidir)/cdsclient-$(cdsclient-version) tarball=scamp-$(scamp-version).tar.lz $(call import-source, $(scamp-url), $(scamp-checksum)) + + # See comment above 'missfits' for '-fcommon'. $(call gbuild, scamp-$(scamp-version), static, \ + CFLAGS="-fcommon" \ --enable-threads \ --enable-openblas \ --enable-plplot=no \ @@ -1209,13 +1234,20 @@ $(ibidir)/scons-$(scons-version): $(ibidir)/python-$(python-version) # libraries. But we can override this issue since we have Openblas # installed, it is just necessary to explicity tell sextractor to use it in # the configuration step. +# +# The '-fcommon' is a necessary C compilation flag for GCC 10 and above. It +# is necessary for astromatic libraries, otherwise their build will crash. $(ibidir)/sextractor-$(sextractor-version): \ $(ibidir)/fftw-$(fftw-version) \ $(ibidir)/openblas-$(openblas-version) tarball=sextractor-$(sextractor-version).tar.lz $(call import-source, $(sextractor-url), $(sextractor-checksum)) + + # See comment above 'missfits' for '-fcommon'. $(call gbuild, sextractor-$(sextractor-version), static, \ - --enable-threads --enable-openblas \ + CFLAGS="-fcommon" \ + --enable-threads \ + --enable-openblas \ --with-openblas-libdir=$(ildir) \ --with-openblas-incdir=$(idir)/include) ln -fs $(ibdir)/sex $(ibdir)/sextractor @@ -1225,7 +1257,10 @@ $(ibidir)/sextractor-$(sextractor-version): \ $(ibidir)/swarp-$(swarp-version): $(ibidir)/fftw-$(fftw-version) tarball=swarp-$(swarp-version).tar.gz $(call import-source, $(swarp-url), $(swarp-checksum)) + + # See comment above 'missfits' for '-fcommon'. $(call gbuild, swarp-$(swarp-version), static, \ + CFLAGS="-fcommon" \ --enable-threads) cp $(dtexdir)/swarp.tex $(ictdir)/ echo "SWarp $(swarp-version) \citep{swarp}" > $@ -- cgit v1.2.1