From 4493acc390a92ba53c7d76766024e7d64c7b31ce Mon Sep 17 00:00:00 2001 From: Boud Roukema Date: Fri, 15 May 2020 04:42:58 +0200 Subject: New software: Valgrind and Patch With this commit, Maneage now includes instructions to build the memory tracing tool Valgrind and the program 'patch' (to apply corrections/patches in text files and in particular the sources of programs). For this version of Valgrind, some patches were necessary for an interface with OpenMPI 2.x (which is the case now). Also note that this version of Valgrind's checks can fail with GCC 10.1.x (when using '--host-cc'), and the failures aren't due to internal problems but due to how the tests are designed (https://bugs.gentoo.org/707598). So currently if any of Valgrind's checks fail, Maneage still assumes that Valgrind was built and installed successfully. While testing on macOS, we noticed that it needs the macOS-specific 'mig' program which we can't build in Maneage. DESCRIPTION: The mig command invokes the Mach Interface Generator to generate Remote Procedure Call (RPC) code for client-server style Mach IPC from specification files. So a symbolic link to the system's 'mig' is now added to the project's programs on macOS systems. This commit's build of Patch and Valgrind has been tested on two GNU/Linux distributions (Debian and ArchLinux) as well as macOS. Work on this commit started by Boud Roukema, but also involved tests and corrections by Mohammad Akhlaghi and Raul Infante-Sainz. --- .file-metadata | Bin 8367 -> 9121 bytes reproduce/software/config/checksums.conf | 2 ++ reproduce/software/config/versions.conf | 7 +++- reproduce/software/make/basic.mk | 1 + reproduce/software/make/high-level.mk | 36 ++++++++++++++++++++ .../patches/valgrind-3.15.0-mpi-fix1.patch | 37 +++++++++++++++++++++ .../patches/valgrind-3.15.0-mpi-fix2.patch | 23 +++++++++++++ 7 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 reproduce/software/patches/valgrind-3.15.0-mpi-fix1.patch create mode 100644 reproduce/software/patches/valgrind-3.15.0-mpi-fix2.patch diff --git a/.file-metadata b/.file-metadata index 1b9ba37..3a4122f 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/software/config/checksums.conf b/reproduce/software/config/checksums.conf index f19d831..643be9c 100644 --- a/reproduce/software/config/checksums.conf +++ b/reproduce/software/config/checksums.conf @@ -53,6 +53,7 @@ sed-checksum = e0be5db4cdf8226b34aaa9071bc5ae0eafde1c52227cee3512eea7fe2520d6c5c tar-checksum = 4be18afeac54aec4af074cf2358cfade5aaebe2041c5075c5764a81114df4d002e90b28f4444bd1430783e7d6bed82abd0440ef5cb244695f2e56a9a41b42fbc texinfo-checksum = 96e0764d0808152d3662e65c3287fb0f86ed918912cdc036380637dbadaacd6a489b516543c07b08105686575e8d495a945f73e23ff0909d5a0f12026e4131e0 unzip-checksum = 0694e403ebc57b37218e00ec1a406cae5cc9c5b52b6798e0d4590840b6cdbf9ddc0d9471f67af783e960f8fa2e620394d51384257dca23d06bcd90224a80ce5d +valgrind-checksum = 5695d1355226fb63b0c80809ed43bb077b6eed4d427792d9d7ed944c38b557a84fe3c783517b921e32f161228e10e4625bea0550faa4685872bb4454450cfa7f wget-checksum = 95fb064f0d79b0a3178a83322f58a85a3a036fb300ed759eb67a538f0bbacdd552f6cbeb60d63b4f0113e8467d923a5ce7ac5570b7a4ce1733b3dfd559bb33b2 which-checksum = d2f04a5c5291f2d7d1226982da7cf999d36cfe24d3f7bda145508efcfb359511251d3c68b860c0ddcedd66b15a0587b648a35ab6d1f173707565305c506dfc61 xz-checksum = e5bf6eb88365d2dbdc774db49261fb9fae0544ed297891fc20f1ed223f4072cb0357cbd98146ac35b6d29410a12b6739bbd111cd57d4a225bef255ed46988578 @@ -110,6 +111,7 @@ netpbm-checksum = 064720f8a9d0a502488e1af4daecdbf3936910996507ca6f311073a0ad8423 openblas-checksum = 91b3074eb922453bf843158b4281cde65db9e8bbdd7590e75e9e6cdcb486157f7973f2936f327bb3eb4f1702ce0ba51ae6729d8d4baf2d986c50771e8f696df0 openmpi-checksum = 760716974cb6b25ad820184622e1ee7926bc6fda87db6b574f76792bc1ca99522e52195866c14b7cb2df5a4981efdaf9f71d2c5533cc0e8e45c2c4b3b74cbacc openssh-checksum = e280fa2d56f550efd37c5d2477670326261aa8b94d991f9eb17aad90e0c6c9c939efa90fe87d33260d0f709485cb05c379f0fd1bd44fc0d5190298b6398c9982 +patch-checksum = 75d4e1544484da12185418cd4a1571994398140a91ac606fa08dd067004187dad77d1413f0eb3319b3fe4df076714615c98b29df06af052bb65960fa8b0c86bf pixman-checksum = 1b0205dbe9d9185c68813ce577a889f3c83e83fbd9955c3a72d411c3b476e6be93fc246b5b6ef4ee17e2bb8eb6fb5559e01dff7feb6a6c4c6314f980e960d690 python-checksum = c25a72ad792f7c1b4c2f79faebbe9608d04b04b2fe58ab804cb4732cdaa75ea93d175f5e52b38e91cb6ae0559ea6b645d802c8b6a869584e8bb9b5018367ce3d R-checksum = 17513e9f4dd27c61c11f7aa45227aeeeefb375bf5d4e193b471724f379a1b2da33e127cbe91aa175cbbbb048b892047e2f610280585c8159242a6c94790b07f9 diff --git a/reproduce/software/config/versions.conf b/reproduce/software/config/versions.conf index 7df4539..e9aad2e 100644 --- a/reproduce/software/config/versions.conf +++ b/reproduce/software/config/versions.conf @@ -104,6 +104,7 @@ missfits-version = 2.8.0 openblas-version = 0.3.5 openmpi-version = 4.0.1 openssh-version = 8.0p1 +patch-version = 2.7.6 pixman-version = 0.38.0 python-version = 3.7.4 R-version = 3.6.2 @@ -216,4 +217,8 @@ minizip-version = $(zlib-version) # From version 1.2 OpenSSL may not need a manual addition, as described in # its comments and `https://savannah.nongnu.org/bugs/?58263'. If it doesn't # cause problems, put it back in the list of "Basic/low-level" tools. -openssl-version = 1.1.1a \ No newline at end of file +openssl-version = 1.1.1a + +# Version 3.15.0 needs two patches, please check if they are necessary on +# any future release. +valgrind-version = 3.15.0 diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 9119930..8b216fe 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -311,6 +311,7 @@ $(ibidir)/low-level-links: | $(ibdir) $(ildir) $(call makelink,clang++) # Mac OS specific + $(call makelink,mig) $(call makelink,sysctl) $(call makelink,sw_vers) $(call makelink,dsymutil) diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 7e9c723..3ad4468 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -42,6 +42,7 @@ ibdir = $(BDIR)/software/installed/bin ildir = $(BDIR)/software/installed/lib iidir = $(BDIR)/software/installed/include dtexdir = $(shell pwd)/reproduce/software/bibtex +patchdir= $(shell pwd)/reproduce/software/patches itidir = $(BDIR)/software/installed/version-info/tex ictdir = $(BDIR)/software/installed/version-info/cite ipydir = $(BDIR)/software/installed/version-info/python @@ -188,6 +189,7 @@ tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ openblas-$(openblas-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ openssh-$(openssh-version).tar.gz \ + patch-$(patch-version).tar.gz \ pixman-$(pixman-version).tar.gz \ R-$(R-version).tar.gz \ scamp-$(scamp-version).tar.lz \ @@ -198,6 +200,7 @@ tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ rpcsvc-proto-$(rpcsvc-proto-version).tar.xz \ tides-$(tides-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ + valgrind-$(valgrind-version).tar.bz2 \ wcslib-$(wcslib-version).tar.bz2 \ xlsxio-$(xlsxio-version).tar.gz \ yaml-$(yaml-version).tar.gz \ @@ -294,6 +297,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = openssh ]; then c=$(openssh-checksum); w=https://artfiles.org/openbsd/OpenSSH/portable + elif [ $$n = patch ]; then c=$(patch-checksum); w=http://ftp.gnu.org/gnu/patch elif [ $$n = pixman ]; then c=$(pixman-checksum); w=https://www.cairographics.org/releases elif [ $$n = R ]; then c=$(R-checksum); majver=$$(echo $(R-version) | sed -e's/\./ /g' | awk '{print $$1}') @@ -309,6 +313,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = swig ]; then c=$(swig-checksum); w=https://sourceforge.net/projects/swig/files/swig/swig-$(swig-version) elif [ $$n = tides ]; then c=$(tides-checksum); w=http://akhlaghi.org/maneage-software elif [ $$n = tiff ]; then c=$(libtiff-checksum); w=https://download.osgeo.org/libtiff + elif [ $$n = valgrind ]; then c=$(valgrind-checksum); w=https://sourceware.org/pub/valgrind elif [ $$n = wcslib ]; then c=$(wcslib-checksum); w=ftp://ftp.atnf.csiro.au/pub/software/wcslib elif [ $$n = xlsxio ]; then mergenames=0 @@ -737,6 +742,33 @@ $(ibidir)/tides: $(tdir)/tides-$(tides-version).tar.gz && cp $(dtexdir)/tides.tex $(ictdir)/ \ && echo "TIDES $(tides-version) \citep{tides}" > $@ +$(ibidir)/valgrind: $(ibidir)/patch \ + $(ibidir)/autoconf \ + $(ibidir)/automake \ + $(tdir)/valgrind-$(valgrind-version).tar.bz2 + # For valgrind-3.15.0, see + # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=946329 for a + # report on an MPI-related compile bug and the two patches + # below. These two patches and `automake` should allow valgrind to + # compile with gcc-9.2.0. + cd $(ddir) \ + && tar -x -f $(word 1,$(filter $(tdir)/%,$^)) \ + && valgrinddir=valgrind-$(valgrind-version) \ + && cd $${valgrinddir} \ + && printf "valgrindir=$${valgrinddir} ; pwd = %s .\n" $$($(ibdir)/pwd) \ + && if [ "x$(valgrind-version)" = "x3.15.0" ]; then \ + patch --verbose -p1 < $(patchdir)/valgrind-3.15.0-mpi-fix1.patch; \ + patch --verbose -p1 < $(patchdir)/valgrind-3.15.0-mpi-fix2.patch; \ + fi \ + && autoreconf \ + && ./configure --prefix=$(idir) \ + && make -j$(numthreads) \ + && if ! make check -j$(numthreads); then \ + echo; echo "Valgrind's 'make check' failed!"; echo; \ + fi \ + && make install \ + && echo "Valgrind $(valgrind-version)" > $@ + $(ibidir)/yaml: $(tdir)/yaml-$(yaml-version).tar.gz $(call gbuild, yaml-$(yaml-version), static) \ && echo "LibYAML $(yaml-version)" > $@ @@ -1099,6 +1131,10 @@ $(ibidir)/netpbm: $(ibidir)/unzip \ && rm -rf $$unpackdir \ && echo "Netpbm $(netpbm-version)" > $@ +$(ibidir)/patch: $(tdir)/patch-$(patch-version).tar.gz + $(call gbuild, patch-$(patch-version), static, ,V=1) \ + && echo "GNU Patch $(patch-version)" > $@ + # R programming language $(ibidir)/R: $(ibidir)/libpng \ $(ibidir)/libjpeg \ diff --git a/reproduce/software/patches/valgrind-3.15.0-mpi-fix1.patch b/reproduce/software/patches/valgrind-3.15.0-mpi-fix1.patch new file mode 100644 index 0000000..94dcab5 --- /dev/null +++ b/reproduce/software/patches/valgrind-3.15.0-mpi-fix1.patch @@ -0,0 +1,37 @@ +--- + mpi/libmpiwrap.c | 12 +++++++++++- + 1 file changed, 11 insertions(+), 1 deletion(-) + Patch by Samuel Thibault: + https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=946329;msg=10 + and hacked further by Boud Roukema 2020-05-10. +--- a/mpi/libmpiwrap.c ++++ b/mpi/libmpiwrap.c +@@ -278,8 +278,12 @@ static void showTy ( FILE* f, MPI_Dataty + else if (ty == MPI_LONG_INT) fprintf(f,"LONG_INT"); + else if (ty == MPI_SHORT_INT) fprintf(f,"SHORT_INT"); + else if (ty == MPI_2INT) fprintf(f,"2INT"); ++# if defined(MPI_UB_ENABLED_IN_MPI1) + else if (ty == MPI_UB) fprintf(f,"UB"); ++# endif ++# if defined(MPI_LB_ENABLED_IN_MPI1) + else if (ty == MPI_LB) fprintf(f,"LB"); ++# endif + # if defined(MPI_WCHAR) + else if (ty == MPI_WCHAR) fprintf(f,"WCHAR"); + # endif +@@ -733,8 +737,14 @@ void walk_type ( void(*f)(void*,long), c + f(base + offsetof(Ty,loc), sizeof(int)); + return; + } +- if (ty == MPI_LB || ty == MPI_UB) ++#if defined(MPI_LB_ENABLED_IN_MPI1) ++ if (ty == MPI_LB) ++ return; /* have zero size, so nothing needs to be done */ ++#endif ++#if defined(MPI_UB_ENABLED_IN_MPI1) ++ if (ty == MPI_UB) + return; /* have zero size, so nothing needs to be done */ ++#endif + goto unhandled; + /*NOTREACHED*/ + } diff --git a/reproduce/software/patches/valgrind-3.15.0-mpi-fix2.patch b/reproduce/software/patches/valgrind-3.15.0-mpi-fix2.patch new file mode 100644 index 0000000..12b50a2 --- /dev/null +++ b/reproduce/software/patches/valgrind-3.15.0-mpi-fix2.patch @@ -0,0 +1,23 @@ +Index: valgrind-3.15.0/mpi/Makefile.am +=================================================================== + Patch by Samuel Thibault: + https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=946329;msg=22 +--- valgrind-3.15.0.orig/mpi/Makefile.am ++++ valgrind-3.15.0/mpi/Makefile.am +@@ -42,14 +42,14 @@ libmpiwrap_@VGCONF_ARCH_PRI@_@VGCONF_OS@ + libmpiwrap_@VGCONF_ARCH_PRI@_@VGCONF_OS@_so_CPPFLAGS = -I$(top_srcdir)/include + libmpiwrap_@VGCONF_ARCH_PRI@_@VGCONF_OS@_so_CFLAGS = \ + $(CFLAGS_MPI) $(MPI_FLAG_M3264_PRI) -Wno-deprecated-declarations +-libmpiwrap_@VGCONF_ARCH_PRI@_@VGCONF_OS@_so_LDFLAGS = $(LDFLAGS_MPI) ++libmpiwrap_@VGCONF_ARCH_PRI@_@VGCONF_OS@_so_LDADD = $(LDFLAGS_MPI) + endif + if BUILD_MPIWRAP_SEC + libmpiwrap_@VGCONF_ARCH_SEC@_@VGCONF_OS@_so_SOURCES = libmpiwrap.c + libmpiwrap_@VGCONF_ARCH_SEC@_@VGCONF_OS@_so_CPPFLAGS = -I$(top_srcdir)/include + libmpiwrap_@VGCONF_ARCH_SEC@_@VGCONF_OS@_so_CFLAGS = \ + $(CFLAGS_MPI) $(MPI_FLAG_M3264_SEC) -Wno-deprecated-declarations +-libmpiwrap_@VGCONF_ARCH_SEC@_@VGCONF_OS@_so_LDFLAGS = $(LDFLAGS_MPI) ++libmpiwrap_@VGCONF_ARCH_SEC@_@VGCONF_OS@_so_LDADD = $(LDFLAGS_MPI) + endif + + #---------------------------------------------------------------------------- -- cgit v1.2.1