From 75571a551f794223bea2995dc7775a49abd63654 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 20 Mar 2019 17:41:50 +0000 Subject: Including ATLAS in the pipeline, not yet complete An initial installation of atlas is now included in the pipeline, but we are still trying to make it compile and build smoothly. In the process, we found that GCC also needs some modifications (for example rpath issues). --- reproduce/src/make/dependencies-basic.mk | 33 +++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) (limited to 'reproduce/src/make/dependencies-basic.mk') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 5965f00..a5cda52 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -744,6 +744,12 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ # On a macOS, we (currently!) won't build GCC because of some # errors we are still trying to find. So, we'll just make a # symbolic link to the host's executables. + # + # GCC builds is own libraries in '$(idir)/lib64'. But all other + # libraries are in '$(idir)/lib'. Since this pipeline is only for a + # single architecture, we can trick GCC into building its libraries + # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic + # link to '$(idir)/lib'. if [ "x$(on_mac_os)" = xyes ]; then \ $(call makelink,gfortran); \ $(call makelink,gcc); \ @@ -754,12 +760,14 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \ rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\ \ + ln -fs $(ildir) $(idir)/lib64; \ + \ cd $(ddir); \ rm -rf gcc-build gcc-$(gcc-version); \ - tar xf $< && \ - mkdir $(ddir)/gcc-build && \ - cd $(ddir)/gcc-build && \ - ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ + tar xf $< \ + && mkdir $(ddir)/gcc-build \ + && cd $(ddir)/gcc-build \ + && ../gcc-$(gcc-version)/configure SHELL=$(ibdir)/bash \ --prefix=$(idir) \ --with-mpc=$(idir) \ --with-mpfr=$(idir) \ @@ -777,9 +785,16 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ --enable-default-pie \ --enable-default-ssp \ --enable-cet=auto \ - --enable-decimal-float && \ - make SHELL=$(ibdir)/bash -j$$(nproc) && \ - make SHELL=$(ibdir)/bash install && \ - cd .. && \ - rm -rf gcc-build gcc-$(gcc-version); \ + --enable-decimal-float \ + && make SHELL=$(ibdir)/bash -j$$(nproc) \ + && make SHELL=$(ibdir)/bash install \ + && cd .. \ + && rm -rf gcc-build gcc-$(gcc-version) \ + \ + && for f in $$(find $(idir)/libexec/gcc); do \ + if ldd $$f &> /dev/null; then \ + patchelf --set-rpath $(ildir) $$f; \ + fi; \ + done; \ fi + -- cgit v1.2.1 From 3e583296cb8ed2784f6cb71b53cbe3be4931fa46 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 21 Mar 2019 15:39:06 +0000 Subject: ATLAS and Scipy working on GNU/Linux Numpy needs ATLAS as shared libraries. So we also need to build Python with shared libraries. We also need to define site.cfg for numpy and scipy so we define a master template: `reproduce/config/pipeline/dependency-numpy-scipy.cfg' Also `Openssl' did not have rpath so we added with this commit. --- reproduce/src/make/dependencies-basic.mk | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'reproduce/src/make/dependencies-basic.mk') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index a5cda52..853f6d7 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -566,8 +566,15 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ --with-zlib-lib=$(ildir) \ --with-zlib-include=$(idir)/include, , , \ ./config ) && \ - cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem && \ - echo "OpenSSL is built and ready" > $@ + cp $(tdir)/cert.pem $(idir)/etc/ssl/cert.pem; \ + if [ $$? = 0 ]; then \ + if [ x$(on_mac_os) = xyes ]; then \ + echo "No need to fix rpath in libssl"; \ + else \ + patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \ + fi; \ + echo "OpenSSL is built and ready" > $@; \ + fi # GNU Wget # -- cgit v1.2.1 From 8748d3e93fbe60511912fbe6ac424a6d6352de4f Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 22 Mar 2019 17:58:45 +0000 Subject: On Mac systems, using a copy of GCC, not a link Until now we were using a symbolic link to replace GCC, but Make doesn't treat symbolic links like files. So it would rebuild the links every time. With this commit, only for GCC on Mac systems, we are actually copying the host's GCC executable to avoid this problem. Also, a wrong comment for cURL was removed. --- reproduce/src/make/dependencies-basic.mk | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'reproduce/src/make/dependencies-basic.mk') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 4d39141..76bfa16 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -206,8 +206,9 @@ makelink = origpath="$$PATH"; \ export PATH=$$(echo $(syspath) | tr : '\n' | grep -v ccache \ | paste -s -d:); \ a=$$(which $(1) 2> /dev/null); \ - if [ -f $(ibdir)/$(1) ]; then rm $(ibdir)/$(1); fi; \ - if [ x$$a != x ]; then ln -s $$a $(ibdir)/$(1); fi; \ + if [ -e $(ibdir)/$(1) ]; then rm $(ibdir)/$(1); fi; \ + if [ x"$(2)" = xcopy ]; then c=cp; else c="ln -s"; fi; \ + if [ x$$a != x ]; then $$c $$a $(ibdir)/$(1); fi; \ export PATH="$$origpath" $(ibdir) $(ildir):; mkdir $@ $(ibdir)/low-level-links: | $(ibdir) $(ildir) @@ -768,7 +769,7 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ if [ "x$(on_mac_os)" = xyes ]; then \ $(call makelink,gfortran); \ $(call makelink,g++); \ - $(call makelink,gcc); \ + $(call makelink,gcc,copy); \ else \ \ rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\ -- cgit v1.2.1 From 7d1d6f88fb28630cc581ea41c487b2b0c0da269f Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Mon, 25 Mar 2019 14:45:47 +0000 Subject: Corrected makelink command to avoid paste command We just noticed that recently the `paste' command on macOS doesn't work with a pipe. So we are now simply using the `tr' command in reverse to re-create the PATH (to find where to link to). --- reproduce/src/make/dependencies-basic.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'reproduce/src/make/dependencies-basic.mk') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 76bfa16..b51f35d 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -202,9 +202,9 @@ $(tarballs): $(tdir)/%: # is very annoying and can cause many complications. We thus remove any # part of PATH of that has `ccache' in it before making symbolic links to # the programs we are not building ourselves. -makelink = origpath="$$PATH"; \ +makelink = origpath="$$PATH"; \ export PATH=$$(echo $(syspath) | tr : '\n' | grep -v ccache \ - | paste -s -d:); \ + | tr '\n' :); \ a=$$(which $(1) 2> /dev/null); \ if [ -e $(ibdir)/$(1) ]; then rm $(ibdir)/$(1); fi; \ if [ x"$(2)" = xcopy ]; then c=cp; else c="ln -s"; fi; \ -- cgit v1.2.1 From 12648438cf9f75738cef045fd02f4093eab7990d Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 27 Mar 2019 17:16:21 +0000 Subject: GCC is now built on a Mac, not yet ATLAS Until now, we were simply using the host's GCC for Mac systems. But we found that except for a single step (to fixing `rpath'), it works on Mac!!! So, GCC is now part of the Mac build as well. However, we are still having some problems in building ATLAS on Mac. It works on GNU/Linux, but not in Mac. So for the time being (just temporarily), we are avoiding ATLAS (and thus Scipy) on Mac systems. We just filed an issue on the ATLAS discussion list to hopefully fix the problem soon. --- reproduce/src/make/dependencies-basic.mk | 37 ++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 16 deletions(-) (limited to 'reproduce/src/make/dependencies-basic.mk') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index b51f35d..69d2d84 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -740,13 +740,13 @@ $(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz # # We are currently having problems installing GCC on macOS, so for the time # being, if the pipeline is being run on a macOS, we'll just set a link. -ifeq ($(on_mac_os),yes) -gcc-prerequisites = -else +#ifeq ($(on_mac_os),yes) +#gcc-prerequisites = +#else gcc-prerequisites = $(tdir)/gcc-$(gcc-version).tar.xz \ $(ilidir)/isl \ $(ilidir)/mpc -endif +#endif $(ibdir)/gcc: $(gcc-prerequisites) \ $(ibdir)/ls \ $(ibdir)/sed \ @@ -766,12 +766,15 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ # single architecture, we can trick GCC into building its libraries # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic # link to '$(idir)/lib'. - if [ "x$(on_mac_os)" = xyes ]; then \ - $(call makelink,gfortran); \ - $(call makelink,g++); \ - $(call makelink,gcc,copy); \ - else \ - \ + +# SO FAR IT SEEMS TO BE WORKING ON MAC, BUT MORE TESTS ARE NEEDED TO TOTALLY +# REMOVE THE STEP WHERE WE JUST USE THE HOST'S GCC. +# if [ "x$(on_mac_os)" = xyesno ]; then \ +# $(call makelink,gfortran); \ +# $(call makelink,g++); \ +# $(call makelink,gcc,copy); \ +# else \ + rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\ rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \ rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\ @@ -807,10 +810,12 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ && cd .. \ && rm -rf gcc-build gcc-$(gcc-version) \ \ - && for f in $$(find $(idir)/libexec/gcc); do \ - if ldd $$f &> /dev/null; then \ - patchelf --set-rpath $(ildir) $$f; \ - fi; \ - done; \ - fi + && if [ "x$(on_mac_os)" != xyes ]; then \ + for f in $$(find $(idir)/libexec/gcc); do \ + if ldd $$f &> /dev/null; then \ + patchelf --set-rpath $(ildir) $$f; \ + fi; \ + done; \ + fi; \ +# fi -- cgit v1.2.1 From ef19dbeb3aa7131754e03ce6d3ccd66db9fd81fd Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Thu, 4 Apr 2019 12:30:33 +0100 Subject: Using host GCC on Mac, defining compilers in HDF5 library We wer not able to build `gcc' on Mac, so we are using links to the host compilers. In this commit we also found that on Mac the HDF5 library needs an explicit definition of the compilers. --- reproduce/src/make/dependencies-basic.mk | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'reproduce/src/make/dependencies-basic.mk') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 69d2d84..74a04ed 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -767,14 +767,15 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic # link to '$(idir)/lib'. -# SO FAR IT SEEMS TO BE WORKING ON MAC, BUT MORE TESTS ARE NEEDED TO TOTALLY -# REMOVE THE STEP WHERE WE JUST USE THE HOST'S GCC. -# if [ "x$(on_mac_os)" = xyesno ]; then \ -# $(call makelink,gfortran); \ -# $(call makelink,g++); \ -# $(call makelink,gcc,copy); \ -# else \ - +# SO FAR WE HAVEN'T BEEN ABLE TO GET A CONSISTENT BUILD OF GCC ON MAC +# (SOMETIMES IT CRASHES IN libiberty with g++) AND SOMETIMES IT FINISHES, +# SO, MORE TESTS ARE NEEDED ON MAC AND WE'LL USE THE HOST'S COMPILER UNTIL +# THEN. + if [ "x$(on_mac_os)" = xyesno ]; then \ + $(call makelink,gfortran); \ + $(call makelink,g++); \ + $(call makelink,gcc,copy); \ + else \ rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\ rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \ rm -rf $(ildir)/libgfortran* $(ildir)/libstdc* rm $(idir)/x86_64*;\ @@ -817,5 +818,5 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ fi; \ done; \ fi; \ -# fi + fi -- cgit v1.2.1