From c2b36cc6b6c62b5f19745703c65df7cfa8b84cc4 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Sun, 5 Apr 2020 17:03:16 +0100 Subject: Building Minizip 1.x instead of Minizip 2.x Minizip is a dependency of XLSX I/O and until now, I was just using the most recent version I found (2.9.2), but XLSX I/O is written for the Minizip 1.x series, not 2.x. Somehow it didn't cause a crash on my computer!!! I think XLSX I/O's CMake is instructed to look into system directories by default when it doesn't find the directories in the given places. And because I had installed Minizip on my operating system, it did't complain. Upon trying the build on their systems, Yahya, Raul and Zahra reported a failure in the build of XLSX I/O which was due the to the problem above (we were installing the wrong version of Minizip!). With this commit, this has been fixed by installing the 1.x series of Minizip (whish is actually installed within zlib!). This commit was original done by Mohammad Akhlaghi. --- reproduce/software/make/high-level.mk | 77 ++++++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 15 deletions(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 274bcdf..43dfaa1 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -184,7 +184,6 @@ tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ libpng-$(libpng-version).tar.xz \ libtirpc-$(libtirpc-version).tar.bz2 \ libxml2-$(libxml2-version).tar.gz \ - minizip-$(minizip-version).tar.gz \ missfits-$(missfits-version).tar.gz \ netpbm-$(netpbm-version).tar.gz \ openblas-$(openblas-version).tar.gz \ @@ -203,6 +202,7 @@ tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ wcslib-$(wcslib-version).tar.bz2 \ xlsxio-$(xlsxio-version).tar.gz \ yaml-$(yaml-version).tar.gz \ + zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: | $(lockdir) @@ -282,10 +282,6 @@ $(tarballs): $(tdir)/%: | $(lockdir) w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = libtirpc ]; then c=$(libtirpc-checksum); w=https://downloads.sourceforge.net/libtirpc elif [ $$n = libxml ]; then c=$(libxml2-checksum); w=ftp://xmlsoft.org/libxml2 - elif [ $$n = minizip ]; then - mergenames=0 - c=$(minizip-checksum); - w=https://github.com/nmoinvaz/minizip/archive/$(minizip-version).tar.gz elif [ $$n = missfits ]; then c=$(missfits-checksum); w=https://www.astromatic.net/download/missfits elif [ $$n = netpbm ]; then c=$(netpbm-checksum); w=http://akhlaghi.org/reproduce-software elif [ $$n = openblas ]; then @@ -319,6 +315,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) c=$(xlsxio-checksum); w=https://github.com/brechtsanders/xlsxio/archive/$(xlsxio-version).tar.gz elif [ $$n = yaml ]; then c=$(yaml-checksum); w=pyyaml.org/download/libyaml + elif [ $$n = zlib ]; then c=$(zlib-checksum); w=https://zlib.net else echo; echo; echo; echo "'$$n' not recognized as a software tarball name to download." @@ -927,8 +924,8 @@ $(ibidir)/ghostscript: $(ibidir)/libpng \ # First we need to make sure some necessary X11 libraries that we # don't yet install in this template are present on the host # system, see https://savannah.nongnu.org/task/?15481 . - # Adding `-L/opt/X11/lib' to LDFLAGS is necessary for macOS systems - # because X11 libraries used to be installed there. + # Adding `-L/opt/X11/lib' to LDFLAGS is necessary for macOS systems + # because X11 libraries used to be installed there. echo; echo "Template: testing necessary X11 libraries for ghostscript" echo "---------------------------------------------------------" @@ -1036,10 +1033,43 @@ $(ibidir)/imfit: $(ibidir)/gsl \ fi \ && echo "Imfit $(imfit-version) \citep{imfit2015}" > $@ -$(ibidir)/minizip: $(ibidir)/cmake \ - | $(tdir)/minizip-$(minizip-version).tar.gz - $(call cbuild, minizip-$(minizip-version), static) \ - && echo "minizip $(minizip-version)" > $@ +# Minizip 1.x is actually distributed within zlib. It doesn't have its own +# independent tarball. So we need a custom build, which include the GNU +# Autotools (Autoconf and Automake). Note that Minizip 2.x isn't like this +# any more and has its own independent tarball, but currently the programs +# that depend on Minizip need Minizip 1.x. The instructions to build +# minizip were taken from ArchLinux. +# +# About deleting the final crypt.h file after installation, see +# https://bugzilla.redhat.com/show_bug.cgi?id=1424609 +$(ibidir)/minizip: $(ibidir)/automake \ + | $(tdir)/zlib-$(zlib-version).tar.gz + cd $(ddir) \ + && unpackdir=minizip-$(minizip-version) \ + && rm -rf $$unpackdir \ + && mkdir $$unpackdir \ + && if ! tar xf $(word 1,$(filter $(tdir)/%,$|)) \ + -C$$unpackdir --strip-components=1; then \ + echo; echo "Tar error"; exit 1; \ + fi \ + && cd $$unpackdir\ + && ./configure --prefix=$(idir) \ + && make \ + && cd contrib/minizip \ + && cp Makefile Makefile.orig \ + && cp ../README.contrib readme.txt \ + && autoreconf --install \ + && ./configure --prefix=$(idir) \ + && make \ + && cd ../../ \ + && make test \ + && cd contrib/minizip \ + && make -f Makefile.orig test \ + && make install \ + && rm $(iidir)/minizip/crypt.h \ + && cd ../../.. \ + && rm -rf $$unpackdir \ + && echo "Minizip $(minizip)" > $@ $(ibidir)/missfits: | $(tdir)/missfits-$(missfits-version).tar.gz $(call gbuild, missfits-$(missfits-version), static) \ @@ -1152,11 +1182,29 @@ $(ibidir)/swig: | $(tdir)/swig-$(swig-version).tar.gz $(call gbuild, swig-$(swig-version), static, --without-pcre) \ && echo "Swig $(swig-version)" > $@ -$(ibidir)/xlsxio: $(ibidir)/expat \ +$(ibidir)/xlsxio: $(ibidir)/cmake \ + $(ibidir)/expat \ $(ibidir)/minizip \ | $(tdir)/xlsxio-$(xlsxio-version).tar.gz - export LDFLAGS="-lbz2 -lbsd"; \ - $(call cbuild, xlsxio-$(xlsxio-version), static) \ + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + export CXX=clang++; \ + export LDFLAGS="-lbz2"; \ + else \ + export LDFLAGS="-lbz2 -lbsd"; \ + fi; \ + $(call cbuild, xlsxio-$(xlsxio-version), static, \ + -DMINIZIP_DIR:PATH=$(idir) \ + -DMINIZIP_LIBRARIES=$(idir) \ + -DMINIZIP_INCLUDE_DIRS=$(iidir)) \ + && if [ "x$(on_mac_os)" != xyes ]; then \ + echo "Adding RPATH to the XLSX I/O executables..."; \ + for f in $(ibdir)/xlsxio_* $(ildir)/libxlsxio_*.so; do \ + patchelf --set-rpath $(ildir) $$f; \ + done; \ + fi \ + && echo "Deleting XLSX I/O example files..." \ + && rm $(ibdir)/example_xlsxio_* \ && echo "XLSX I/O $(xlsxio-version)" > $@ @@ -1166,7 +1214,6 @@ $(ibidir)/xlsxio: $(ibidir)/expat \ - # Since we want to avoid complicating the PATH, we are putting a symbolic # link of all the TeX Live executables in $(ibdir). But symbolic links are # hard to track for Make (as a target). Also, TeX in general is optional -- cgit v1.2.1 From 7201b0f0daf484ee326dcd13f3c1348b3f96f626 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Sun, 5 Apr 2020 17:38:07 +0100 Subject: Including version number of Minizip in its installation target Until this commit, only the word `Minizip' was written into the Minizip installation target (without the version number of Minizip). With this commit, this minor bug has been fixed by using the appropiate Make variable: `$(minizip-version)'. --- reproduce/software/make/high-level.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 43dfaa1..916cb0a 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1069,7 +1069,7 @@ $(ibidir)/minizip: $(ibidir)/automake \ && rm $(iidir)/minizip/crypt.h \ && cd ../../.. \ && rm -rf $$unpackdir \ - && echo "Minizip $(minizip)" > $@ + && echo "Minizip $(minizip-version)" > $@ $(ibidir)/missfits: | $(tdir)/missfits-$(missfits-version).tar.gz $(call gbuild, missfits-$(missfits-version), static) \ -- cgit v1.2.1 From a4e2ec81d57eae3ec88c6f5af2f801f3b94bea24 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Sun, 5 Apr 2020 17:54:39 +0100 Subject: The build of M4 and XLSX I/O on macOS has been fixed Until now, the M4 that was built on macOS had internal problems (as discussed in #1): it would simply print `Abort trap: 6' in the output and abort. After looking at the build of Homebrew, I noticed that they apply a patch (correct one line) to fix this problem. To be able to apply that patch on macOS systems, I had fully open up the build recipe of M4 and atleast on the testing system, it was built successfully. Also, after successfully building M4, and thus Autoconf and thus Minizip, we were able to build XLSX I/O on a macOS and found out that the internal library's full address wasn't being put in the libraries and executables. With this commit, we now use macOS's `install_name_tool' to correct the positions of the two `libxlsxio_*' libraries in all its executables. This commit was originally written by Mohammad Akhlaghi --- reproduce/software/make/high-level.mk | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 916cb0a..603e710 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1069,7 +1069,7 @@ $(ibidir)/minizip: $(ibidir)/automake \ && rm $(iidir)/minizip/crypt.h \ && cd ../../.. \ && rm -rf $$unpackdir \ - && echo "Minizip $(minizip-version)" > $@ + && echo "Minizip $(minizip)" > $@ $(ibidir)/missfits: | $(tdir)/missfits-$(missfits-version).tar.gz $(call gbuild, missfits-$(missfits-version), static) \ @@ -1197,10 +1197,17 @@ $(ibidir)/xlsxio: $(ibidir)/cmake \ -DMINIZIP_DIR:PATH=$(idir) \ -DMINIZIP_LIBRARIES=$(idir) \ -DMINIZIP_INCLUDE_DIRS=$(iidir)) \ - && if [ "x$(on_mac_os)" != xyes ]; then \ - echo "Adding RPATH to the XLSX I/O executables..."; \ + && echo "Correcting internal linking of XLSX I/O executables..." \ + && if [ "x$(on_mac_os)" = xyes ]; then \ + for f in $(ibdir)/xlsxio_* $(ildir)/libxlsxio_*.dylib; do \ + install_name_tool -change libxlsxio_read.dylib \ + $(ildir)/libxlsxio_read.dylib $$f; \ + install_name_tool -change libxlsxio_write.dylib \ + $(ildir)/libxlsxio_write.dylib $$f; \ + done; \ + else \ for f in $(ibdir)/xlsxio_* $(ildir)/libxlsxio_*.so; do \ - patchelf --set-rpath $(ildir) $$f; \ + patchelf --set-rpath $(ildir) $$f; \ done; \ fi \ && echo "Deleting XLSX I/O example files..." \ -- cgit v1.2.1 From d0a51f78d4705c2496c42a02c49b0bc70e931cf7 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Mon, 6 Apr 2020 09:12:47 +0100 Subject: Astropy now depending on the Expat library to fix internal conflict Until now, Astropy was instructed to build its own internal copy of the Expat library. However, with the recent commits before, Maneage now includes an installation of Expat and Astropy can't keep the two (its internal version and the project's version) separate, so they conflict and don't let Astropy get built. With this commit, the problem is fixed by setting the Expat library as an explicit dependency of Astropy and asking Astropy to ignore its internal copy. While doing this, I recognized that it is much easier and elegant to add steps in various stages of the `pybuild' function through hooks instead of variables. So the fifth argument of the `pybuild' function was removed and now it actually checks if hooks are defined as functions and if so, they will be called. The `pyhook_after' function was also implemented in the installation of `pybind11' (which needed it, given that the 5th argument of `pybuild' was removed) and after doing a test-build, I noticed that two lines were not ending with a `\' in `boost' (a dependency of `pybind11'). Commit written originally by Mohammad Akghlaghi --- reproduce/software/make/high-level.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 603e710..7d2577a 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -520,8 +520,8 @@ $(ibidir)/boost: $(ibidir)/openmpi \ vstr=$$(echo $(boost-version) | sed -e's/\./_/g') rm -rf $(ddir)/boost_$$vstr topdir=$(pwd); cd $(ddir); - tar xf $(word 1,$(filter $(tdir)/%,$|)) - && cd boost_$$vstr + tar xf $(word 1,$(filter $(tdir)/%,$|)) \ + && cd boost_$$vstr \ && ./bootstrap.sh --prefix=$(idir) --with-libraries=all \ --with-python=python3 \ && echo "using mpi ;" > project-config.jam \ @@ -1069,7 +1069,7 @@ $(ibidir)/minizip: $(ibidir)/automake \ && rm $(iidir)/minizip/crypt.h \ && cd ../../.. \ && rm -rf $$unpackdir \ - && echo "Minizip $(minizip)" > $@ + && echo "Minizip $(minizip-version)" > $@ $(ibidir)/missfits: | $(tdir)/missfits-$(missfits-version).tar.gz $(call gbuild, missfits-$(missfits-version), static) \ -- cgit v1.2.1 From 7523206bf425829865179375120773cfa23716bf Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 13 Apr 2020 21:39:55 +0100 Subject: Configure (TeXLive): Year of distribution no longer in directory It is this time of year again: TeXLive has transitioned to its 2020 release and the year is imprinted into the installation directory of TeXLive. Until now, we have had to manually change this year and it caused complications and was very annoying. With this commit, the explicit year has been removed from TeXLive's installation and we now simply put a `maneage' instead of the year. I tried this on another system and it worked nicely. Until the time that we can fully install LaTeX packages from source tarballs, this is the best thing we could do for now. --- reproduce/software/make/high-level.mk | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 7d2577a..46b79ca 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1259,12 +1259,10 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/installation/texlive.co # don't want the configure script to fail if it can't run. if ./install-tl --profile=texlive.conf -repository $(tlmirror); then - # Put a symbolic link of the TeX Live executables in `ibdir'. The - # main problem is that the year and build system (for example - # `x86_64-linux') are also in the directory names, making it hard - # to be generic. We are using wildcards here, but only in this - # Makefile, not in any other. - ln -fs $(idir)/texlive/20*/bin/*/* $(ibdir)/ + # Put a symbolic link of the TeX Live executables in `ibdir' to + # avoid all the complexities of its sub-directories and additions + # to PATH. + ln -fs $(idir)/texlive/maneage/bin/*/* $(ibdir)/ # Register that the build was successful. echo "TeX Live is ready." > $@ -- cgit v1.2.1 From e0a57efde9206c0c7420e03e5fb2a354d568c3ad Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 13 Apr 2020 21:48:59 +0100 Subject: Configure (TeXLive): year correction also in high-level packages In the previous commit, I removed the year from the basic installation of TeXLive packages, but I forgot to correct this in the high-level TeXLive packages! This is corrected with this commit. --- reproduce/software/make/high-level.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 46b79ca..6e01354 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1322,7 +1322,7 @@ $(itidir)/texlive: reproduce/software/config/installation/texlive-packages.conf # Make a symbolic link of all the TeX Live executables in the bin # directory so we don't have to modify `PATH'. - ln -fs $(idir)/texlive/20*/bin/*/* $(ibdir)/ + ln -fs $(idir)/texlive/maneage/bin/*/* $(ibdir)/ # Get all the necessary versions. texlive=$$(pdflatex --version | awk 'NR==1' | sed 's/.*(\(.*\))/\1/' \ -- cgit v1.2.1 From 8eb0892e179b4970a9835d29b5f57dd912ca4464 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 17 Apr 2020 01:06:20 +0100 Subject: IMPORTANT: software config directly under reproduce/software/config Until now the software configuration parameters were defined under the `reproduce/software/config/installation/' directory. This was because the configuration parameters of analysis software (for example Gnuastro's configurations) were placed under there too. But this was terribly confusing, because the run-time options of programs falls under the "analysis" phase of the project. With this commit, the Gnuastro configuration files have been moved under the new `reproduce/analysis/config/gnuastro' directory and the software configuration files are directly under `reproduce/software/config'. A clean build was done with this change and it didn't crash, but it may cause crashes in derived projects, so after merging with Maneage, please re-configure your project to see if anything has been missed. Please let us know if there is a problem. --- reproduce/software/make/high-level.mk | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 6e01354..e62f02f 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -27,12 +27,12 @@ # Top level environment +include reproduce/software/config/LOCAL.conf include reproduce/software/make/build-rules.mk -include reproduce/software/config/installation/LOCAL.conf -include reproduce/software/config/installation/TARGETS.conf -include reproduce/software/config/installation/versions.conf -include reproduce/software/config/installation/checksums.conf -include reproduce/software/config/installation/texlive-packages.conf +include reproduce/software/config/TARGETS.conf +include reproduce/software/config/versions.conf +include reproduce/software/config/checksums.conf +include reproduce/software/config/texlive-packages.conf lockdir = $(BDIR)/locks tdir = $(BDIR)/software/tarballs @@ -1240,7 +1240,7 @@ $(ibidir)/xlsxio: $(ibidir)/cmake \ tlmirror=http://mirrors.rit.edu/CTAN/systems/texlive/tlnet # The core TeX Live system. -$(itidir)/texlive-ready-tlmgr: reproduce/software/config/installation/texlive.conf \ +$(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf \ | $(tdir)/install-tl-unx.tar.gz # Unpack, enter the directory, and install based on the given @@ -1251,7 +1251,7 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/installation/texlive.co tar xf $(tdir)/install-tl-unx.tar.gz cd install-tl-* sed -e's|@installdir[@]|$(idir)|g' \ - $$topdir/reproduce/software/config/installation/texlive.conf \ + $$topdir/reproduce/software/config/texlive.conf \ > texlive.conf # TeX Live's installation may fail due to any reason. But TeX Live @@ -1292,7 +1292,7 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/installation/texlive.co #else #forbiber = $(ibidir)/libnsl #endif -$(itidir)/texlive: reproduce/software/config/installation/texlive-packages.conf \ +$(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ $(itidir)/texlive-ready-tlmgr \ $(forbiber) -- cgit v1.2.1