From 22d915e020643797eab6f3f3017752cb1f6b0178 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 6 May 2020 00:45:22 +0100 Subject: Software are rebuilt automatically with change of version Until now, when you changed the version of a software in an already-built system, its tarball would be downloaded, but it wouldn't actually build. The only way would be to force the build by deleting the main target of that file (under `.local/version-info/TYPE/PROGRAM'). This was because the tarballs were an order-only prerequisite which was implemented some time ago based on some theoretical argument that if the tarball dates changes, the software should not be rebuilt (because we check the checksum). However, the problems this causes are more than those it solves: Users may forget to delete the main target of the program and mistakenly think that they are using the new version. The fact that all the numbers going into the paper also contain this number further hides this. With this commit, tarballs are no longer order-only and any time a version of a software is updated, it will be automatically built and not cause confusion and manual intervention by the users. As a result of this change, I also had to correct the way we find the tarball from the list of prerequisites. --- reproduce/software/make/high-level.mk | 154 +++++++++++++++++----------------- 1 file changed, 78 insertions(+), 76 deletions(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index eccc073..21cc4ea 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -401,14 +401,14 @@ $(tarballs): $(tdir)/%: | $(lockdir) # build it because it will complain about the version of libtool, so until # the version 0.11.0 of log4cxx, we'll have to run `autogen.sh' on the # unpacked source also. -$(ibidir)/apachelog4cxx: | $(ibidir)/automake \ - $(tdir)/apachelog4cxx-$(apachelog4cxx-version).tar.lz +$(ibidir)/apachelog4cxx: $(ibidir)/automake \ + $(tdir)/apachelog4cxx-$(apachelog4cxx-version).tar.lz pdir=apachelog4cxx-$(apachelog4cxx-version) rm -rf $(ddir)/$$pdir topdir=$(pwd) cd $(ddir) - tar xf $(word 1,$(filter $(tdir)/%,$|)) + tar xf $(word 1,$(filter $(tdir)/%,$^)) cd $$pdir ./autogen.sh \ && ./configure SHELL=$(ibdir)/bash --prefix=$(idir) \ @@ -419,12 +419,12 @@ $(ibidir)/apachelog4cxx: | $(ibidir)/automake \ && cd $$topdir \ && echo "Apache log4cxx $(apachelog4cxx-version)" > $@ -$(ibidir)/apr: | $(tdir)/apr-$(apr-version).tar.gz +$(ibidir)/apr: $(tdir)/apr-$(apr-version).tar.gz $(call gbuild, apr-$(apr-version), ,--disable-static) \ && echo "Apache Portable Runtime $(apr-version)" > $@ $(ibidir)/apr-util: $(ibidir)/apr \ - | $(tdir)/apr-util-$(apr-util-version).tar.gz + $(tdir)/apr-util-$(apr-util-version).tar.gz $(call gbuild, apr-util-$(apr-util-version), , \ --disable-static \ --with-apr=$(idir) \ @@ -432,8 +432,8 @@ $(ibidir)/apr-util: $(ibidir)/apr \ --with-crypto ) \ && echo "Apache Portable Runtime Utility $(apr-util-version)" > $@ -$(ibidir)/atlas: | $(tdir)/atlas-$(atlas-version).tar.bz2 \ - $(tdir)/lapack-$(lapack-version).tar.gz +$(ibidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ + $(tdir)/lapack-$(lapack-version).tar.gz # Get the operating system specific features (how to get # CPU frequency and the library suffixes). To make the steps @@ -514,12 +514,12 @@ $(ibidir)/atlas: | $(tdir)/atlas-$(atlas-version).tar.bz2 \ # Boost doesn't use the standard GNU Build System. $(ibidir)/boost: $(ibidir)/openmpi \ - | $(ibidir)/python \ - $(tdir)/boost-$(boost-version).tar.gz + $(ibidir)/python \ + $(tdir)/boost-$(boost-version).tar.gz vstr=$$(echo $(boost-version) | sed -e's/\./_/g') rm -rf $(ddir)/boost_$$vstr topdir=$(pwd); cd $(ddir); - tar xf $(word 1,$(filter $(tdir)/%,$|)) \ + tar xf $(word 1,$(filter $(tdir)/%,$^)) \ && cd boost_$$vstr \ && ./bootstrap.sh --prefix=$(idir) --with-libraries=all \ --with-python=python3 \ @@ -531,13 +531,13 @@ $(ibidir)/boost: $(ibidir)/openmpi \ && echo "Boost $(boost-version)" > $@ $(ibidir)/cfitsio: $(ibidir)/curl \ - | $(tdir)/cfitsio-$(cfitsio-version).tar.gz + $(tdir)/cfitsio-$(cfitsio-version).tar.gz # CFITSIO hard-codes '@rpath' inside the shared library on # Mac systems. So we need to change it to our library # installation path. It doesn't affect GNU/Linux, so we'll # just do it in any case to keep things clean. - topdir=$(pwd); cd $(ddir); tar xf $(word 1,$(filter $(tdir)/%,$|)) + topdir=$(pwd); cd $(ddir); tar xf $(word 1,$(filter $(tdir)/%,$^)) customtar=cfitsio-$(cfitsio-version)-custom.tar.gz cd cfitsio-$(cfitsio-version) sed configure -e's|@rpath|$(ildir)|g' > configure_tmp @@ -560,7 +560,7 @@ $(ibidir)/cfitsio: $(ibidir)/curl \ $(ibidir)/cairo: $(ibidir)/freetype \ $(ibidir)/libpng \ $(ibidir)/pixman \ - | $(tdir)/cairo-$(cairo-version).tar.xz + $(tdir)/cairo-$(cairo-version).tar.xz $(call gbuild, cairo-$(cairo-version), static, \ --with-x=no, -j$(numthreads) V=1) \ && echo "Cairo $(cairo-version)" > $@ @@ -568,20 +568,20 @@ $(ibidir)/cairo: $(ibidir)/freetype \ # Eigen is just headers! So it doesn't need to be compiled. Once unpacked # it has a checksum after `eigen-eigen', so we'll just use a `*' to choose # the unpacked directory. -$(ibidir)/eigen: | $(tdir)/eigen-$(eigen-version).tar.gz +$(ibidir)/eigen: $(tdir)/eigen-$(eigen-version).tar.gz rm -rf $(ddir)/eigen-eigen-* - topdir=$(pwd); cd $(ddir); tar xf $(word 1,$(filter $(tdir)/%,$|)) + topdir=$(pwd); cd $(ddir); tar xf $(word 1,$(filter $(tdir)/%,$^)) cd eigen-eigen-* cp -r Eigen $(iidir)/eigen3 \ && cd $$topdir \ && rm -rf $(ddir)/eigen-eigen-* \ && echo "Eigen $(eigen-version)" > $@ -$(ibidir)/expat: | $(tdir)/expat-$(expat-version).tar.lz +$(ibidir)/expat: $(tdir)/expat-$(expat-version).tar.lz $(call gbuild, expat-$(expat-version), static) \ && echo "Expat $(expat-version)" > $@ -$(ibidir)/fftw: | $(tdir)/fftw-$(fftw-version).tar.gz +$(ibidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz # FFTW's single and double precission libraries must be built # independently: for the the single-precision library, we need to # add the `--enable-float' option. We will build this first, then @@ -596,16 +596,16 @@ $(ibidir)/fftw: | $(tdir)/fftw-$(fftw-version).tar.gz # Freetype is necessary to install matplotlib $(ibidir)/freetype: $(ibidir)/libpng \ - | $(tdir)/freetype-$(freetype-version).tar.gz + $(tdir)/freetype-$(freetype-version).tar.gz $(call gbuild, freetype-$(freetype-version), static) \ && echo "FreeType $(freetype-version)" > $@ -$(ibidir)/gsl: | $(tdir)/gsl-$(gsl-version).tar.gz +$(ibidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, gsl-$(gsl-version), static) \ && echo "GNU Scientific Library $(gsl-version)" > $@ $(ibidir)/hdf5: $(ibidir)/openmpi \ - | $(tdir)/hdf5-$(hdf5-version).tar.gz + $(tdir)/hdf5-$(hdf5-version).tar.gz export CC=mpicc; \ export FC=mpif90; \ $(call gbuild, hdf5-$(hdf5-version), static, \ @@ -629,10 +629,10 @@ else healpix-python-dep = $(ipydir)/matplotlib $(ipydir)/astropy endif $(ibidir)/healpix: $(ibidir)/cfitsio \ + $(ibidir)/autoconf \ + $(ibidir)/automake \ $(healpix-python-dep) \ - | $(ibidir)/autoconf \ - $(ibidir)/automake \ - $(tdir)/healpix-$(healpix-version).tar.gz + $(tdir)/healpix-$(healpix-version).tar.gz if [ x"$(healpix-python-dep)" = x ]; then pycommand1="echo no-healpy-because-no-other-python" pycommand2="echo no-healpy-because-no-other-python" @@ -642,7 +642,7 @@ $(ibidir)/healpix: $(ibidir)/cfitsio \ fi rm -rf $(ddir)/Healpix_$(healpix-version) topdir=$(pwd); cd $(ddir); - tar xf $(word 1,$(filter $(tdir)/%,$|)) + tar xf $(word 1,$(filter $(tdir)/%,$^)) && cd Healpix_$(healpix-version)/src/C/autotools/ \ && autoreconf --install \ && ./configure --prefix=$(idir) \ @@ -661,30 +661,30 @@ $(ibidir)/healpix: $(ibidir)/cfitsio \ && cp $(dtexdir)/healpix.tex $(ictdir)/ \ && echo "HEALPix $(healpix-version) \citep{healpix}" > $@ -$(ibidir)/libjpeg: | $(tdir)/jpegsrc.$(libjpeg-version).tar.gz +$(ibidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz $(call gbuild, jpeg-9b, static,,V=1) \ && echo "Libjpeg $(libjpeg-version)" > $@ $(ibidir)/libnsl: $(ibidir)/libtirpc \ $(ibidir)/rpcsvc-proto \ - | $(tdir)/libnsl-$(libnsl-version).tar.gz + $(tdir)/libnsl-$(libnsl-version).tar.gz $(call gbuild, libnsl-$(libnsl-version), static, \ --sysconfdir=$(idir)/etc) \ && echo "Libnsl $(libnsl-version)" > $@ -$(ibidir)/libpng: | $(tdir)/libpng-$(libpng-version).tar.xz +$(ibidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz $(call gbuild, libpng-$(libpng-version), static) \ && echo "Libpng $(libpng-version)" > $@ $(ibidir)/libtiff: $(ibidir)/libjpeg \ - | $(tdir)/tiff-$(libtiff-version).tar.gz + $(tdir)/tiff-$(libtiff-version).tar.gz $(call gbuild, tiff-$(libtiff-version), static, \ --disable-jbig \ --disable-webp \ --disable-zstd) \ && echo "Libtiff $(libtiff-version)" > $@ -$(ibidir)/libtirpc: | $(tdir)/libtirpc-$(libtirpc-version).tar.bz2 +$(ibidir)/libtirpc: $(tdir)/libtirpc-$(libtirpc-version).tar.bz2 $(call gbuild, libtirpc-$(libtirpc-version), static, \ --disable-gssapi, V=1) \ echo "libtirpc $(libtirpc-version)" > $@ @@ -700,12 +700,12 @@ $(ibidir)/libxml2: | $(tdir)/libxml2-$(libxml2-version).tar.gz --without-python) \ && echo "Libxml2 $(libxml2-version)" > $@ -$(ibidir)/openblas: | $(tdir)/openblas-$(openblas-version).tar.gz +$(ibidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz if [ x$(on_mac_os) = xyes ]; then \ export CC=clang; \ fi; \ cd $(ddir) \ - && tar xf $(word 1,$(filter $(tdir)/%,$|)) \ + && tar xf $(word 1,$(filter $(tdir)/%,$^)) \ && cd OpenBLAS-$(openblas-version) \ && make \ && make PREFIX=$(idir) install \ @@ -713,7 +713,7 @@ $(ibidir)/openblas: | $(tdir)/openblas-$(openblas-version).tar.gz && rm -rf OpenBLAS-$(openblas-version) \ && echo "OpenBLAS $(openblas-version)" > $@ -$(ibidir)/openmpi: | $(tdir)/openmpi-$(openmpi-version).tar.gz +$(ibidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz $(call gbuild, openmpi-$(openmpi-version), static, , \ -j$(numthreads) V=1) \ && echo "Open MPI $(openmpi-version)" > $@ @@ -723,7 +723,7 @@ $(ibidir)/openmpi: | $(tdir)/openmpi-$(openmpi-version).tar.gz # within the project because of all the security issues it may cause. Only # enable/build it in a project with caution, and if there is no other # solution (for example to disable SSH in a program that may ask for it. -$(ibidir)/openssh: | $(tdir)/openssh-$(openssh-version).tar.gz +$(ibidir)/openssh: $(tdir)/openssh-$(openssh-version).tar.gz $(call gbuild, openssh-$(openssh-version), static, \ --with-privsep-path=$(ibdir)/.ssh_privsep \ --with-privsep-user=nobody \ @@ -732,22 +732,22 @@ $(ibidir)/openssh: | $(tdir)/openssh-$(openssh-version).tar.gz , -j$(numthreads) V=1) \ && echo "OpenSSH $(openssh-version)" > $@ -$(ibidir)/pixman: | $(tdir)/pixman-$(pixman-version).tar.gz +$(ibidir)/pixman: $(tdir)/pixman-$(pixman-version).tar.gz $(call gbuild, pixman-$(pixman-version), static, , \ -j$(numthreads) V=1) \ && echo "Pixman $(pixman-version)" > $@ -$(ibidir)/rpcsvc-proto: | $(tdir)/rpcsvc-proto-$(rpcsvc-proto-version).tar.xz +$(ibidir)/rpcsvc-proto: $(tdir)/rpcsvc-proto-$(rpcsvc-proto-version).tar.xz $(call gbuild, rpcsvc-proto-$(rpcsvc-proto-version), static) \ && echo "rpcsvc $(rpcsvc-proto-version)" > $@ -$(ibidir)/tides: | $(tdir)/tides-$(tides-version).tar.gz +$(ibidir)/tides: $(tdir)/tides-$(tides-version).tar.gz $(call gbuild, tides-$(tides-version), static,\ --with-gmp=$(idir) --with-mpfr=$(idir)) \ && cp $(dtexdir)/tides.tex $(ictdir)/ \ && echo "TIDES $(tides-version) \citep{tides}" > $@ -$(ibidir)/yaml: | $(tdir)/yaml-$(yaml-version).tar.gz +$(ibidir)/yaml: $(tdir)/yaml-$(yaml-version).tar.gz $(call gbuild, yaml-$(yaml-version), static) \ && echo "LibYAML $(yaml-version)" > $@ @@ -777,8 +777,8 @@ $(ibidir)/yaml: | $(tdir)/yaml-$(yaml-version).tar.gz # that it uses if it can't find libiconv on macOS. So, to fix this problem # it is necessary to use the option `-DUSE_ICONV=OFF` in the configure step. $(ibidir)/libgit2: $(ibidir)/curl \ - | $(ibidir)/cmake \ - $(tdir)/libgit2-$(libgit2-version).tar.gz + $(ibidir)/cmake \ + $(tdir)/libgit2-$(libgit2-version).tar.gz $(call cbuild, libgit2-$(libgit2-version), static, \ -DUSE_SSH=OFF -DBUILD_CLAR=OFF \ -DTHREADSAFE=ON -DUSE_ICONV=OFF ) \ @@ -789,7 +789,7 @@ $(ibidir)/libgit2: $(ibidir)/curl \ && echo "Libgit2 $(libgit2-version)" > $@ $(ibidir)/wcslib: $(ibidir)/cfitsio \ - | $(tdir)/wcslib-$(wcslib-version).tar.bz2 + $(tdir)/wcslib-$(wcslib-version).tar.bz2 $(call gbuild, wcslib-$(wcslib-version), , \ LIBS="-pthread -lcurl -lm" \ --with-cfitsiolib=$(ildir) \ @@ -825,14 +825,14 @@ $(ibidir)/astrometrynet: $(ibidir)/gsl \ $(ibidir)/wcslib \ $(ibidir)/cfitsio \ $(ibidir)/libjpeg \ - | $(tdir)/astrometry.net-$(astrometrynet-version).tar.gz + $(tdir)/astrometry.net-$(astrometrynet-version).tar.gz # We are modifying the Makefile in two steps because on Mac OS # system we do not have `/proc/cpuinfo' nor `free'. Since this is # only for the `report.txt', this changes do not causes problems in # running `astrometrynet' cd $(ddir) \ && rm -rf astrometry.net-$(astrometrynet-version) \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$|)); then \ + && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)); then \ echo; echo "Tar error"; exit 1; \ fi \ && cd astrometry.net-$(astrometrynet-version) \ @@ -848,17 +848,17 @@ $(ibidir)/astrometrynet: $(ibidir)/gsl \ && cp $(dtexdir)/astrometrynet.tex $(ictdir)/ \ && echo "Astrometry.net $(astrometrynet-version) \citep{astrometrynet}" > $@ -$(ibidir)/autoconf: | $(tdir)/autoconf-$(autoconf-version).tar.lz +$(ibidir)/autoconf: $(tdir)/autoconf-$(autoconf-version).tar.lz $(call gbuild, autoconf-$(autoconf-version), static, ,V=1) \ && echo "GNU Autoconf $(autoconf-version)" > $@ -$(ibidir)/automake: | $(ibidir)/autoconf \ - $(tdir)/automake-$(automake-version).tar.gz +$(ibidir)/automake: $(ibidir)/autoconf \ + $(tdir)/automake-$(automake-version).tar.gz $(call gbuild, automake-$(automake-version), static, ,V=1) \ && echo "GNU Automake $(automake-version)" > $@ -$(ibidir)/bison: | $(ibidir)/help2man \ - $(tdir)/bison-$(bison-version).tar.xz +$(ibidir)/bison: $(ibidir)/help2man \ + $(tdir)/bison-$(bison-version).tar.xz $(call gbuild, bison-$(bison-version), static, ,V=1) \ && echo "GNU Bison $(bison-version)" > $@ @@ -869,9 +869,9 @@ $(ibidir)/bison: | $(ibidir)/help2man \ # programs are scripts and we need to touch them before installing. # Otherwise this software will be re-built each time the configure step is # invoked. -$(ibidir)/cdsclient: | $(tdir)/cdsclient-$(cdsclient-version).tar.gz +$(ibidir)/cdsclient: $(tdir)/cdsclient-$(cdsclient-version).tar.gz cd $(ddir) \ - && tar xf $(word 1,$(filter $(tdir)/%,$|)) \ + && tar xf $(word 1,$(filter $(tdir)/%,$^)) \ && cd cdsclient-$(cdsclient-version) \ && touch * \ && ./configure --prefix=$(idir) \ @@ -883,7 +883,7 @@ $(ibidir)/cdsclient: | $(tdir)/cdsclient-$(cdsclient-version).tar.gz # CMake can be built with its custom `./bootstrap' script. $(ibidir)/cmake: $(ibidir)/curl \ - | $(tdir)/cmake-$(cmake-version).tar.gz + $(tdir)/cmake-$(cmake-version).tar.gz # After searching in `bootstrap', I couldn't find `LIBS', only # `LDFLAGS'. So the extra libraries are being added to `LDFLAGS', # not `LIBS'. @@ -896,7 +896,7 @@ $(ibidir)/cmake: $(ibidir)/curl \ fi; \ cd $(ddir) \ && rm -rf cmake-$(cmake-version) \ - && tar xf $(word 1,$(filter $(tdir)/%,$|)) \ + && tar xf $(word 1,$(filter $(tdir)/%,$^)) \ && cd cmake-$(cmake-version) \ && ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ --system-bzip2 --system-liblzma --no-qt-gui \ @@ -908,18 +908,18 @@ $(ibidir)/cmake: $(ibidir)/curl \ && echo "CMake $(cmake-version)" > $@ $(ibidir)/flex: $(ibidir)/bison \ - | $(tdir)/flex-$(flex-version).tar.gz + $(tdir)/flex-$(flex-version).tar.gz $(call gbuild, flex-$(flex-version), static, ,V=1) \ && echo "Flex $(flex-version)" > $@ -$(ibidir)/gdb: | $(ibidir)/python \ - $(tdir)/gdb-$(gdb-version).tar.gz +$(ibidir)/gdb: $(ibidir)/python \ + $(tdir)/gdb-$(gdb-version).tar.gz $(call gbuild, gdb-$(gdb-version),,,V=1) \ && echo "GNU Project Debugger (GDB) $(gdb-version)" > $@ $(ibidir)/ghostscript: $(ibidir)/libpng \ $(ibidir)/libtiff \ - | $(tdir)/ghostscript-$(ghostscript-version).tar.gz + $(tdir)/ghostscript-$(ghostscript-version).tar.gz # First we need to make sure some necessary X11 libraries that we # don't yet install in this template are present on the host # system, see https://savannah.nongnu.org/task/?15481 . @@ -965,7 +965,7 @@ $(ibidir)/gnuastro: $(ibidir)/gsl \ $(ibidir)/libtiff \ $(ibidir)/libgit2 \ $(ibidir)/ghostscript \ - | $(tdir)/gnuastro-$(gnuastro-version).tar.lz + $(tdir)/gnuastro-$(gnuastro-version).tar.lz ifeq ($(static_build),yes) staticopts="--enable-static=yes --enable-shared=no"; endif @@ -974,14 +974,14 @@ endif && cp $(dtexdir)/gnuastro.tex $(ictdir)/ \ && echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ -$(ibidir)/help2man: | $(tdir)/help2man-$(help2man-version).tar.xz +$(ibidir)/help2man: $(tdir)/help2man-$(help2man-version).tar.xz $(call gbuild, help2man-$(help2man-version), static, ,V=1) \ && echo "Help2man $(Help2man-version)" > $@ $(ibidir)/imagemagick: $(ibidir)/zlib \ $(ibidir)/libjpeg \ $(ibidir)/libtiff \ - | $(tdir)/imagemagick-$(imagemagick-version).tar.xz + $(tdir)/imagemagick-$(imagemagick-version).tar.xz $(call gbuild, ImageMagick-$(imagemagick-version), static, \ --without-x --disable-openmp, V=1 -j$(numthreads)) \ && echo "ImageMagick $(imagemagick-version)" > $@ @@ -999,11 +999,11 @@ $(ibidir)/imfit: $(ibidir)/gsl \ $(ibidir)/fftw \ $(ibidir)/scons \ $(ibidir)/cfitsio \ - | $(tdir)/imfit-$(imfit-version).tar.gz + $(tdir)/imfit-$(imfit-version).tar.gz cd $(ddir) \ && unpackdir=imfit-$(imfit-version) \ && rm -rf $$unpackdir \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$|)); then \ + && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)); then \ echo; echo "Tar error"; exit 1; \ fi \ && cd $$unpackdir \ @@ -1042,12 +1042,12 @@ $(ibidir)/imfit: $(ibidir)/gsl \ # About deleting the final crypt.h file after installation, see # https://bugzilla.redhat.com/show_bug.cgi?id=1424609 $(ibidir)/minizip: $(ibidir)/automake \ - | $(tdir)/zlib-$(zlib-version).tar.gz + $(tdir)/zlib-$(zlib-version).tar.gz cd $(ddir) \ && unpackdir=minizip-$(minizip-version) \ && rm -rf $$unpackdir \ && mkdir $$unpackdir \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$|)) \ + && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)) \ -C$$unpackdir --strip-components=1; then \ echo; echo "Tar error"; exit 1; \ fi \ @@ -1070,7 +1070,7 @@ $(ibidir)/minizip: $(ibidir)/automake \ && rm -rf $$unpackdir \ && echo "Minizip $(minizip-version)" > $@ -$(ibidir)/missfits: | $(tdir)/missfits-$(missfits-version).tar.gz +$(ibidir)/missfits: $(tdir)/missfits-$(missfits-version).tar.gz $(call gbuild, missfits-$(missfits-version), static) \ && cp $(dtexdir)/missfits.tex $(ictdir)/ \ && echo "MissFITS $(missfits-version) \citep{missfits}" > $@ @@ -1086,7 +1086,7 @@ $(ibidir)/netpbm: $(ibidir)/unzip \ $(ibidir)/libjpeg \ $(ibidir)/libtiff \ $(ibidir)/libxml2 \ - | $(tdir)/netpbm-$(netpbm-version).tar.gz + $(tdir)/netpbm-$(netpbm-version).tar.gz if [ x$(on_mac_os) = xyes ]; then \ answers='\n\n$(ildir)\n\n\n\n\n\n$(ildir)/include\n\n$(ildir)/include\n\n$(ildir)/include\nnone\n\n'; \ else \ @@ -1095,7 +1095,7 @@ $(ibidir)/netpbm: $(ibidir)/unzip \ cd $(ddir) \ && unpackdir=netpbm-$(netpbm-version) \ && rm -rf $$unpackdir \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$|)); then \ + && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)); then \ echo; echo "Tar error"; exit 1; \ fi \ && cd $$unpackdir \ @@ -1113,7 +1113,7 @@ $(ibidir)/netpbm: $(ibidir)/unzip \ $(ibidir)/R: $(ibidir)/libpng \ $(ibidir)/libjpeg \ $(ibidir)/libtiff \ - | $(tdir)/R-$(R-version).tar.gz + $(tdir)/R-$(R-version).tar.gz export R_SHELL=$(SHELL); \ $(call gbuild, R-$(R-version), static, \ --without-x --with-readline \ @@ -1128,7 +1128,7 @@ $(ibidir)/R: $(ibidir)/libpng \ $(ibidir)/scamp: $(ibidir)/fftw \ $(ibidir)/openblas \ $(ibidir)/cdsclient \ - | $(tdir)/scamp-$(scamp-version).tar.lz + $(tdir)/scamp-$(scamp-version).tar.lz $(call gbuild, scamp-$(scamp-version), static, \ --enable-threads --enable-openblas \ --with-fftw-libdir=$(idir) \ @@ -1140,12 +1140,12 @@ $(ibidir)/scamp: $(ibidir)/fftw \ # Since `scons' doesn't use the traditional GNU installation with # `configure' and `make' it is installed manually using `python'. -$(ibidir)/scons: | $(ibidir)/python \ - $(tdir)/scons-$(scons-version).tar.gz +$(ibidir)/scons: $(ibidir)/python \ + $(tdir)/scons-$(scons-version).tar.gz cd $(ddir) \ && unpackdir=scons-$(scons-version) \ && rm -rf $$unpackdir \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$|)); then \ + && if ! tar xf $(word 1,$(filter $(tdir)/%,$^)); then \ echo; echo "Tar error"; exit 1; \ fi \ && cd $$unpackdir \ @@ -1158,7 +1158,7 @@ $(ibidir)/scons: | $(ibidir)/python \ # the configuration step. $(ibidir)/sextractor: $(ibidir)/fftw \ $(ibidir)/openblas \ - | $(tdir)/sextractor-$(sextractor-version).tar.lz + $(tdir)/sextractor-$(sextractor-version).tar.lz $(call gbuild, sextractor-$(sextractor-version), static, \ --enable-threads --enable-openblas \ --with-openblas-libdir=$(ildir) \ @@ -1168,13 +1168,13 @@ $(ibidir)/sextractor: $(ibidir)/fftw \ && echo "SExtractor $(sextractor-version) \citep{sextractor}" > $@ $(ibidir)/swarp: $(ibidir)/fftw \ - | $(tdir)/swarp-$(swarp-version).tar.gz + $(tdir)/swarp-$(swarp-version).tar.gz $(call gbuild, swarp-$(swarp-version), static, \ --enable-threads) \ && cp $(dtexdir)/swarp.tex $(ictdir)/ \ && echo "SWarp $(swarp-version) \citep{swarp}" > $@ -$(ibidir)/swig: | $(tdir)/swig-$(swig-version).tar.gz +$(ibidir)/swig: $(tdir)/swig-$(swig-version).tar.gz # Option --without-pcre was a suggestion once the configure step # was tried and it failed. It was not recommended but it works! # pcr is a dependency of swig @@ -1184,7 +1184,7 @@ $(ibidir)/swig: | $(tdir)/swig-$(swig-version).tar.gz $(ibidir)/xlsxio: $(ibidir)/cmake \ $(ibidir)/expat \ $(ibidir)/minizip \ - | $(tdir)/xlsxio-$(xlsxio-version).tar.gz + $(tdir)/xlsxio-$(xlsxio-version).tar.gz if [ x$(on_mac_os) = xyes ]; then \ export CC=clang; \ export CXX=clang++; \ @@ -1240,7 +1240,7 @@ tlmirror=http://mirrors.rit.edu/CTAN/systems/texlive/tlnet # The core TeX Live system. $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf \ - | $(tdir)/install-tl-unx.tar.gz + $(tdir)/install-tl-unx.tar.gz # Unpack, enter the directory, and install based on the given # configuration (prerequisite of this rule). @@ -1324,7 +1324,9 @@ $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ ln -fs $(idir)/texlive/maneage/bin/*/* $(ibdir)/ # Get all the necessary versions. - texlive=$$(pdflatex --version | awk 'NR==1' | sed 's/.*(\(.*\))/\1/' \ + texlive=$$(pdflatex --version \ + | awk 'NR==1' \ + | sed 's/.*(\(.*\))/\1/' \ | awk '{print $$NF}'); # Package names and versions. Note that all TeXLive packages -- cgit v1.2.1 From b1e1522a5a5b6d0800ea207f2d93a4f36bffa68d Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 1 May 2020 12:09:18 +0100 Subject: GNU Gettext built as a dependency of Bash Until now Maneage used the host's GNU Gettext if it was present. Gettext is a relatively low-level software that enables programs to print messages in different languages based on the host environment. Even though it has not direct effect on the running of the software for Maneage and the lanugage environment in Maneage is pre-determined, it is necessary to have it because if the basic programs see it in the host they will link with it and will have problems if/when the host's Gettext is updated. With this commit (which is actually a squashed rebase of 9 commits by Raul and Mohammad), Gettext and its two extra dependencies (libxml2 and libunistring) are now installed within Maneage as a basic software and built before GNU Bash. As a result, all programs built afterwards will successfully link with our own internal version of Gettext and libraries. To get this working, some of the basic software dependencies had to updated and re-ordered and it has been tested in both GNU/Linux and macoS. Some other minor issues that are fixed with this commit - Until this commit, when TeX was not installed, the warning message saying how to run the configure step in order to re-configure the project was not showing the option `-e'. However, the use of this option is more convenient than entering the top-build directory and etc every time. So with this commit, the warning message has been changed in order use the option `-e' in the re-configure of the project. - Until now, on macOS systems, Bash was not linking with our internally built `libncurses'. With this commit, this has been fixed by setting `--withcurses=yes' for Bash's configure script. --- reproduce/software/make/high-level.mk | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 21cc4ea..8caa968 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -180,7 +180,6 @@ tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ libnsl-$(libnsl-version).tar.gz \ libpng-$(libpng-version).tar.xz \ libtirpc-$(libtirpc-version).tar.bz2 \ - libxml2-$(libxml2-version).tar.gz \ missfits-$(missfits-version).tar.gz \ netpbm-$(netpbm-version).tar.gz \ openblas-$(openblas-version).tar.gz \ @@ -280,7 +279,6 @@ $(tarballs): $(tdir)/%: | $(lockdir) c=$(libgit2-checksum) w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = libtirpc ]; then c=$(libtirpc-checksum); w=https://downloads.sourceforge.net/libtirpc - elif [ $$n = libxml2 ]; then c=$(libxml2-checksum); w=ftp://xmlsoft.org/libxml2 elif [ $$n = missfits ]; then c=$(missfits-checksum); w=https://www.astromatic.net/download/missfits elif [ $$n = netpbm ]; then c=$(netpbm-checksum); w=http://akhlaghi.org/maneage-software elif [ $$n = openblas ]; then @@ -689,17 +687,6 @@ $(ibidir)/libtirpc: $(tdir)/libtirpc-$(libtirpc-version).tar.bz2 --disable-gssapi, V=1) \ echo "libtirpc $(libtirpc-version)" > $@ -$(ibidir)/libxml2: | $(tdir)/libxml2-$(libxml2-version).tar.gz - # The libxml2 tarball also contains Python bindings which are built and - # installed to a system directory by default. If you don't need the Python - # bindings, the easiest solution is to compile without Python support: - # ./configure --without-python - # If you really need the Python bindings, try the - # --with-python-install-dir=DIR option - $(call gbuild, libxml2-$(libxml2-version), static, \ - --without-python) \ - && echo "Libxml2 $(libxml2-version)" > $@ - $(ibidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz if [ x$(on_mac_os) = xyes ]; then \ export CC=clang; \ -- cgit v1.2.1 From 221f2884fdacf31d60745e58fffffef75844f6b5 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 6 May 2020 15:24:28 +0100 Subject: Installing Scamp with the option --enable-plplot=no Until this commit, Scamp was installed with the option `--enable-plplot=yes' (the default). However, Maneage does not have PLplot included. As it is possible to install Scamp without PLplot (in that case it won't generate plots), with this commit this option has been set to `no'. As a consequence, Scamp will be installed even if the host system does not have PLplot without crashing (but it won't make any plot). --- reproduce/software/make/high-level.mk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 8caa968..7269c3d 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1117,7 +1117,9 @@ $(ibidir)/scamp: $(ibidir)/fftw \ $(ibidir)/cdsclient \ $(tdir)/scamp-$(scamp-version).tar.lz $(call gbuild, scamp-$(scamp-version), static, \ - --enable-threads --enable-openblas \ + --enable-threads \ + --enable-openblas \ + --enable-plplot=no \ --with-fftw-libdir=$(idir) \ --with-fftw-incdir=$(idir)/include \ --with-openblas-libdir=$(ildir) \ -- cgit v1.2.1 From c0a512e0631a024f18bec847125a9a03e6aa23a1 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 22 May 2020 23:35:21 +0100 Subject: Corrected copyright notices to fit GPL suggested format In time, some of the copyright license description had been mistakenly shortened to two paragraphs instead of the original three that is recommended in the GPL. With this commit, they are corrected to be exactly in the same three paragraph format suggested by GPL. The following files also didn't have a copyright notice, so one was added for them: reproduce/software/make/README.md reproduce/software/bibtex/healpix.tex reproduce/analysis/config/delete-me-num.conf reproduce/analysis/config/verify-outputs.conf --- reproduce/software/make/high-level.mk | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 7269c3d..7e9c723 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -11,15 +11,18 @@ # Copyright (C) 2018-2020 Mohammad Akhlaghi # Copyright (C) 2019-2020 Raul Infante-Sainz # -# This Makefile is part of Maneage. Maneage is free software: you can -# redistribute it and/or modify it under the terms of the GNU General -# Public License as published by the Free Software Foundation, either -# version 3 of the License, or (at your option) any later version. +# This Makefile is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. # -# Maneage is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. See . +# This Makefile is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this Makefile. If not, see . -- cgit v1.2.1 From 4493acc390a92ba53c7d76766024e7d64c7b31ce Mon Sep 17 00:00:00 2001 From: Boud Roukema Date: Fri, 15 May 2020 04:42:58 +0200 Subject: New software: Valgrind and Patch With this commit, Maneage now includes instructions to build the memory tracing tool Valgrind and the program 'patch' (to apply corrections/patches in text files and in particular the sources of programs). For this version of Valgrind, some patches were necessary for an interface with OpenMPI 2.x (which is the case now). Also note that this version of Valgrind's checks can fail with GCC 10.1.x (when using '--host-cc'), and the failures aren't due to internal problems but due to how the tests are designed (https://bugs.gentoo.org/707598). So currently if any of Valgrind's checks fail, Maneage still assumes that Valgrind was built and installed successfully. While testing on macOS, we noticed that it needs the macOS-specific 'mig' program which we can't build in Maneage. DESCRIPTION: The mig command invokes the Mach Interface Generator to generate Remote Procedure Call (RPC) code for client-server style Mach IPC from specification files. So a symbolic link to the system's 'mig' is now added to the project's programs on macOS systems. This commit's build of Patch and Valgrind has been tested on two GNU/Linux distributions (Debian and ArchLinux) as well as macOS. Work on this commit started by Boud Roukema, but also involved tests and corrections by Mohammad Akhlaghi and Raul Infante-Sainz. --- reproduce/software/make/high-level.mk | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 7e9c723..3ad4468 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -42,6 +42,7 @@ ibdir = $(BDIR)/software/installed/bin ildir = $(BDIR)/software/installed/lib iidir = $(BDIR)/software/installed/include dtexdir = $(shell pwd)/reproduce/software/bibtex +patchdir= $(shell pwd)/reproduce/software/patches itidir = $(BDIR)/software/installed/version-info/tex ictdir = $(BDIR)/software/installed/version-info/cite ipydir = $(BDIR)/software/installed/version-info/python @@ -188,6 +189,7 @@ tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ openblas-$(openblas-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ openssh-$(openssh-version).tar.gz \ + patch-$(patch-version).tar.gz \ pixman-$(pixman-version).tar.gz \ R-$(R-version).tar.gz \ scamp-$(scamp-version).tar.lz \ @@ -198,6 +200,7 @@ tarballs = $(foreach t, apachelog4cxx-$(apachelog4cxx-version).tar.lz \ rpcsvc-proto-$(rpcsvc-proto-version).tar.xz \ tides-$(tides-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ + valgrind-$(valgrind-version).tar.bz2 \ wcslib-$(wcslib-version).tar.bz2 \ xlsxio-$(xlsxio-version).tar.gz \ yaml-$(yaml-version).tar.gz \ @@ -294,6 +297,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = openssh ]; then c=$(openssh-checksum); w=https://artfiles.org/openbsd/OpenSSH/portable + elif [ $$n = patch ]; then c=$(patch-checksum); w=http://ftp.gnu.org/gnu/patch elif [ $$n = pixman ]; then c=$(pixman-checksum); w=https://www.cairographics.org/releases elif [ $$n = R ]; then c=$(R-checksum); majver=$$(echo $(R-version) | sed -e's/\./ /g' | awk '{print $$1}') @@ -309,6 +313,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = swig ]; then c=$(swig-checksum); w=https://sourceforge.net/projects/swig/files/swig/swig-$(swig-version) elif [ $$n = tides ]; then c=$(tides-checksum); w=http://akhlaghi.org/maneage-software elif [ $$n = tiff ]; then c=$(libtiff-checksum); w=https://download.osgeo.org/libtiff + elif [ $$n = valgrind ]; then c=$(valgrind-checksum); w=https://sourceware.org/pub/valgrind elif [ $$n = wcslib ]; then c=$(wcslib-checksum); w=ftp://ftp.atnf.csiro.au/pub/software/wcslib elif [ $$n = xlsxio ]; then mergenames=0 @@ -737,6 +742,33 @@ $(ibidir)/tides: $(tdir)/tides-$(tides-version).tar.gz && cp $(dtexdir)/tides.tex $(ictdir)/ \ && echo "TIDES $(tides-version) \citep{tides}" > $@ +$(ibidir)/valgrind: $(ibidir)/patch \ + $(ibidir)/autoconf \ + $(ibidir)/automake \ + $(tdir)/valgrind-$(valgrind-version).tar.bz2 + # For valgrind-3.15.0, see + # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=946329 for a + # report on an MPI-related compile bug and the two patches + # below. These two patches and `automake` should allow valgrind to + # compile with gcc-9.2.0. + cd $(ddir) \ + && tar -x -f $(word 1,$(filter $(tdir)/%,$^)) \ + && valgrinddir=valgrind-$(valgrind-version) \ + && cd $${valgrinddir} \ + && printf "valgrindir=$${valgrinddir} ; pwd = %s .\n" $$($(ibdir)/pwd) \ + && if [ "x$(valgrind-version)" = "x3.15.0" ]; then \ + patch --verbose -p1 < $(patchdir)/valgrind-3.15.0-mpi-fix1.patch; \ + patch --verbose -p1 < $(patchdir)/valgrind-3.15.0-mpi-fix2.patch; \ + fi \ + && autoreconf \ + && ./configure --prefix=$(idir) \ + && make -j$(numthreads) \ + && if ! make check -j$(numthreads); then \ + echo; echo "Valgrind's 'make check' failed!"; echo; \ + fi \ + && make install \ + && echo "Valgrind $(valgrind-version)" > $@ + $(ibidir)/yaml: $(tdir)/yaml-$(yaml-version).tar.gz $(call gbuild, yaml-$(yaml-version), static) \ && echo "LibYAML $(yaml-version)" > $@ @@ -1099,6 +1131,10 @@ $(ibidir)/netpbm: $(ibidir)/unzip \ && rm -rf $$unpackdir \ && echo "Netpbm $(netpbm-version)" > $@ +$(ibidir)/patch: $(tdir)/patch-$(patch-version).tar.gz + $(call gbuild, patch-$(patch-version), static, ,V=1) \ + && echo "GNU Patch $(patch-version)" > $@ + # R programming language $(ibidir)/R: $(ibidir)/libpng \ $(ibidir)/libjpeg \ -- cgit v1.2.1 From 0043ba8d9032048d62b36f42f143e88ad330af27 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 25 May 2020 21:00:10 +0100 Subject: Unified reference to GNU/Linux and free software One of the main reasons to building Maneage is to properly acknowledge/attribute the authors of software in research. So we have adopted a standard of never referring to the GNU-based operating systems running the Linux kernel simply as "Linux", we avoid terms like "Open Sourse" and use Free Software instead (in the same spirit). With this commit, a few instances of the cases above have been corrected, they had slipped through our fingers when we initially imported them into the project. In the special case of the "Journal for Open Source Software", we simply replaced it with its abbreviation (JOSS). This was done because in effect we were generally using journal name abbreviations in almost all the citations already. To avoid any inconsistancies, the names of the three other journals that weren't abbreviated are also abbreviated. --- reproduce/software/make/high-level.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 3ad4468..96de036 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1059,7 +1059,7 @@ $(ibidir)/imfit: $(ibidir)/gsl \ # Autotools (Autoconf and Automake). Note that Minizip 2.x isn't like this # any more and has its own independent tarball, but currently the programs # that depend on Minizip need Minizip 1.x. The instructions to build -# minizip were taken from ArchLinux. +# minizip were taken from Arch GNU/Linux. # # About deleting the final crypt.h file after installation, see # https://bugzilla.redhat.com/show_bug.cgi?id=1424609 -- cgit v1.2.1 From 213dac320133571bca871275c1bf6446ddee33cb Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 27 May 2020 12:00:50 +0100 Subject: Fixed TeXLive crash because of differing local and server versions Until this commit, when the user had a previous TeXLive tarball already present (in their software-tarball directory) compared to the CTAN server, the project crashed in the configure phase. This was because TeXLive is updated yearly and we don't yet install TeXLive from source (currently we use its own package manager, but we plan to fix this in task #15267). With this commit, we fix the problem by checking the cause of the crash during the installation of TeX. If the crash is due to this particular error, we ignore the old tarball and download the new one and install it (the old one is still kept in '.build/software/tarballs', but will get a '-OLD' in its name. This probem was recurrent, and every year that TeXLive is updated, the previous tarball had to be removed manually! But with this commit, this is done automatically. The detection and fix of this bug has been possible with the help of Mohammad Akhlaghi, thanks! --- reproduce/software/make/high-level.mk | 83 +++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 96de036..19cb52f 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1255,7 +1255,7 @@ $(ibidir)/xlsxio: $(ibidir)/cmake \ # the final PDF). So we'll make a simple ASCII file called # `texlive-ready-tlmgr' and use its contents to mark if we can use it or # not. - +# # TeX Live mirror # --------------- # @@ -1284,7 +1284,33 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf \ # TeX Live's installation may fail due to any reason. But TeX Live # is optional (only necessary for building the final PDF). So we # don't want the configure script to fail if it can't run. - if ./install-tl --profile=texlive.conf -repository $(tlmirror); then + # Possible error messages will be saved into `log.txt' and if it + # fails, 'log.txt' will be checked to see if the error is due to + # the different version of the current tarball and the TeXLive + # server or something else. + # + # The problem with versions is this: each installer tarball (that + # is downloaded and a user may backup) is for a specific version of + # TeXLive (specified by year, usually around April). So if a user + # has an old tarball, but the CTAN server has been updated, the + # script will fail with a message like this: + # + # ============================================================= + # ./install-tl: The TeX Live versions of the local installation + # and the repository being accessed are not compatible: + # local: 2019 + # repository: 2020 + # Perhaps you need to use a different CTAN mirror? + # (For more, see the output of install-tl --help, especially the + # -repository option. Online via https://tug.org/texlive/doc.) + # ============================================================= + # + # To address this problem, when this happens, we simply download a + # the most recent tarball, and if it succeeds, we will build + # TeXLive using that. The old tarball will be preserved, but will + # have an '-OLD' suffix after it. + if ./install-tl --profile=texlive.conf -repository \ + $(tlmirror) 2> log.txt; then # Put a symbolic link of the TeX Live executables in `ibdir' to # avoid all the complexities of its sub-directories and additions @@ -1293,8 +1319,59 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf \ # Register that the build was successful. echo "TeX Live is ready." > $@ + + # The build failed! else - echo "NOT!" > $@ + # Print on the command line the error messages during the + # installation. + cat log.txt + + # Look for words `repository:' and `local:' in `log.txt' and make + # sure that two lines are returned. Note that we need to check + # for two lines because one of them may exist, but another may + # not (in this case, its not a version conflict scenario). + version_check=$$(grep -w 'repository:\|local:' log.txt | wc -l) + + # If these words exists and two lines are found, there is a + # conflict with the main TeXLive version in the tarball and on + # the server. So it is necessary to move the old tarball and + # download the new one to install it. + if [ x"$$version_check" = x2 ]; then + # Go back to the top project directory, don't remove the + # tarball, just rename it. + cd $$topdir + mv $(tdir)/install-tl-unx.tar.gz $(tdir)/install-tl-unx-OLD.tar.gz + + # Download using the script specially defined for this job. If + # the download of new tarball success, install it (same lines + # than above). If not, record the fail into the target. + url=http://mirror.ctan.org/systems/texlive/tlnet + tarballurl=$$url/install-tl-unx.tar.gz + touch $(lockdir)/download + downloader="wget --no-use-server-timestamps -O" + if $(downloadwrapper) "$$downloader" $(lockdir)/download \ + $$tarballurl "$(tdir)/install-tl-unx.tar.gz" \ + "$(backupservers)"; then + cd $(ddir) + rm -rf install-tl-* + tar xf $(tdir)/install-tl-unx.tar.gz + cd install-tl-* + sed -e's|@installdir[@]|$(idir)|g' \ + $$topdir/reproduce/software/config/texlive.conf \ + > texlive.conf + if ./install-tl --profile=texlive.conf -repository \ + $(tlmirror); then + ln -fs $(idir)/texlive/maneage/bin/*/* $(ibdir)/ + echo "TeX Live is ready." > $@ + else + echo "NOT!" > $@ # Building failed. + fi + else + echo "NOT!" > $@ # Download failed. + fi + else + echo "NOT!" > $@ # Error was not version. + fi fi # Clean up -- cgit v1.2.1 From 3d8aa5953c4e0b79278ab2e27ec4e1051310d04f Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 1 Jun 2020 03:05:37 +0100 Subject: Core software build before using Make to build other software Until now, Maneage would only build Flock before building everything else using Make (calling 'basic.mk') in parallel. Flock was necessary to avoid parallel downloads during the building of software (which could cause network problems). But after recently trying Maneage on FreeBSD (which is not yet complete, see bug #58465), we noticed that the BSD implemenation of Make couldn't parse 'basic.mk' (in particular, complaining with the 'ifeq' parts) and its shell also had some peculiarities. It was thus decided to also install our own minimalist shell, Make and compressor program before calling 'basic.mk'. In this way, 'basic.mk' can now assume the same GNU Make features that high-level.mk and python.mk assume. The pre-make building of software is now organized in 'reproduce/software/shell/pre-make-build.sh'. Another nice feature of this commit is for macOS users: until now the default macOS Make had problems for parallel building of software, so 'basic.mk' was built in one thread. But now that we can build the core tools with GNU Make on macOS too, it uses all threads. Furthermore, since we now run 'basic.mk' with GNU Make, we can use '.ONESHELL' and don't have to finish every line of a long rule with a backslash to keep variables and such. Generally, the pre-make software are now organized like this: first we build Lzip before anything else: it is downloaded as a simple '.tar' file that is not compressed (only ~400kb). Once Lzip is built, the pre-make phase continues with building GNU Make, Dash (a minimalist shell) and Flock. All of their tarballs are in '.tar.lz'. Maneage then enters 'basic.mk' and the first program it builds is GNU Gzip (itself packaged as '.tar.lz'). Once Gzip is built, we build all the other compression software (all downloaded as '.tar.gz'). Afterwards, any compression standard for other software is fine because we have it. In the process, a bug related to using backup servers was found in 'reproduce/analysis/bash/download-multi-try' for calling outside of 'basic.mk' and removed Bash-specific features. As a result of that bug-fix, because we now have multiple servers for software tarballs, the backup servers now have their own configuration file in 'reproduce/software/config/servers-backup.conf'. This makes it much easier to maintain the backup server list across the multiple places that we need it. Some other minor fixes: - In building Bzip2, we need to specify 'CC' so it doesn't use 'gcc'. - In building Zip, the 'generic_gcc' Make option caused a crash on FreeBSD (which doesn't have GCC). - We are now using 'uname -s' to specify if we are on a Linux kernel or not, if not, we are still using the old 'on_mac_os' variable. - While I was trying to build on FreeBSD, I noticed some further corrections that could help. For example the 'makelink' Make-function now takes a third argument which can be a different name compared to the actual program (used for examle to make a link to '/usr/bin/cc' from 'gcc'. - Until now we didn't know if the host's Make implementation supports placing a '@' at the start of the recipe (to avoid printing the actual commands to standard output). Especially in the tarball download phase, there are many lines that are printed for each download which was really annoying. We already used '@' in 'high-level.mk' and 'python.mk' before, but now that we also know that 'basic.mk' is called with our custom GNU Make, we can use it at the start for a cleaner stdout. - Until now, WCSLIB assumed a Fortran compiler, but when the user is on a system where we can't install GCC (or has activated the '--host-cc' option), it may not be present and the project shouldn't break because of this. So with this commit, when a Fortran compiler isn't present, WCSLIB will be built with the '--disable-fortran' configuration option. This commit (task #15667) was completed with help/checks by Raul Infante-Sainz and Boud Roukema. --- reproduce/software/make/high-level.mk | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'reproduce/software/make/high-level.mk') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 19cb52f..7cc2d51 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -104,7 +104,8 @@ export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh # Servers to use as backup, later this should go in a file that is not # under version control (the actual server that the tarbal comes from is # irrelevant). -backupservers = http://akhlaghi.org/maneage-software +backupservers := $(shell awk '!/^#/{printf "%s ", $$1}' \ + reproduce/software/config/servers-backup.conf) # Building flags: # @@ -335,6 +336,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) # storing all the tarballs in one directory, we want it to have the # same naming convention, so we'll download it to a temporary name, # then rename that. + rm -f "$@.unchecked" if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* "$@.unchecked" else @@ -812,11 +814,17 @@ $(ibidir)/libgit2: $(ibidir)/curl \ $(ibidir)/wcslib: $(ibidir)/cfitsio \ $(tdir)/wcslib-$(wcslib-version).tar.bz2 + # If Fortran isn't present, don't build WCSLIB with it. + if type gfortran &> /dev/null; then fortranopt=""; + else fortranopt="--disable-fortran" + fi + + # Build WCSLIB. $(call gbuild, wcslib-$(wcslib-version), , \ LIBS="-pthread -lcurl -lm" \ --with-cfitsiolib=$(ildir) \ --with-cfitsioinc=$(idir)/include \ - --without-pgplot) \ + --without-pgplot $$fortranopt) \ && if [ x$(on_mac_os) = xyes ]; then \ install_name_tool -id $(ildir)/libwcs.6.4.dylib \ $(ildir)/libwcs.6.4.dylib; \ -- cgit v1.2.1