diff options
| -rwxr-xr-x | configure | 36 | ||||
| -rw-r--r-- | reproduce/config/pipeline/LOCAL.mk.in | 39 | ||||
| -rw-r--r-- | reproduce/config/pipeline/dependency-versions.mk | 7 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies.mk | 203 | ||||
| -rw-r--r-- | reproduce/src/make/initialize.mk | 29 |
5 files changed, 188 insertions, 126 deletions
@@ -76,25 +76,29 @@ echo # Identify the downloader tool # ---------------------------- # -# If cURL is already present, that will be used, otherwise, we'll use -# Wget. Since the options specifying the output filename are different -# between the two, we'll also specify the output option within the -# `downloader' variable. So it is important to first give the output -# filename after calling `DOWNLOADER' within the Makefiles, and finish the -# command with the web address. +# We use Wget for the downloading. As of November 12th, cURL couldn't +# download Ghostscript's source from its standard address, but Wget +# could. Until November 12th, we would check for cURL or Wget, so we +# defined this check system here (explained in the next paragraph of this +# comment). For now its redundant, but if the fix for cURL is found later, +# we can add it back (or add other downloaders). So we'll keep it. +# +# Since the options specifying the output filename are different between +# the two, we'll also specify the output option within the `downloader' +# variable. So it is important to first give the output filename after +# calling `DOWNLOADER' within the Makefiles, and finish the command with +# the web address. print_downloader_notice=1 -if type curl > /dev/null; then - downloader="curl -o" -elif type wget > /dev/null; then +if type wget > /dev/null; then downloader="wget -O"; else echo "=======" echo "Warning" echo "=======" - echo "Couldn't find any of the 'curl' or 'wget' programs. They are used" - echo "for downloading necessary programs and data if they aren't already" - echo "present in the specified directories. Therefore the pipeline will" - echo "crash if the necessary files are not already present on the system." + echo "Couldn't find GNU Wget. It is used for downloading necessary " + echo "programs and data if they aren't already present in the specified " + echo "directories. Therefore the pipeline will crash if the necessary " + echo "files are not already present on the system." echo "=======" echo downloader="no-downloader-found" @@ -256,7 +260,7 @@ echo # Notice on build Make and Bash, build top directories # ---------------------------------------------------- -tsec=10 +tsec=0 echo; echo; echo "----------------"; @@ -268,7 +272,7 @@ echo sleep $tsec export USE_LOCAL_BASH=no bindir=$bdir/dependencies/installed/bin -make -j2 -f reproduce/src/make/dependencies.mk $bindir/bash $bindir/make +make -f reproduce/src/make/dependencies.mk $bindir/bash $bindir/make -j2 @@ -313,7 +317,7 @@ echo "Please run the following command to start." echo echo "(Replace '8' with the number of CPU threads)" echo -echo " ./local/bin/make -j8" +echo " .local/bin/make -j8" echo echo "To change the configuration later, please re-run './configure'," echo "DO NOT manually edit the relevant files." diff --git a/reproduce/config/pipeline/LOCAL.mk.in b/reproduce/config/pipeline/LOCAL.mk.in index 02f8b11..b74d9f6 100644 --- a/reproduce/config/pipeline/LOCAL.mk.in +++ b/reproduce/config/pipeline/LOCAL.mk.in @@ -22,21 +22,6 @@ -# (OPTIONAL) Dependencies directory -# --------------------------------- -# -# To ensure an identical build environment, the pipeline will use its own -# build of the programs it needs. Therefore the tarball of the relevant -# programs are necessary for this pipeline. If a tarball isn't present in -# the specified directory, it will be downloaded by the pipeline. Therefore -# an internet connection will be mandatory. -# -# Important note: Keep atleast one blank space before and after `='. -DEPENDENCIES-DIR = /optional/path/to/directory/containing/tarballs - - - - # Build directory # --------------- # @@ -55,8 +40,24 @@ BDIR = /path/of/directory/for/building -# Memory mapping minimum size -# --------------------------- +# (OPTIONAL) Dependencies directory +# --------------------------------- +# +# To ensure an identical build environment, the pipeline will use its own +# build of the programs it needs. Therefore the tarball of the relevant +# programs are necessary for this pipeline. If a tarball isn't present in +# the specified directory, it will be downloaded by the pipeline. Therefore +# an internet connection will be mandatory. +# +# Important note: Keep atleast one blank space before and after `='. +DEPENDENCIES-DIR = /optional/path/to/directory/containing/tarballs + + + + + +# (OPTIONAL) Memory mapping minimum size +# -------------------------------------- # # Some programs (for example Gnuastro) can deal with cases where the local # system doesn't have enough memory (RAM) to keep large files there. For @@ -75,8 +76,8 @@ MINMAPSIZE = 1000000000 -# Downloader program -# ------------------ +# (OPTIONAL) Downloader program +# ----------------------------- # # The downloder program (and its output option name, for example `wget -O' # or `curl -o') that will be used if any of the necessary datasets aren't diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 3d9d8b3..dac56b1 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -1,15 +1,20 @@ # Versions of the various dependnecies bash-version = 4.4.18 cfitsio-version = 3450 +cmake-version = 3.12.4 coreutils-version = 8.30 +curl-version = 7.62.0 gawk-version = 4.2.1 ghostscript-version = 9.25 -gnuastro-version = 0.7.58-e72a +git-version = 2.19.1 +gnuastro-version = 0.7.60-4c9eb grep-version = 3.1 gsl-version = 2.5 libjpeg-version = v9b libgit2-version = 0.26.0 +libtiff-version = 4.0.10 libtool-version = 2.4.6 make-version = 4.2.90 sed-version = 4.5 wcslib-version = 6.2 +zlib-version = 1.2.11 diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 671d6a5..f4d2e88 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -45,7 +45,7 @@ ildir = $(BDIR)/dependencies/installed/lib # Define the top-level programs to build (installed in `.local/bin', so for # Coreutils, only one of its executables is enough). -top-level-programs = ls gawk gs grep libtool sed astnoisechisel +top-level-programs = ls gawk gs grep libtool sed git astnoisechisel all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # This Makefile will be called to also build Bash locally. So when we don't @@ -74,18 +74,23 @@ LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) # All the necessary tarballs are defined and prepared with this rule. tarballs = $(foreach t, bash-$(bash-version).tar.gz \ cfitsio$(cfitsio-version).tar.gz \ + cmake-$(cmake-version).tar.gz \ coreutils-$(coreutils-version).tar.xz \ + curl-$(curl-version).tar.gz \ gawk-$(gawk-version).tar.gz \ ghostscript-$(ghostscript-version).tar.gz \ - gnuastro-$(gnuastro-version).tar.gz \ + git-$(git-version).tar.xz \ + gnuastro-$(gnuastro-version).tar.lz \ grep-$(grep-version).tar.xz \ gsl-$(gsl-version).tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ + tiff-$(libtiff-version).tar.gz \ libtool-$(libtool-version).tar.gz \ libgit2-$(libgit2-version).tar.gz \ sed-$(sed-version).tar.xz \ make-$(make-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ + zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then @@ -97,23 +102,28 @@ $(tarballs): $(tdir)/%: | awk '{print $$1}' ) # Set the top download link of the requested tarball. - prefix="" + mergenames=1 if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash elif [ $$n = cfitsio ]; then w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c + elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils + elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs925 - elif [ $$n = gnuastro ]; then w=http://akhlaghi.org + elif [ $$n = git ]; then w=https://mirrors.edge.kernel.org/pub/software/scm/git + elif [ $$n = gnuastro ]; then w=http://akhlaghi.org/src elif [ $$n = grep ]; then w=http://ftp.gnu.org/gnu/grep elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl elif [ $$n = jpegsrc ]; then w=http://ijg.org/files elif [ $$n = libtool ]; then w=ftp://ftp.gnu.org/gnu/libtool elif [ $$n = libgit ]; then - w=https://github.com/libgit2/libgit2/archive/v0.27.7.tar.gz - prefix=v + mergenames=0 + w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz + elif [ $$n = make ]; then w=http://akhlaghi.org/src elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed - elif [ $$n = make ]; then w=http://akhlaghi.org + elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib + elif [ $$n = zlib ]; then w=https://www.zlib.net else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." @@ -121,74 +131,111 @@ $(tarballs): $(tdir)/%: exit 1 fi - # Download the requested tarball. - $(DOWNLOADER) $@ $$w/$$prefix$* + # Download the requested tarball. Note that some packages may not + # follow our naming convention (where the package name is merged + # with its version number). In such cases, `w' will be the full + # address, not just the top directory address. But since we are + # storing all the tarballs in one directory, we want it to have + # the same naming convention, so we'll download it to a temporary + # name, then rename that. + if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*" + else tarballurl=$$w + fi + echo "Downloading $$tarballurl" + if [ $$mergenames = 1 ]; then + $(DOWNLOADER) $@ $$tarballurl + else + $(DOWNLOADER) $@_tmp $$tarballurl + mv $@_tmp $@ + fi fi -# Customized build -# ---------------- -# -# Programs that need some customization on their build. -# For CFITSIO we'll need to intervene manually to remove the check on -# libcurl (which can be real trouble in this controlled environment). -$(ildir)/libcfitsio.a: $(ibdir)/ls \ - $(tdir)/cfitsio$(cfitsio-version).tar.gz - # Same as before - cd $(ddir) - tar xf $(tdir)/cfitsio$(cfitsio-version).tar.gz - cd cfitsio - - # Remove the part that checks for the CURL library, so it assumes - # that the CURL library wasn't found. - awk 'NR<4785 || NR>4847' configure > new_configure - mv new_configure configure - chmod +x configure - - # Do the standard configuring and building - ./configure CFLAGS=--static --disable-shared --prefix=$(idir) - make; make install; - cd ..; rm -rf cfitsio - - -# Why not shared: Gnuastro's configure can't link with it in static mode. -$(ildir)/libgit2.a: $(tdir)/libgit2-$(libgit2-version).tar.gz - cd $(ddir) - tar xf $(tdir)/libgit2-$(libgit2-version).tar.gz - cd libgit2-$(libgit2-version) - mkdir build - cd build - export CFLAGS="--static $$CFLAGS" - cmake .. -DUSE_SSH=OFF -DUSE_OPENSSL=OFF -DBUILD_SHARED_LIBS=OFF \ - -DBUILD_CLAR=OFF -DTHREADSAFE=ON - cmake --build . - cmake .. -DCMAKE_INSTALL_PREFIX=$(idir) - cmake --build . --target install - cd ../.. - rm -rf libgit2-$(libgit2-version) - - - - - -# GNU Build system programs -# ------------------------- -# -# Programs that use the basic GNU build system. -gbuild = cd $(ddir); tar xf $(tdir)/$(1); cd $(2); \ - if [ $(3)x = staticx ]; then \ - opts="CFLAGS=--static --disable-shared"; \ - fi; \ - ./configure $$opts $(4) --prefix=$(idir); make $(5); \ - check="$(6)"; if [ x"$$check" != x ]; then $$check; fi; \ +# Build system rules +# ------------------ +gbuild = cd $(ddir); rm -rf $(2); tar xf $(tdir)/$(1); cd $(2); \ + if [ $(3)x = staticx ]; then \ + opts="CFLAGS=--static --disable-shared"; \ + fi; \ + ./configure $$opts $(4) --prefix=$(idir); make $(5); \ + check="$(6)"; if [ x"$$check" != x ]; then $$check; fi; \ make install; cd ..; rm -rf $(2) + +cbuild = cd $(ddir); rm -rf $(2); tar xf $(tdir)/$(1); cd $(2); \ + rm -rf my-build; mkdir my-build; cd my-build; opts=""; \ + if [ $(3)x = staticx ]; then \ + export CFLAGS="--static $$CFLAGS"; \ + opts="-DBUILD_SHARED_LIBS=OFF"; \ + fi; \ + cmake .. $$opts $(4); \ + cmake --build .; \ + cmake .. -DCMAKE_INSTALL_PREFIX=$(idir); \ + cmake --build . --target install; cd ../..; rm -rf $(2) + + + + + +# Libraries +# --------- +$(ildir)/libcfitsio.a: $(tdir)/cfitsio$(cfitsio-version).tar.gz \ + $(ildir)/libcurl.a \ + $(ibdir)/ls + $(call gbuild,$(subst $(tdir),,$<), gsl-$(gsl-version), static, \ + --enable-sse2 --enable-reentrant) + +$(ildir)/libcurl.a: $(tdir)/curl-$(curl-version).tar.gz \ + $(ildir)/libz.a \ + $(ibdir)/ls + $(call gbuild,$(subst $(tdir),,$<), curl-$(curl-version), static, \ + --without-brotli) + +$(ildir)/libgit2.a: $(tdir)/libgit2-$(libgit2-version).tar.gz \ + $(ildir)/libcurl.a \ + $(ibdir)/cmake + $(call cbuild,$(subst $(tdir),,$<), libgit2-$(libgit2-version), \ + static, -DUSE_SSH=OFF -DUSE_OPENSSL=OFF \ + -DBUILD_CLAR=OFF -DTHREADSAFE=ON, ) + +$(ildir)/libgsl.a: $(tdir)/gsl-$(gsl-version).tar.gz \ + $(ibdir)/ls + $(call gbuild,$(subst $(tdir),,$<), gsl-$(gsl-version), static) + +$(ildir)/libjpeg.a: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz + $(call gbuild,$(subst $(tdir),,$<), jpeg-9b, static) + +$(ildir)/libtiff.a: $(tdir)/tiff-$(libtiff-version).tar.gz \ + $(ibdir)/ls + $(call gbuild,$(subst $(tdir),,$<), tiff-$(libtiff-version), static) + +$(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ + $(ildir)/libcfitsio.a + $(call gbuild,$(subst $(tdir),,$<), wcslib-$(wcslib-version), \ + static, LIBS="-pthread -lcurl -lm" --without-pgplot \ + --disable-fortran) + +# Zlib's `./configure' doesn't use Autoconf's configure script, it just +# accepts a direct `--static' option. +$(ildir)/libz.a: $(tdir)/zlib-$(zlib-version).tar.gz + $(call gbuild,$(subst $(tdir),,$<), zlib-$(zlib-version), , \ + --static) + + + + + +# Programs +# -------- $(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz $(call gbuild,$(subst $(tdir),,$<), bash-$(bash-version), static) +$(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ + $(ibdir)/ls + $(call cbuild,$(subst $(tdir),,$<), cmake-$(cmake-version)) # Unfortunately GNU Make needs dynamic linking in two instances: when # loading objects (dynamically linked libraries), or when using the @@ -198,59 +245,41 @@ $(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz $(ibdir)/make: $(tdir)/make-$(make-version).tar.gz $(call gbuild,$(subst $(tdir),,$<), make-$(make-version)) - $(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz $(call gbuild,$(subst $(tdir),,$<), coreutils-$(coreutils-version), \ static) - $(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.gz \ $(ibdir)/ls $(call gbuild,$(subst $(tdir),,$<), gawk-$(gawk-version), static) - $(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \ $(ibdir)/ls $(call gbuild,$(subst $(tdir),,$<), sed-$(sed-version), static) - $(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \ $(ibdir)/ls $(call gbuild,$(subst $(tdir),,$<), grep-$(grep-version), static) - $(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.gz \ $(ibdir)/ls $(call gbuild,$(subst $(tdir),,$<), libtool-$(libtool-version), static) - -$(ildir)/libgsl.a: $(tdir)/gsl-$(gsl-version).tar.gz \ - $(ibdir)/ls - $(call gbuild,$(subst $(tdir),,$<), gsl-$(gsl-version), static) - - -$(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ - $(ildir)/libcfitsio.a - $(call gbuild,$(subst $(tdir),,$<), wcslib-$(wcslib-version), , \ - LIBS="-pthread -lcurl -lm" --without-pgplot \ - --disable-fortran) - - $(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz \ $(ibdir)/ls $(call gbuild,$(subst $(tdir),,$<), ghostscript-$(ghostscript-version)) +$(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \ + $(ibdir)/ls + $(call gbuild,$(subst $(tdir),,$<), git-$(git-version), static) -$(ildir)/libjpeg.a: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz - $(call gbuild,$(subst $(tdir),,$<), jpeg-9b, static) - - -$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.gz \ +$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \ $(ildir)/libgsl.a \ $(ildir)/libcfitsio.a \ $(ildir)/libwcs.a \ $(ibdir)/gs \ $(ildir)/libjpeg.a \ + $(ildir)/libtiff.a \ $(ildir)/libgit2.a \ $(call gbuild,$(subst $(tdir),,$<), gnuastro-$(gnuastro-version), \ diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 165db78..5fd87c9 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -47,6 +47,16 @@ pconfdir = reproduce/config/pipeline +# System's environment +# -------------------- +# +# Before defining the local sub-environment here, we'll need to save the +# system's environment for some scenarios (for example after `clean'ing the +# built programs). +sys-rm := $(shell which rm) + + + # High level environment # ---------------------- # @@ -113,10 +123,23 @@ $(mtexdir) $(texbdir): | $(texdir); mkdir $@ clean-mmap:; rm -f reproduce/config/gnuastro/mmap* # ------------------------------------------ clean: clean-mmap - rm -rf $(BDIR) - rm -f reproduce/build *.pdf *.log *.out *.aux *.auxlock + # Delete the top-level PDF file. + rm -f *.pdf + + # Delete all the built outputs except the dependency + # programs. We'll use Bash's extended options builtin (`shopt') to + # enable "extended glob" (for listing of files). It allows extended + # features like ignoring the listing of a file with `!()' that we + # are using afterwards. + shopt -s extglob + rm -rf $(BDIR)/!(dependencies) distclean: clean - rm -f Makefile $(pconfdir)/LOCAL.mk .gnuastro + # We'll be deleting the built environent programs and just need the + # `rm' program. So for this recipe, we'll use the host system's + # `rm', not our own. + $(sys-rm) -rf $(BDIR) reproduce/build + $(sys-rm) -f Makefile $(pconfdir)/LOCAL.mk .gnuastro .local + |