aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMohammad Akhlaghi <mohammad@akhlaghi.org>2018-11-12 20:28:21 +0000
committerMohammad Akhlaghi <mohammad@akhlaghi.org>2018-11-12 20:31:10 +0000
commit393bc1e84addf5c17b056a86c27356eb8d2ea7d9 (patch)
tree2370ea0b65b6c0dc3b9c7ec73dbd17e1ba28842c
parent7d8d0b0855826308b1dc63c7c6f2ac43e45c4a33 (diff)
Libcurl, Git, CMake, TIFF, Zlib also built at configure time
During the configuration step several new programs that were necessary for a more complete controlled environment are now also downloaded and built statically.
-rwxr-xr-xconfigure36
-rw-r--r--reproduce/config/pipeline/LOCAL.mk.in39
-rw-r--r--reproduce/config/pipeline/dependency-versions.mk7
-rw-r--r--reproduce/src/make/dependencies.mk203
-rw-r--r--reproduce/src/make/initialize.mk29
5 files changed, 188 insertions, 126 deletions
diff --git a/configure b/configure
index d9c2ef6..72d7e1c 100755
--- a/configure
+++ b/configure
@@ -76,25 +76,29 @@ echo
# Identify the downloader tool
# ----------------------------
#
-# If cURL is already present, that will be used, otherwise, we'll use
-# Wget. Since the options specifying the output filename are different
-# between the two, we'll also specify the output option within the
-# `downloader' variable. So it is important to first give the output
-# filename after calling `DOWNLOADER' within the Makefiles, and finish the
-# command with the web address.
+# We use Wget for the downloading. As of November 12th, cURL couldn't
+# download Ghostscript's source from its standard address, but Wget
+# could. Until November 12th, we would check for cURL or Wget, so we
+# defined this check system here (explained in the next paragraph of this
+# comment). For now its redundant, but if the fix for cURL is found later,
+# we can add it back (or add other downloaders). So we'll keep it.
+#
+# Since the options specifying the output filename are different between
+# the two, we'll also specify the output option within the `downloader'
+# variable. So it is important to first give the output filename after
+# calling `DOWNLOADER' within the Makefiles, and finish the command with
+# the web address.
print_downloader_notice=1
-if type curl > /dev/null; then
- downloader="curl -o"
-elif type wget > /dev/null; then
+if type wget > /dev/null; then
downloader="wget -O";
else
echo "======="
echo "Warning"
echo "======="
- echo "Couldn't find any of the 'curl' or 'wget' programs. They are used"
- echo "for downloading necessary programs and data if they aren't already"
- echo "present in the specified directories. Therefore the pipeline will"
- echo "crash if the necessary files are not already present on the system."
+ echo "Couldn't find GNU Wget. It is used for downloading necessary "
+ echo "programs and data if they aren't already present in the specified "
+ echo "directories. Therefore the pipeline will crash if the necessary "
+ echo "files are not already present on the system."
echo "======="
echo
downloader="no-downloader-found"
@@ -256,7 +260,7 @@ echo
# Notice on build Make and Bash, build top directories
# ----------------------------------------------------
-tsec=10
+tsec=0
echo;
echo;
echo "----------------";
@@ -268,7 +272,7 @@ echo
sleep $tsec
export USE_LOCAL_BASH=no
bindir=$bdir/dependencies/installed/bin
-make -j2 -f reproduce/src/make/dependencies.mk $bindir/bash $bindir/make
+make -f reproduce/src/make/dependencies.mk $bindir/bash $bindir/make -j2
@@ -313,7 +317,7 @@ echo "Please run the following command to start."
echo
echo "(Replace '8' with the number of CPU threads)"
echo
-echo " ./local/bin/make -j8"
+echo " .local/bin/make -j8"
echo
echo "To change the configuration later, please re-run './configure',"
echo "DO NOT manually edit the relevant files."
diff --git a/reproduce/config/pipeline/LOCAL.mk.in b/reproduce/config/pipeline/LOCAL.mk.in
index 02f8b11..b74d9f6 100644
--- a/reproduce/config/pipeline/LOCAL.mk.in
+++ b/reproduce/config/pipeline/LOCAL.mk.in
@@ -22,21 +22,6 @@
-# (OPTIONAL) Dependencies directory
-# ---------------------------------
-#
-# To ensure an identical build environment, the pipeline will use its own
-# build of the programs it needs. Therefore the tarball of the relevant
-# programs are necessary for this pipeline. If a tarball isn't present in
-# the specified directory, it will be downloaded by the pipeline. Therefore
-# an internet connection will be mandatory.
-#
-# Important note: Keep atleast one blank space before and after `='.
-DEPENDENCIES-DIR = /optional/path/to/directory/containing/tarballs
-
-
-
-
# Build directory
# ---------------
#
@@ -55,8 +40,24 @@ BDIR = /path/of/directory/for/building
-# Memory mapping minimum size
-# ---------------------------
+# (OPTIONAL) Dependencies directory
+# ---------------------------------
+#
+# To ensure an identical build environment, the pipeline will use its own
+# build of the programs it needs. Therefore the tarball of the relevant
+# programs are necessary for this pipeline. If a tarball isn't present in
+# the specified directory, it will be downloaded by the pipeline. Therefore
+# an internet connection will be mandatory.
+#
+# Important note: Keep atleast one blank space before and after `='.
+DEPENDENCIES-DIR = /optional/path/to/directory/containing/tarballs
+
+
+
+
+
+# (OPTIONAL) Memory mapping minimum size
+# --------------------------------------
#
# Some programs (for example Gnuastro) can deal with cases where the local
# system doesn't have enough memory (RAM) to keep large files there. For
@@ -75,8 +76,8 @@ MINMAPSIZE = 1000000000
-# Downloader program
-# ------------------
+# (OPTIONAL) Downloader program
+# -----------------------------
#
# The downloder program (and its output option name, for example `wget -O'
# or `curl -o') that will be used if any of the necessary datasets aren't
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index 3d9d8b3..dac56b1 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -1,15 +1,20 @@
# Versions of the various dependnecies
bash-version = 4.4.18
cfitsio-version = 3450
+cmake-version = 3.12.4
coreutils-version = 8.30
+curl-version = 7.62.0
gawk-version = 4.2.1
ghostscript-version = 9.25
-gnuastro-version = 0.7.58-e72a
+git-version = 2.19.1
+gnuastro-version = 0.7.60-4c9eb
grep-version = 3.1
gsl-version = 2.5
libjpeg-version = v9b
libgit2-version = 0.26.0
+libtiff-version = 4.0.10
libtool-version = 2.4.6
make-version = 4.2.90
sed-version = 4.5
wcslib-version = 6.2
+zlib-version = 1.2.11
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index 671d6a5..f4d2e88 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -45,7 +45,7 @@ ildir = $(BDIR)/dependencies/installed/lib
# Define the top-level programs to build (installed in `.local/bin', so for
# Coreutils, only one of its executables is enough).
-top-level-programs = ls gawk gs grep libtool sed astnoisechisel
+top-level-programs = ls gawk gs grep libtool sed git astnoisechisel
all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
# This Makefile will be called to also build Bash locally. So when we don't
@@ -74,18 +74,23 @@ LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH)
# All the necessary tarballs are defined and prepared with this rule.
tarballs = $(foreach t, bash-$(bash-version).tar.gz \
cfitsio$(cfitsio-version).tar.gz \
+ cmake-$(cmake-version).tar.gz \
coreutils-$(coreutils-version).tar.xz \
+ curl-$(curl-version).tar.gz \
gawk-$(gawk-version).tar.gz \
ghostscript-$(ghostscript-version).tar.gz \
- gnuastro-$(gnuastro-version).tar.gz \
+ git-$(git-version).tar.xz \
+ gnuastro-$(gnuastro-version).tar.lz \
grep-$(grep-version).tar.xz \
gsl-$(gsl-version).tar.gz \
jpegsrc.$(libjpeg-version).tar.gz \
+ tiff-$(libtiff-version).tar.gz \
libtool-$(libtool-version).tar.gz \
libgit2-$(libgit2-version).tar.gz \
sed-$(sed-version).tar.xz \
make-$(make-version).tar.gz \
wcslib-$(wcslib-version).tar.bz2 \
+ zlib-$(zlib-version).tar.gz \
, $(tdir)/$(t) )
$(tarballs): $(tdir)/%:
if [ -f $(DEPENDENCIES-DIR)/$* ]; then
@@ -97,23 +102,28 @@ $(tarballs): $(tdir)/%:
| awk '{print $$1}' )
# Set the top download link of the requested tarball.
- prefix=""
+ mergenames=1
if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash
elif [ $$n = cfitsio ]; then w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c
+ elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12
elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils
+ elif [ $$n = curl ]; then w=https://curl.haxx.se/download
elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk
elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs925
- elif [ $$n = gnuastro ]; then w=http://akhlaghi.org
+ elif [ $$n = git ]; then w=https://mirrors.edge.kernel.org/pub/software/scm/git
+ elif [ $$n = gnuastro ]; then w=http://akhlaghi.org/src
elif [ $$n = grep ]; then w=http://ftp.gnu.org/gnu/grep
elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl
elif [ $$n = jpegsrc ]; then w=http://ijg.org/files
elif [ $$n = libtool ]; then w=ftp://ftp.gnu.org/gnu/libtool
elif [ $$n = libgit ]; then
- w=https://github.com/libgit2/libgit2/archive/v0.27.7.tar.gz
- prefix=v
+ mergenames=0
+ w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
+ elif [ $$n = make ]; then w=http://akhlaghi.org/src
elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed
- elif [ $$n = make ]; then w=http://akhlaghi.org
+ elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff
elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib
+ elif [ $$n = zlib ]; then w=https://www.zlib.net
else
echo; echo; echo;
echo "'$$n' not recognized as a dependency name to download."
@@ -121,74 +131,111 @@ $(tarballs): $(tdir)/%:
exit 1
fi
- # Download the requested tarball.
- $(DOWNLOADER) $@ $$w/$$prefix$*
+ # Download the requested tarball. Note that some packages may not
+ # follow our naming convention (where the package name is merged
+ # with its version number). In such cases, `w' will be the full
+ # address, not just the top directory address. But since we are
+ # storing all the tarballs in one directory, we want it to have
+ # the same naming convention, so we'll download it to a temporary
+ # name, then rename that.
+ if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*"
+ else tarballurl=$$w
+ fi
+ echo "Downloading $$tarballurl"
+ if [ $$mergenames = 1 ]; then
+ $(DOWNLOADER) $@ $$tarballurl
+ else
+ $(DOWNLOADER) $@_tmp $$tarballurl
+ mv $@_tmp $@
+ fi
fi
-# Customized build
-# ----------------
-#
-# Programs that need some customization on their build.
-# For CFITSIO we'll need to intervene manually to remove the check on
-# libcurl (which can be real trouble in this controlled environment).
-$(ildir)/libcfitsio.a: $(ibdir)/ls \
- $(tdir)/cfitsio$(cfitsio-version).tar.gz
- # Same as before
- cd $(ddir)
- tar xf $(tdir)/cfitsio$(cfitsio-version).tar.gz
- cd cfitsio
-
- # Remove the part that checks for the CURL library, so it assumes
- # that the CURL library wasn't found.
- awk 'NR<4785 || NR>4847' configure > new_configure
- mv new_configure configure
- chmod +x configure
-
- # Do the standard configuring and building
- ./configure CFLAGS=--static --disable-shared --prefix=$(idir)
- make; make install;
- cd ..; rm -rf cfitsio
-
-
-# Why not shared: Gnuastro's configure can't link with it in static mode.
-$(ildir)/libgit2.a: $(tdir)/libgit2-$(libgit2-version).tar.gz
- cd $(ddir)
- tar xf $(tdir)/libgit2-$(libgit2-version).tar.gz
- cd libgit2-$(libgit2-version)
- mkdir build
- cd build
- export CFLAGS="--static $$CFLAGS"
- cmake .. -DUSE_SSH=OFF -DUSE_OPENSSL=OFF -DBUILD_SHARED_LIBS=OFF \
- -DBUILD_CLAR=OFF -DTHREADSAFE=ON
- cmake --build .
- cmake .. -DCMAKE_INSTALL_PREFIX=$(idir)
- cmake --build . --target install
- cd ../..
- rm -rf libgit2-$(libgit2-version)
-
-
-
-
-
-# GNU Build system programs
-# -------------------------
-#
-# Programs that use the basic GNU build system.
-gbuild = cd $(ddir); tar xf $(tdir)/$(1); cd $(2); \
- if [ $(3)x = staticx ]; then \
- opts="CFLAGS=--static --disable-shared"; \
- fi; \
- ./configure $$opts $(4) --prefix=$(idir); make $(5); \
- check="$(6)"; if [ x"$$check" != x ]; then $$check; fi; \
+# Build system rules
+# ------------------
+gbuild = cd $(ddir); rm -rf $(2); tar xf $(tdir)/$(1); cd $(2); \
+ if [ $(3)x = staticx ]; then \
+ opts="CFLAGS=--static --disable-shared"; \
+ fi; \
+ ./configure $$opts $(4) --prefix=$(idir); make $(5); \
+ check="$(6)"; if [ x"$$check" != x ]; then $$check; fi; \
make install; cd ..; rm -rf $(2)
+
+cbuild = cd $(ddir); rm -rf $(2); tar xf $(tdir)/$(1); cd $(2); \
+ rm -rf my-build; mkdir my-build; cd my-build; opts=""; \
+ if [ $(3)x = staticx ]; then \
+ export CFLAGS="--static $$CFLAGS"; \
+ opts="-DBUILD_SHARED_LIBS=OFF"; \
+ fi; \
+ cmake .. $$opts $(4); \
+ cmake --build .; \
+ cmake .. -DCMAKE_INSTALL_PREFIX=$(idir); \
+ cmake --build . --target install; cd ../..; rm -rf $(2)
+
+
+
+
+
+# Libraries
+# ---------
+$(ildir)/libcfitsio.a: $(tdir)/cfitsio$(cfitsio-version).tar.gz \
+ $(ildir)/libcurl.a \
+ $(ibdir)/ls
+ $(call gbuild,$(subst $(tdir),,$<), gsl-$(gsl-version), static, \
+ --enable-sse2 --enable-reentrant)
+
+$(ildir)/libcurl.a: $(tdir)/curl-$(curl-version).tar.gz \
+ $(ildir)/libz.a \
+ $(ibdir)/ls
+ $(call gbuild,$(subst $(tdir),,$<), curl-$(curl-version), static, \
+ --without-brotli)
+
+$(ildir)/libgit2.a: $(tdir)/libgit2-$(libgit2-version).tar.gz \
+ $(ildir)/libcurl.a \
+ $(ibdir)/cmake
+ $(call cbuild,$(subst $(tdir),,$<), libgit2-$(libgit2-version), \
+ static, -DUSE_SSH=OFF -DUSE_OPENSSL=OFF \
+ -DBUILD_CLAR=OFF -DTHREADSAFE=ON, )
+
+$(ildir)/libgsl.a: $(tdir)/gsl-$(gsl-version).tar.gz \
+ $(ibdir)/ls
+ $(call gbuild,$(subst $(tdir),,$<), gsl-$(gsl-version), static)
+
+$(ildir)/libjpeg.a: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz
+ $(call gbuild,$(subst $(tdir),,$<), jpeg-9b, static)
+
+$(ildir)/libtiff.a: $(tdir)/tiff-$(libtiff-version).tar.gz \
+ $(ibdir)/ls
+ $(call gbuild,$(subst $(tdir),,$<), tiff-$(libtiff-version), static)
+
+$(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
+ $(ildir)/libcfitsio.a
+ $(call gbuild,$(subst $(tdir),,$<), wcslib-$(wcslib-version), \
+ static, LIBS="-pthread -lcurl -lm" --without-pgplot \
+ --disable-fortran)
+
+# Zlib's `./configure' doesn't use Autoconf's configure script, it just
+# accepts a direct `--static' option.
+$(ildir)/libz.a: $(tdir)/zlib-$(zlib-version).tar.gz
+ $(call gbuild,$(subst $(tdir),,$<), zlib-$(zlib-version), , \
+ --static)
+
+
+
+
+
+# Programs
+# --------
$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz
$(call gbuild,$(subst $(tdir),,$<), bash-$(bash-version), static)
+$(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
+ $(ibdir)/ls
+ $(call cbuild,$(subst $(tdir),,$<), cmake-$(cmake-version))
# Unfortunately GNU Make needs dynamic linking in two instances: when
# loading objects (dynamically linked libraries), or when using the
@@ -198,59 +245,41 @@ $(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz
$(ibdir)/make: $(tdir)/make-$(make-version).tar.gz
$(call gbuild,$(subst $(tdir),,$<), make-$(make-version))
-
$(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz
$(call gbuild,$(subst $(tdir),,$<), coreutils-$(coreutils-version), \
static)
-
$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.gz \
$(ibdir)/ls
$(call gbuild,$(subst $(tdir),,$<), gawk-$(gawk-version), static)
-
$(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \
$(ibdir)/ls
$(call gbuild,$(subst $(tdir),,$<), sed-$(sed-version), static)
-
$(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \
$(ibdir)/ls
$(call gbuild,$(subst $(tdir),,$<), grep-$(grep-version), static)
-
$(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.gz \
$(ibdir)/ls
$(call gbuild,$(subst $(tdir),,$<), libtool-$(libtool-version), static)
-
-$(ildir)/libgsl.a: $(tdir)/gsl-$(gsl-version).tar.gz \
- $(ibdir)/ls
- $(call gbuild,$(subst $(tdir),,$<), gsl-$(gsl-version), static)
-
-
-$(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
- $(ildir)/libcfitsio.a
- $(call gbuild,$(subst $(tdir),,$<), wcslib-$(wcslib-version), , \
- LIBS="-pthread -lcurl -lm" --without-pgplot \
- --disable-fortran)
-
-
$(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz \
$(ibdir)/ls
$(call gbuild,$(subst $(tdir),,$<), ghostscript-$(ghostscript-version))
+$(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \
+ $(ibdir)/ls
+ $(call gbuild,$(subst $(tdir),,$<), git-$(git-version), static)
-$(ildir)/libjpeg.a: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz
- $(call gbuild,$(subst $(tdir),,$<), jpeg-9b, static)
-
-
-$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.gz \
+$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \
$(ildir)/libgsl.a \
$(ildir)/libcfitsio.a \
$(ildir)/libwcs.a \
$(ibdir)/gs \
$(ildir)/libjpeg.a \
+ $(ildir)/libtiff.a \
$(ildir)/libgit2.a \
$(call gbuild,$(subst $(tdir),,$<), gnuastro-$(gnuastro-version), \
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index 165db78..5fd87c9 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -47,6 +47,16 @@ pconfdir = reproduce/config/pipeline
+# System's environment
+# --------------------
+#
+# Before defining the local sub-environment here, we'll need to save the
+# system's environment for some scenarios (for example after `clean'ing the
+# built programs).
+sys-rm := $(shell which rm)
+
+
+
# High level environment
# ----------------------
#
@@ -113,10 +123,23 @@ $(mtexdir) $(texbdir): | $(texdir); mkdir $@
clean-mmap:; rm -f reproduce/config/gnuastro/mmap*
# ------------------------------------------
clean: clean-mmap
- rm -rf $(BDIR)
- rm -f reproduce/build *.pdf *.log *.out *.aux *.auxlock
+ # Delete the top-level PDF file.
+ rm -f *.pdf
+
+ # Delete all the built outputs except the dependency
+ # programs. We'll use Bash's extended options builtin (`shopt') to
+ # enable "extended glob" (for listing of files). It allows extended
+ # features like ignoring the listing of a file with `!()' that we
+ # are using afterwards.
+ shopt -s extglob
+ rm -rf $(BDIR)/!(dependencies)
distclean: clean
- rm -f Makefile $(pconfdir)/LOCAL.mk .gnuastro
+ # We'll be deleting the built environent programs and just need the
+ # `rm' program. So for this recipe, we'll use the host system's
+ # `rm', not our own.
+ $(sys-rm) -rf $(BDIR) reproduce/build
+ $(sys-rm) -f Makefile $(pconfdir)/LOCAL.mk .gnuastro .local
+