Metastore package now installed to allow keeping file meta-data

The pipeline heavily depends on file meta data (and in particular the
modification dates), for example the configuration-Makefiles within the
pipeline are set as prerequisites to the rules of the pipeline.

However, when Git checks out a branch, it doesn't preserve the meta-data of
the files unique to that branch (for example program source files or
configuration-Makefiles). As a result, the rules that depend on them will
be re-done.

This is especially troublesome in the scenario of this reproducible paper
project because we commonly need to switch between branches (for example to
import recent work in the pipeline into the projects). After some
searching, I think the Metastore program is the best solution. Metastore is
now built as part of the pipeline and through two Git hooks, it is called
by Git to store the original meta-data of files into a binary file that is
version controlled (and managed by Metastore).

3 files changed, 50 insertions, 19 deletions

diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk
index 1c30a55..0c7262e 100644
--- a/reproduce/src/make/dependencies-build-rules.mk
+++ b/reproduce/src/make/dependencies-build-rules.mk
@@ -49,10 +49,15 @@
 #  5: Extra options/arguments to pass to Make.
 #  6: Step to run between `make' and `make install': usually `make check'.
 #  7: The configuration script (`configure' by default).
+#  8: Arguments for `make install'.
 #
 # NOTE: Unfortunately the configure script of `zlib' doesn't recognize
 # `SHELL'. So we'll have to remove it from the call to the configure
 # script.
+#
+# NOTE: A program might not contain any configure script. In this case,
+# we'll just pass a non-relevant function like `pwd'. So SED should be used
+# to modify `confscript' or to set `configop'.
 gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then        \
 	   export LDFLAGS="$$LDFLAGS -static";                                \
 	 fi;                                                                  \
@@ -67,19 +72,23 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then        \
 	 fi;                                                                  \
                                                                               \
 	 if   [ -f $(ibdir)/bash ]; then                                      \
-	   sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|'                         \
-	       -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|'                          \
-	       $$confscript > $$confscript-tmp;                               \
-	   mv $$confscript-tmp $$confscript;                                  \
-	   chmod +x $$confscript;                                             \
+	   if [ -f $$confscript ]; then                                       \
+	     sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|'                       \
+	         -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|'                        \
+	         $$confscript > $$confscript-tmp;                             \
+	     mv $$confscript-tmp $$confscript;                                \
+	     chmod +x $$confscript;                                           \
+	   fi;                                                                \
 	   shellop="SHELL=$(ibdir)/bash";                                     \
 	 elif [ -f /bin/bash ]; then shellop="SHELL=/bin/bash";               \
 	 else shellop="SHELL=/bin/sh";                                        \
 	 fi;                                                                  \
                                                                               \
-	 if [ x"$(strip $(2))" = x"zlib-$(zlib-version)" ]; then              \
-	    configop="--prefix=$(idir)";                                      \
-	 else configop="$$shellop --prefix=$(idir)";                          \
+	 if [ -f $$confscript ]; then                                         \
+	   if [ x"$(strip $(2))" = x"zlib-$(zlib-version)" ]; then            \
+	     configop="--prefix=$(idir)";                                     \
+	   else configop="$$shellop --prefix=$(idir)";                        \
+	   fi;                                                                \
 	 fi;                                                                  \
                                                                               \
 	 echo; echo "Using '$$confscript' to configure:"; echo;               \
@@ -87,7 +96,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then        \
 	 $$confscript $(4) $$configop  &&                                     \
 	 make "$$shellop" $(5) &&                                             \
 	 $$check &&                                                           \
-	 make "$$shellop" install &&                                          \
+	 make "$$shellop" install $(8) &&                                     \
 	 cd .. && rm -rf $(2)
 
 
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index b4a2ec1..b42968b 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -43,7 +43,7 @@ ildir  = $(BDIR)/dependencies/installed/lib
 ilidir = $(BDIR)/dependencies/installed/lib/built
 
 # Define the top-level programs to build (installed in `.local/bin').
-top-level-programs = astnoisechisel git flock
+top-level-programs = astnoisechisel metastore flock
 all: $(ddir)/texlive-versions.tex \
      $(foreach p, $(top-level-programs), $(ibdir)/$(p))
 
@@ -97,9 +97,10 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz             \
 	                gsl-$(gsl-version).tar.gz                     \
                         install-tl-unx.tar.gz                         \
 	                jpegsrc.$(libjpeg-version).tar.gz             \
-                        tiff-$(libtiff-version).tar.gz                \
                         libtool-$(libtool-version).tar.xz             \
                         libgit2-$(libgit2-version).tar.gz             \
+                        metastore-$(metastore-version).tar.gz         \
+                        tiff-$(libtiff-version).tar.gz                \
 	                wcslib-$(wcslib-version).tar.bz2              \
                       , $(tdir)/$(t) )
 $(tarballs): $(tdir)/%:
@@ -135,6 +136,7 @@ $(tarballs): $(tdir)/%:
 	  elif [ $$n = libgit      ]; then
 	    mergenames=0
 	    w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
+	  elif [ $$n = metastore   ]; then w=http://ftp.przemoc.net/pub/software/utils/metastore
 	  elif [ $$n = tiff        ]; then w=https://download.osgeo.org/libtiff
 	  elif [ $$n = wcslib      ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib
 	  else
@@ -346,6 +348,28 @@ $(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \
                        --without-tcltk --with-shell=$(ibdir)/bash, \
 	               V=1)
 
+# Metastore is used to keep file modification dates (and generally many
+# meta-data) within the Git history.
+$(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
+                    $(ibdir)/git
+        # Metastore doesn't have any `./configure' script. So we'll just
+        # call `pwd' as a place-holder for the `./configure' command.
+	current_dir=$$(pwd)
+	$(call gbuild, $<, metastore-$(metastore-version), static,,V=1,, \
+	               pwd, PREFIX=$(idir))
+
+        # Write the relevant hooks into this system's Git hooks, so Git
+        # calls metastore properly on every commit and every checkout.
+	if [ -f $@ ]; then
+	  cd $$current_dir
+	  rm -f .git/hooks/pre-commit .git/hooks/post-checkout
+	  sed -e's|@BINDIR[@]|$(ibdir)|g' \
+	      reproduce/src/bash/git-pre-commit    > .git/hooks/pre-commit
+	  sed -e's|@BINDIR[@]|$(ibdir)|g' \
+	      reproduce/src/bash/git-post-checkout > .git/hooks/post-checkout
+	  chmod +x .git/hooks/pre-commit .git/hooks/post-checkout
+	fi
+
 # The order of dependencies is based on how long they take to build (how
 # large they are): Libgit2 depends on CMake which takes a VERY long time to
 # build. Also, Ghostscript and GSL are relatively large packages. So when
@@ -360,14 +384,11 @@ $(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \
                          $(ilidir)/libtiff \
                          $(ilidir)/wcslib
 ifeq ($(static_build),yes)
-	$(call gbuild, $<, gnuastro-$(gnuastro-version), static,     \
-	               --enable-static=yes --enable-shared=no,       \
-	               -j$(numthreads), make check -j$(numthreads))
-else
-	$(call gbuild, $<, gnuastro-$(gnuastro-version), , ,         \
-	               -j$(numthreads), make check -j$(numthreads))
+	staticopts="--enable-static=yes --enable-shared=no";
 endif
-
+	$(call gbuild, $<, gnuastro-$(gnuastro-version), static,     \
+	               $$staticopts, -j$(numthreads),                \
+	               make check -j$(numthreads))
 
 
 
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index 545f113..7aa9f4b 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -248,7 +248,8 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
 	                coreutilsversion)
 	$(call pvcheck, lzip, $(lzip-version), Lzip, lzipversion)
 	$(call pvcheck, make, $(make-version), GNU Make, makeversion)
-
+	$(call pvcheck, metastore, $(metastore-version), Metastore,    \
+	                metastoreversion)
 	$(call pvcheck, pkg-config, $(pkgconfig-version), pkg-config,  \
 	                pkgconfigversion)
 	$(call pvcheck, sed, $(sed-version), GNU SED, sedversion)