From 15d32a7df7b2d4420bebd18b855f83606d872f30 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 21 Jan 2019 19:24:49 +0000 Subject: Metastore package now installed to allow keeping file meta-data The pipeline heavily depends on file meta data (and in particular the modification dates), for example the configuration-Makefiles within the pipeline are set as prerequisites to the rules of the pipeline. However, when Git checks out a branch, it doesn't preserve the meta-data of the files unique to that branch (for example program source files or configuration-Makefiles). As a result, the rules that depend on them will be re-done. This is especially troublesome in the scenario of this reproducible paper project because we commonly need to switch between branches (for example to import recent work in the pipeline into the projects). After some searching, I think the Metastore program is the best solution. Metastore is now built as part of the pipeline and through two Git hooks, it is called by Git to store the original meta-data of files into a binary file that is version controlled (and managed by Metastore). --- .file-metadata | Bin 0 -> 4492 bytes paper.tex | 32 +++++++-------- reproduce/config/pipeline/dependency-versions.mk | 1 + reproduce/src/bash/git-post-checkout | 30 ++++++++++++++ reproduce/src/bash/git-pre-commit | 50 +++++++++++++++++++++++ reproduce/src/make/dependencies-build-rules.mk | 27 ++++++++---- reproduce/src/make/dependencies.mk | 39 ++++++++++++++---- reproduce/src/make/initialize.mk | 3 +- 8 files changed, 147 insertions(+), 35 deletions(-) create mode 100644 .file-metadata create mode 100644 reproduce/src/bash/git-post-checkout create mode 100644 reproduce/src/bash/git-pre-commit diff --git a/.file-metadata b/.file-metadata new file mode 100644 index 0000000..1eb83fa Binary files /dev/null and b/.file-metadata differ diff --git a/paper.tex b/paper.tex index 27589dc..eb0c122 100644 --- a/paper.tex +++ b/paper.tex @@ -213,22 +213,22 @@ Libtool \libtoolversion, GNU Make \makeversion, GNU NCURSES Scientific Library (GSL) \gslversion, GNU Tar \tarversion, GNU Wget \wgetversion, GNU Which \whichversion, Lzip \lzipversion, GPL Ghostscript \ghostscriptversion, Libgit2 \libgitwoversion, Libtiff \libtiffversion, -OpenSSL \opensslversion, Pkg-config \pkgconfigversion, WCSLIB -\wcslibversion, XZ Utils \xzversion, and ZLib \zlibversion. The final paper -was produced with \TeX{} Live \texliveversion, using the following -packages: \TeX{} \textexversion, EC \texecversion, NewTX \texnewtxversion, -Fontaxes \texfontaxesversion, Keyval, \texxkeyvalversion, Etoolbox -\texetoolboxversion, Xcolor \texxcolorversion, Setspace -\texsetspaceversion, Caption \texcaptionversion, Footmisc -\texfootmiscversion, Datetime \texdatetimeversion, Fmtcount -\texfmtcountversion, Titlesec \textitlesecversion, Preprint -\texpreprintversion, Ulem \texulemversion, Bib\LaTeX{} \texbiblatexversion, -Biber \texbiberversion, Logreq \texlogreqversion, PGF/TiKZ \texpgfversion, -PGFPlots \texpgfplotsversion, FP \texfpversion, Courier \texcourierversion, -\TeX-gyre \textexgyreversion, TXFonts \textxfontsversion, Times -\textimesversion. We are very grateful to all their creators for freely -providing this necessary infrastructure. This research would not be -possible without them. +Metastore \metastoreversion, OpenSSL \opensslversion, Pkg-config +\pkgconfigversion, WCSLIB \wcslibversion, XZ Utils \xzversion, and ZLib +\zlibversion. The final paper was produced with \TeX{} Live +\texliveversion, using the following packages: \TeX{} \textexversion, EC +\texecversion, NewTX \texnewtxversion, Fontaxes \texfontaxesversion, +Keyval, \texxkeyvalversion, Etoolbox \texetoolboxversion, Xcolor +\texxcolorversion, Setspace \texsetspaceversion, Caption +\texcaptionversion, Footmisc \texfootmiscversion, Datetime +\texdatetimeversion, Fmtcount \texfmtcountversion, Titlesec +\textitlesecversion, Preprint \texpreprintversion, Ulem \texulemversion, +Bib\LaTeX{} \texbiblatexversion, Biber \texbiberversion, Logreq +\texlogreqversion, PGF/TiKZ \texpgfversion, PGFPlots \texpgfplotsversion, +FP \texfpversion, Courier \texcourierversion, \TeX-gyre \textexgyreversion, +TXFonts \textxfontsversion, Times \textimesversion. We are very grateful to +all their creators for freely providing this necessary infrastructure. This +research would not be possible without them. %% Tell BibLaTeX to put the bibliography list here. \printbibliography diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 9f9306f..86102dd 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -21,6 +21,7 @@ isl-version = 0.18 libtool-version = 2.4.6 lzip-version = 1.20 make-version = 4.2.90 +metastore-version = 1.1.2 mpfr-version = 4.0.1 mpc-version = 1.1.0 ncurses-version = 6.1 diff --git a/reproduce/src/bash/git-post-checkout b/reproduce/src/bash/git-post-checkout new file mode 100644 index 0000000..4ec2fa6 --- /dev/null +++ b/reproduce/src/bash/git-post-checkout @@ -0,0 +1,30 @@ +#!.local/bin/bash +# +# The example hook script to store the metadata information of version +# controlled files (with each commit) using the `metastore' program. +# +# This script is taken from the `examples/hooks/pre-commit' file of the +# `metastore' package (installed within the pipeline, with an MIT +# license). We have just changed the name of the `MSFILE' and also set +# special characters for the installation location of meta-store so our own +# installation is found by Git. + +MSFILE=".file-metadata" + +exit_on_fail() { + "$@" + if [ $? -ne 0 ]; then + echo "Failed to execute: $@" >&2 + exit 1 + fi +} + +if [ ! -e "$MSFILE" ]; then + echo "\"$MSFILE\" missing" >&2 + exit 1 +fi + +exit_on_fail \ + @BINDIR@/metastore -a -m -e -E -q -f "$MSFILE" + +exit 0 diff --git a/reproduce/src/bash/git-pre-commit b/reproduce/src/bash/git-pre-commit new file mode 100644 index 0000000..295c033 --- /dev/null +++ b/reproduce/src/bash/git-pre-commit @@ -0,0 +1,50 @@ +#!.local/bin/bash +# +# The example hook script to store the metadata information of version +# controlled files (with each commit) using the `metastore' program. +# +# This script is taken from the `examples/hooks/pre-commit' file of the +# `metastore' package (installed within the pipeline, with an MIT +# license). We have just changed the name of the `MSFILE' and also set +# special characters for the installation location of meta-store so our own +# installation is found by Git. +# +# WARNING: +# +# If the commit is aborted (e.g. by not entering any synopsis), +# then updated metastore file (.metadata by default) is not reverted, +# so its new version remains in the index. +# To undo any changes in metastore file written since HEAD commit, +# you may want to reset and checkout HEAD version of the file: +# +# git reset HEAD -- .metadata +# git checkout HEAD -- .metadata + +MSFILE=".file-metadata" + +exit_on_fail() { + "$@" + if [ $? -ne 0 ]; then + echo "Failed to execute: $@" >&2 + exit 1 + fi +} + +exit_on_fail \ + @BINDIR@/metastore -s -f "$MSFILE" + +# If it's first metastore commit, store again to include $MSFILE in $MSFILE. +if ! git-ls-tree --name-only HEAD 2>/dev/null | grep -Fqx "$MSFILE"; then + exit_on_fail \ + @BINDIR@/metastore -s -f "$MSFILE" +fi + +if [ ! -e "$MSFILE" ]; then + echo "\"$MSFILE\" missing" >&2 + exit 1 +fi + +exit_on_fail \ + git-add "$MSFILE" + +exit 0 diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk index 1c30a55..0c7262e 100644 --- a/reproduce/src/make/dependencies-build-rules.mk +++ b/reproduce/src/make/dependencies-build-rules.mk @@ -49,10 +49,15 @@ # 5: Extra options/arguments to pass to Make. # 6: Step to run between `make' and `make install': usually `make check'. # 7: The configuration script (`configure' by default). +# 8: Arguments for `make install'. # # NOTE: Unfortunately the configure script of `zlib' doesn't recognize # `SHELL'. So we'll have to remove it from the call to the configure # script. +# +# NOTE: A program might not contain any configure script. In this case, +# we'll just pass a non-relevant function like `pwd'. So SED should be used +# to modify `confscript' or to set `configop'. gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then \ export LDFLAGS="$$LDFLAGS -static"; \ fi; \ @@ -67,19 +72,23 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then \ fi; \ \ if [ -f $(ibdir)/bash ]; then \ - sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ - $$confscript > $$confscript-tmp; \ - mv $$confscript-tmp $$confscript; \ - chmod +x $$confscript; \ + if [ -f $$confscript ]; then \ + sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ + -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ + $$confscript > $$confscript-tmp; \ + mv $$confscript-tmp $$confscript; \ + chmod +x $$confscript; \ + fi; \ shellop="SHELL=$(ibdir)/bash"; \ elif [ -f /bin/bash ]; then shellop="SHELL=/bin/bash"; \ else shellop="SHELL=/bin/sh"; \ fi; \ \ - if [ x"$(strip $(2))" = x"zlib-$(zlib-version)" ]; then \ - configop="--prefix=$(idir)"; \ - else configop="$$shellop --prefix=$(idir)"; \ + if [ -f $$confscript ]; then \ + if [ x"$(strip $(2))" = x"zlib-$(zlib-version)" ]; then \ + configop="--prefix=$(idir)"; \ + else configop="$$shellop --prefix=$(idir)"; \ + fi; \ fi; \ \ echo; echo "Using '$$confscript' to configure:"; echo; \ @@ -87,7 +96,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(3)" = xstatic ]; then \ $$confscript $(4) $$configop && \ make "$$shellop" $(5) && \ $$check && \ - make "$$shellop" install && \ + make "$$shellop" install $(8) && \ cd .. && rm -rf $(2) diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index b4a2ec1..b42968b 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,7 +43,7 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). -top-level-programs = astnoisechisel git flock +top-level-programs = astnoisechisel metastore flock all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) @@ -97,9 +97,10 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ gsl-$(gsl-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ - tiff-$(libtiff-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ + metastore-$(metastore-version).tar.gz \ + tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: @@ -135,6 +136,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = libgit ]; then mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz + elif [ $$n = metastore ]; then w=http://ftp.przemoc.net/pub/software/utils/metastore elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib else @@ -346,6 +348,28 @@ $(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \ --without-tcltk --with-shell=$(ibdir)/bash, \ V=1) +# Metastore is used to keep file modification dates (and generally many +# meta-data) within the Git history. +$(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ + $(ibdir)/git + # Metastore doesn't have any `./configure' script. So we'll just + # call `pwd' as a place-holder for the `./configure' command. + current_dir=$$(pwd) + $(call gbuild, $<, metastore-$(metastore-version), static,,V=1,, \ + pwd, PREFIX=$(idir)) + + # Write the relevant hooks into this system's Git hooks, so Git + # calls metastore properly on every commit and every checkout. + if [ -f $@ ]; then + cd $$current_dir + rm -f .git/hooks/pre-commit .git/hooks/post-checkout + sed -e's|@BINDIR[@]|$(ibdir)|g' \ + reproduce/src/bash/git-pre-commit > .git/hooks/pre-commit + sed -e's|@BINDIR[@]|$(ibdir)|g' \ + reproduce/src/bash/git-post-checkout > .git/hooks/post-checkout + chmod +x .git/hooks/pre-commit .git/hooks/post-checkout + fi + # The order of dependencies is based on how long they take to build (how # large they are): Libgit2 depends on CMake which takes a VERY long time to # build. Also, Ghostscript and GSL are relatively large packages. So when @@ -360,14 +384,11 @@ $(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \ $(ilidir)/libtiff \ $(ilidir)/wcslib ifeq ($(static_build),yes) - $(call gbuild, $<, gnuastro-$(gnuastro-version), static, \ - --enable-static=yes --enable-shared=no, \ - -j$(numthreads), make check -j$(numthreads)) -else - $(call gbuild, $<, gnuastro-$(gnuastro-version), , , \ - -j$(numthreads), make check -j$(numthreads)) + staticopts="--enable-static=yes --enable-shared=no"; endif - + $(call gbuild, $<, gnuastro-$(gnuastro-version), static, \ + $$staticopts, -j$(numthreads), \ + make check -j$(numthreads)) diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 545f113..7aa9f4b 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -248,7 +248,8 @@ $(mtexdir)/initialize.tex: | $(mtexdir) coreutilsversion) $(call pvcheck, lzip, $(lzip-version), Lzip, lzipversion) $(call pvcheck, make, $(make-version), GNU Make, makeversion) - + $(call pvcheck, metastore, $(metastore-version), Metastore, \ + metastoreversion) $(call pvcheck, pkg-config, $(pkgconfig-version), pkg-config, \ pkgconfigversion) $(call pvcheck, sed, $(sed-version), GNU SED, sedversion) -- cgit v1.2.1