From e670140b509d4940ab6435b02cc73d8cfeac821f Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 22 Apr 2019 15:31:20 +0100 Subject: Git, Metastore, cURL and Libbsd moved to basic software Git and Metastore are very basic and fundamental tools for the template, so to keep things clean (let the `high-level.mk' software only represent optional software in the template), these two software (and their dependencies: `cURL' and `Libbsd') will now be built in `basic.mk'. --- reproduce/software/make/basic.mk | 131 ++++++++++++++++++++++++++++++++-- reproduce/software/make/high-level.mk | 114 +---------------------------- 2 files changed, 126 insertions(+), 119 deletions(-) diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 6e6550e..43ccfa1 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -59,8 +59,8 @@ export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) -# Define the top-level programs (that don't depend on any other). -top-level-programs = low-level-links wget gcc +# Define the top-level basic programs (that don't depend on any other). +top-level-programs = low-level-links wget metastore gcc all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) @@ -96,19 +96,23 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \ bzip2-$(bzip2-version).tar.gz \ cert.pem \ coreutils-$(coreutils-version).tar.xz \ + curl-$(curl-version).tar.gz \ diffutils-$(diffutils-version).tar.xz \ file-$(file-version).tar.gz \ findutils-$(findutils-version).tar.lz \ gawk-$(gawk-version).tar.lz \ gcc-$(gcc-version).tar.xz \ + git-$(git-version).tar.xz \ gmp-$(gmp-version).tar.lz \ grep-$(grep-version).tar.xz \ gzip-$(gzip-version).tar.gz \ isl-$(isl-version).tar.bz2 \ + libbsd-$(libbsd-version).tar.xz \ libtool-$(libtool-version).tar.xz \ lzip-$(lzip-version).tar.gz \ m4-$(m4-version).tar.gz \ make-$(make-version).tar.lz \ + metastore-$(metastore-version).tar.gz \ mpfr-$(mpfr-version).tar.xz \ mpc-$(mpc-version).tar.gz \ ncurses-$(ncurses-version).tar.gz \ @@ -139,21 +143,25 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \ elif [ $$n = cert ]; then w=http://akhlaghi.org/src; \ elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils;\ + elif [ $$n = curl ]; then w=https://curl.haxx.se/download; \ elif [ $$n = diffutils ]; then w=http://ftp.gnu.org/gnu/diffutils;\ elif [ $$n = file ]; then w=ftp://ftp.astron.com/pub/file; \ elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \ elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk; \ elif [ $$n = gcc ]; then w=http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version); \ + elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git; \ elif [ $$n = gmp ]; then w=https://gmplib.org/download/gmp; \ elif [ $$n = grep ]; then w=http://ftp.gnu.org/gnu/grep; \ elif [ $$n = gzip ]; then w=http://ftp.gnu.org/gnu/gzip; \ elif [ $$n = isl ]; then w=ftp://gcc.gnu.org/pub/gcc/infrastructure; \ + elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases; \ elif [ $$n = libtool ]; then w=http://ftp.gnu.org/gnu/libtool; \ elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \ elif [ $$n = m ]; then \ mergenames=0; \ w=http://akhlaghi.org/src/m4-1.4.18-patched.tar.gz; \ elif [ $$n = make ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = metastore ]; then w=http://akhlaghi.org/src; \ elif [ $$n = mpfr ]; then w=http://www.mpfr.org/mpfr-current;\ elif [ $$n = mpc ]; then w=http://ftp.gnu.org/gnu/mpc; \ elif [ $$n = ncurses ]; then w=http://ftp.gnu.org/gnu/ncurses; \ @@ -590,11 +598,43 @@ $(ibidir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ -# Downloader -# ---------- +# Downloaders +# ----------- + +# cURL +# +# cURL can optionally link with many different network-related libraries on +# the host system that we are not yet building in the template. Many of +# these are not relevant to most science projects, so we are explicitly +# using `--without-XXX' or `--disable-XXX' so cURL doesn't link with +# them. Note that if it does link with them, the configuration will crash +# when the library is updated/changed by the host, and the whole purpose of +# this project is avoid dependency on the host as much as possible. +$(ibidir)/curl: $(tdir)/curl-$(curl-version).tar.gz \ + $(ilidir)/openssl + $(call gbuild, $<, curl-$(curl-version), , \ + LIBS="-pthread" \ + --with-zlib=$(ildir) \ + --with-ssl=$(idir) \ + --without-mesalink \ + --with-ca-fallback \ + --without-librtmp \ + --without-libidn2 \ + --without-wolfssl \ + --without-brotli \ + --without-gnutls \ + --without-cyassl \ + --without-libpsl \ + --without-axtls \ + --disable-ldaps \ + --disable-ldap \ + --without-nss, V=1) \ + && echo "cURL $(curl-version)" > $@ + +# OpenSSL # -# OpenSSL: Some programs/libraries later need dynamic linking. So we'll -# build libssl (and libcrypto) dynamically also. +# Some programs/libraries later need dynamic linking. So we'll build libssl +# (and libcrypto) dynamically also. # # Until we find a nice and generic way to create an updated CA file in the # project, the certificates will be available in a file for this project @@ -675,6 +715,8 @@ $(ibidir)/wget: $(tdir)/wget-$(wget-version).tar.lz \ + + # Basic command-line tools and their dependencies # ----------------------------------------------- # @@ -724,6 +766,13 @@ $(ibidir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \ fi \ && echo "GNU AWK $(gawk-version)" > $@ +$(ibidir)/git: $(tdir)/git-$(git-version).tar.xz \ + $(ibidir)/curl + $(call gbuild, $<, git-$(git-version), static, \ + --without-tcltk --with-shell=$(ibdir)/bash, \ + V=1) \ + && echo "Git $(git-version)" > $@ + $(ilidir)/gmp: $(tdir)/gmp-$(gmp-version).tar.lz \ $(ibidir)/bash $(call gbuild, $<, gmp-$(gmp-version), static, , , make check) \ @@ -742,11 +791,81 @@ $(ibidir)/grep: $(tdir)/grep-$(grep-version).tar.xz \ $(call gbuild, $<, grep-$(grep-version), static) \ && echo "GNU Grep $(grep-version)" > $@ +$(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz \ + $(ibidir)/bash + $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \ + && echo "Libbsd $(libbsd-version)" > $@ + $(ibidir)/m4: $(tdir)/m4-$(m4-version).tar.gz \ $(ibidir)/bash $(call gbuild, $<, m4-$(m4-version), static) \ && echo "GNU M4 $(m4-version)" > $@ +# Metastore is used (through a Git hook) to restore the source modification +# dates of files after a Git checkout. Another Git hook saves all file +# metadata just before a commit (to allow restoration after a +# checkout). Since this project is managed in Makefiles, file modification +# dates are critical to not having to redo the whole analysis after +# checking out between branches. +# +# Note that we aren't using the standard version of Metastore, but a fork +# of it that is maintained in this repository: +# https://gitlab.com/makhlaghi/metastore-fork +# +# Libbsd is not necessary on macOS systems, because macOS is already a +# BSD-based distribution. But on GNU/Linux systems, it is necessary. +ifeq ($(on_mac_os),yes) +needlibbsd = +else +needlibbsd = $(ilidir)/libbsd +endif +$(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ + $(needlibbsd) \ + $(ibidir)/git + + # The build command below will change the current directory of this + # build, so we'll fix its value here. + current_dir=$$(pwd) + + # Metastore doesn't have any `./configure' script. So we'll just + # call `pwd' as a place-holder for the `./configure' command. + # + # File attributes are also not available on some systems, since the + # main purpose here is modification dates (and not attributes), + # we'll also set the `NO_XATTR' flag. + $(call gbuild, $<, metastore-$(metastore-version), static,, \ + NO_XATTR=1 V=1,,pwd,PREFIX=$(idir)) + + # Write the relevant hooks into this system's Git hooks, so Git + # calls metastore properly on every commit and every checkout. + # + # Note that the -O and -G options used here are currently only in a + # fork of `metastore' currently hosted at: + # https://github.com/mohammad-akhlaghi/metastore + user=$$(whoami) + group=$$(groups | awk '{print $$1}') + cd $$current_dir + if [ -f $(ibdir)/metastore ]; then + for f in pre-commit post-checkout; do + sed -e's|@USER[@]|'$$user'|g' \ + -e's|@GROUP[@]|'$$group'|g' \ + -e's|@BINDIR[@]|$(ibdir)|g' \ + -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ + reproduce/software/bash/git-$$f > .git/hooks/$$f + chmod +x .git/hooks/$$f + echo "Metastore (forked) $(metastore-version)" > $@ + done + else + echo; echo; echo; + echo "*****************" + echo "metastore couldn't be installed!" + echo + echo "Its used for preserving timestamps on Git commits." + echo "Its useful for development, not simple running of the project." + echo "So we won't stop the configuration because it wasn't built." + echo "*****************" + fi + $(ilidir)/mpfr: $(tdir)/mpfr-$(mpfr-version).tar.xz \ $(ilidir)/gmp $(call gbuild, $<, mpfr-$(mpfr-version), static, , , make check) \ diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index cff2ff8..3370bfb 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -60,7 +60,7 @@ ipydir = $(BDIR)/software/installed/version-info/python # Makefile). If that software is needed, just remove the comment `#' to # install it. top-level-libraries = # atlas -top-level-programs = gnuastro metastore # astrometrynet scamp sextractor swarp +top-level-programs = gnuastro # astrometrynet scamp sextractor swarp top-level-python = numpy # astropy astroquery matplotlib scipy all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ @@ -129,22 +129,18 @@ tarballs = $(foreach t, astrometry.net-$(astrometrynet-version).tar.gz \ cdsclient-$(cdsclient-version).tar.gz \ cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ - curl-$(curl-version).tar.gz \ freetype-$(freetype-version).tar.gz \ fftw-$(fftw-version).tar.gz \ ghostscript-$(ghostscript-version).tar.gz \ - git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ gsl-$(gsl-version).tar.gz \ hdf5-$(hdf5-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ lapack-$(lapack-version).tar.gz \ - libbsd-$(libbsd-version).tar.xz \ libpng-$(libpng-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ libxml2-$(libxml2-version).tar.gz \ - metastore-$(metastore-version).tar.gz \ netpbm-$(netpbm-version).tgz \ openmpi-$(openmpi-version).tar.gz \ openblas-$(openblas-version).tar.gz \ @@ -188,7 +184,6 @@ $(tarballs): $(tdir)/%: | $(lockdir) | sed -e's/\./ /' \ | awk '{printf("%d.%d", $$1, $$2)}') w=https://cmake.org/files/v$$majv/cmake-$(cmake-version).tar.gz - elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype elif [ $$n = hdf ]; then @@ -196,19 +191,16 @@ $(tarballs): $(tdir)/%: | $(lockdir) majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$* elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 - elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://ftp.gnu.org/gnu/gnuastro elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files elif [ $$n = lapack ]; then w=http://www.netlib.org/lapack - elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng elif [ $$n = libgit ]; then mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = libxml ]; then w=ftp://xmlsoft.org/libxml2 - elif [ $$n = metastore ]; then w=http://akhlaghi.org/src elif [ $$n = netpbm ]; then mergenames=0 w=https://sourceforge.net/projects/netpbm/files/super_stable/$(netpbm-version)/netpbm-$(netpbm-version).tgz/download @@ -324,10 +316,6 @@ $(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ --enable-fortran, V=1) \ && echo "HDF5 library $(hdf5-version)" > $@ -$(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz - $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \ - && echo "Libbsd $(libbsd-version)" > $@ - $(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz $(call gbuild, $<, jpeg-9b, static) \ && echo "Libjpeg $(libjpeg-version)" > $@ @@ -586,110 +574,10 @@ $(ibidir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ && rm -rf cmake-$(cmake-version) \ && echo "CMake $(cmake-version)" > $@ -# cURL (and its library, which is needed by several programs here) can -# optionally link with many different network-related libraries on the host -# system that we are not yet building in the template. Many of these are -# not relevant to most science projects, so we are explicitly using -# `--without-XXX' or `--disable-XXX' so cURL doesn't link with them. Note -# that if it does link with them, the configuration will crash when the -# library is updated/changed by the host, and the whole purpose of this -# project is avoid dependency on the host as much as possible. -$(ibidir)/curl: $(tdir)/curl-$(curl-version).tar.gz - $(call gbuild, $<, curl-$(curl-version), , \ - LIBS="-pthread" \ - --with-zlib=$(ildir) \ - --with-ssl=$(idir) \ - --without-mesalink \ - --with-ca-fallback \ - --without-librtmp \ - --without-libidn2 \ - --without-wolfssl \ - --without-brotli \ - --without-gnutls \ - --without-cyassl \ - --without-libpsl \ - --without-axtls \ - --disable-ldaps \ - --disable-ldap \ - --without-nss, V=1) \ - && echo "cURL $(curl-version)" > $@ - $(ibidir)/ghostscript: $(tdir)/ghostscript-$(ghostscript-version).tar.gz $(call gbuild, $<, ghostscript-$(ghostscript-version)) \ && echo "GPL Ghostscript $(ghostscript-version)" > $@ -$(ibidir)/git: $(tdir)/git-$(git-version).tar.xz \ - $(ibidir)/curl - $(call gbuild, $<, git-$(git-version), static, \ - --without-tcltk --with-shell=$(ibdir)/bash, \ - V=1) \ - && echo "Git $(git-version)" > $@ - -# Metastore is used (through a Git hook) to restore the source modification -# dates of files after a Git checkout. Another Git hook saves all file -# metadata just before a commit (to allow restoration after a -# checkout). Since this project is managed in Makefiles, file modification -# dates are critical to not having to redo the whole analysis after -# checking out between branches. -# -# Note that we aren't using the standard version of Metastore, but a fork -# of it that is maintained in this repository: -# https://gitlab.com/makhlaghi/metastore-fork -# -# Libbsd is not necessary on macOS systems, because macOS is already a -# BSD-based distribution. But on GNU/Linux systems, it is necessary. -ifeq ($(on_mac_os),yes) -needlibbsd = -else -needlibbsd = $(ilidir)/libbsd -endif -$(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ - $(needlibbsd) \ - $(ibidir)/git - - # The build command below will change the current directory of this - # build, so we'll fix its value here. - current_dir=$$(pwd) - - # Metastore doesn't have any `./configure' script. So we'll just - # call `pwd' as a place-holder for the `./configure' command. - # - # File attributes are also not available on some systems, since the - # main purpose here is modification dates (and not attributes), - # we'll also set the `NO_XATTR' flag. - $(call gbuild, $<, metastore-$(metastore-version), static,, \ - NO_XATTR=1 V=1,,pwd,PREFIX=$(idir)) - - # Write the relevant hooks into this system's Git hooks, so Git - # calls metastore properly on every commit and every checkout. - # - # Note that the -O and -G options used here are currently only in a - # fork of `metastore' currently hosted at: - # https://github.com/mohammad-akhlaghi/metastore - user=$$(whoami) - group=$$(groups | awk '{print $$1}') - cd $$current_dir - if [ -f $(ibdir)/metastore ]; then - for f in pre-commit post-checkout; do - sed -e's|@USER[@]|'$$user'|g' \ - -e's|@GROUP[@]|'$$group'|g' \ - -e's|@BINDIR[@]|$(ibdir)|g' \ - -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ - reproduce/software/bash/git-$$f > .git/hooks/$$f - chmod +x .git/hooks/$$f - echo "Metastore (forked) $(metastore-version)" > $@ - done - else - echo; echo; echo; - echo "*****************" - echo "metastore couldn't be installed!" - echo - echo "Its used for preserving timestamps on Git commits." - echo "Its useful for development, not simple running of the project." - echo "So we won't stop the configuration because it wasn't built." - echo "*****************" - fi - # The order of dependencies is based on how long they take to build (how # large they are): Libgit2 depends on CMake which takes a VERY long time to # build. Also, Ghostscript and GSL are relatively large packages. So when -- cgit v1.2.1