aboutsummaryrefslogtreecommitdiff
path: root/reproduce/software/make
diff options
context:
space:
mode:
authorMohammad Akhlaghi <mohammad@akhlaghi.org>2019-04-22 15:31:20 +0100
committerMohammad Akhlaghi <mohammad@akhlaghi.org>2019-04-22 15:31:20 +0100
commite670140b509d4940ab6435b02cc73d8cfeac821f (patch)
treea919d62eb2a440ddcb9dbc9db134e39a30687c66 /reproduce/software/make
parentaf681d01b3837ed205308a959834f6a5aa07307a (diff)
Git, Metastore, cURL and Libbsd moved to basic software
Git and Metastore are very basic and fundamental tools for the template, so to keep things clean (let the `high-level.mk' software only represent optional software in the template), these two software (and their dependencies: `cURL' and `Libbsd') will now be built in `basic.mk'.
Diffstat (limited to 'reproduce/software/make')
-rw-r--r--reproduce/software/make/basic.mk131
-rw-r--r--reproduce/software/make/high-level.mk114
2 files changed, 126 insertions, 119 deletions
diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk
index 6e6550e..43ccfa1 100644
--- a/reproduce/software/make/basic.mk
+++ b/reproduce/software/make/basic.mk
@@ -59,8 +59,8 @@ export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH)
export CPPFLAGS := -I$(idir)/include $(CPPFLAGS)
export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS)
-# Define the top-level programs (that don't depend on any other).
-top-level-programs = low-level-links wget gcc
+# Define the top-level basic programs (that don't depend on any other).
+top-level-programs = low-level-links wget metastore gcc
all: $(foreach p, $(top-level-programs), $(ibidir)/$(p))
@@ -96,19 +96,23 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \
bzip2-$(bzip2-version).tar.gz \
cert.pem \
coreutils-$(coreutils-version).tar.xz \
+ curl-$(curl-version).tar.gz \
diffutils-$(diffutils-version).tar.xz \
file-$(file-version).tar.gz \
findutils-$(findutils-version).tar.lz \
gawk-$(gawk-version).tar.lz \
gcc-$(gcc-version).tar.xz \
+ git-$(git-version).tar.xz \
gmp-$(gmp-version).tar.lz \
grep-$(grep-version).tar.xz \
gzip-$(gzip-version).tar.gz \
isl-$(isl-version).tar.bz2 \
+ libbsd-$(libbsd-version).tar.xz \
libtool-$(libtool-version).tar.xz \
lzip-$(lzip-version).tar.gz \
m4-$(m4-version).tar.gz \
make-$(make-version).tar.lz \
+ metastore-$(metastore-version).tar.gz \
mpfr-$(mpfr-version).tar.xz \
mpc-$(mpc-version).tar.gz \
ncurses-$(ncurses-version).tar.gz \
@@ -139,21 +143,25 @@ $(tarballs): $(tdir)/%: | $(lockdir)
elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \
elif [ $$n = cert ]; then w=http://akhlaghi.org/src; \
elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils;\
+ elif [ $$n = curl ]; then w=https://curl.haxx.se/download; \
elif [ $$n = diffutils ]; then w=http://ftp.gnu.org/gnu/diffutils;\
elif [ $$n = file ]; then w=ftp://ftp.astron.com/pub/file; \
elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \
elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk; \
elif [ $$n = gcc ]; then w=http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version); \
+ elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git; \
elif [ $$n = gmp ]; then w=https://gmplib.org/download/gmp; \
elif [ $$n = grep ]; then w=http://ftp.gnu.org/gnu/grep; \
elif [ $$n = gzip ]; then w=http://ftp.gnu.org/gnu/gzip; \
elif [ $$n = isl ]; then w=ftp://gcc.gnu.org/pub/gcc/infrastructure; \
+ elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases; \
elif [ $$n = libtool ]; then w=http://ftp.gnu.org/gnu/libtool; \
elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \
elif [ $$n = m ]; then \
mergenames=0; \
w=http://akhlaghi.org/src/m4-1.4.18-patched.tar.gz; \
elif [ $$n = make ]; then w=http://akhlaghi.org/src; \
+ elif [ $$n = metastore ]; then w=http://akhlaghi.org/src; \
elif [ $$n = mpfr ]; then w=http://www.mpfr.org/mpfr-current;\
elif [ $$n = mpc ]; then w=http://ftp.gnu.org/gnu/mpc; \
elif [ $$n = ncurses ]; then w=http://ftp.gnu.org/gnu/ncurses; \
@@ -590,11 +598,43 @@ $(ibidir)/bash: $(tdir)/bash-$(bash-version).tar.gz \
-# Downloader
-# ----------
+# Downloaders
+# -----------
+
+# cURL
+#
+# cURL can optionally link with many different network-related libraries on
+# the host system that we are not yet building in the template. Many of
+# these are not relevant to most science projects, so we are explicitly
+# using `--without-XXX' or `--disable-XXX' so cURL doesn't link with
+# them. Note that if it does link with them, the configuration will crash
+# when the library is updated/changed by the host, and the whole purpose of
+# this project is avoid dependency on the host as much as possible.
+$(ibidir)/curl: $(tdir)/curl-$(curl-version).tar.gz \
+ $(ilidir)/openssl
+ $(call gbuild, $<, curl-$(curl-version), , \
+ LIBS="-pthread" \
+ --with-zlib=$(ildir) \
+ --with-ssl=$(idir) \
+ --without-mesalink \
+ --with-ca-fallback \
+ --without-librtmp \
+ --without-libidn2 \
+ --without-wolfssl \
+ --without-brotli \
+ --without-gnutls \
+ --without-cyassl \
+ --without-libpsl \
+ --without-axtls \
+ --disable-ldaps \
+ --disable-ldap \
+ --without-nss, V=1) \
+ && echo "cURL $(curl-version)" > $@
+
+# OpenSSL
#
-# OpenSSL: Some programs/libraries later need dynamic linking. So we'll
-# build libssl (and libcrypto) dynamically also.
+# Some programs/libraries later need dynamic linking. So we'll build libssl
+# (and libcrypto) dynamically also.
#
# Until we find a nice and generic way to create an updated CA file in the
# project, the certificates will be available in a file for this project
@@ -675,6 +715,8 @@ $(ibidir)/wget: $(tdir)/wget-$(wget-version).tar.lz \
+
+
# Basic command-line tools and their dependencies
# -----------------------------------------------
#
@@ -724,6 +766,13 @@ $(ibidir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \
fi \
&& echo "GNU AWK $(gawk-version)" > $@
+$(ibidir)/git: $(tdir)/git-$(git-version).tar.xz \
+ $(ibidir)/curl
+ $(call gbuild, $<, git-$(git-version), static, \
+ --without-tcltk --with-shell=$(ibdir)/bash, \
+ V=1) \
+ && echo "Git $(git-version)" > $@
+
$(ilidir)/gmp: $(tdir)/gmp-$(gmp-version).tar.lz \
$(ibidir)/bash
$(call gbuild, $<, gmp-$(gmp-version), static, , , make check) \
@@ -742,11 +791,81 @@ $(ibidir)/grep: $(tdir)/grep-$(grep-version).tar.xz \
$(call gbuild, $<, grep-$(grep-version), static) \
&& echo "GNU Grep $(grep-version)" > $@
+$(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz \
+ $(ibidir)/bash
+ $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \
+ && echo "Libbsd $(libbsd-version)" > $@
+
$(ibidir)/m4: $(tdir)/m4-$(m4-version).tar.gz \
$(ibidir)/bash
$(call gbuild, $<, m4-$(m4-version), static) \
&& echo "GNU M4 $(m4-version)" > $@
+# Metastore is used (through a Git hook) to restore the source modification
+# dates of files after a Git checkout. Another Git hook saves all file
+# metadata just before a commit (to allow restoration after a
+# checkout). Since this project is managed in Makefiles, file modification
+# dates are critical to not having to redo the whole analysis after
+# checking out between branches.
+#
+# Note that we aren't using the standard version of Metastore, but a fork
+# of it that is maintained in this repository:
+# https://gitlab.com/makhlaghi/metastore-fork
+#
+# Libbsd is not necessary on macOS systems, because macOS is already a
+# BSD-based distribution. But on GNU/Linux systems, it is necessary.
+ifeq ($(on_mac_os),yes)
+needlibbsd =
+else
+needlibbsd = $(ilidir)/libbsd
+endif
+$(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
+ $(needlibbsd) \
+ $(ibidir)/git
+
+ # The build command below will change the current directory of this
+ # build, so we'll fix its value here.
+ current_dir=$$(pwd)
+
+ # Metastore doesn't have any `./configure' script. So we'll just
+ # call `pwd' as a place-holder for the `./configure' command.
+ #
+ # File attributes are also not available on some systems, since the
+ # main purpose here is modification dates (and not attributes),
+ # we'll also set the `NO_XATTR' flag.
+ $(call gbuild, $<, metastore-$(metastore-version), static,, \
+ NO_XATTR=1 V=1,,pwd,PREFIX=$(idir))
+
+ # Write the relevant hooks into this system's Git hooks, so Git
+ # calls metastore properly on every commit and every checkout.
+ #
+ # Note that the -O and -G options used here are currently only in a
+ # fork of `metastore' currently hosted at:
+ # https://github.com/mohammad-akhlaghi/metastore
+ user=$$(whoami)
+ group=$$(groups | awk '{print $$1}')
+ cd $$current_dir
+ if [ -f $(ibdir)/metastore ]; then
+ for f in pre-commit post-checkout; do
+ sed -e's|@USER[@]|'$$user'|g' \
+ -e's|@GROUP[@]|'$$group'|g' \
+ -e's|@BINDIR[@]|$(ibdir)|g' \
+ -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \
+ reproduce/software/bash/git-$$f > .git/hooks/$$f
+ chmod +x .git/hooks/$$f
+ echo "Metastore (forked) $(metastore-version)" > $@
+ done
+ else
+ echo; echo; echo;
+ echo "*****************"
+ echo "metastore couldn't be installed!"
+ echo
+ echo "Its used for preserving timestamps on Git commits."
+ echo "Its useful for development, not simple running of the project."
+ echo "So we won't stop the configuration because it wasn't built."
+ echo "*****************"
+ fi
+
$(ilidir)/mpfr: $(tdir)/mpfr-$(mpfr-version).tar.xz \
$(ilidir)/gmp
$(call gbuild, $<, mpfr-$(mpfr-version), static, , , make check) \
diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk
index cff2ff8..3370bfb 100644
--- a/reproduce/software/make/high-level.mk
+++ b/reproduce/software/make/high-level.mk
@@ -60,7 +60,7 @@ ipydir = $(BDIR)/software/installed/version-info/python
# Makefile). If that software is needed, just remove the comment `#' to
# install it.
top-level-libraries = # atlas
-top-level-programs = gnuastro metastore # astrometrynet scamp sextractor swarp
+top-level-programs = gnuastro # astrometrynet scamp sextractor swarp
top-level-python = numpy # astropy astroquery matplotlib scipy
all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \
$(foreach p, $(top-level-programs), $(ibidir)/$(p)) \
@@ -129,22 +129,18 @@ tarballs = $(foreach t, astrometry.net-$(astrometrynet-version).tar.gz \
cdsclient-$(cdsclient-version).tar.gz \
cfitsio-$(cfitsio-version).tar.gz \
cmake-$(cmake-version).tar.gz \
- curl-$(curl-version).tar.gz \
freetype-$(freetype-version).tar.gz \
fftw-$(fftw-version).tar.gz \
ghostscript-$(ghostscript-version).tar.gz \
- git-$(git-version).tar.xz \
gnuastro-$(gnuastro-version).tar.lz \
gsl-$(gsl-version).tar.gz \
hdf5-$(hdf5-version).tar.gz \
install-tl-unx.tar.gz \
jpegsrc.$(libjpeg-version).tar.gz \
lapack-$(lapack-version).tar.gz \
- libbsd-$(libbsd-version).tar.xz \
libpng-$(libpng-version).tar.xz \
libgit2-$(libgit2-version).tar.gz \
libxml2-$(libxml2-version).tar.gz \
- metastore-$(metastore-version).tar.gz \
netpbm-$(netpbm-version).tgz \
openmpi-$(openmpi-version).tar.gz \
openblas-$(openblas-version).tar.gz \
@@ -188,7 +184,6 @@ $(tarballs): $(tdir)/%: | $(lockdir)
| sed -e's/\./ /' \
| awk '{printf("%d.%d", $$1, $$2)}')
w=https://cmake.org/files/v$$majv/cmake-$(cmake-version).tar.gz
- elif [ $$n = curl ]; then w=https://curl.haxx.se/download
elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw
elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype
elif [ $$n = hdf ]; then
@@ -196,19 +191,16 @@ $(tarballs): $(tdir)/%: | $(lockdir)
majorver=$$(echo $(hdf5-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}')
w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$*
elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926
- elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git
elif [ $$n = gnuastro ]; then w=http://ftp.gnu.org/gnu/gnuastro
elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl
elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet
elif [ $$n = jpegsrc ]; then w=http://ijg.org/files
elif [ $$n = lapack ]; then w=http://www.netlib.org/lapack
- elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases
elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng
elif [ $$n = libgit ]; then
mergenames=0
w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
elif [ $$n = libxml ]; then w=ftp://xmlsoft.org/libxml2
- elif [ $$n = metastore ]; then w=http://akhlaghi.org/src
elif [ $$n = netpbm ]; then
mergenames=0
w=https://sourceforge.net/projects/netpbm/files/super_stable/$(netpbm-version)/netpbm-$(netpbm-version).tgz/download
@@ -324,10 +316,6 @@ $(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \
--enable-fortran, V=1) \
&& echo "HDF5 library $(hdf5-version)" > $@
-$(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz
- $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \
- && echo "Libbsd $(libbsd-version)" > $@
-
$(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz
$(call gbuild, $<, jpeg-9b, static) \
&& echo "Libjpeg $(libjpeg-version)" > $@
@@ -586,110 +574,10 @@ $(ibidir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
&& rm -rf cmake-$(cmake-version) \
&& echo "CMake $(cmake-version)" > $@
-# cURL (and its library, which is needed by several programs here) can
-# optionally link with many different network-related libraries on the host
-# system that we are not yet building in the template. Many of these are
-# not relevant to most science projects, so we are explicitly using
-# `--without-XXX' or `--disable-XXX' so cURL doesn't link with them. Note
-# that if it does link with them, the configuration will crash when the
-# library is updated/changed by the host, and the whole purpose of this
-# project is avoid dependency on the host as much as possible.
-$(ibidir)/curl: $(tdir)/curl-$(curl-version).tar.gz
- $(call gbuild, $<, curl-$(curl-version), , \
- LIBS="-pthread" \
- --with-zlib=$(ildir) \
- --with-ssl=$(idir) \
- --without-mesalink \
- --with-ca-fallback \
- --without-librtmp \
- --without-libidn2 \
- --without-wolfssl \
- --without-brotli \
- --without-gnutls \
- --without-cyassl \
- --without-libpsl \
- --without-axtls \
- --disable-ldaps \
- --disable-ldap \
- --without-nss, V=1) \
- && echo "cURL $(curl-version)" > $@
-
$(ibidir)/ghostscript: $(tdir)/ghostscript-$(ghostscript-version).tar.gz
$(call gbuild, $<, ghostscript-$(ghostscript-version)) \
&& echo "GPL Ghostscript $(ghostscript-version)" > $@
-$(ibidir)/git: $(tdir)/git-$(git-version).tar.xz \
- $(ibidir)/curl
- $(call gbuild, $<, git-$(git-version), static, \
- --without-tcltk --with-shell=$(ibdir)/bash, \
- V=1) \
- && echo "Git $(git-version)" > $@
-
-# Metastore is used (through a Git hook) to restore the source modification
-# dates of files after a Git checkout. Another Git hook saves all file
-# metadata just before a commit (to allow restoration after a
-# checkout). Since this project is managed in Makefiles, file modification
-# dates are critical to not having to redo the whole analysis after
-# checking out between branches.
-#
-# Note that we aren't using the standard version of Metastore, but a fork
-# of it that is maintained in this repository:
-# https://gitlab.com/makhlaghi/metastore-fork
-#
-# Libbsd is not necessary on macOS systems, because macOS is already a
-# BSD-based distribution. But on GNU/Linux systems, it is necessary.
-ifeq ($(on_mac_os),yes)
-needlibbsd =
-else
-needlibbsd = $(ilidir)/libbsd
-endif
-$(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \
- $(needlibbsd) \
- $(ibidir)/git
-
- # The build command below will change the current directory of this
- # build, so we'll fix its value here.
- current_dir=$$(pwd)
-
- # Metastore doesn't have any `./configure' script. So we'll just
- # call `pwd' as a place-holder for the `./configure' command.
- #
- # File attributes are also not available on some systems, since the
- # main purpose here is modification dates (and not attributes),
- # we'll also set the `NO_XATTR' flag.
- $(call gbuild, $<, metastore-$(metastore-version), static,, \
- NO_XATTR=1 V=1,,pwd,PREFIX=$(idir))
-
- # Write the relevant hooks into this system's Git hooks, so Git
- # calls metastore properly on every commit and every checkout.
- #
- # Note that the -O and -G options used here are currently only in a
- # fork of `metastore' currently hosted at:
- # https://github.com/mohammad-akhlaghi/metastore
- user=$$(whoami)
- group=$$(groups | awk '{print $$1}')
- cd $$current_dir
- if [ -f $(ibdir)/metastore ]; then
- for f in pre-commit post-checkout; do
- sed -e's|@USER[@]|'$$user'|g' \
- -e's|@GROUP[@]|'$$group'|g' \
- -e's|@BINDIR[@]|$(ibdir)|g' \
- -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \
- reproduce/software/bash/git-$$f > .git/hooks/$$f
- chmod +x .git/hooks/$$f
- echo "Metastore (forked) $(metastore-version)" > $@
- done
- else
- echo; echo; echo;
- echo "*****************"
- echo "metastore couldn't be installed!"
- echo
- echo "Its used for preserving timestamps on Git commits."
- echo "Its useful for development, not simple running of the project."
- echo "So we won't stop the configuration because it wasn't built."
- echo "*****************"
- fi
-
# The order of dependencies is based on how long they take to build (how
# large they are): Libgit2 depends on CMake which takes a VERY long time to
# build. Also, Ghostscript and GSL are relatively large packages. So when