aboutsummaryrefslogtreecommitdiff
path: root/reproduce
diff options
context:
space:
mode:
authorMohammad Akhlaghi <mohammad@akhlaghi.org>2018-11-26 18:09:35 +0000
committerMohammad Akhlaghi <mohammad@akhlaghi.org>2018-11-26 19:09:49 +0000
commit5bb72e98b312d22191b13f3821f258449e548516 (patch)
tree045e4a33502581208a86f271d3ab39f6ed10ee5f /reproduce
parent00b4893b2829ddb0e9050ae7de6af7cb4d4d8304 (diff)
High-level dependencies build without system's PATH
The high-level dependencies are now built without having access to the system's PATH. To do this, all the necessary software that we aren't building ourselves are now brought into the installed `bin/' directory using a symbolic link to the corresponding software on the host. To do this, it was also necessary to increase the number of basic/low-level packages that we are building, and add several more (Diffutils and Findutils). With this process in place, we now have a list of the exact software packages that we are not building our selves, enabling easy building of all such dependencies in the future.
Diffstat (limited to 'reproduce')
-rw-r--r--reproduce/config/pipeline/dependency-versions.mk2
-rw-r--r--reproduce/src/make/dependencies-basic.mk143
-rw-r--r--reproduce/src/make/dependencies-build-rules.mk2
-rw-r--r--reproduce/src/make/dependencies.mk38
-rw-r--r--reproduce/src/make/initialize.mk10
5 files changed, 129 insertions, 66 deletions
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index e5b4bbb..9c6f18f 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -5,6 +5,8 @@ bash-version = 4.4.18
bzip2-version = 1.0.6
cmake-version = 3.12.4
coreutils-version = 8.30
+diffutils-version = 3.6
+findutils-version = 4.6.0.199-e3fc
flock-version = 0.2.3
gawk-version = 4.2.1
ghostscript-version = 9.26
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index 17bd73f..a3d679e 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -49,6 +49,10 @@ idir = $(BDIR)/dependencies/installed
ibdir = $(BDIR)/dependencies/installed/bin
ildir = $(BDIR)/dependencies/installed/lib
+# We'll need the system's PATH for making links to low-level programs we
+# won't be building ourselves.
+syspath := $(PATH)
+
# As we build more programs, we want to use our own pipeline's built
# programs and libraries, not the host's.
PATH := $(ibdir):$(PATH)
@@ -56,7 +60,7 @@ LDFLAGS := -L$(ildir) $(LDFLAGS)
CPPFLAGS := -I$(idir)/include $(CPPFLAGS)
LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH)
-top-level-programs = bash which ls
+top-level-programs = bash which ls sed gawk grep diff find
all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
@@ -88,9 +92,14 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
tarballs = $(foreach t, bash-$(bash-version).tar.gz \
bzip2-$(bzip2-version).tar.gz \
coreutils-$(coreutils-version).tar.xz \
- gzip-$(gzip-version).tar.gz \
+ diffutils-$(diffutils-version).tar.xz \
+ findutils-$(findutils-version).tar.lz \
+ gawk-$(gawk-version).tar.lz \
+ grep-$(grep-version).tar.xz \
+ gzip-$(gzip-version).tar.gz \
lzip-$(lzip-version).tar.gz \
- make-$(make-version).tar.lz \
+ make-$(make-version).tar.lz \
+ sed-$(sed-version).tar.xz \
tar-$(tar-version).tar.gz \
which-$(which-version).tar.gz \
xz-$(xz-version).tar.gz \
@@ -107,9 +116,14 @@ $(tarballs): $(tdir)/%:
if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash; \
elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \
elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils;\
+ elif [ $$n = diffutils ]; then w=http://ftp.gnu.org/gnu/diffutils;\
+ elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \
+ elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk; \
+ elif [ $$n = grep ]; then w=http://ftp.gnu.org/gnu/grep; \
elif [ $$n = gzip ]; then w=http://akhlaghi.org/src; \
elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \
elif [ $$n = make ]; then w=http://akhlaghi.org/src; \
+ elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed; \
elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \
elif [ $$n = which ]; then w=http://ftp.gnu.org/gnu/which; \
elif [ $$n = xz ]; then w=http://tukaani.org/xz; \
@@ -131,37 +145,75 @@ $(tarballs): $(tdir)/%:
-# GNU Gzip.
-$(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz
- $(call gbuild, $<, gzip-$(gzip-version), static)
+# Low-level (not built) programs
+# ------------------------------
+#
+# For the time being, we aren't building a local C compiler, but we'll use
+# any C compiler that the system already has and just make a symbolic link
+# to it.
+makelink = a=$$(which $(1) 2> /dev/null); \
+ if [ x$$a != x ]; then ln -s $$a $(ibdir)/$(1); fi
+$(ibdir):; mkdir $@
+$(ibdir)/low-level: | $(ibdir)
+ # We aren't building these low-levels tools yet ourselves. We'll
+ # thus just use what the host operating system has available.
+ PATH=$(syspath)
+ # The Assembler
+ $(call makelink,as)
+ # The compiler
+ $(call makelink,clang)
+ $(call makelink,gcc)
+ $(call makelink,g++)
+ $(call makelink,cc)
+ # The linker
+ $(call makelink,ar)
+ $(call makelink,ld)
+ # GNU Gettext (translate messages)
+ $(call makelink,msgfmt)
-# GNU Lzip: For a static build, the `-static' flag should be given to
-# LDFLAGS on the command-line (not from the environment).
-$(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz
-ifeq ($(static_build),yes)
- $(call gbuild, $<, lzip-$(lzip-version), , LDFLAGS="-static")
-else
- $(call gbuild, $<, lzip-$(lzip-version))
-endif
+ # GNU M4 (for managing building macros)
+ $(call makelink,m4)
+ # Needed by TeXLive specifically
+ $(call makelink,perl)
+ $(call makelink,wget)
+ echo "Low-level program links are setup" > $@
-# XZ Utils
-$(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz
- $(call gbuild, $<, xz-$(xz-version), static)
+# Compression programs
+# --------------------
+#
+# The first set of programs to be built are those that we need to unpack
+# the source code tarballs of each program. First, we'll build the
+# necessary programs, then we'll build GNU Tar.
+$(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz \
+ $(ibdir)/low-level
+ $(call gbuild, $<, gzip-$(gzip-version), static)
+# GNU Lzip: For a static build, the `-static' flag should be given to
+# LDFLAGS on the command-line (not from the environment).
+$(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz \
+ $(ibdir)/low-level
+ifeq ($(static_build),yes)
+ $(call gbuild, $<, lzip-$(lzip-version), , LDFLAGS="-static")
+else
+ $(call gbuild, $<, lzip-$(lzip-version))
+endif
+$(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz \
+ $(ibdir)/low-level
+ $(call gbuild, $<, xz-$(xz-version), static)
-# Bzip2: Bzip2 doesn't have a configure script.
-$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
+$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz \
+ $(ibdir)/low-level
tdir=bzip2-$(bzip2-version); \
if [ $(static_build) = yes ]; then \
makecommand="make LDFLAGS=-static"; \
@@ -172,10 +224,6 @@ $(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz
$$makecommand && make install PREFIX=$(idir) && \
cd .. && rm -rf $$tdir
-
-
-
-
# GNU Tar: When built statically, tar gives a segmentation fault on
# unpacking Bash. So we'll build it dynamically.
$(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
@@ -189,9 +237,15 @@ $(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \
-# GNU Make: Unfortunately it needs dynamic linking in two instances: when
-# loading objects (dynamically linked libraries), or when using the
-# `getpwnam' function (for tilde expansion). The first can be disabled with
+# GNU Make
+# --------
+#
+# GNU Make is the second layer that we'll need to build the basic
+# dependencies.
+#
+# Unfortunately it needs dynamic linking in two instances: when loading
+# objects (dynamically linked libraries), or when using the `getpwnam'
+# function (for tilde expansion). The first can be disabled with
# `--disable-load', but unfortunately I don't know any way to fix the
# second. So, we'll have to build it dynamically for now.
$(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \
@@ -202,7 +256,33 @@ $(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \
-# GNU Which:
+# Basic command-line programs necessary in build process of the
+# higher-level dependencies: Note that during the building of those
+# programs, there is no access to the system's PATH.
+$(ibdir)/diff: $(tdir)/diffutils-$(diffutils-version).tar.xz \
+ $(ibdir)/make
+ $(call gbuild, $<, diffutils-$(diffutils-version), static)
+
+$(ibdir)/find: $(tdir)/findutils-$(findutils-version).tar.lz \
+ $(ibdir)/make
+ $(call gbuild, $<, findutils-$(findutils-version), static)
+
+$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \
+ $(ibdir)/make
+ $(call gbuild, $<, gawk-$(gawk-version), static)
+
+$(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \
+ $(ibdir)/make
+ $(call gbuild, $<, grep-$(grep-version), static)
+
+$(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz \
+ $(ibdir)/make
+ $(call gbuild, $<, coreutils-$(coreutils-version), static)
+
+$(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \
+ $(ibdir)/make
+ $(call gbuild, $<, sed-$(sed-version), static)
+
$(ibdir)/which: $(tdir)/which-$(which-version).tar.gz \
$(ibdir)/make
$(call gbuild, $<, which-$(which-version), static)
@@ -234,12 +314,3 @@ endif
# before making the link, we'll see if the file actually exists
# there.
if [ -f $@ ]; then ln -s $@ $(ibdir)/sh; fi
-
-
-
-
-
-# GNU Coreutils
-$(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz \
- $(ibdir)/make
- $(call gbuild, $<, coreutils-$(coreutils-version), static)
diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk
index 25e2444..7b3a419 100644
--- a/reproduce/src/make/dependencies-build-rules.mk
+++ b/reproduce/src/make/dependencies-build-rules.mk
@@ -53,6 +53,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then \
fi; \
check="$(6)"; \
if [ x"$$check" = x ]; then check="echo Skipping-check"; fi; \
+ export SHELL=$(ibdir)/bash; \
cd $(ddir) && rm -rf $(2) && tar xf $(1) && cd $(2) && \
./configure $(4) --prefix=$(idir) && \
make $(5) && \
@@ -70,6 +71,7 @@ cbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then \
export LDFLAGS="$$LDFLAGS -static"; \
opts="-DBUILD_SHARED_LIBS=OFF"; \
fi; \
+ export SHELL=$(ibdir)/bash; \
cd $(ddir) && rm -rf $(2) && tar xf $(1) && cd $(2) && \
rm -rf pipeline-build && mkdir pipeline-build && \
cd pipeline-build && \
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index a784883..c795b34 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -43,7 +43,7 @@ ildir = $(BDIR)/dependencies/installed/lib
ilidir = $(BDIR)/dependencies/installed/lib/built
# Define the top-level programs to build (installed in `.local/bin').
-top-level-programs = gawk gs grep sed git flock astnoisechisel texlive-ready
+top-level-programs = gs git flock astnoisechisel texlive-ready
all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
# Other basic environment settings: We are only including the host
@@ -52,11 +52,11 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
# using our internally built libraries.
.ONESHELL:
.SHELLFLAGS := -ec
-SHELL := $(ibdir)/bash
-PATH := $(ibdir):$(PATH)
+PATH := $(ibdir)
+LD_LIBRARY_PATH := $(ildir)
LDFLAGS := -L$(ildir)
+SHELL := $(ibdir)/bash
CPPFLAGS := -I$(idir)/include
-LD_LIBRARY_PATH := $(ildir)
@@ -76,17 +76,14 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \
cmake-$(cmake-version).tar.gz \
curl-$(curl-version).tar.gz \
flock-$(flock-version).tar.xz \
- gawk-$(gawk-version).tar.lz \
ghostscript-$(ghostscript-version).tar.gz \
git-$(git-version).tar.xz \
gnuastro-$(gnuastro-version).tar.lz \
- grep-$(grep-version).tar.xz \
gsl-$(gsl-version).tar.gz \
jpegsrc.$(libjpeg-version).tar.gz \
tiff-$(libtiff-version).tar.gz \
libtool-$(libtool-version).tar.xz \
libgit2-$(libgit2-version).tar.gz \
- sed-$(sed-version).tar.xz \
wcslib-$(wcslib-version).tar.bz2 \
zlib-$(zlib-version).tar.gz \
, $(tdir)/$(t) )
@@ -113,18 +110,15 @@ $(tarballs): $(tdir)/%:
elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12
elif [ $$n = curl ]; then w=https://curl.haxx.se/download
elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version)
- elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk
elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926
elif [ $$n = git ]; then w=https://mirrors.edge.kernel.org/pub/software/scm/git
elif [ $$n = gnuastro ]; then w=http://akhlaghi.org/src
- elif [ $$n = grep ]; then w=http://ftp.gnu.org/gnu/grep
elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl
elif [ $$n = jpegsrc ]; then w=http://ijg.org/files
elif [ $$n = libtool ]; then w=ftp://ftp.gnu.org/gnu/libtool
elif [ $$n = libgit ]; then
mergenames=0
w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
- elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed
elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff
elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib
elif [ $$n = zlib ]; then w=http://www.zlib.net
@@ -231,15 +225,6 @@ $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz \
$(ilidir)/zlib
$(call gbuild, $<, curl-$(curl-version), static, --without-brotli)
-$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz
- $(call gbuild, $<, gawk-$(gawk-version), static)
-
-$(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz
- $(call gbuild, $<, sed-$(sed-version), static)
-
-$(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz
- $(call gbuild, $<, grep-$(grep-version), static)
-
$(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.xz
$(call gbuild, $<, libtool-$(libtool-version), static)
@@ -251,7 +236,8 @@ $(ibdir)/flock: $(tdir)/flock-$(flock-version).tar.xz
$(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \
$(ilidir)/zlib
- $(call gbuild, $<, git-$(git-version), static)
+ $(call gbuild, $<, git-$(git-version), static, \
+ --without-tcltk --with-shell=$(ibdir)/bash)
$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \
$(ibdir)/gs \
@@ -281,10 +267,9 @@ endif
$(ibdir)/texlive-ready-tlmgr: reproduce/config/pipeline/texlive.conf
# To work with TeX live installation, we'll need the internet.
- if ping -c1 ctan.org; then
- # Download the TeX Live installation tarball.
- $(DOWNLOADER) $(tdir)/install-tl-unx.tar.gz \
- http://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz
+ if $(DOWNLOADER) $(tdir)/install-tl-unx.tar.gz \
+ http://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz; \
+ then
# Unpack, enter the directory, and install based on the given
# configuration (prerequisite of this rule).
@@ -321,9 +306,8 @@ $(ibdir)/texlive-ready: reproduce/config/pipeline/dependency-texlive.mk \
# To work with TeX live installation, we'll need the internet.
res=$(cat $(ibdir)/texlive-ready-tlmgr)
- if ping -c1 ctan.org \
- && [ -f $(ibdir)/texlive-ready-tlmgr ] \
- && [ x$$res != x"NOT" ]; then
+ if [ -f $(ibdir)/texlive-ready-tlmgr ]; then
+
# The current directory is necessary later.
topdir=$$(pwd)
diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk
index 41a5e05..61f28cf 100644
--- a/reproduce/src/make/initialize.mk
+++ b/reproduce/src/make/initialize.mk
@@ -191,18 +191,22 @@ $(mtexdir)/initialize.tex: | $(mtexdir)
@echo "\newcommand{\bdir}{$(BDIR)}" >> $@
# Versions of programs (same order as 'dependency-versions.mk').
+ $(call pvcheck, awk, $(gawk-version), GNU AWK, gawkversion)
$(call pvcheck, bash, $(bash-version), GNU Bash, bashversion)
$(call pvcheck, cmake, $(cmake-version), CMake, cmakeversion)
$(call pvcheck, curl, $(curl-version), cURL, curlversion)
- $(call pvcheck, ls, $(coreutils-version), GNU Coreutils, \
- coreutilsversion)
- $(call pvcheck, awk, $(gawk-version), GNU AWK, gawkversion)
+ $(call pvcheck, diff, $(diffutils-version), GNU Diffutils, \
+ diffutilsversion)
+ $(call pvcheck, find, $(findutils-version), GNU Findutils, \
+ findutilsversion)
$(call pvcheck, gs, $(ghostscript-version), GPL Ghostscript, \
ghostscriptversion)
$(call pvcheck, git, $(git-version), Git, gitversion)
$(call pvcheck, grep, $(grep-version), GNU Grep, grepversion)
$(call pvcheck, libtool, $(libtool-version), GNU Libtool, \
libtoolversion)
+ $(call pvcheck, ls, $(coreutils-version), GNU Coreutils, \
+ coreutilsversion)
$(call pvcheck, lzip, $(lzip-version), Lzip, lzipversion)
$(call pvcheck, make, $(make-version), GNU Make, makeversion)
$(call pvcheck, sed, $(sed-version), GNU SED, sedversion)