From db97ea9a8f32050f74d8c981a9efa21f9e0dc512 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 14 Nov 2018 01:33:16 +0000 Subject: ./configure and building of Bash and Make with more basic tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a test by Raúl Infante Sainz, we found out that the configure script and the Make script for Bash and Make are making too many assumptions on more recent versions of both. As a result, it couldn't be built. Therefore, the `configure' script was modified to not use more recent tools like `readlink' (to find the absolute address of a relative one). It was also re-organized to not have to read the configuration parameters from a text file. The parameters are directly read from the command-line and are written into the proper file afterwards. This removes the need to opening a text editor by the user (which also caused problems on Raúl's system). To fix the Make version issue, the building of Bash and Make are now done in a new Makefile (`reproduce/src/make/dependencies-basic.mk'). This file doesn't make many of the assumptions that were made in `dependencies.mk'. So it should hopefully work on any version of Make. To help in debugging, for now, the Makefile of configure, are asked to work on one thread (the `-j' option is commented in the `configure'). But after checks, we'll fix this. --- reproduce/config/pipeline/LOCAL.mk.in | 93 +--------------------- reproduce/src/make/dependencies-basic.mk | 128 +++++++++++++++++++++++++++++++ reproduce/src/make/dependencies.mk | 34 +------- 3 files changed, 136 insertions(+), 119 deletions(-) create mode 100644 reproduce/src/make/dependencies-basic.mk (limited to 'reproduce') diff --git a/reproduce/config/pipeline/LOCAL.mk.in b/reproduce/config/pipeline/LOCAL.mk.in index b74d9f6..d6bf2c0 100644 --- a/reproduce/config/pipeline/LOCAL.mk.in +++ b/reproduce/config/pipeline/LOCAL.mk.in @@ -1,89 +1,4 @@ -# Top-level user specific directories. Note the points below: -# -# - The VALUES to these directories are initially JUST PLACE-HOLDERS! -# Please correct them based on your system. -# -# - The directories don't need to necessarily exist. If they do not exist, -# they will be created and the necessary data will be downloaded into -# them. Ofcourse provided that you have write permissions and an internet -# connection. -# -# - Do not use the tilde expansion `~' or variables for your home -# directory. Please use the full address, for example -# `/home/your-user-name'. -# -# - An ending forward-slash `/' is NOT necessary. In the pipeline, all -# these variables will be followed by a `/', so if you put a `/' at the -# end of the value here, you will see a `//' in the printed outputs -# during the processing. This has no technical problem, but can make -# reading the outputs harder and is thus not recommended. - - - - - -# Build directory -# --------------- -# -# This is where the intermediate outputs of each step are kept. -# -# Why a separate build directory? So the source and configuration files for -# this reproduction pipeline do not get crowded by all the -# intermediate/derivative files. Also to make synchronization and backups -# more easy: the contents of the build directory do not need to be backed -# up since they can be reproduced and they can be large. -# -# IMPORTANT NOTE: Keep atleast one blank space before and after `='. -BDIR = /path/of/directory/for/building - - - - - -# (OPTIONAL) Dependencies directory -# --------------------------------- -# -# To ensure an identical build environment, the pipeline will use its own -# build of the programs it needs. Therefore the tarball of the relevant -# programs are necessary for this pipeline. If a tarball isn't present in -# the specified directory, it will be downloaded by the pipeline. Therefore -# an internet connection will be mandatory. -# -# Important note: Keep atleast one blank space before and after `='. -DEPENDENCIES-DIR = /optional/path/to/directory/containing/tarballs - - - - - -# (OPTIONAL) Memory mapping minimum size -# -------------------------------------- -# -# Some programs (for example Gnuastro) can deal with cases where the local -# system doesn't have enough memory (RAM) to keep large files there. For -# example, they will create memory-mapped (`mmap' for short) files on the -# HDD or SSD and read/write to them instead of RAM. This will ofcourse, -# slow down the processing, but atleast the program won't crash. -# -# Since the memory requirements of different systems are different and it -# has no effect on the software's final result, the minimum size of an -# allocated array to warrant a mapping to HDD/SSD instead of RAM must also -# be defined here. This value will be used in the programs that support -# this feature. -MINMAPSIZE = 1000000000 - - - - - -# (OPTIONAL) Downloader program -# ----------------------------- -# -# The downloder program (and its output option name, for example `wget -O' -# or `curl -o') that will be used if any of the necessary datasets aren't -# already available on the system. -# -# This is usually set at an early stage of the configuration system -# automatically before the file is opened for editing by the user. It is -# thus recommended to not modify it manually. -DOWNLOADER = @downloader@ +# Local pipeline configuration. +BDIR = @bdir@ +DEPENDENCIES-DIR = @ddir@ +DOWNLOADER = @downloader@ diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk new file mode 100644 index 0000000..dad9c5f --- /dev/null +++ b/reproduce/src/make/dependencies-basic.mk @@ -0,0 +1,128 @@ +# Build the VERY BASIC reproduction pipeline dependencies (programs and +# libraries). +# +# ------------------------------------------------------------------------ +# !!!!! IMPORTANT NOTES !!!!! +# +# This Makefile will be run by the initial `./configure' script. It is not +# included into the reproduction pipe after that. +# +# This Makefile builds very low-level and basic tools like GNU Bash and GNU +# Make. Therefore this is the only Makefile in the reproduction pipeline +# where you MUST NOT assume that modern GNU Bash or GNU Make are used. +# +# This Makefile is a very simplified version of `dependencies.mk' in the +# same directory. See there for more comments. +# +# ------------------------------------------------------------------------ +# +# Original author: +# Mohammad Akhlaghi +# Contributing author(s): +# Your name +# Copyright (C) 2018, Your Name. +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# . + + +# Top level environment +include reproduce/config/pipeline/LOCAL.mk +include reproduce/config/pipeline/dependency-versions.mk + +ddir = $(BDIR)/dependencies +tdir = $(BDIR)/dependencies/tarballs +idir = $(BDIR)/dependencies/installed +ibdir = $(BDIR)/dependencies/installed/bin +ildir = $(BDIR)/dependencies/installed/lib + +top-level-programs = bash make +all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) + + + + + +# Tarballs +# -------- +# +# Prepare tarballs. Difference with that in `dependencies.mk': `.ONESHELL' +# is not recognized by some versions of Make (even older GNU Makes). So +# we'll have to make sure the recipe doesn't break into multiple shell +# calls (so we can preserve the variables). +tarballs = $(foreach t, bash-$(bash-version).tar.gz \ + make-$(make-version).tar.gz \ + , $(tdir)/$(t) ) +$(tarballs): $(tdir)/%: + if [ -f $(DEPENDENCIES-DIR)/$* ]; then \ + cp $(DEPENDENCIES-DIR)/$* $@; \ + else \ + n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ + | awk '{print $$1}' ); \ + \ + mergenames=1; \ + if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash; \ + elif [ $$n = make ]; then w=http://akhlaghi.org/src; \ + else \ + echo; echo; echo; \ + echo "'$$n' not recognized as a dependency name to download." \ + echo; echo; echo; \ + exit 1; \ + fi; \ + \ + if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*"; \ + else tarballurl=$$w; \ + fi; \ + echo "Downloading $$tarballurl"; \ + $(DOWNLOADER) $@ $$tarballurl; \ + fi + + + + + +# Build system rules +# ------------------ +gbuild = cd $(ddir); rm -rf $(2); tar xf $(tdir)/$(1); cd $(2); \ + if [ $(3)x = staticx ]; then \ + opts="CFLAGS=--static --disable-shared"; \ + fi; \ + ./configure $$opts $(4) --prefix=$(idir); make $(5); \ + check="$(6)"; if [ x"$$check" != x ]; then $$check; fi; \ + make install; cd ..; rm -rf $(2) + + + + + +# GNU Bash +# -------- +# +# Everything is standard and we can make it statically. +$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz + $(call gbuild,$(subst $(tdir),,$<), bash-$(bash-version), static) + + + + + +# GNU Make +# -------- +# +# Unfortunately GNU Make needs dynamic linking in two instances: when +# loading objects (dynamically linked libraries), or when using the +# `getpwnam' function (for tilde expansion). The first can be disabled with +# `--disable-load', but unfortunately I don't know any way to fix the +# second. So, we'll have to build it dynamically for now. +$(ibdir)/make: $(tdir)/make-$(make-version).tar.gz + $(call gbuild,$(subst $(tdir),,$<), make-$(make-version)) diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 1a0a5d1..1e29973 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -6,10 +6,6 @@ # This Makefile will be run by the initial `./configure' script. It is not # included into the reproduction pipe after that. # -# This Makefile also builds GNU Bash and GNU Make. Therefore this is the -# only Makefile in the reproduction pipeline where you MUST NOT assume that -# GNU Bash or GNU Make are to be used. -# # ------------------------------------------------------------------------ # # Original author: @@ -48,17 +44,10 @@ ildir = $(BDIR)/dependencies/installed/lib top-level-programs = ls gawk gs grep libtool sed git astnoisechisel all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) -# This Makefile will be called to also build Bash locally. So when we don't -# have it yet, we'll have to use the system's bash. -ifeq ($(USE_LOCAL_BASH),yes) -SHELL := $(ibdir)/bash -else -SHELL := /bin/sh -endif - # Other basic environment settings. .ONESHELL: -.SHELLFLAGS = -ec +.SHELLFLAGS := -ec +SHELL := $(ibdir)/bash PATH := $(ibdir):$(PATH) LDFLAGS := -L$(ildir) $(LDFLAGS) CPPFLAGS := -I$(idir)/include $(CPPFLAGS) @@ -78,8 +67,7 @@ LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. -tarballs = $(foreach t, bash-$(bash-version).tar.gz \ - cfitsio-$(cfitsio-version).tar.gz \ +tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ coreutils-$(coreutils-version).tar.xz \ curl-$(curl-version).tar.gz \ @@ -94,7 +82,6 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \ libtool-$(libtool-version).tar.gz \ libgit2-$(libgit2-version).tar.gz \ sed-$(sed-version).tar.xz \ - make-$(make-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) @@ -109,8 +96,7 @@ $(tarballs): $(tdir)/%: # Set the top download link of the requested tarball. mergenames=1 - if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash - elif [ $$n = cfitsio ]; then + if [ $$n = cfitsio ]; then mergenames=0 v=$$(echo $(cfitsio-version) | sed -e's/\.//' \ | awk '{l=length($1); \ @@ -133,7 +119,6 @@ $(tarballs): $(tdir)/%: elif [ $$n = libgit ]; then mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz - elif [ $$n = make ]; then w=http://akhlaghi.org/src elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib @@ -239,21 +224,10 @@ $(ildir)/libz.a: $(tdir)/zlib-$(zlib-version).tar.gz # Programs # -------- -$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz - $(call gbuild,$(subst $(tdir),,$<), bash-$(bash-version), static) - $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ $(ibdir)/ls $(call cbuild,$(subst $(tdir),,$<), cmake-$(cmake-version)) -# Unfortunately GNU Make needs dynamic linking in two instances: when -# loading objects (dynamically linked libraries), or when using the -# `getpwnam' function (for tilde expansion). The first can be disabled with -# `--disable-load', but unfortunately I don't know any way to fix the -# second. So, we'll have to build it dynamically for now. -$(ibdir)/make: $(tdir)/make-$(make-version).tar.gz - $(call gbuild,$(subst $(tdir),,$<), make-$(make-version)) - $(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz $(call gbuild,$(subst $(tdir),,$<), coreutils-$(coreutils-version), \ static) -- cgit v1.2.1