From fddd7ee92f2fcbaf5b0880c9077df1302b1cf068 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 14 Jan 2019 01:36:29 +0000 Subject: GNU NCURSES and GNU Readline also built before GNU Bash Readline is a prerequisite of Bash and AWK, while NCURSES is a prerequisite of Readline. With the recent update of GNU Bash (and thus GNU Readline) on my host operating system, the pipeline crashed and I noticed this hole in the pipeline. In particular, AWK (which linked with Readline 7.0) would complain about not finding it and abort. --- paper.tex | 39 ++--- reproduce/config/pipeline/dependency-versions.mk | 2 + reproduce/src/make/dependencies-basic.mk | 199 +++++++++++++++++------ reproduce/src/make/initialize.mk | 4 + 4 files changed, 179 insertions(+), 65 deletions(-) diff --git a/paper.tex b/paper.tex index 4d148e7..499e461 100644 --- a/paper.tex +++ b/paper.tex @@ -208,25 +208,26 @@ research (in alphabetical order): Bzip2 \bziptwoversion, CFITSIO \bashversion, GNU Coreutils \coreutilsversion, GNU Diffutils \diffutilsversion, GNU Findutils \findutilsversion, GNU Grep \grepversion, GNU Gzip \gzipversion, GNU Libtool \libtoolversion, GNU Make \makeversion, -GNU Sed \sedversion, GNU Scientific Library (GSL) \gslversion, GNU Tar -\tarversion, GNU Wget \wgetversion, GNU Which \whichversion, Lzip -\lzipversion, GPL Ghostscript \ghostscriptversion, Libgit2 -\libgitwoversion, Libtiff \libtiffversion, OpenSSL \opensslversion, -Pkg-config \pkgconfigversion, WCSLIB \wcslibversion, XZ Utils \xzversion, -and ZLib \zlibversion. The final paper was produced with \TeX{} Live -\texliveversion, using the following packages: \TeX{} \textexversion, EC -\texecversion, NewTX \texnewtxversion, Fontaxes \texfontaxesversion, -Keyval, \texxkeyvalversion, Etoolbox \texetoolboxversion, Xcolor -\texxcolorversion, Setspace \texsetspaceversion, Caption -\texcaptionversion, Footmisc \texfootmiscversion, Datetime -\texdatetimeversion, Fmtcount \texfmtcountversion, Titlesec -\textitlesecversion, Preprint \texpreprintversion, Ulem \texulemversion, -Bib\LaTeX{} \texbiblatexversion, Biber \texbiberversion, Logreq -\texlogreqversion, PGF/TiKZ \texpgfversion, PGFPlots \texpgfplotsversion, -FP \texfpversion, Courier \texcourierversion, \TeX-gyre \textexgyreversion, -TXFonts \textxfontsversion, Times \textimesversion. We are very grateful to -all their creators for freely providing this necessary infrastructure. This -research would not be possible without them. +GNU NCURSES \ncursesversion, GNU Readline \readlineversion, GNU Sed +\sedversion, GNU Scientific Library (GSL) \gslversion, GNU Tar \tarversion, +GNU Wget \wgetversion, GNU Which \whichversion, Lzip \lzipversion, GPL +Ghostscript \ghostscriptversion, Libgit2 \libgitwoversion, Libtiff +\libtiffversion, OpenSSL \opensslversion, Pkg-config \pkgconfigversion, +WCSLIB \wcslibversion, XZ Utils \xzversion, and ZLib \zlibversion. The +final paper was produced with \TeX{} Live \texliveversion, using the +following packages: \TeX{} \textexversion, EC \texecversion, NewTX +\texnewtxversion, Fontaxes \texfontaxesversion, Keyval, \texxkeyvalversion, +Etoolbox \texetoolboxversion, Xcolor \texxcolorversion, Setspace +\texsetspaceversion, Caption \texcaptionversion, Footmisc +\texfootmiscversion, Datetime \texdatetimeversion, Fmtcount +\texfmtcountversion, Titlesec \textitlesecversion, Preprint +\texpreprintversion, Ulem \texulemversion, Bib\LaTeX{} \texbiblatexversion, +Biber \texbiberversion, Logreq \texlogreqversion, PGF/TiKZ \texpgfversion, +PGFPlots \texpgfplotsversion, FP \texfpversion, Courier \texcourierversion, +\TeX-gyre \textexgyreversion, TXFonts \textxfontsversion, Times +\textimesversion. We are very grateful to all their creators for freely +providing this necessary infrastructure. This research would not be +possible without them. %% Tell BibLaTeX to put the bibliography list here. \printbibliography diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk index 19120d9..6fbdaf7 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/config/pipeline/dependency-versions.mk @@ -23,8 +23,10 @@ lzip-version = 1.20 make-version = 4.2.90 mpfr-version = 4.0.1 mpc-version = 1.1.0 +ncurses-version = 6.1 openssl-version = 1.1.1a pkgconfig-version = 0.29.2 +readline-version = 8.0 sed-version = 4.7 tar-version = 1.31 wget-version = 1.20.1 diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 9ae52a0..d74ac7d 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -64,8 +64,9 @@ export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) +# Define the programs that don't depend on any other. top-level-programs = low-level-links ls sed gawk grep diff find \ - bash wget which + wget which all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) @@ -111,8 +112,10 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \ make-$(make-version).tar.lz \ mpfr-$(mpfr-version).tar.xz \ mpc-$(mpc-version).tar.gz \ + ncurses-$(ncurses-version).tar.gz \ openssl-$(openssl-version).tar.gz \ pkg-config-$(pkgconfig-version).tar.gz \ + readline-$(readline-version).tar.gz \ sed-$(sed-version).tar.xz \ tar-$(tar-version).tar.gz \ wget-$(wget-version).tar.lz \ @@ -146,8 +149,10 @@ $(tarballs): $(tdir)/%: elif [ $$n = make ]; then w=http://akhlaghi.org/src; \ elif [ $$n = mpfr ]; then w=http://www.mpfr.org/mpfr-current;\ elif [ $$n = mpc ]; then w=http://ftpmirror.gnu.org/gnu/mpc;\ + elif [ $$n = ncurses ]; then w=http://ftpmirror.gnu.org/gnu/ncurses;\ elif [ $$n = openssl ]; then w=http://www.openssl.org/source; \ elif [ $$n = pkg ]; then w=http://pkg-config.freedesktop.org/releases; \ + elif [ $$n = readline ]; then w=http://ftpmirror.gnu.org/gnu/readline; \ elif [ $$n = sed ]; then w=http://ftpmirror.gnu.org/gnu/sed;\ elif [ $$n = tar ]; then w=http://ftpmirror.gnu.org/gnu/tar;\ elif [ $$n = wget ]; then w=http://ftpmirror.gnu.org/gnu/wget;\ @@ -253,8 +258,13 @@ $(ibdir)/low-level-links: | $(ibdir) $(ildir) -# Compression programs -# -------------------- + + + + + +# Level 1 (MOST BASIC): Compression programs +# ------------------------------------------ # # The first set of programs to be built are those that we need to unpack # the source code tarballs of each program. First, we'll build the @@ -302,13 +312,18 @@ $(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \ -# GNU Make -# -------- + + + + + +# Level 2 (SECOND MOST BASIC): Bash and Make +# ------------------------------------------ # -# GNU Make is the second layer that we'll need to build the basic -# dependencies. +# GNU Make and GNU Bash are the second layer that we'll need to build the +# basic dependencies. # -# Unfortunately it needs dynamic linking in two instances: when loading +# Unfortunately Make needs dynamic linking in two instances: when loading # objects (dynamically linked libraries), or when using the `getpwnam' # function (for tilde expansion). The first can be disabled with # `--disable-load', but unfortunately I don't know any way to fix the @@ -318,6 +333,120 @@ $(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \ # See Tar's comments for the `-j' option. $(call gbuild, $<, make-$(make-version), , , -j$(numthreads)) +$(ilidir)/ncurses: $(tdir)/ncurses-$(ncurses-version).tar.gz \ + $(ibdir)/make | $(ilidir) + + # Delete the library that will be installed (so we can make sure + # the build process completed afterwards). + if [ x$(on_mac_os) = xyes ]; then rm -f $(ildir)/libncursesw.dylib;\ + else rm -f $(ildir)/libncursesw.so; \ + fi + + # Standard build process. + $(call gbuild, $<, ncurses-$(ncurses-version), static, \ + --with-shared --enable-rpath --without-normal \ + --without-debug --with-cxx-binding \ + --with-cxx-shared --enable-widec --enable-pc-files \ + --with-pkg-config=$(ildir)/pkgconfig ) + + # Unfortunately there are many problems with `ncurses' using + # "normal" (or 8-bit) characters. The standard way that will work + # is to build it with wide character mode as you see above in the + # configuration (or the `w' prefix you see below). Also, most + # programs (and in particular Bash and AWK), first look for other + # (mostly obsolete) libraries like tinfo, which define the same + # symbols. The links below address both situations: we need to fool + # higher-level packages to find this library even if they aren't + # explicitly mentioning its name correctly (as a value to `-l' at + # link time in their configure scripts). + # + # This part is taken from the Arch Linux build script[1], then + # extended to Mac thanks to Homebrew's script [2]. + # + # [1] https://git.archlinux.org/svntogit/packages.git/tree/trunk/PKGBUILD?h=packages/ncurses + # [2] https://github.com/Homebrew/homebrew-core/blob/master/Formula/ncurses.rb + if [ x$(on_mac_os) = xyes ]; then oname=$(ildir)/libncursesw.dylib;\ + else oname=$(ildir)/libncursesw.so; \ + fi; \ + if [ -f $$oname ]; then \ + cd "$(ildir)"; \ + for lib in ncurses ncurses++ form panel menu; do \ + if [ x$(on_mac_os) = xyes ]; then \ + linkname=lib$$lib.dylib; \ + target=lib"$$lib"w.$(ncurses-version).dylib; \ + else \ + linkname=lib$$lib.so; \ + target=lib"$$lib"w.so.$(ncurses-version); \ + fi; \ + ln -fs $$target $$linkname; \ + ln -fs pkgconfig/"$$lib"w.pc pkgconfig/$$lib.pc; \ + done; \ + for lib in tic tinfo; do \ + if [ x$(on_mac_os) = xyes ]; then \ + linka=lib$$lib.dylib; \ + linkb=lib$$lib.$(ncurses-version).dylib; \ + target=libncursesw.$(ncurses-version).dylib; \ + else \ + linka=lib$$lib.so; \ + linkb=lib$$lib.so.$(ncurses-version); \ + target=libncursesw.so.$(ncurses-version); \ + fi; \ + ln -fs $$target $$linka; \ + ln -fs $$target $$linkb; \ + ln -fs pkgconfig/ncursesw.pc pkgconfig/$$lib.pc; \ + done; \ + if [ x$(on_mac_os) = xyes ]; then \ + ln -fs $$target libcurses.dylib; \ + ln -fs $$target libcursesw.dylib; \ + else \ + ln -fs $$target libcurses.so; \ + ln -fs $$target libcursesw.so; \ + fi; \ + ln -fs pkgconfig/ncursesw.pc pkgconfig/curses.pc; \ + ln -fs pkgconfig/ncursesw.pc pkgconfig/cursesw.pc; \ + echo "GNU ncurses is built and ready" > $@; \ + else \ + exit 1; \ + fi + +$(ilidir)/readline: $(tdir)/readline-$(readline-version).tar.gz \ + $(ilidir)/ncurses + $(call gbuild, $<, readline-$(readline-version), static, \ + --with-curses --disable-install-examples, \ + SHLIB_LIBS="-lncursesw" ) && \ + echo "GNU Readline is built and ready" > $@ + +# IMPORTANT: Even though we have enabled `rpath', Bash doesn't write the +# absolute adddress of the libraries it depends on. So if you run `ldd +# $(ibdir)/bash' on the command-line, it will say that it is linking with +# the system's `readline'. But if you run that same command within a rule +# in this reproduction pipeline, you'll see that it is indeed linking with +# our own built readline. +$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ + $(ilidir)/readline + + # Delete any possibly existing output + rm -f $@ + + # Build Bash. +ifeq ($(static_build),yes) + $(call gbuild, $<, bash-$(bash-version), , --enable-static-link \ + --with-installed-readline=$(idir)) +else + $(call gbuild, $<, bash-$(bash-version), , \ + --with-installed-readline=$(idir)) +endif + + # To be generic, some systems use the `sh' command to call the + # shell. By convention, `sh' is just a symbolic link to the + # preferred shell executable. So we'll define `$(ibdir)/sh' as a + # symbolic link to the Bash that we just built and installed. + # + # Just to be sure that the installation step above went well, + # before making the link, we'll see if the file actually exists + # there. + if [ -f $@ ]; then ln -fs $@ $(ibdir)/sh; else exit 1; fi + @@ -333,7 +462,7 @@ $(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \ $(idir)/etc:; mkdir $@ $(ilidir): | $(ildir); mkdir $@ $(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \ - $(ibdir)/make | $(ilidir) + $(ibdir)/bash $(call gbuild, $<, zlib-$(zlib-version)) && echo "Zlib is built" > $@ # OpenSSL: Some programs/libraries later need dynamic linking. So we'll @@ -399,19 +528,20 @@ $(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \ # higher-level dependencies: Note that during the building of those # programs, there is no access to the system's PATH. $(ibdir)/diff: $(tdir)/diffutils-$(diffutils-version).tar.xz \ - $(ibdir)/make + $(ibdir)/bash $(call gbuild, $<, diffutils-$(diffutils-version), static) $(ibdir)/find: $(tdir)/findutils-$(findutils-version).tar.lz \ - $(ibdir)/make + $(ibdir)/bash $(call gbuild, $<, findutils-$(findutils-version), static) $(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \ - $(ibdir)/make - $(call gbuild, $<, gawk-$(gawk-version), static) + $(ibdir)/bash + $(call gbuild, $<, gawk-$(gawk-version), static, \ + --with-readline=$(idir)); $(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \ - $(ibdir)/make + $(ibdir)/bash $(call gbuild, $<, grep-$(grep-version), static) $(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz \ @@ -424,50 +554,22 @@ $(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz \ -j$(numthreads)) $(ibdir)/pkg-config: $(tdir)/pkg-config-$(pkgconfig-version).tar.gz \ - $(ibdir)/make + $(ibdir)/bash $(call gbuild, $<, pkg-config-$(pkgconfig-version), static, \ --with-internal-glib --with-pc-path=$(ildir)/pkgconfig) $(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \ - $(ibdir)/make + $(ibdir)/bash $(call gbuild, $<, sed-$(sed-version), static) $(ibdir)/which: $(tdir)/which-$(which-version).tar.gz \ - $(ibdir)/make + $(ibdir)/bash $(call gbuild, $<, which-$(which-version), static) -# GNU Bash -$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ - $(ibdir)/make - - # Delete any possibly existing output (so it doesn't interfere with - # the build: we are building bash itself!) - if [ -f $@ ]; then rm $@; fi; - - # Build Bash. -ifeq ($(static_build),yes) - $(call gbuild, $<, bash-$(bash-version), , --enable-static-link) -else - $(call gbuild, $<, bash-$(bash-version)) -endif - - # To be generic, some systems use the `sh' command to call the - # shell. By convention, `sh' is just a symbolic link to the - # preferred shell executable. So we'll define `$(ibdir)/sh' as a - # symbolic link to the Bash that we just built and installed. - # - # Just to be sure that the installation step above went well, - # before making the link, we'll see if the file actually exists - # there and remove any possibly existing link that might already be - # there. - if [ -f $(ibdir)/sh ]; then rm $(ibdir)/sh; fi - if [ -f $@ ]; then ln -s $@ $(ibdir)/sh; \ - else echo "Bash not build!"; exit 1; fi - @@ -476,7 +578,7 @@ endif # (CURRENTLY IGNORED) GCC prerequisites # ------------------------------------- $(ilidir)/gmp: $(tdir)/gmp-$(gmp-version).tar.lz \ - $(ibdir)/make | $(ilidir) + $(ibdir)/bash $(call gbuild, $<, gmp-$(gmp-version), static, , , make check) \ && echo "GNU multiple precision arithmetic library is built" > $@ @@ -515,6 +617,11 @@ $(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz \ + + + + + # (CURRENTLY IGNORED) Build GCC # ----------------------------- # diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index a6ad00b..35d365b 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -255,6 +255,10 @@ $(mtexdir)/initialize.tex: | $(mtexdir) libgitwoversion) $(call lvcheck, openssl/opensslv.h, $(openssl-version), OpenSSL, \ opensslversion) + $(call lvcheck, ncursesw/curses.h, $(ncurses-version), GNU NCURSES, \ + ncursesversion) + $(call lvcheck, readline/readline.h, $(readline-version), GNU Readline, \ + readlineversion) $(call lvcheck, tiffvers.h, $(libtiff-version), Libtiff, \ libtiffversion) $(call lvcheck, wcslib/wcsconfig.h, $(wcslib-version), WCSLIB, \ -- cgit v1.2.1