From 2a3304cfb832d2f9548bb869f5a44a3412a1c3f9 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 2 Dec 2018 01:25:57 +0000 Subject: Wget and OpenSSL now installed as a basic dependency The TeX Live installer needs Wget to operate smoothly, especially on recent Mac OS systems that don't have Wget pre-installed. Also, it would be good for the pipeline to have its own downloader. So with this commit, the pipeline also installs Wget and OpenSSL which is a dependency. Many other small changes/fixes were done in this process. --- configure | 36 ++++++- paper.tex | 37 +++---- reproduce/config/pipeline/dependency-versions.mk | 2 + reproduce/src/make/dependencies-basic.mk | 126 +++++++++++++++++------ reproduce/src/make/dependencies-build-rules.mk | 15 ++- reproduce/src/make/dependencies.mk | 70 +++++-------- reproduce/src/make/initialize.mk | 6 +- 7 files changed, 188 insertions(+), 104 deletions(-) diff --git a/configure b/configure index c3c11f7..c660e82 100755 --- a/configure +++ b/configure @@ -566,7 +566,32 @@ rm -f $oprog $cprog -# Inform the user that the build process is starting +# See if we need the dynamic-linker (-ldl) +# ---------------------------------------- +# +# Some programs (like Wget) need dynamic loading (using `libdl'). On +# GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But +# Mac OS doesn't need any explicit calling. So we'll check here to use in +# the building of programs. +oprog=$depdir/ldl-test +cprog=$depdir/ldl-test.c +cat > $cprog < +#include +int +main(void) { + void *handle=dlopen ("/lib/CEDD_LIB.so.6", RTLD_LAZY); + return 0; +} +EOF +if gcc $cprog -o$oprog &> /dev/null; then needs_ldl=no; else needs_ldl=yes; fi +rm -f $oprog $cprog + + + + + +# inform the user that the build process is starting # ------------------------------------------------- if [ $printnotice = yes ]; then tsec=10 @@ -603,8 +628,13 @@ fi if which nproc > /dev/null 2>/dev/null; then numthreads=$(nproc --all); else numthreads=2; fi +####################################### +#static_build=no +####################################### make -f reproduce/src/make/dependencies-basic.mk \ - static_build=$static_build -j$numthreads + static_build=$static_build \ + needs_ldl=$needs_ldl \ + -j$numthreads @@ -637,7 +667,7 @@ numthreads=$($instdir/bin/nproc) # and it is only necessary to build the PDF. So we don't want to stop the # pipeline if its not present. texlive_result=$(cat $instdir/bin/texlive-ready-tlmgr) -if [ x$texlive_result = x"NOT!" ]; then +if [ x"$texlive_result" = x"NOT!" ]; then cat < /dev/null); \ if [ x$$a != x ]; then ln -s $$a $(ibdir)/$(1); fi -$(ibdir):; mkdir $@ -$(ibdir)/low-level: | $(ibdir) +$(ibdir) $(ildir):; mkdir $@ +$(ibdir)/low-level: | $(ibdir) $(ildir) # The Assembler $(call makelink,as) @@ -202,11 +212,12 @@ $(ibdir)/low-level: | $(ibdir) # Needed by TeXLive specifically. $(call makelink,perl) - # Downloaders. - $(call makelink,wget) - $(call makelink,curl) + # Libdl (for dynamic loading libraries at runtime) + if [ -f /usr/lib/libdl.a ]; then \ + ln -s /usr/lib/libdl.* $(ildir)/; \ + fi; - echo "Low-level program links are setup" > $@ + echo "Low-level symbolic links are setup" > $@ @@ -280,6 +291,57 @@ $(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \ +# Downloader +# ---------- +# +# Zlib's `./configure' doesn't use Autoconf's configure script, it just +# accepts a direct `--static' option. +$(idir)/etc:; mkdir $@ +$(ilidir): | $(ildir); mkdir $@ +$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \ + $(ibdir)/make | $(ilidir) + # IMPORTANT, the second argument to `gbuild', must not have any + # spaces before or after it: it is going to be checked. +ifeq ($(static_build),yes) + $(call gbuild, $<,zlib-$(zlib-version), , --static) \ + && echo "Zlib is built" > $@ +else + $(call gbuild, $<,zlib-$(zlib-version)) && echo "Zlib is built" > $@ +endif + +# OpenSSL +ifeq ($(static_build),yes) +openssl-static = no-dso no-dynamic-engine no-shared +endif +$(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ + $(ilidir)/zlib | $(idir)/etc + $(call gbuild, $<, openssl-$(openssl-version), static, \ + --openssldir=$(idir)/etc/ssl \ + --with-zlib-lib=$(ildir) \ + --with-zlib-include=$(idir)/include zlib \ + $(openssl-static) ) \ + && echo "OpenSSL is built" > $@ + +# GNU Wget +# +# Note that on some systems (for example GNU/Linux) Wget needs to explicity +# link with `libdl', but on others (for example Mac OS) it doesn't. We +# check this at configure time and define the `needs_ldl' variable. +# +# Also note that since Wget needs to load outside libraries dynamically, it +# gives a segmentation fault when built statically. +$(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \ + $(ilidir)/openssl + libs="-pthread"; \ + if [ x$(needs_ldl) = xyes ]; then libs="$$libs -ldl"; fi; \ + $(call gbuild, $<, wget-$(wget-version), , \ + LIBS="$$LIBS $$libs" --with-ssl=openssl \ + --with-openssl=yes) + + + + + # Basic command-line programs necessary in build process of the # higher-level dependencies: Note that during the building of those # programs, there is no access to the system's PATH. @@ -350,8 +412,6 @@ endif # (CURRENTLY IGNORED) GCC prerequisites # ------------------------------------- -$(ildir): | $(idir); mkdir $@ -$(ilidir): | $(ildir); mkdir $@ $(ilidir)/gmp: $(tdir)/gmp-$(gmp-version).tar.lz \ $(ibdir)/make | $(ilidir) $(call gbuild, $<, gmp-$(gmp-version), static, , , make check) \ diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk index f613975..d1cc5dc 100644 --- a/reproduce/src/make/dependencies-build-rules.mk +++ b/reproduce/src/make/dependencies-build-rules.mk @@ -61,11 +61,16 @@ gbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then \ if ! tar xf $(1); then echo; echo "Tar error"; exit 1; fi; \ cd $(2); \ \ + if [ -f configure ]; then confscript=configure; \ + elif [ -f config ]; then confscript=config; \ + fi; \ + \ if [ -f $(ibdir)/bash ]; then \ - sed configure -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|'> configure-t; \ - mv configure-t configure; \ - chmod +x configure; \ + sed $$confscript -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ + -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ + > tmp-$$confscript; \ + mv tmp-$$confscript $$confscript; \ + chmod +x $$confscript; \ shellop="SHELL=$(ibdir)/bash"; \ elif [ -f /bin/bash ]; then shellop="SHELL=/bin/bash"; \ else shellop="SHELL=/bin/sh"; \ @@ -76,7 +81,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then \ else configop="$$shellop --prefix=$(idir)"; \ fi; \ \ - ./configure $(4) $$configop && \ + ./$$confscript $(4) $$configop && \ make "$$shellop" $(5) && \ $$check && \ make "$$shellop" install && \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 0d37146..b65d8e8 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -62,7 +62,7 @@ all: $(ddir)/texlive-versions.tex \ # 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the # user's environment. .ONESHELL: -.SHELLFLAGS := -ec +.SHELLFLAGS := --noprofile --norc -ec export PATH := $(ibdir) export LD_RUN_PATH := $(ildir) export LD_LIBRARY_PATH := $(ildir) @@ -100,7 +100,6 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ - zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then @@ -126,18 +125,17 @@ $(tarballs): $(tdir)/%: elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 - elif [ $$n = git ]; then w=https://mirrors.edge.kernel.org/pub/software/scm/git + elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://akhlaghi.org/src - elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl + elif [ $$n = gsl ]; then w=http://ftpmirror.gnu.org/gnu/gsl elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files - elif [ $$n = libtool ]; then w=ftp://ftp.gnu.org/gnu/libtool + elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool elif [ $$n = libgit ]; then mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib - elif [ $$n = zlib ]; then w=http://www.zlib.net else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." @@ -156,7 +154,7 @@ $(tarballs): $(tdir)/%: else tarballurl=$$w fi echo "Downloading $$tarballurl" - $(DOWNLOADER) $@ $$tarballurl + $(ibdir)/wget --no-use-server-timestamps -O$@ $$tarballurl fi @@ -177,9 +175,8 @@ $(tarballs): $(tdir)/%: # for us here. So, we'll make an `$(ildir)/built' directory and make a # simple plain text file in it with the basic library name (an no prefix) # and create/write into it when the library is successfully built. -$(ilidir): | $(ildir); mkdir $@ $(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ - $(ibdir)/curl | $(ilidir) + $(ibdir)/curl $(call gbuild, $<,cfitsio, static, --enable-sse2 --enable-reentrant) \ && echo "CFITSIO is built" > $@ @@ -193,48 +190,33 @@ $(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ # https://libgit2.org/docs/guides/build-and-link $(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \ $(ibdir)/cmake \ - $(ibdir)/curl | $(ilidir) - $(call cbuild, $<, libgit2-$(libgit2-version), static, \ - -DUSE_SSH=OFF -DUSE_OPENSSL=OFF -DBUILD_CLAR=OFF \ - -DTHREADSAFE=ON -DBUILD_SHARED_LIBS=OFF) \ + $(ibdir)/curl + $(call cbuild, $<, libgit2-$(libgit2-version), static, \ + -DUSE_SSH=OFF -DBUILD_CLAR=OFF \ + -DTHREADSAFE=ON -DBUILD_SHARED_LIBS=OFF) \ && echo "Libgit2 is built" > $@ -$(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz | $(ilidir) +$(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, $<, gsl-$(gsl-version), static) \ && echo "GNU Scientific Library is built" > $@ -$(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz | $(ilidir) +$(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz $(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ - $(ilidir)/libjpeg | $(ilidir) + $(ilidir)/libjpeg $(call gbuild, $<, tiff-$(libtiff-version), static) \ && echo "Libtiff is built" > $@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ - $(ilidir)/cfitsio | $(ilidir) + $(ilidir)/cfitsio # Unfortunately WCSLIB forces the building of shared libraries. So # we'll just delete any shared library that is produced afterwards. $(call gbuild, $<, wcslib-$(wcslib-version), , \ LIBS="-pthread -lcurl -lm" --without-pgplot \ --disable-fortran) && \ - rm -f $(ildir)/libwcs.so* $(ildir)/libwcs*.dylib && \ echo "WCSLIB is built" > $@ -# Zlib: its `./configure' doesn't use Autoconf's configure script, it just -# accepts a direct `--static' option. -$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz | $(ilidir) - - # IMPORTANT, the second argument to `gbuild', must not have any - # spaces before or after it: it is going to be checked. -ifeq ($(static_build),yes) - $(call gbuild, $<,zlib-$(zlib-version), , --static) \ - && echo "Zlib is built" > $@ -else - $(call gbuild, $<,zlib-$(zlib-version)) && echo "Zlib is built" > $@ -endif - - @@ -242,14 +224,16 @@ endif # -------- # # CMake can be built with its custom `./bootstrap' script. -$(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz - cd $(ddir) && rm -rf cmake-$(cmake-version) && \ - tar xf $< && cd cmake-$(cmake-version) && \ - ./bootstrap --prefix=$(idir) && make && make install && \ +$(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ + $(ibdir)/curl + cd $(ddir) && rm -rf cmake-$(cmake-version) && \ + tar xf $< && cd cmake-$(cmake-version) && \ + ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ + --system-bzip2 --system-liblzma && \ + make && make install && \ cd ..&& rm -rf cmake-$(cmake-version) -$(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz \ - $(ilidir)/zlib +$(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz $(call gbuild, $<, curl-$(curl-version), static, --without-brotli) # On Mac OS, libtool does different things, so to avoid confusion, we'll @@ -264,8 +248,7 @@ $(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz $(ibdir)/flock: $(tdir)/flock-$(flock-version).tar.xz $(call gbuild, $<, flock-$(flock-version), static) -$(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \ - $(ilidir)/zlib +$(ibdir)/git: $(tdir)/git-$(git-version).tar.xz $(call gbuild, $<, git-$(git-version), static, \ --without-tcltk --with-shell=$(ibdir)/bash) @@ -280,10 +263,10 @@ $(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \ ifeq ($(static_build),yes) $(call gbuild, $<, gnuastro-$(gnuastro-version), static, \ --enable-static=yes --enable-shared=no, -j8, \ - make check -j8) + cp config.log ../gnuastro-config.log; make check -j8) else $(call gbuild, $<, gnuastro-$(gnuastro-version), , , -j8, \ - make check -j8) + cp config.log ../gnuastro-config.log; make check -j8) endif @@ -344,10 +327,9 @@ $(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \ # To work with TeX live installation, we'll need the internet. @res=$$(cat $(ibdir)/texlive-ready-tlmgr) - if [ x$$res = x"NOT!" ]; then + if [ x"$$res" = x"NOT!" ]; then echo "" > $@ else - # The current directory is necessary later. topdir=$$(pwd) diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index db27289..b2b5407 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -167,7 +167,7 @@ lvcheck = idir=$(BDIR)/dependencies/installed/include; \ ver="$(strip $(2))"; \ name="$(strip $(3))"; \ macro="$(strip $(4))"; \ - v=$$(awk '$$1=="\#define" && /'$$ver'/{print "y";exit 0}' $$f);\ + v=$$(awk '/^\#/&&/define/&&/'$$ver'/{print "y";exit 0}' $$f); \ if [ x$$v != xy ]; then \ echo; echo "PIPELINE ERROR: Not linking with $$name $$ver"; \ echo; exit 1; \ @@ -210,10 +210,12 @@ $(mtexdir)/initialize.tex: | $(mtexdir) coreutilsversion) $(call pvcheck, lzip, $(lzip-version), Lzip, lzipversion) $(call pvcheck, make, $(make-version), GNU Make, makeversion) + $(call pvcheck, pkg-config, $(pkgconfig-version), pkg-config, \ pkgconfigversion) $(call pvcheck, sed, $(sed-version), GNU SED, sedversion) $(call pvcheck, tar, $(tar-version), GNU Tar, tarversion) + $(call pvcheck, wget, $(wget-version), GNU Wget, wgetversion) $(call pvcheck, which, $(which-version), GNU Which, whichversion) $(call pvcheck, xz, $(xz-version), XZ Utils, xzversion) @@ -248,6 +250,8 @@ $(mtexdir)/initialize.tex: | $(mtexdir) ########## libjpeg not yet checked. $(call lvcheck, git2/version.h, $(libgit2-version), Libgit2, \ libgitwoversion) + $(call lvcheck, openssl/opensslv.h, $(openssl-version), OpenSSL, \ + opensslversion) $(call lvcheck, tiffvers.h, $(libtiff-version), Libtiff, \ libtiffversion) $(call lvcheck, wcslib/wcsconfig.h, $(wcslib-version), WCSLIB, \ -- cgit v1.2.1