diff options
author | Mohammad Akhlaghi <mohammad@akhlaghi.org> | 2018-12-02 01:25:57 +0000 |
---|---|---|
committer | Mohammad Akhlaghi <mohammad@akhlaghi.org> | 2018-12-02 06:14:26 +0000 |
commit | 2a3304cfb832d2f9548bb869f5a44a3412a1c3f9 (patch) | |
tree | b23f2887bd06ad44771e6818aef6fe341b890701 /reproduce/src/make | |
parent | ccfb8d41ad851432e73a82b486a29707d0faa405 (diff) |
Wget and OpenSSL now installed as a basic dependency
The TeX Live installer needs Wget to operate smoothly, especially on recent
Mac OS systems that don't have Wget pre-installed. Also, it would be good
for the pipeline to have its own downloader. So with this commit, the
pipeline also installs Wget and OpenSSL which is a dependency.
Many other small changes/fixes were done in this process.
Diffstat (limited to 'reproduce/src/make')
-rw-r--r-- | reproduce/src/make/dependencies-basic.mk | 126 | ||||
-rw-r--r-- | reproduce/src/make/dependencies-build-rules.mk | 15 | ||||
-rw-r--r-- | reproduce/src/make/dependencies.mk | 70 | ||||
-rw-r--r-- | reproduce/src/make/initialize.mk | 6 |
4 files changed, 134 insertions, 83 deletions
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 889ed0b..77a47c4 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -63,7 +63,7 @@ export LDFLAGS := -L$(ildir) $(LDFLAGS) export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) -top-level-programs = ls sed gawk grep diff find bash which pkg-config +top-level-programs = ls sed gawk grep diff find bash wget which pkg-config all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) @@ -108,11 +108,14 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \ make-$(make-version).tar.lz \ mpfr-$(mpfr-version).tar.xz \ mpc-$(mpc-version).tar.gz \ + openssl-$(openssl-version).tar.gz \ pkg-config-$(pkgconfig-version).tar.gz \ sed-$(sed-version).tar.xz \ - tar-$(tar-version).tar.gz \ + tar-$(tar-version).tar.gz \ + wget-$(wget-version).tar.lz \ which-$(which-version).tar.gz \ xz-$(xz-version).tar.gz \ + zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then \ @@ -123,39 +126,46 @@ $(tarballs): $(tdir)/%: | awk '{print $$1}' ); \ \ mergenames=1; \ - if [ $$n = bash ]; then w=http://ftpmirror.gnu.org/gnu/bash; \ - elif [ $$n = binutils ]; then w=http://ftpmirror.gnu.org/gnu/binutils; \ - elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \ - elif [ $$n = coreutils ]; then w=http://ftpmirror.gnu.org/gnu/coreutils;\ - elif [ $$n = diffutils ]; then w=http://ftpmirror.gnu.org/gnu/diffutils;\ - elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \ - elif [ $$n = gawk ]; then w=http://ftpmirror.gnu.org/gnu/gawk; \ + if [ $$n = bash ]; then w=http://ftpmirror.gnu.org/gnu/bash; \ + elif [ $$n = binutils ]; then w=http://ftpmirror.gnu.org/gnu/binutils; \ + elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = coreutils ]; then w=http://ftpmirror.gnu.org/gnu/coreutils;\ + elif [ $$n = diffutils ]; then w=http://ftpmirror.gnu.org/gnu/diffutils;\ + elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = gawk ]; then w=http://ftpmirror.gnu.org/gnu/gawk; \ elif [ $$n = gcc ]; then w=http://ftpmirror.gnu.org/gcc/gcc-$(gcc-version); \ elif [ $$n = gmp ]; then w=https://gmplib.org/download/gmp; \ - elif [ $$n = grep ]; then w=http://ftpmirror.gnu.org/gnu/grep; \ - elif [ $$n = gzip ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = grep ]; then w=http://ftpmirror.gnu.org/gnu/grep; \ + elif [ $$n = gzip ]; then w=http://akhlaghi.org/src; \ elif [ $$n = isl ]; then w=ftp://gcc.gnu.org/pub/gcc/infrastructure; \ - elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \ - elif [ $$n = make ]; then w=http://akhlaghi.org/src; \ + elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \ + elif [ $$n = make ]; then w=http://akhlaghi.org/src; \ elif [ $$n = mpfr ]; then w=http://www.mpfr.org/mpfr-current;\ elif [ $$n = mpc ]; then w=http://ftpmirror.gnu.org/gnu/mpc;\ - elif [ $$n = pkg ]; then w=http://pkg-config.freedesktop.org/releases; \ - elif [ $$n = sed ]; then w=http://ftpmirror.gnu.org/gnu/sed;\ - elif [ $$n = tar ]; then w=http://ftpmirror.gnu.org/gnu/tar;\ - elif [ $$n = which ]; then w=http://ftpmirror.gnu.org/gnu/which;\ - elif [ $$n = xz ]; then w=http://tukaani.org/xz; \ - else \ - echo; echo; echo; \ - echo "'$$n' not a basic dependency name (for downloading)." \ - echo; echo; echo; \ - exit 1; \ - fi; \ + elif [ $$n = openssl ]; then w=http://www.openssl.org/source; \ + elif [ $$n = pkg ]; then w=http://pkg-config.freedesktop.org/releases; \ + elif [ $$n = sed ]; then w=http://ftpmirror.gnu.org/gnu/sed;\ + elif [ $$n = tar ]; then w=http://ftpmirror.gnu.org/gnu/tar;\ + elif [ $$n = wget ]; then w=http://ftpmirror.gnu.org/gnu/wget;\ + elif [ $$n = which ]; then w=http://ftpmirror.gnu.org/gnu/which;\ + elif [ $$n = xz ]; then w=http://tukaani.org/xz; \ + elif [ $$n = zlib ]; then w=http://www.zlib.net; \ + else \ + echo; echo; echo; \ + echo "'$$n' not a basic dependency name (for downloading)." \ + echo; echo; echo; \ + exit 1; \ + fi; \ \ if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*"; \ else tarballurl=$$w; \ fi; \ echo "Downloading $$tarballurl"; \ - $(DOWNLOADER) $@ $$tarballurl; \ + if [ -f $(ibdir)/wget ]; then \ + $(ibdir)/wget --no-use-server-timestamps -O$@ $$tarballurl; \ + else \ + $(DOWNLOADER) $@ $$tarballurl; \ + fi; \ fi @@ -170,8 +180,8 @@ $(tarballs): $(tdir)/%: # to it. makelink = export PATH=$(syspath); a=$$(which $(1) 2> /dev/null); \ if [ x$$a != x ]; then ln -s $$a $(ibdir)/$(1); fi -$(ibdir):; mkdir $@ -$(ibdir)/low-level: | $(ibdir) +$(ibdir) $(ildir):; mkdir $@ +$(ibdir)/low-level: | $(ibdir) $(ildir) # The Assembler $(call makelink,as) @@ -202,11 +212,12 @@ $(ibdir)/low-level: | $(ibdir) # Needed by TeXLive specifically. $(call makelink,perl) - # Downloaders. - $(call makelink,wget) - $(call makelink,curl) + # Libdl (for dynamic loading libraries at runtime) + if [ -f /usr/lib/libdl.a ]; then \ + ln -s /usr/lib/libdl.* $(ildir)/; \ + fi; - echo "Low-level program links are setup" > $@ + echo "Low-level symbolic links are setup" > $@ @@ -280,6 +291,57 @@ $(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \ +# Downloader +# ---------- +# +# Zlib's `./configure' doesn't use Autoconf's configure script, it just +# accepts a direct `--static' option. +$(idir)/etc:; mkdir $@ +$(ilidir): | $(ildir); mkdir $@ +$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \ + $(ibdir)/make | $(ilidir) + # IMPORTANT, the second argument to `gbuild', must not have any + # spaces before or after it: it is going to be checked. +ifeq ($(static_build),yes) + $(call gbuild, $<,zlib-$(zlib-version), , --static) \ + && echo "Zlib is built" > $@ +else + $(call gbuild, $<,zlib-$(zlib-version)) && echo "Zlib is built" > $@ +endif + +# OpenSSL +ifeq ($(static_build),yes) +openssl-static = no-dso no-dynamic-engine no-shared +endif +$(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ + $(ilidir)/zlib | $(idir)/etc + $(call gbuild, $<, openssl-$(openssl-version), static, \ + --openssldir=$(idir)/etc/ssl \ + --with-zlib-lib=$(ildir) \ + --with-zlib-include=$(idir)/include zlib \ + $(openssl-static) ) \ + && echo "OpenSSL is built" > $@ + +# GNU Wget +# +# Note that on some systems (for example GNU/Linux) Wget needs to explicity +# link with `libdl', but on others (for example Mac OS) it doesn't. We +# check this at configure time and define the `needs_ldl' variable. +# +# Also note that since Wget needs to load outside libraries dynamically, it +# gives a segmentation fault when built statically. +$(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \ + $(ilidir)/openssl + libs="-pthread"; \ + if [ x$(needs_ldl) = xyes ]; then libs="$$libs -ldl"; fi; \ + $(call gbuild, $<, wget-$(wget-version), , \ + LIBS="$$LIBS $$libs" --with-ssl=openssl \ + --with-openssl=yes) + + + + + # Basic command-line programs necessary in build process of the # higher-level dependencies: Note that during the building of those # programs, there is no access to the system's PATH. @@ -350,8 +412,6 @@ endif # (CURRENTLY IGNORED) GCC prerequisites # ------------------------------------- -$(ildir): | $(idir); mkdir $@ -$(ilidir): | $(ildir); mkdir $@ $(ilidir)/gmp: $(tdir)/gmp-$(gmp-version).tar.lz \ $(ibdir)/make | $(ilidir) $(call gbuild, $<, gmp-$(gmp-version), static, , , make check) \ diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk index f613975..d1cc5dc 100644 --- a/reproduce/src/make/dependencies-build-rules.mk +++ b/reproduce/src/make/dependencies-build-rules.mk @@ -61,11 +61,16 @@ gbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then \ if ! tar xf $(1); then echo; echo "Tar error"; exit 1; fi; \ cd $(2); \ \ + if [ -f configure ]; then confscript=configure; \ + elif [ -f config ]; then confscript=config; \ + fi; \ + \ if [ -f $(ibdir)/bash ]; then \ - sed configure -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|'> configure-t; \ - mv configure-t configure; \ - chmod +x configure; \ + sed $$confscript -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ + -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ + > tmp-$$confscript; \ + mv tmp-$$confscript $$confscript; \ + chmod +x $$confscript; \ shellop="SHELL=$(ibdir)/bash"; \ elif [ -f /bin/bash ]; then shellop="SHELL=/bin/bash"; \ else shellop="SHELL=/bin/sh"; \ @@ -76,7 +81,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then \ else configop="$$shellop --prefix=$(idir)"; \ fi; \ \ - ./configure $(4) $$configop && \ + ./$$confscript $(4) $$configop && \ make "$$shellop" $(5) && \ $$check && \ make "$$shellop" install && \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 0d37146..b65d8e8 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -62,7 +62,7 @@ all: $(ddir)/texlive-versions.tex \ # 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the # user's environment. .ONESHELL: -.SHELLFLAGS := -ec +.SHELLFLAGS := --noprofile --norc -ec export PATH := $(ibdir) export LD_RUN_PATH := $(ildir) export LD_LIBRARY_PATH := $(ildir) @@ -100,7 +100,6 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ - zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then @@ -126,18 +125,17 @@ $(tarballs): $(tdir)/%: elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 - elif [ $$n = git ]; then w=https://mirrors.edge.kernel.org/pub/software/scm/git + elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://akhlaghi.org/src - elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl + elif [ $$n = gsl ]; then w=http://ftpmirror.gnu.org/gnu/gsl elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files - elif [ $$n = libtool ]; then w=ftp://ftp.gnu.org/gnu/libtool + elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool elif [ $$n = libgit ]; then mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib - elif [ $$n = zlib ]; then w=http://www.zlib.net else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." @@ -156,7 +154,7 @@ $(tarballs): $(tdir)/%: else tarballurl=$$w fi echo "Downloading $$tarballurl" - $(DOWNLOADER) $@ $$tarballurl + $(ibdir)/wget --no-use-server-timestamps -O$@ $$tarballurl fi @@ -177,9 +175,8 @@ $(tarballs): $(tdir)/%: # for us here. So, we'll make an `$(ildir)/built' directory and make a # simple plain text file in it with the basic library name (an no prefix) # and create/write into it when the library is successfully built. -$(ilidir): | $(ildir); mkdir $@ $(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ - $(ibdir)/curl | $(ilidir) + $(ibdir)/curl $(call gbuild, $<,cfitsio, static, --enable-sse2 --enable-reentrant) \ && echo "CFITSIO is built" > $@ @@ -193,48 +190,33 @@ $(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ # https://libgit2.org/docs/guides/build-and-link $(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \ $(ibdir)/cmake \ - $(ibdir)/curl | $(ilidir) - $(call cbuild, $<, libgit2-$(libgit2-version), static, \ - -DUSE_SSH=OFF -DUSE_OPENSSL=OFF -DBUILD_CLAR=OFF \ - -DTHREADSAFE=ON -DBUILD_SHARED_LIBS=OFF) \ + $(ibdir)/curl + $(call cbuild, $<, libgit2-$(libgit2-version), static, \ + -DUSE_SSH=OFF -DBUILD_CLAR=OFF \ + -DTHREADSAFE=ON -DBUILD_SHARED_LIBS=OFF) \ && echo "Libgit2 is built" > $@ -$(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz | $(ilidir) +$(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, $<, gsl-$(gsl-version), static) \ && echo "GNU Scientific Library is built" > $@ -$(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz | $(ilidir) +$(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz $(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ - $(ilidir)/libjpeg | $(ilidir) + $(ilidir)/libjpeg $(call gbuild, $<, tiff-$(libtiff-version), static) \ && echo "Libtiff is built" > $@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ - $(ilidir)/cfitsio | $(ilidir) + $(ilidir)/cfitsio # Unfortunately WCSLIB forces the building of shared libraries. So # we'll just delete any shared library that is produced afterwards. $(call gbuild, $<, wcslib-$(wcslib-version), , \ LIBS="-pthread -lcurl -lm" --without-pgplot \ --disable-fortran) && \ - rm -f $(ildir)/libwcs.so* $(ildir)/libwcs*.dylib && \ echo "WCSLIB is built" > $@ -# Zlib: its `./configure' doesn't use Autoconf's configure script, it just -# accepts a direct `--static' option. -$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz | $(ilidir) - - # IMPORTANT, the second argument to `gbuild', must not have any - # spaces before or after it: it is going to be checked. -ifeq ($(static_build),yes) - $(call gbuild, $<,zlib-$(zlib-version), , --static) \ - && echo "Zlib is built" > $@ -else - $(call gbuild, $<,zlib-$(zlib-version)) && echo "Zlib is built" > $@ -endif - - @@ -242,14 +224,16 @@ endif # -------- # # CMake can be built with its custom `./bootstrap' script. -$(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz - cd $(ddir) && rm -rf cmake-$(cmake-version) && \ - tar xf $< && cd cmake-$(cmake-version) && \ - ./bootstrap --prefix=$(idir) && make && make install && \ +$(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ + $(ibdir)/curl + cd $(ddir) && rm -rf cmake-$(cmake-version) && \ + tar xf $< && cd cmake-$(cmake-version) && \ + ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ + --system-bzip2 --system-liblzma && \ + make && make install && \ cd ..&& rm -rf cmake-$(cmake-version) -$(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz \ - $(ilidir)/zlib +$(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz $(call gbuild, $<, curl-$(curl-version), static, --without-brotli) # On Mac OS, libtool does different things, so to avoid confusion, we'll @@ -264,8 +248,7 @@ $(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz $(ibdir)/flock: $(tdir)/flock-$(flock-version).tar.xz $(call gbuild, $<, flock-$(flock-version), static) -$(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \ - $(ilidir)/zlib +$(ibdir)/git: $(tdir)/git-$(git-version).tar.xz $(call gbuild, $<, git-$(git-version), static, \ --without-tcltk --with-shell=$(ibdir)/bash) @@ -280,10 +263,10 @@ $(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \ ifeq ($(static_build),yes) $(call gbuild, $<, gnuastro-$(gnuastro-version), static, \ --enable-static=yes --enable-shared=no, -j8, \ - make check -j8) + cp config.log ../gnuastro-config.log; make check -j8) else $(call gbuild, $<, gnuastro-$(gnuastro-version), , , -j8, \ - make check -j8) + cp config.log ../gnuastro-config.log; make check -j8) endif @@ -344,10 +327,9 @@ $(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \ # To work with TeX live installation, we'll need the internet. @res=$$(cat $(ibdir)/texlive-ready-tlmgr) - if [ x$$res = x"NOT!" ]; then + if [ x"$$res" = x"NOT!" ]; then echo "" > $@ else - # The current directory is necessary later. topdir=$$(pwd) diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index db27289..b2b5407 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -167,7 +167,7 @@ lvcheck = idir=$(BDIR)/dependencies/installed/include; \ ver="$(strip $(2))"; \ name="$(strip $(3))"; \ macro="$(strip $(4))"; \ - v=$$(awk '$$1=="\#define" && /'$$ver'/{print "y";exit 0}' $$f);\ + v=$$(awk '/^\#/&&/define/&&/'$$ver'/{print "y";exit 0}' $$f); \ if [ x$$v != xy ]; then \ echo; echo "PIPELINE ERROR: Not linking with $$name $$ver"; \ echo; exit 1; \ @@ -210,10 +210,12 @@ $(mtexdir)/initialize.tex: | $(mtexdir) coreutilsversion) $(call pvcheck, lzip, $(lzip-version), Lzip, lzipversion) $(call pvcheck, make, $(make-version), GNU Make, makeversion) + $(call pvcheck, pkg-config, $(pkgconfig-version), pkg-config, \ pkgconfigversion) $(call pvcheck, sed, $(sed-version), GNU SED, sedversion) $(call pvcheck, tar, $(tar-version), GNU Tar, tarversion) + $(call pvcheck, wget, $(wget-version), GNU Wget, wgetversion) $(call pvcheck, which, $(which-version), GNU Which, whichversion) $(call pvcheck, xz, $(xz-version), XZ Utils, xzversion) @@ -248,6 +250,8 @@ $(mtexdir)/initialize.tex: | $(mtexdir) ########## libjpeg not yet checked. $(call lvcheck, git2/version.h, $(libgit2-version), Libgit2, \ libgitwoversion) + $(call lvcheck, openssl/opensslv.h, $(openssl-version), OpenSSL, \ + opensslversion) $(call lvcheck, tiffvers.h, $(libtiff-version), Libtiff, \ libtiffversion) $(call lvcheck, wcslib/wcsconfig.h, $(wcslib-version), WCSLIB, \ |