From 63dd3b9936c22270c7f21da8965a01445ade4973 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 19 Nov 2018 17:41:41 +0000 Subject: Removed GNU Binutils, CMake's built with its own bootstrap When the C compiler is not GNU GCC, linking with GNU Binutils is going to cause problems. So until the time that we can include GCC into this pipeline, its best to avoid Binutils also. Also, for building CMake, we were relying on an installed CMake, but now, we are using its own `./bootstrap' script, so it can be built even if the host system doesn't have CMake. Also, for TeX Live, we are now setting a custom file as main target to avoid complications with symbolic links as targets in Make. Finally, when the user says they don't want to re-write an existing configuration file, no extra notices will be printed and the configure script will immediately start building programs. --- reproduce/src/make/dependencies-basic.mk | 53 ++------ reproduce/src/make/dependencies-build-rules.mk | 4 +- reproduce/src/make/dependencies.mk | 172 +++++++++++++++---------- reproduce/src/make/initialize.mk | 1 - 4 files changed, 118 insertions(+), 112 deletions(-) (limited to 'reproduce/src') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 0372d7d..c9bb4e1 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -86,7 +86,6 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # However, downloading from this link is slow (because its just a link). So # its easier to just keep a with the others. tarballs = $(foreach t, bash-$(bash-version).tar.gz \ - binutils-$(binutils-version).tar.lz \ bzip2-$(bzip2-version).tar.gz \ gzip-$(gzip-version).tar.gz \ lzip-$(lzip-version).tar.gz \ @@ -94,7 +93,6 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \ tar-$(tar-version).tar.gz \ which-$(which-version).tar.gz \ xz-$(xz-version).tar.gz \ - zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then \ @@ -106,7 +104,6 @@ $(tarballs): $(tdir)/%: \ mergenames=1; \ if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash; \ - elif [ $$n = binutils ]; then w=http://ftp.gnu.org/gnu/binutils; \ elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \ elif [ $$n = gzip ]; then w=http://akhlaghi.org/src; \ elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \ @@ -114,7 +111,6 @@ $(tarballs): $(tdir)/%: elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \ elif [ $$n = which ]; then w=http://ftp.gnu.org/gnu/which; \ elif [ $$n = xz ]; then w=http://tukaani.org/xz; \ - elif [ $$n = zlib ]; then w=http://www.zlib.net; \ else \ echo; echo; echo; \ echo "'$$n' not a basic dependency name (for downloading)." \ @@ -135,7 +131,7 @@ $(tarballs): $(tdir)/%: # GNU Gzip. $(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz - $(call gbuild,$(subst $(tdir)/,,$<), gzip-$(gzip-version), static) + $(call gbuild, $<, gzip-$(gzip-version), static) @@ -145,24 +141,9 @@ $(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz # LDFLAGS on the command-line (not from the environment). $(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz ifeq ($(static_build),yes) - $(call gbuild,$(subst $(tdir)/,,$<), lzip-$(lzip-version), , \ - LDFLAGS="-static") + $(call gbuild, $<, lzip-$(lzip-version), , LDFLAGS="-static") else - $(call gbuild,$(subst $(tdir)/,,$<), lzip-$(lzip-version)) -endif - - - - - -# Zlib: its `./configure' doesn't use Autoconf's configure script, it just -# accepts a direct `--static' option. -$(ildir)/libz.a: $(tdir)/zlib-$(zlib-version).tar.gz -ifeq ($(static_build),yes) - $(call gbuild,$(subst $(tdir)/,,$<), zlib-$(zlib-version), , \ - --static) -else - $(call gbuild,$(subst $(tdir)/,,$<), zlib-$(zlib-version)) + $(call gbuild, $<, lzip-$(lzip-version)) endif @@ -171,7 +152,7 @@ endif # XZ Utils $(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz - $(call gbuild,$(subst $(tdir)/,,$<), xz-$(xz-version), static) + $(call gbuild, $<, xz-$(xz-version), static) @@ -200,18 +181,7 @@ $(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \ $(ibdir)/lzip \ $(ibdir)/gzip \ $(ibdir)/xz - $(call gbuild,$(subst $(tdir)/,,$<), tar-$(tar-version)) - - - - - -# GNU Binutils: -$(ibdir)/nm: $(tdir)/binutils-$(binutils-version).tar.lz \ - $(ibdir)/tar \ - $(ildir)/libz.a - $(call gbuild,$(subst $(tdir)/,,$<), binutils-$(binutils-version), \ - static) + $(call gbuild, $<, tar-$(tar-version)) @@ -219,9 +189,8 @@ $(ibdir)/nm: $(tdir)/binutils-$(binutils-version).tar.lz \ # GNU Which: $(ibdir)/which: $(tdir)/which-$(which-version).tar.gz \ - $(ibdir)/tar \ - $(ibdir)/nm - $(call gbuild,$(subst $(tdir)/,,$<), which-$(which-version), static) + $(ibdir)/tar + $(call gbuild, $<, which-$(which-version), static) @@ -233,9 +202,8 @@ $(ibdir)/which: $(tdir)/which-$(which-version).tar.gz \ # `--disable-load', but unfortunately I don't know any way to fix the # second. So, we'll have to build it dynamically for now. $(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \ - $(ibdir)/tar \ - $(ibdir)/nm - $(call gbuild,$(subst $(tdir)/,,$<), make-$(make-version)) + $(ibdir)/tar + $(call gbuild, $<, make-$(make-version)) @@ -246,8 +214,7 @@ $(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ $(ibdir)/which \ $(ibdir)/make ifeq ($(static_build),yes) - $(call gbuild,$(subst $(tdir)/,,$<), bash-$(bash-version), , \ - --enable-static-link) + $(call gbuild, $<, bash-$(bash-version), , --enable-static-link) else $(call gbuild,$(subst $(tdir)/,,$<), bash-$(bash-version)) endif diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk index e0d91ef..25e2444 100644 --- a/reproduce/src/make/dependencies-build-rules.mk +++ b/reproduce/src/make/dependencies-build-rules.mk @@ -53,7 +53,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then \ fi; \ check="$(6)"; \ if [ x"$$check" = x ]; then check="echo Skipping-check"; fi; \ - cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) && \ + cd $(ddir) && rm -rf $(2) && tar xf $(1) && cd $(2) && \ ./configure $(4) --prefix=$(idir) && \ make $(5) && \ $$check && \ @@ -70,7 +70,7 @@ cbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then \ export LDFLAGS="$$LDFLAGS -static"; \ opts="-DBUILD_SHARED_LIBS=OFF"; \ fi; \ - cd $(ddir) && rm -rf $(2) && tar xf $(tdir)/$(1) && cd $(2) && \ + cd $(ddir) && rm -rf $(2) && tar xf $(1) && cd $(2) && \ rm -rf pipeline-build && mkdir pipeline-build && \ cd pipeline-build && \ cmake .. $$opts $(4) && \ diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index 156f254..010f3a4 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -35,15 +35,16 @@ include reproduce/src/make/dependencies-build-rules.mk include reproduce/config/pipeline/dependency-texlive.mk include reproduce/config/pipeline/dependency-versions.mk -ddir = $(BDIR)/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib +ddir = $(BDIR)/dependencies +tdir = $(BDIR)/dependencies/tarballs +idir = $(BDIR)/dependencies/installed +ibdir = $(BDIR)/dependencies/installed/bin +ildir = $(BDIR)/dependencies/installed/lib +ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin', so for # Coreutils, only one of its executables is enough). -top-level-programs = ls gawk gs grep libtool sed git latex astnoisechisel +top-level-programs = ls gawk gs grep libtool sed git astnoisechisel texlive-ready all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # Other basic environment settings. @@ -85,6 +86,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libgit2-$(libgit2-version).tar.gz \ sed-$(sed-version).tar.xz \ wcslib-$(wcslib-version).tar.bz2 \ + zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then @@ -123,6 +125,7 @@ $(tarballs): $(tdir)/%: elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib + elif [ $$n = zlib ]; then w=http://www.zlib.net else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." @@ -150,41 +153,66 @@ $(tarballs): $(tdir)/%: # Libraries # --------- -$(ildir)/libcfitsio.a: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ - $(ibdir)/curl \ - $(ibdir)/ls - $(call gbuild,$(subst $(tdir)/,,$<), cfitsio, static, \ - --enable-sse2 --enable-reentrant) - - -$(ildir)/libgit2.a: $(tdir)/libgit2-$(libgit2-version).tar.gz \ - $(ibdir)/cmake \ - $(ibdir)/curl - $(call cbuild,$(subst $(tdir)/,,$<), libgit2-$(libgit2-version), \ - static, -DUSE_SSH=OFF -DUSE_OPENSSL=OFF \ - -DBUILD_CLAR=OFF -DTHREADSAFE=ON) - -$(ildir)/libgsl.a: $(tdir)/gsl-$(gsl-version).tar.gz \ - $(ibdir)/ls - $(call gbuild,$(subst $(tdir)/,,$<), gsl-$(gsl-version), static) - -$(ildir)/libjpeg.a: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz - $(call gbuild,$(subst $(tdir)/,,$<), jpeg-9b, static) - -$(ildir)/libtiff.a: $(tdir)/tiff-$(libtiff-version).tar.gz \ - $(ibdir)/ls - $(call gbuild,$(subst $(tdir)/,,$<), tiff-$(libtiff-version), \ - static) - -$(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ - $(ildir)/libcfitsio.a +# +# We would prefer to build static libraries, but some compilers like LLVM +# don't have static capabilities, so they'll only build dynamic/shared +# libraries. Therefore, we can't use the easy `.a' suffix for static +# libraries as targets and there are different conventions for shared +# library names. +# +# For the actual build, the same compiler that built the library will build +# the programs, so exact knowledge of the suffix is ultimately irrelevant +# for us here. So, we'll make an `$(ildir)/built' directory and make a +# simple plain text file in it with the basic library name (an no prefix) +# and create/write into it when the library is successfully built. +$(ilidir): | $(ildir); mkdir -p $@ +$(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ + $(ibdir)/curl \ + $(ibdir)/ls | $(ilidir) + $(call gbuild, $<,cfitsio, static, --enable-sse2 --enable-reentrant) \ + && echo "CFITSIO is built" > $@ + + +$(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \ + $(ibdir)/cmake \ + $(ibdir)/curl | $(ilidir) + $(call cbuild, $<, libgit2-$(libgit2-version), static, \ + -DUSE_SSH=OFF -DUSE_OPENSSL=OFF -DBUILD_CLAR=OFF \ + -DTHREADSAFE=ON) \ + && echo "Libgit2 is built" > $@ + +$(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz \ + $(ibdir)/ls | $(ilidir) + $(call gbuild, $<, gsl-$(gsl-version), static) \ + && echo "GNU Scientific Library is built" > $@ + +$(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz | $(ilidir) + $(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@ + +$(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ + $(ibdir)/ls | $(ilidir) + $(call gbuild, $<, tiff-$(libtiff-version), static) \ + && echo "Libtiff is built" > $@ + +$(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ + $(ilidir)/cfitsio | $(ilidir) # Unfortunately WCSLIB forces the building of shared libraries. So # we'll allow it to finish, then remove the shared libraries # afterwards. - $(call gbuild,$(subst $(tdir)/,,$<), wcslib-$(wcslib-version), , \ - LIBS="-pthread -lcurl -lm" --without-pgplot \ - --disable-fortran) - rm -f $(ildir)/libwcs.so* + $(call gbuild, $<, wcslib-$(wcslib-version), , \ + LIBS="-pthread -lcurl -lm" --without-pgplot \ + --disable-fortran) \ + && echo "WCSLIB is built" > $@ + +# Zlib: its `./configure' doesn't use Autoconf's configure script, it just +# accepts a direct `--static' option. +$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz | $(ilidir) +ifeq ($(static_build),yes) + $(call gbuild, $<, zlib-$(zlib-version), , --static) \ + && echo "Zlib is built" > $@ +else + $(call gbuild, $<, zlib-$(zlib-version)) && echo "Zlib is built" > $@ +endif @@ -192,58 +220,69 @@ $(ildir)/libwcs.a: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ # Programs # -------- +# +# CMake can be built with its custom `./bootstrap' script. $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ $(ibdir)/ls - $(call cbuild,$(subst $(tdir)/,,$<), cmake-$(cmake-version)) + cd $(ddir) && rm -rf cmake-$(cmake-version) && \ + tar xf $< && cd cmake-$(cmake-version) && \ + ./bootstrap --prefix=$(idir) && make && make install && \ + cd ..&& rm -rf cmake-$(cmake-version) $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz \ - $(ildir)/libz.a \ + $(ilidir)/zlib \ $(ibdir)/ls - $(call gbuild,$(subst $(tdir)/,,$<), curl-$(curl-version), static, \ - --without-brotli) + $(call gbuild, $<, curl-$(curl-version), static, --without-brotli) $(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz - $(call gbuild,$(subst $(tdir)/,,$<), coreutils-$(coreutils-version), \ - static) + $(call gbuild, $<, coreutils-$(coreutils-version), static) $(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \ $(ibdir)/ls - $(call gbuild,$(subst $(tdir)/,,$<), gawk-$(gawk-version), static) + $(call gbuild, $<, gawk-$(gawk-version), static) $(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \ $(ibdir)/ls - $(call gbuild,$(subst $(tdir)/,,$<), sed-$(sed-version), static) + $(call gbuild, $<, sed-$(sed-version), static) $(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \ $(ibdir)/ls - $(call gbuild,$(subst $(tdir)/,,$<), grep-$(grep-version), static) + $(call gbuild, $<, grep-$(grep-version), static) $(ibdir)/libtool: $(tdir)/libtool-$(libtool-version).tar.xz \ $(ibdir)/ls - $(call gbuild,$(subst $(tdir)/,,$<), libtool-$(libtool-version), static) + $(call gbuild, $<, libtool-$(libtool-version), static) $(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz \ $(ibdir)/ls - $(call gbuild,$(subst $(tdir)/,,$<), ghostscript-$(ghostscript-version)) + $(call gbuild, $<, ghostscript-$(ghostscript-version)) $(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \ $(ibdir)/ls - $(call gbuild,$(subst $(tdir)/,,$<), git-$(git-version), static) + $(call gbuild, $<, git-$(git-version), static) $(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \ - $(ildir)/libgsl.a \ - $(ildir)/libcfitsio.a \ - $(ildir)/libwcs.a \ $(ibdir)/gs \ - $(ildir)/libjpeg.a \ - $(ildir)/libtiff.a \ - $(ildir)/libgit2.a \ - - $(call gbuild,$(subst $(tdir)/,,$<), gnuastro-$(gnuastro-version), \ - static, --enable-static=yes --enable-shared=no, -j8, \ - make check -j8) - -$(ibdir)/latex: reproduce/config/pipeline/dependency-texlive.mk + $(ilidir)/gsl \ + $(ilidir)/wcslib \ + $(ilidir)/libjpeg \ + $(ilidir)/libtiff \ + $(ilidir)/libgit2 +ifeq ($(static_build),yes) + $(call gbuild, $<, gnuastro-$(gnuastro-version), static, \ + --enable-static=yes --enable-shared=no, -j8, \ + make check -j8) +else + $(call gbuild, $<, gnuastro-$(gnuastro-version), , , -j8, \ + make check -j8) +endif + +# Since we want to avoid complicating the PATH, we are putting a symbolic +# link of all the TeX Live executables in $(ibdir). Therefore, since the +# symbolic link is hard to track for Make (as a target), we'll make a +# simple ASCII file called `texlive-ready' when it is complete and use that +# as a target. +$(ibdir)/texlive-ready: reproduce/config/pipeline/dependency-texlive.mk # We'll need the current directory later down. topdir=$$(pwd) @@ -273,10 +312,6 @@ $(ibdir)/latex: reproduce/config/pipeline/dependency-texlive.mk # `ibdir'. For `latex' do a copy, because it is the target of # this rule and it won't cause problems. ln -fs $(idir)/texlive/20*/bin/*/* $(ibdir)/ - rm $@ - cp $(idir)/texlive/20*/bin/*/latex $@ - else - echo "Not able to download TeX Live installer" > $@ fi fi @@ -314,3 +349,8 @@ $(ibdir)/latex: reproduce/config/pipeline/dependency-texlive.mk printf("\\newcommand{\\tex%sversion}{%s}\n",\ name, version)}' >> $$tv fi + + # Write the target if TeX live was actually installed. + if [ -f $(idir)/texlive/20*/bin/*/latex ]; then + echo "TeX Live is installed." > $@ + fi; diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 2cc393c..694aca0 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -191,7 +191,6 @@ $(mtexdir)/initialize.tex: | $(mtexdir) # Versions of programs (same order as 'dependency-versions.mk'). $(call pvcheck, bash, $(bash-version), GNU Bash, bashversion) - $(call pvcheck, nm, $(binutils-version), GNU Binutils, binutilsversion) $(call pvcheck, cmake, $(cmake-version), CMake, cmakeversion) $(call pvcheck, curl, $(curl-version), cURL, curlversion) $(call pvcheck, ls, $(coreutils-version), GNU Coreutils, \ -- cgit v1.2.1