# Build the reproduction pipeline dependencies (programs and libraries). # # ------------------------------------------------------------------------ # !!!!! IMPORTANT NOTES !!!!! # # This Makefile will be run by the initial `./configure' script. It is not # included into the reproduction pipe after that. # # ------------------------------------------------------------------------ # # Original author: # Mohammad Akhlaghi # Contributing author(s): # Your name # Copyright (C) 2018-2019, Your Name. # # This Makefile is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the # Free Software Foundation, either version 3 of the License, or (at your # option) any later version. # # This Makefile is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General # Public License for more details. # # A copy of the GNU General Public License is available at # . # Top level environment include reproduce/config/pipeline/LOCAL.mk include reproduce/src/make/dependencies-build-rules.mk include reproduce/config/pipeline/dependency-texlive.mk include reproduce/config/pipeline/dependency-versions.mk ddir = $(BDIR)/dependencies tdir = $(BDIR)/dependencies/tarballs idir = $(BDIR)/dependencies/installed ibdir = $(BDIR)/dependencies/installed/bin ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). top-level-programs = astnoisechisel metastore flock all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # Other basic environment settings: We are only including the host # operating system's PATH environment variable (after our own!) for the # compiler and linker. For the library binaries and headers, we are only # using our internally built libraries. # # To investigate: # # 1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all # the parameters defined bellow as `NAME=VALUE' statements before # calling Bash. This will enable us to completely ignore the user's # native environment. # # 2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the # user's environment. .ONESHELL: .SHELLFLAGS := --noprofile --norc -ec export CCACHE_DISABLE := 1 export PATH := $(ibdir) export LD_RUN_PATH := $(ildir) export LD_LIBRARY_PATH := $(ildir) export SHELL := $(ibdir)/bash export CPPFLAGS := -I$(idir)/include export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig export LDFLAGS := $(rpath_command) -L$(ildir) # Tarballs # -------- # # All the necessary tarballs are defined and prepared with this rule. # # Note that we want the tarballs to follow the convention of NAME-VERSION # before the `tar.XX' prefix. For those programs that don't follow this # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ flock-$(flock-version).tar.xz \ ghostscript-$(ghostscript-version).tar.gz \ git-$(git-version).tar.xz \ gnuastro-$(gnuastro-version).tar.lz \ gsl-$(gsl-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ libtool-$(libtool-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* $@ else # Remove all numbers, `-' and `.' from the tarball name so we can # search more easily only with the program name. n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g' \ | awk '{print $$1}' ) # Set the top download link of the requested tarball. mergenames=1 if [ $$n = cfitsio ]; then mergenames=0 v=$$(echo $(cfitsio-version) | sed -e's/\.//' \ | awk '{l=length($$1); \ printf (l==4 ? "%d\n" \ : (l==3 ? "%d0\n" \ : (l==2 ? "%d00\n" \ : "%d000\n") ), $$1)}') w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro elif [ $$n = gsl ]; then w=http://ftpmirror.gnu.org/gnu/gsl elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool elif [ $$n = libgit ]; then mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz elif [ $$n = metastore ]; then w=http://ftp.przemoc.net/pub/software/utils/metastore elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." echo; echo; echo; exit 1 fi # Download the requested tarball. Note that some packages may not # follow our naming convention (where the package name is merged # with its version number). In such cases, `w' will be the full # address, not just the top directory address. But since we are # storing all the tarballs in one directory, we want it to have # the same naming convention, so we'll download it to a temporary # name, then rename that. if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*" else tarballurl=$$w fi # If the download fails, Wget will write the error message in the # target file, so Make will think that its done! To avoid this # problem, we'll rename the output. echo "Downloading $$tarballurl" if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then rm -f $@ echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1 fi fi # Libraries # --------- # # We would prefer to build static libraries, but some compilers like LLVM # don't have static capabilities, so they'll only build dynamic/shared # libraries. Therefore, we can't use the easy `.a' suffix for static # libraries as targets and there are different conventions for shared # library names. # # For the actual build, the same compiler that built the library will build # the programs, so exact knowledge of the suffix is ultimately irrelevant # for us here. So, we'll make an `$(ildir)/built' directory and make a # simple plain text file in it with the basic library name (an no prefix) # and create/write into it when the library is successfully built. $(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ $(ibdir)/curl # CFITSIO hard-codes the absolute address of cURL's `curl-config' # program (which gives the necessary header and linking # information) into the configure script. So we'll have to modify # it manually before doing the standard build. topdir=$(pwd); cd $(ddir); tar xf $< customtar=cfitsio-$(cfitsio-version)-custom.tar.gz sed cfitsio/configure \ -e's|/usr/bin/curl-config|$(ibdir)/curl-config|g' \ > cfitsio/configure_tmp mv cfitsio/configure_tmp cfitsio/configure chmod +x cfitsio/configure tar cf $$customtar cfitsio cd $$topdir # Continue the standard build on the customized tarball. $(call gbuild, $$customtar, cfitsio, static, \ --enable-sse2 --enable-reentrant) \ && rm $$customtar \ && echo "CFITSIO is built" > $@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, $<, gsl-$(gsl-version), static) \ && echo "GNU Scientific Library is built" > $@ $(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz $(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/libjpeg $(call gbuild, $<, tiff-$(libtiff-version), static, \ --disable-webp --disable-zstd) \ && echo "Libtiff is built" > $@ # Libraries with special attention on Mac OS # ------------------------------------------ # # Libgit2 and WCSLIB don't set their installation path, or don't do it # properly, in their finally installed shared libraries. But since we are # linking everything (including OpenSSL and its dependencies) dynamically, # we need to also make a shared libraries and can't use static # libraries. So for Mac OS systems we have to correct their addresses # manually. # # For example, Libgit2 page recommends doing a static build, especially for # Mac systems (with `-DBUILD_SHARED_LIBS=OFF'): "It’s highly recommended # that you build libgit2 as a static library for Xcode projects. This # simplifies distribution significantly, as the resolution of dynamic # libraries at runtime can be extremely problematic.". This is a major # problem we have been having so far with Mac systems: # https://libgit2.org/docs/guides/build-and-link $(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \ $(ibdir)/cmake \ $(ibdir)/curl # Build and install the library. $(call cbuild, $<, libgit2-$(libgit2-version), static, \ -DUSE_SSH=OFF -DBUILD_CLAR=OFF \ -DTHREADSAFE=ON ) # Correct the shared library absolute address if necessary. if [ x$(on_mac_os) = xyes ]; then install_name_tool -id $(ildir)/libgit2.26.dylib \ $(ildir)/libgit2.26.dylib fi # Write the target file. echo "Libgit2 is built" > $@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ $(ilidir)/cfitsio # Build and install the library. $(call gbuild, $<, wcslib-$(wcslib-version), , \ LIBS="-pthread -lcurl -lm" \ --with-cfitsiolib=$(ildir) \ --with-cfitsioinc=$(idir)/include \ --without-pgplot --disable-fortran) # Correct the shared library absolute address if necessary. if [ x$(on_mac_os) = xyes ]; then install_name_tool -id $(ildir)/libwcs.6.2.dylib \ $(ildir)/libwcs.6.2.dylib; fi # Write the target file. echo "WCSLIB is built" > $@ # Programs # -------- # # CMake can be built with its custom `./bootstrap' script. $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ $(ibdir)/curl # After searching in `bootstrap', I couldn't find `LIBS', only # `LDFLAGS'. So the extra libraries are being added to `LDFLAGS', # not `LIBS'. cd $(ddir) && rm -rf cmake-$(cmake-version) && \ tar xf $< && cd cmake-$(cmake-version) && \ ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ --system-bzip2 --system-liblzma --no-qt-gui && \ make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 && \ make install && \ cd ..&& rm -rf cmake-$(cmake-version) # Some programs that depend on cURL (in particular CMake) don't necessarily # have easiy ways to explicity tell them to also link with libcurl's # dependencies (libssl, libcrypto, and libz). So we won't force curl to # only be static. # # cURL (and its library, which is needed by several programs here) can # optionally link with many different network-related libraries on the host # system that we are not yet building in the pipeline. Many of these are # not relevant to most science projects, so we are explicitly using # `--without-XXX' or `--disable-XXX' so cURL doesn't link with them. Note # that if it does link with them, the pipeline will crash when the library # is updated/changed by the host, and the whole purpose of this pipeline is # avoid dependency on the host as much as possible. $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz $(call gbuild, $<, curl-$(curl-version), , \ LIBS="-pthread" \ --with-zlib=$(ildir) \ --with-ssl=$(idir) \ --without-mesalink \ --with-ca-fallback \ --without-librtmp \ --without-libidn2 \ --without-wolfssl \ --without-brotli \ --without-gnutls \ --without-cyassl \ --without-libpsl \ --without-axtls \ --disable-ldaps \ --disable-ldap \ --without-nss ) # On Mac OS, libtool does different things, so to avoid confusion, we'll # prefix GNU's libtool executables with `glibtool'. $(ibdir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz $(call gbuild, $<, libtool-$(libtool-version), static, \ --program-prefix=g) $(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz $(call gbuild, $<, ghostscript-$(ghostscript-version)) $(ibdir)/flock: $(tdir)/flock-$(flock-version).tar.xz $(call gbuild, $<, flock-$(flock-version), static) $(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \ $(ibdir)/curl $(call gbuild, $<, git-$(git-version), static, \ --without-tcltk --with-shell=$(ibdir)/bash, \ V=1) # Metastore is used to keep file modification dates (and generally many # meta-data) within the Git history. $(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ $(ibdir)/git # Metastore doesn't have any `./configure' script. So we'll just # call `pwd' as a place-holder for the `./configure' command. current_dir=$$(pwd) $(call gbuild, $<, metastore-$(metastore-version), static,,V=1,, \ pwd, PREFIX=$(idir)) # Write the relevant hooks into this system's Git hooks, so Git # calls metastore properly on every commit and every checkout. if [ -f $@ ]; then cd $$current_dir rm -f .git/hooks/pre-commit .git/hooks/post-checkout sed -e's|@BINDIR[@]|$(ibdir)|g' \ reproduce/src/bash/git-pre-commit > .git/hooks/pre-commit sed -e's|@BINDIR[@]|$(ibdir)|g' \ reproduce/src/bash/git-post-checkout > .git/hooks/post-checkout chmod +x .git/hooks/pre-commit .git/hooks/post-checkout fi # The order of dependencies is based on how long they take to build (how # large they are): Libgit2 depends on CMake which takes a VERY long time to # build. Also, Ghostscript and GSL are relatively large packages. So when # building in parallel, its better to have these packages start building # early. $(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \ $(ilidir)/libgit2 \ $(ibdir)/gs \ $(ilidir)/gsl \ $(ibdir)/glibtool \ $(ilidir)/libjpeg \ $(ilidir)/libtiff \ $(ilidir)/wcslib ifeq ($(static_build),yes) staticopts="--enable-static=yes --enable-shared=no"; endif $(call gbuild, $<, gnuastro-$(gnuastro-version), static, \ $$staticopts, -j$(numthreads), \ make check -j$(numthreads)) # Since we want to avoid complicating the PATH, we are putting a symbolic # link of all the TeX Live executables in $(ibdir). But symbolic links are # hard to track for Make (as a target). Also, TeX in general is optional # for the pipeline (the processing is the main target, not the generation # of the final PDF). So we'll make a simple ASCII file called # `texlive-ready-tlmgr' and use its contents to mark if we can use it or # not. $(ibdir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \ reproduce/config/pipeline/texlive.conf # Unpack, enter the directory, and install based on the given # configuration (prerequisite of this rule). @topdir=$$(pwd) cd $(ddir) rm -rf install-tl-* tar xf $(tdir)/install-tl-unx.tar.gz cd install-tl-* sed -e's|@installdir[@]|$(idir)|g' -e's|@topdir[@]|'"$$topdir"'|g' \ $$topdir/reproduce/config/pipeline/texlive.conf > texlive.conf # TeX Live's installation may fail due to any reason. But TeX Live # is optional (only necessary for building the final PDF). So we # don't want the configure script to fail if it can't run. if ./install-tl --profile=texlive.conf; then # Put a symbolic link of the TeX Live executables in `ibdir'. The # main problem is that the year and build system (for example # `x86_64-linux') are also in the directory names, making it hard # to be generic. We are using wildcards here, but only in this # Makefile, not in any other. ln -fs $(idir)/texlive/20*/bin/*/* $(ibdir)/ # Register that the build was successful. echo "TeX Live is ready." > $@ else echo "NOT!" > $@ fi # Clean up cd .. rm -rf install-tl-* # To keep things modular and simple, we'll break up the installation of TeX # Live itself (only very basic TeX and LaTeX) and the installation of its # necessary packages into two packages. $(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \ $(ibdir)/texlive-ready-tlmgr # To work with TeX live installation, we'll need the internet. @res=$$(cat $(ibdir)/texlive-ready-tlmgr) if [ x"$$res" = x"NOT!" ]; then echo "" > $@ else # The current directory is necessary later. topdir=$$(pwd) # Install all the extra necessary packages. If LaTeX complains # about not finding a command/file/what-ever/XXXXXX, simply run # the following command to find which package its in, then add it # to the `texlive-packages' variable of the first prerequisite. # # ./.local/bin/tlmgr info XXXXXX # # We are putting a notice, because if there is no internet, # `tlmgr' just hangs waiting. echo; echo; echo "Downloading necessary TeX packages..."; echo; tlmgr install $(texlive-packages) # Make a symbolic link of all the TeX Live executables in the bin # directory so we don't have to modify `PATH'. ln -fs $(idir)/texlive/20*/bin/*/* $(ibdir)/ # Get all the necessary versions. texlive=$$(pdflatex --version | awk 'NR==1' | sed 's/.*(\(.*\))/\1/' \ | awk '{print $$NF}'); echo "\newcommand{\\texliveversion}{$$texlive}" > $@ # LaTeX Package versions. tlmgr info $(texlive-packages) --only-installed | awk \ '$$1=="package:" {version=0; \ if($$NF=="tex-gyre") name="texgyre"; \ else name=$$NF} \ $$1=="cat-version:" {version=$$NF} \ $$1=="cat-date:" {if(version==0) version=$$2; \ printf("\\newcommand{\\tex%sversion}{%s}\n", \ name, version)}' >> $@ fi