path: root/reproduce/src/make/dependencies.mk

# Build the reproduction pipeline dependencies (programs and libraries).
#
# ------------------------------------------------------------------------
#                      !!!!! IMPORTANT NOTES !!!!!
#
# This Makefile will be run by the initial `./configure' script. It is not
# included into the reproduction pipe after that.
#
# ------------------------------------------------------------------------
#
# Original author:
#     Mohammad Akhlaghi <mohammad@akhlaghi.org>
# Contributing author(s):
#     Your name <your@email.address>
# Copyright (C) 2018-2019, Your Name.
#
# This Makefile is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# This Makefile is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
# Public License for more details.
#
# A copy of the GNU General Public License is available at
# <http://www.gnu.org/licenses/>.



# Top level environment
include reproduce/config/pipeline/LOCAL.mk
include reproduce/src/make/dependencies-build-rules.mk
include reproduce/config/pipeline/dependency-texlive.mk
include reproduce/config/pipeline/dependency-versions.mk

ddir   = $(BDIR)/dependencies
tdir   = $(BDIR)/dependencies/tarballs
idir   = $(BDIR)/dependencies/installed
ibdir  = $(BDIR)/dependencies/installed/bin
ildir  = $(BDIR)/dependencies/installed/lib
ilidir = $(BDIR)/dependencies/installed/lib/built

# Define the top-level programs to build (installed in `.local/bin').
top-level-programs = astnoisechisel git flock
all: $(ddir)/texlive-versions.tex \
     $(foreach p, $(top-level-programs), $(ibdir)/$(p))

# Other basic environment settings: We are only including the host
# operating system's PATH environment variable (after our own!) for the
# compiler and linker. For the library binaries and headers, we are only
# using our internally built libraries.
#
# To investigate:
#
#    1) Set SHELL to `$(ibdir)/env - NAME=VALUE $(ibdir)/bash' and set all
#       the parameters defined bellow as `NAME=VALUE' statements before
#       calling Bash. This will enable us to completely ignore the user's
#       native environment.
#
#    2) Add `--noprofile --norc' to `.SHELLFLAGS' so doesn't load the
#       user's environment.
.ONESHELL:
.SHELLFLAGS              := --noprofile --norc -ec
export PATH              := $(ibdir)
export LD_RUN_PATH       := $(ildir)
export LD_LIBRARY_PATH   := $(ildir)
export SHELL             := $(ibdir)/bash
export CPPFLAGS          := -I$(idir)/include
export PKG_CONFIG_PATH   := $(ildir)/pkgconfig
export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig
export LDFLAGS           := $(rpath_command) -L$(ildir)





# Tarballs
# --------
#
# All the necessary tarballs are defined and prepared with this rule.
#
# Note that we want the tarballs to follow the convention of NAME-VERSION
# before the `tar.XX' prefix. For those programs that don't follow this
# convention, but include the name/version in their tarball names with
# another format, we'll do the modification before the download so the
# downloaded file has our desired format.
tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz             \
                        cmake-$(cmake-version).tar.gz                 \
                        curl-$(curl-version).tar.gz                   \
	                flock-$(flock-version).tar.xz                 \
	                ghostscript-$(ghostscript-version).tar.gz     \
	                git-$(git-version).tar.xz                     \
	                gnuastro-$(gnuastro-version).tar.lz           \
	                gsl-$(gsl-version).tar.gz                     \
                        install-tl-unx.tar.gz                         \
	                jpegsrc.$(libjpeg-version).tar.gz             \
                        tiff-$(libtiff-version).tar.gz                \
                        libtool-$(libtool-version).tar.xz             \
                        libgit2-$(libgit2-version).tar.gz             \
	                wcslib-$(wcslib-version).tar.bz2              \
                      , $(tdir)/$(t) )
$(tarballs): $(tdir)/%:
	if [ -f $(DEPENDENCIES-DIR)/$* ]; then
	  cp $(DEPENDENCIES-DIR)/$* $@
	else
          # Remove all numbers, `-' and `.' from the tarball name so we can
          # search more easily only with the program name.
	  n=$$(echo $* | sed -e's/[0-9\-]/ /g' -e's/\./ /g'           \
	               | awk '{print $$1}' )

          # Set the top download link of the requested tarball.
	  mergenames=1
	  if [ $$n = cfitsio     ]; then
	    mergenames=0
	    v=$$(echo $(cfitsio-version) | sed -e's/\.//'             \
	              | awk '{l=length($$1);                          \
	                      printf (l==4 ? "%d\n"                   \
	                              : (l==3 ? "%d0\n"               \
	                                 : (l==2 ? "%d00\n"           \
                                            : "%d000\n") ), $$1)}')
	    w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz
	  elif [ $$n = cmake       ]; then w=https://cmake.org/files/v3.12
	  elif [ $$n = curl        ]; then w=https://curl.haxx.se/download
	  elif [ $$n = flock       ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version)
	  elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926
	  elif [ $$n = git         ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git
	  elif [ $$n = gnuastro    ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro
	  elif [ $$n = gsl         ]; then w=http://ftpmirror.gnu.org/gnu/gsl
	  elif [ $$n = install     ]; then w=http://mirror.ctan.org/systems/texlive/tlnet
	  elif [ $$n = jpegsrc     ]; then w=http://ijg.org/files
	  elif [ $$n = libtool     ]; then w=http://ftpmirror.gnu.org/gnu/libtool
	  elif [ $$n = libgit      ]; then
	    mergenames=0
	    w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
	  elif [ $$n = tiff        ]; then w=https://download.osgeo.org/libtiff
	  elif [ $$n = wcslib      ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib
	  else
	    echo; echo; echo;
	    echo "'$$n' not recognized as a dependency name to download."
	    echo; echo; echo;
	    exit 1
	  fi

          # Download the requested tarball. Note that some packages may not
          # follow our naming convention (where the package name is merged
          # with its version number). In such cases, `w' will be the full
          # address, not just the top directory address. But since we are
          # storing all the tarballs in one directory, we want it to have
          # the same naming convention, so we'll download it to a temporary
          # name, then rename that.
	  if [ $$mergenames = 1 ]; then  tarballurl=$$w/"$*"
	  else                           tarballurl=$$w
	  fi

          # If the download fails, Wget will write the error message in the
          # target file, so Make will think that its done! To avoid this
          # problem, we'll rename the output.
	  echo "Downloading $$tarballurl"
	  if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then
	     rm -f $@
	     echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1
	  fi
	fi





# Libraries
# ---------
#
# We would prefer to build static libraries, but some compilers like LLVM
# don't have static capabilities, so they'll only build dynamic/shared
# libraries. Therefore, we can't use the easy `.a' suffix for static
# libraries as targets and there are different conventions for shared
# library names.
#
# For the actual build, the same compiler that built the library will build
# the programs, so exact knowledge of the suffix is ultimately irrelevant
# for us here. So, we'll make an `$(ildir)/built' directory and make a
# simple plain text file in it with the basic library name (an no prefix)
# and create/write into it when the library is successfully built.
$(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \
                   $(ibdir)/curl

        # CFITSIO hard-codes the absolute address of cURL's `curl-config'
        # program (which gives the necessary header and linking
        # information) into the configure script. So we'll have to modify
        # it manually before doing the standard build.
	topdir=$(pwd); cd $(ddir); tar xf $<
	customtar=cfitsio-$(cfitsio-version)-custom.tar.gz
	sed cfitsio/configure                                 \
	    -e's|/usr/bin/curl-config|$(ibdir)/curl-config|g' \
	    > cfitsio/configure_tmp
	mv cfitsio/configure_tmp cfitsio/configure
	chmod +x cfitsio/configure
	tar cf $$customtar cfitsio
	cd $$topdir

        # Continue the standard build on the customized tarball.
	$(call gbuild, $$customtar, cfitsio, static,     \
	               --enable-sse2 --enable-reentrant) \
	&& rm $$customtar                                \
	&& echo "CFITSIO is built" > $@

$(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz
	$(call gbuild, $<, gsl-$(gsl-version), static) \
	&& echo "GNU Scientific Library is built" > $@

$(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz
	$(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@

$(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \
                   $(ilidir)/libjpeg
	$(call gbuild, $<, tiff-$(libtiff-version), static) \
	&& echo "Libtiff is built" > $@





# Libraries with special attention on Mac OS
# ------------------------------------------
#
# Libgit2 and WCSLIB don't set their installation path, or don't do it
# properly, in their finally installed shared libraries. But since we are
# linking everything (including OpenSSL and its dependencies) dynamically,
# we need to also make a shared libraries and can't use static
# libraries. So for Mac OS systems we have to correct their addresses
# manually.
#
# For example, Libgit2 page recommends doing a static build, especially for
# Mac systems (with `-DBUILD_SHARED_LIBS=OFF'): "It’s highly recommended
# that you build libgit2 as a static library for Xcode projects. This
# simplifies distribution significantly, as the resolution of dynamic
# libraries at runtime can be extremely problematic.". This is a major
# problem we have been having so far with Mac systems:
# https://libgit2.org/docs/guides/build-and-link
$(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \
                   $(ibdir)/cmake                            \
                   $(ibdir)/curl
        # Build and install the library.
	$(call cbuild, $<, libgit2-$(libgit2-version), static,  \
	              -DUSE_SSH=OFF -DBUILD_CLAR=OFF            \
	              -DTHREADSAFE=ON )

        # Correct the shared library absolute address if necessary.
	if [ x$(on_mac_os) = xyes ]; then
	  install_name_tool -id $(ildir)/libgit2.26.dylib \
	                        $(ildir)/libgit2.26.dylib
	fi

        # Write the target file.
	echo "Libgit2 is built" > $@

$(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
                  $(ilidir)/cfitsio
        # Build and install the library.
	$(call gbuild, $<, wcslib-$(wcslib-version), ,               \
	               LIBS="-pthread -lcurl -lm"                    \
                       --with-cfitsiolib=$(ildir)                    \
                       --with-cfitsioinc=$(idir)/include             \
                       --without-pgplot --disable-fortran)

        # Correct the shared library absolute address if necessary.
	if [ x$(on_mac_os) = xyes ]; then
	  install_name_tool -id $(ildir)/libwcs.6.2.dylib \
	                        $(ildir)/libwcs.6.2.dylib;
	fi

        # Write the target file.
	echo "WCSLIB is built" > $@





# Programs
# --------
#
# CMake can be built with its custom `./bootstrap' script.
$(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
                $(ibdir)/curl
        # After searching in `bootstrap', I couldn't find `LIBS', only
        # `LDFLAGS'. So the extra libraries are being added to `LDFLAGS',
        # not `LIBS'.
	cd $(ddir) && rm -rf cmake-$(cmake-version) &&             \
	tar xf $< && cd cmake-$(cmake-version) &&                  \
	./bootstrap --prefix=$(idir) --system-curl --system-zlib   \
	            --system-bzip2 --system-liblzma --no-qt-gui && \
	make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 &&         \
	make install &&                                            \
	cd ..&& rm -rf cmake-$(cmake-version)

# Some programs that depend on cURL (in particular CMake) don't necessarily
# have easiy ways to explicity tell them to also link with libcurl's
# dependencies (libssl, libcrypto, and libz). So we won't force curl to
# only be static.
$(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz
	$(call gbuild, $<, curl-$(curl-version), ,       \
                       --with-zlib=$(ildir)              \
                       --with-ssl=$(idir)                \
                       --without-brotli                  \
                       LIBS="-pthread" )

# On Mac OS, libtool does different things, so to avoid confusion, we'll
# prefix GNU's libtool executables with `glibtool'.
$(ibdir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz
	$(call gbuild, $<, libtool-$(libtool-version), static, \
                       --program-prefix=g)

$(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz
	$(call gbuild, $<, ghostscript-$(ghostscript-version))

$(ibdir)/flock: $(tdir)/flock-$(flock-version).tar.xz
	$(call gbuild, $<, flock-$(flock-version), static)

$(ibdir)/git: $(tdir)/git-$(git-version).tar.xz
	$(call gbuild, $<, git-$(git-version), static,             \
                       --without-tcltk --with-shell=$(ibdir)/bash, \
	               V=1)

# The order of dependencies is based on how long they take to build (how
# large they are): Libgit2 depends on CMake which takes a VERY long time to
# build. Also, Ghostscript and GSL are relatively large packages. So when
# building in parallel, its better to have these packages start building
# early.
$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \
                         $(ilidir)/libgit2 \
                         $(ibdir)/gs       \
                         $(ilidir)/gsl     \
                         $(ibdir)/glibtool \
                         $(ilidir)/libjpeg \
                         $(ilidir)/libtiff \
                         $(ilidir)/wcslib
ifeq ($(static_build),yes)
	$(call gbuild, $<, gnuastro-$(gnuastro-version), static,     \
	               --enable-static=yes --enable-shared=no,       \
	               -j$(numthreads), make check -j$(numthreads))
else
	$(call gbuild, $<, gnuastro-$(gnuastro-version), , ,         \
	               -j$(numthreads), make check -j$(numthreads))
endif





# Since we want to avoid complicating the PATH, we are putting a symbolic
# link of all the TeX Live executables in $(ibdir). But symbolic links are
# hard to track for Make (as a target). Also, TeX in general is optional
# for the pipeline (the processing is the main target, not the generation
# of the final PDF). So we'll make a simple ASCII file called
# `texlive-ready-tlmgr' and use its contents to mark if we can use it or
# not.
$(ibdir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \
                              reproduce/config/pipeline/texlive.conf

        # Unpack, enter the directory, and install based on the given
        # configuration (prerequisite of this rule).
	@topdir=$$(pwd)
	cd $(ddir)
	rm -rf install-tl-*
	tar xf $(tdir)/install-tl-unx.tar.gz
	cd install-tl-*
	sed -e's|@installdir[@]|$(idir)|g' -e's|@topdir[@]|'"$$topdir"'|g' \
	    $$topdir/reproduce/config/pipeline/texlive.conf > texlive.conf

        # TeX Live's installation may fail due to any reason. But TeX Live
        # is optional (only necessary for building the final PDF). So we
        # don't want the configure script to fail if it can't run.
	if ./install-tl --profile=texlive.conf; then

          # Put a symbolic link of the TeX Live executables in `ibdir'. The
          # main problem is that the year and build system (for example
          # `x86_64-linux') are also in the directory names, making it hard
          # to be generic. We are using wildcards here, but only in this
          # Makefile, not in any other.
	  ln -fs $(idir)/texlive/20*/bin/*/* $(ibdir)/

          # Register that the build was successful.
	  echo "TeX Live is ready." > $@
	else
	  echo "NOT!" > $@
	fi

        # Clean up
	cd ..
	rm -rf install-tl-*





# To keep things modular and simple, we'll break up the installation of TeX
# Live itself (only very basic TeX and LaTeX) and the installation of its
# necessary packages into two packages.
$(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \
	                      $(ibdir)/texlive-ready-tlmgr

        # To work with TeX live installation, we'll need the internet.
	@res=$$(cat $(ibdir)/texlive-ready-tlmgr)
	if [ x"$$res" = x"NOT!" ]; then
	  echo "" > $@
	else
          # The current directory is necessary later.
	  topdir=$$(pwd)

          # Install all the extra necessary packages. If LaTeX complains
          # about not finding a command/file/what-ever/XXXXXX, simply run
          # the following command to find which package its in, then add it
          # to the `texlive-packages' variable of the first prerequisite.
          #
          #     ./.local/bin/tlmgr info XXXXXX
          #
          # We are putting a notice, because if there is no internet,
          # `tlmgr' just hangs waiting.
	  echo; echo; echo "Downloading necessary TeX packages..."; echo;
	  tlmgr install $(texlive-packages)

          # Make a symbolic link of all the TeX Live executables in the bin
          # directory so we don't have to modify `PATH'.
	  ln -fs $(idir)/texlive/20*/bin/*/* $(ibdir)/

          # Get all the necessary versions.
	  texlive=$$(pdflatex --version | awk 'NR==1' | sed 's/.*(\(.*\))/\1/' \
	                      | awk '{print $$NF}');
	  echo "\newcommand{\\texliveversion}{$$texlive}" > $@

          # LaTeX Package versions.
	  tlmgr info $(texlive-packages) --only-installed | awk                \
	       '$$1=="package:" {version=0;                                    \
	                         if($$NF=="tex-gyre") name="texgyre";          \
	                         else                 name=$$NF}               \
	        $$1=="cat-version:" {version=$$NF}                             \
	        $$1=="cat-date:" {if(version==0) version=$$2;                  \
	                          printf("\\newcommand{\\tex%sversion}{%s}\n", \
	                          name, version)}' >> $@
	fi