Preference for shared library linking

Some high-level programs like Wget and cURL need to be built in shared mode
because they also include dynamic loading of libraries. Therefore, if we
only build the lower-level libraries in static mode, our own build will be
ignored and they will go and find the system's shared libraries to link
with. Because of this, for now, we have manually set the `static_build'
variable in the configure script to `no'.

Also, if the downloader fails, we'll delete the output (an empty file in
the case of Wget) because it interefers with a target definition.

5 files changed, 132 insertions, 87 deletions

diff --git a/configure b/configure
index fc2adc7..5fddc73 100755
--- a/configure
+++ b/configure
@@ -502,43 +502,56 @@ ln -s $(pwd)/reproduce/config/gnuastro .gnuastro
 
 # See if the C compiler can build static libraries
 # ------------------------------------------------
-oprog=$depdir/static-test
-cprog=$depdir/static-test.c
-echo "#include <stdio.h>"          > $cprog
-echo "int main(void) {return 0;}" >> $cprog
-if [ x$CC = x ]; then CC=gcc; fi;
-if $CC $cprog -o$oprog -static &> /dev/null; then
-    export static_build="yes"
-else
-    export static_build="no"
-fi
-rm -f $oprog $cprog
-if [ $printnotice = yes ] && [ $static_build = "no" ]; then
-    cat <<EOF
-_________________________________________________________________________
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!      WARNING      !!!!!!!!!!!!!!!!!!!!!!!!!!
-
-Your system's C compiler ('$CC') doesn't support building static
-libraries. Therefore the dependencies will be built dynamically. This means
-that they will depend more strongly on changes/updates in the host
-system. For high-level applications (like most research projects in natural
-sciences), this shouldn't be a significant problem.
-
-But generally, for reproducibility, its better to build static libraries
-and programs. For more on their difference (and generally an introduction
-on linking), please see the link below:
-
-https://www.gnu.org/software/gnuastro/manual/html_node/Linking.html
-
-If you have other compilers on your system, you can select a different
-compiler by setting the 'CC' environment variable before running
-'./configure'.
 
-!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-
-EOF
-    sleep 5
-fi
+# We are manually only working with shared libraries: because some
+# high-level programs like Wget and cURL need dynamic linking and if we
+# build the libraries statically, our own builds will be ignored and these
+# programs will go and find their necessary libraries on the host system.
+#
+# Another good advantage of shared libraries is that we can actually use
+# the shared library tool of the system (`ldd' with GNU C Library) and see
+# exactly where each linked library comes from. But in static building,
+# unless you follow the build closely, its not easy to see if the source of
+# the library came from the system or our build.
+static_build=no
+
+#oprog=$depdir/static-test
+#cprog=$depdir/static-test.c
+#echo "#include <stdio.h>"          > $cprog
+#echo "int main(void) {return 0;}" >> $cprog
+#if [ x$CC = x ]; then CC=gcc; fi;
+#if $CC $cprog -o$oprog -static &> /dev/null; then
+#    export static_build="yes"
+#else
+#    export static_build="no"
+#fi
+#rm -f $oprog $cprog
+#if [ $printnotice = yes ] && [ $static_build = "no" ]; then
+#    cat <<EOF
+#_________________________________________________________________________
+#!!!!!!!!!!!!!!!!!!!!!!!!!!!!      WARNING      !!!!!!!!!!!!!!!!!!!!!!!!!!
+#
+#Your system's C compiler ('$CC') doesn't support building static
+#libraries. Therefore the dependencies will be built dynamically. This means
+#that they will depend more strongly on changes/updates in the host
+#system. For high-level applications (like most research projects in natural
+#sciences), this shouldn't be a significant problem.
+#
+#But generally, for reproducibility, its better to build static libraries
+#and programs. For more on their difference (and generally an introduction
+#on linking), please see the link below:
+#
+#https://www.gnu.org/software/gnuastro/manual/html_node/Linking.html
+#
+#If you have other compilers on your system, you can select a different
+#compiler by setting the 'CC' environment variable before running
+#'./configure'.
+#
+#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+#
+#EOF
+#    sleep 5
+#fi
 
 
 
@@ -628,13 +641,10 @@ fi
 if which nproc > /dev/null 2>/dev/null; then numthreads=$(nproc --all);
 else                                         numthreads=2;
 fi
-#######################################
-#static_build=no
-#######################################
 make -f reproduce/src/make/dependencies-basic.mk \
      static_build=$static_build                  \
      needs_ldl=$needs_ldl                        \
-     #-j$numthreads
+     -j$numthreads
 
 
 
@@ -651,7 +661,7 @@ numthreads=$($instdir/bin/nproc)
 ./.local/bin/make -f reproduce/src/make/dependencies.mk    \
                   static_build=$static_build               \
                   rpath_command=$rpath_command             \
-                  #-j$numthreads
+                  -j$numthreads
 
 
 
diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/config/pipeline/dependency-versions.mk
index 25aa277..99d55ee 100644
--- a/reproduce/config/pipeline/dependency-versions.mk
+++ b/reproduce/config/pipeline/dependency-versions.mk
@@ -14,7 +14,7 @@ gcc-version         = 8.2.0
 ghostscript-version = 9.26
 git-version         = 2.19.1
 gmp-version         = 6.1.2
-gnuastro-version    = 0.7.66-0ad3
+gnuastro-version    = 0.7.67-4a59
 grep-version        = 3.1
 gzip-version        = 1.9.10-051e
 isl-version         = 0.18
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index 77a47c4..0f66b06 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -63,7 +63,7 @@ export LDFLAGS           := -L$(ildir) $(LDFLAGS)
 export CPPFLAGS          := -I$(idir)/include $(CPPFLAGS)
 export LD_LIBRARY_PATH   := $(ildir):$(LD_LIBRARY_PATH)
 
-top-level-programs = ls sed gawk grep diff find bash wget which pkg-config
+top-level-programs = ls sed gawk grep diff find bash wget which
 all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
 
 
@@ -160,11 +160,17 @@ $(tarballs): $(tdir)/%:
 	  if [ $$mergenames = 1 ]; then  tarballurl=$$w/"$*";               \
 	  else                           tarballurl=$$w;                    \
 	  fi;                                                               \
+                                                                            \
 	  echo "Downloading $$tarballurl";                                  \
 	  if [ -f $(ibdir)/wget ]; then                                     \
-	    $(ibdir)/wget --no-use-server-timestamps -O$@ $$tarballurl;     \
+	    downloader="wget --no-use-server-timestamps -O";                \
 	  else                                                              \
-	    $(DOWNLOADER) $@ $$tarballurl;                                  \
+	    downloader="$(DOWNLOADER)";                                     \
+	  fi;                                                               \
+                                                                            \
+	  if ! $$downloader $@ $$tarballurl; then                           \
+	     rm -f $@;                                                      \
+	     echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1;      \
 	  fi;                                                               \
 	fi
 
@@ -212,10 +218,14 @@ $(ibdir)/low-level: | $(ibdir) $(ildir)
         # Needed by TeXLive specifically.
 	$(call makelink,perl)
 
-        # Libdl (for dynamic loading libraries at runtime)
-	if [ -f /usr/lib/libdl.a ]; then    \
-	  ln -s /usr/lib/libdl.* $(ildir)/;        \
-	fi;
+        # Necessary libraries:
+        #   Libdl (for dynamic loading libraries at runtime)
+        #   POSIX Threads library for multi-threaded programs.
+	for l in dl pthread; do                    \
+	  if [ -f /usr/lib/lib$$l.a ]; then        \
+	    ln -s /usr/lib/lib$$l.* $(ildir)/;     \
+	  fi;                                      \
+	done
 
 	echo "Low-level symbolic links are setup" > $@
 
@@ -294,32 +304,32 @@ $(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \
 # Downloader
 # ----------
 #
-# Zlib's `./configure' doesn't use Autoconf's configure script, it just
-# accepts a direct `--static' option.
+# Some programs (like Wget and CMake) that use zlib need it to be dynamic
+# so they use our custom build. So we won't force a static-only build.
+#
+# Note for a static-only build: Zlib's `./configure' doesn't use Autoconf's
+# configure script, it just accepts a direct `--static' option.
 $(idir)/etc:; mkdir $@
 $(ilidir): | $(ildir); mkdir $@
 $(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \
                 $(ibdir)/make | $(ilidir)
-        # IMPORTANT, the second argument to `gbuild', must not have any
-        # spaces before or after it: it is going to be checked.
-ifeq ($(static_build),yes)
-	$(call gbuild, $<,zlib-$(zlib-version), , --static) \
-	&& echo "Zlib is built" > $@
-else
 	$(call gbuild, $<,zlib-$(zlib-version)) && echo "Zlib is built" > $@
-endif
 
-# OpenSSL
-ifeq ($(static_build),yes)
-openssl-static = no-dso no-dynamic-engine no-shared
-endif
+# OpenSSL: Some programs/libraries later need dynamic linking. So we'll
+# build libssl (and libcrypto) dynamically also.
+#
+# In case you do want a static OpenSSL and libcrypto, then uncomment the
+# following conditional and put $(openssl-static) in the configure options.
+#
+#ifeq ($(static_build),yes)
+#openssl-static = no-dso no-dynamic-engine no-shared
+#endif
 $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz             \
                    $(ilidir)/zlib | $(idir)/etc
-	$(call gbuild, $<, openssl-$(openssl-version), static,           \
+	$(call gbuild, $<, openssl-$(openssl-version), ,                 \
                        --openssldir=$(idir)/etc/ssl                      \
 	               --with-zlib-lib=$(ildir)                          \
-                       --with-zlib-include=$(idir)/include zlib          \
-                       $(openssl-static) )                               \
+                       --with-zlib-include=$(idir)/include zlib  )       \
 	&& echo "OpenSSL is built" > $@
 
 # GNU Wget
@@ -331,12 +341,13 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz             \
 # Also note that since Wget needs to load outside libraries dynamically, it
 # gives a segmentation fault when built statically.
 $(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \
+	       $(ibdir)/pkg-config                 \
                $(ilidir)/openssl
 	libs="-pthread";                                                 \
 	if [ x$(needs_ldl) = xyes ]; then libs="$$libs -ldl"; fi;        \
 	$(call gbuild, $<, wget-$(wget-version), ,                       \
                        LIBS="$$LIBS $$libs" --with-ssl=openssl           \
-	               --with-openssl=yes)
+	               --with-openssl=yes --with-libssl-prefix=$(idir))
 
 
 
diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/src/make/dependencies-build-rules.mk
index d1cc5dc..457d5fe 100644
--- a/reproduce/src/make/dependencies-build-rules.mk
+++ b/reproduce/src/make/dependencies-build-rules.mk
@@ -106,7 +106,8 @@ cbuild = if [ x$(static_build) = xyes ] && [ $(3)x = staticx ]; then          \
 	 rm -rf pipeline-build && mkdir pipeline-build &&                     \
 	 cd pipeline-build &&                                                 \
 	 cmake .. -DCMAKE_LIBRARY_PATH=$(ildir)                               \
-	          -DCMAKE_INSTALL_PREFIX=$(idir) $$opts $(4) &&               \
+	          -DCMAKE_INSTALL_PREFIX=$(idir)                              \
+	          -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON $$opts $(4) &&             \
 	 make && make install &&                                              \
 	 cd ../.. &&                                                          \
 	 rm -rf $(2)
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index b65d8e8..d705d1e 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -153,8 +153,15 @@ $(tarballs): $(tdir)/%:
 	  if [ $$mergenames = 1 ]; then  tarballurl=$$w/"$*"
 	  else                           tarballurl=$$w
 	  fi
+
+          # If the download fails, Wget will write the error message in the
+          # target file, so Make will think that its done! To avoid this
+          # problem, we'll rename the output.
 	  echo "Downloading $$tarballurl"
-	  $(ibdir)/wget --no-use-server-timestamps -O$@ $$tarballurl
+	  if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then
+	     rm -f $@
+	     echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1
+	  fi
 	fi
 
 
@@ -177,23 +184,25 @@ $(tarballs): $(tdir)/%:
 # and create/write into it when the library is successfully built.
 $(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \
                    $(ibdir)/curl
-	$(call gbuild, $<,cfitsio, static, --enable-sse2 --enable-reentrant) \
+	$(call gbuild, $<, cfitsio, static, LIBS="-lssl -lcrypto -lz" \
+                       --enable-sse2 --enable-reentrant) \
 	&& echo "CFITSIO is built" > $@
 
 
 # The libgit2 page recommends doing a static build, especially for Mac
-# systems. Under XCode, the following link has written "It’s highly
-# recommended that you build libgit2 as a static library for Xcode
-# projects. This simplifies distribution significantly, as the resolution
-# of dynamic libraries at runtime can be extremely problematic.". This is a
-# major problem we have been having so far with Mac systems:
-# https://libgit2.org/docs/guides/build-and-link
+# systems (with `-DBUILD_SHARED_LIBS=OFF'). Under XCode, the following link
+# has written "It’s highly recommended that you build libgit2 as a static
+# library for Xcode projects. This simplifies distribution significantly,
+# as the resolution of dynamic libraries at runtime can be extremely
+# problematic.". This is a major problem we have been having so far with
+# Mac systems: https://libgit2.org/docs/guides/build-and-link
 $(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \
                    $(ibdir)/cmake                            \
                    $(ibdir)/curl
+	export LDFLAGS="$$LDFLAGS -lssl -lcrypto -lz";          \
 	$(call cbuild, $<, libgit2-$(libgit2-version), static,  \
 	              -DUSE_SSH=OFF -DBUILD_CLAR=OFF            \
-	              -DTHREADSAFE=ON -DBUILD_SHARED_LIBS=OFF)  \
+	              -DTHREADSAFE=ON )                         \
 	&& echo "Libgit2 is built" > $@
 
 $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz
@@ -212,9 +221,11 @@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
                   $(ilidir)/cfitsio
         # Unfortunately WCSLIB forces the building of shared libraries. So
         # we'll just delete any shared library that is produced afterwards.
-	$(call gbuild, $<, wcslib-$(wcslib-version), ,            \
-	              LIBS="-pthread -lcurl -lm" --without-pgplot \
-	              --disable-fortran) &&                       \
+	$(call gbuild, $<, wcslib-$(wcslib-version), ,              \
+	               LIBS="-pthread -lcurl -lssl -lcrypto -lz -lm" \
+                       --with-cfitsiolib=$(ildir)                    \
+                       --with-cfitsioinc=$(idir)/include             \
+                       --without-pgplot --disable-fortran) &&        \
 	echo "WCSLIB is built" > $@
 
 
@@ -226,15 +237,27 @@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \
 # CMake can be built with its custom `./bootstrap' script.
 $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \
                 $(ibdir)/curl
-	cd $(ddir) && rm -rf cmake-$(cmake-version) &&           \
-	tar xf $< && cd cmake-$(cmake-version) &&                \
-	./bootstrap --prefix=$(idir) --system-curl --system-zlib \
-	             --system-bzip2 --system-liblzma &&          \
-	make && make install &&                                  \
+        # After searching in `bootstrap', I couldn't find `LIBS', only
+        # `LDFLAGS'. So the extra libraries are being added to `LDFLAGS',
+        # not `LIBS'.
+	cd $(ddir) && rm -rf cmake-$(cmake-version) &&             \
+	tar xf $< && cd cmake-$(cmake-version) &&                  \
+	./bootstrap --prefix=$(idir) --system-curl --system-zlib   \
+	            --system-bzip2 --system-liblzma --no-qt-gui && \
+	make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 &&         \
+	make install &&                                            \
 	cd ..&& rm -rf cmake-$(cmake-version)
 
+# Some programs that depend on cURL (in particular CMake) don't necessarily
+# have easiy ways to explicity tell them to also link with libcurl's
+# dependencies (libssl, libcrypto, and libz). So we won't force curl to
+# only be static.
 $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz
-	$(call gbuild, $<, curl-$(curl-version), static, --without-brotli)
+	$(call gbuild, $<, curl-$(curl-version), ,       \
+                       --with-zlib=$(ildir)              \
+                       --with-ssl=$(idir)                \
+                       --without-brotli                  \
+                       LIBS="-pthread" )
 
 # On Mac OS, libtool does different things, so to avoid confusion, we'll
 # prefix GNU's libtool executables with `glibtool'.
@@ -263,10 +286,10 @@ $(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \
 ifeq ($(static_build),yes)
 	$(call gbuild, $<, gnuastro-$(gnuastro-version), static,     \
 	               --enable-static=yes --enable-shared=no, -j8,  \
-	               cp config.log ../gnuastro-config.log; make check -j8)
+	               make check -j8)
 else
 	$(call gbuild, $<, gnuastro-$(gnuastro-version), , , -j8,    \
-	               cp config.log ../gnuastro-config.log; make check -j8)
+	               make check -j8)
 endif