flock is now built in configure, to allow serial downloads

Until now, we were using `flock' (file-lock) for downloading the input
datasets in series. But we couldn't do this when downloading the software
tarballs because `flock' wasn't yet available. Generally, unlike
processing, downloading is much better done in series than in parallel.

To enable serial downloads of the software also, with this commit we are
installing `flock' in the configure script (not in a Makefile). As a
result, besides `flock', we can also benefit from the other good features
of the `reproduce/src/bash/download-multi-try' script *(for example
attempting download again after some time).

Some GNU mirrors may have problems at the time of download, so with this
commit, we are using the main GNU FTP server for GNU programs.

3 files changed, 33 insertions, 35 deletions

diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk
index 2e6caf4..cdb4a8c 100644
--- a/reproduce/src/make/dependencies-basic.mk
+++ b/reproduce/src/make/dependencies-basic.mk
@@ -43,12 +43,13 @@ include reproduce/config/pipeline/LOCAL.mk
 include reproduce/src/make/dependencies-build-rules.mk
 include reproduce/config/pipeline/dependency-versions.mk
 
-ddir  = $(BDIR)/dependencies
-tdir  = $(BDIR)/dependencies/tarballs
-idir  = $(BDIR)/dependencies/installed
-ibdir = $(BDIR)/dependencies/installed/bin
-ildir = $(BDIR)/dependencies/installed/lib
-ilidir = $(BDIR)/dependencies/installed/lib/built
+lockdir = $(BDIR)/locks
+ddir    = $(BDIR)/dependencies
+tdir    = $(BDIR)/dependencies/tarballs
+idir    = $(BDIR)/dependencies/installed
+ibdir   = $(BDIR)/dependencies/installed/bin
+ildir   = $(BDIR)/dependencies/installed/lib
+ilidir  = $(BDIR)/dependencies/installed/lib/built
 
 # We'll need the system's PATH for making links to low-level programs we
 # won't be building ourselves.
@@ -94,6 +95,8 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p))
 #
 # However, downloading from this link is slow (because its just a link). So
 # its easier to just keep a with the others.
+$(lockdir): | $(BDIR); mkdir $@
+downloadwrapper = ./reproduce/src/bash/download-multi-try
 tarballs = $(foreach t, bash-$(bash-version).tar.gz                         \
                         binutils-$(binutils-version).tar.lz                 \
                         bzip2-$(bzip2-version).tar.gz                       \
@@ -123,7 +126,7 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz                         \
                         xz-$(xz-version).tar.gz                             \
                         zlib-$(zlib-version).tar.gz                         \
                       , $(tdir)/$(t) )
-$(tarballs): $(tdir)/%:
+$(tarballs): $(tdir)/%: $(lockdir)
 	if [ -f $(DEPENDENCIES-DIR)/$* ]; then                              \
 	  cp $(DEPENDENCIES-DIR)/$* $@;                                     \
 	else                                                                \
@@ -132,32 +135,32 @@ $(tarballs): $(tdir)/%:
 	               | awk '{print $$1}' );                               \
 	                                                                    \
 	  mergenames=1;                                                     \
-          if   [ $$n = bash      ]; then w=http://ftpmirror.gnu.org/gnu/bash; \
-          elif [ $$n = binutils  ]; then w=http://ftpmirror.gnu.org/gnu/binutils; \
+          if   [ $$n = bash      ]; then w=http://ftp.gnu.org/gnu/bash;     \
+          elif [ $$n = binutils  ]; then w=http://ftp.gnu.org/gnu/binutils; \
           elif [ $$n = bzip      ]; then w=http://akhlaghi.org/src;         \
           elif [ $$n = cert      ]; then w=http://akhlaghi.org/src;         \
-          elif [ $$n = coreutils ]; then w=http://ftpmirror.gnu.org/gnu/coreutils;\
-          elif [ $$n = diffutils ]; then w=http://ftpmirror.gnu.org/gnu/diffutils;\
+          elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils;\
+          elif [ $$n = diffutils ]; then w=http://ftp.gnu.org/gnu/diffutils;\
           elif [ $$n = findutils ]; then w=http://akhlaghi.org/src;         \
-          elif [ $$n = gawk      ]; then w=http://ftpmirror.gnu.org/gnu/gawk; \
+          elif [ $$n = gawk      ]; then w=http://ftp.gnu.org/gnu/gawk;     \
           elif [ $$n = gcc       ]; then w=http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version); \
           elif [ $$n = gmp       ]; then w=https://gmplib.org/download/gmp; \
-          elif [ $$n = grep      ]; then w=http://ftpmirror.gnu.org/gnu/grep; \
-          elif [ $$n = gzip      ]; then w=http://ftpmirror.gnu.org/gnu/gzip; \
+          elif [ $$n = grep      ]; then w=http://ftp.gnu.org/gnu/grep;     \
+          elif [ $$n = gzip      ]; then w=http://ftp.gnu.org/gnu/gzip;     \
           elif [ $$n = isl       ]; then w=ftp://gcc.gnu.org/pub/gcc/infrastructure; \
           elif [ $$n = lzip      ]; then w=http://download.savannah.gnu.org/releases/lzip; \
           elif [ $$n = make      ]; then w=http://akhlaghi.org/src;         \
           elif [ $$n = mpfr      ]; then w=http://www.mpfr.org/mpfr-current;\
-          elif [ $$n = mpc       ]; then w=http://ftpmirror.gnu.org/gnu/mpc;\
-          elif [ $$n = ncurses   ]; then w=http://ftpmirror.gnu.org/gnu/ncurses;\
+          elif [ $$n = mpc       ]; then w=http://ftp.gnu.org/gnu/mpc;      \
+          elif [ $$n = ncurses   ]; then w=http://ftp.gnu.org/gnu/ncurses;  \
           elif [ $$n = openssl   ]; then w=http://www.openssl.org/source;   \
           elif [ $$n = patchelf  ]; then w=http://nixos.org/releases/patchelf/patchelf-$(patchelf-version); \
           elif [ $$n = pkg       ]; then w=http://pkg-config.freedesktop.org/releases; \
-          elif [ $$n = readline  ]; then w=http://ftpmirror.gnu.org/gnu/readline; \
-          elif [ $$n = sed       ]; then w=http://ftpmirror.gnu.org/gnu/sed;\
-          elif [ $$n = tar       ]; then w=http://ftpmirror.gnu.org/gnu/tar;\
-          elif [ $$n = wget      ]; then w=http://ftpmirror.gnu.org/gnu/wget;\
-          elif [ $$n = which     ]; then w=http://ftpmirror.gnu.org/gnu/which;\
+          elif [ $$n = readline  ]; then w=http://ftp.gnu.org/gnu/readline; \
+          elif [ $$n = sed       ]; then w=http://ftp.gnu.org/gnu/sed;      \
+          elif [ $$n = tar       ]; then w=http://ftp.gnu.org/gnu/tar;      \
+          elif [ $$n = wget      ]; then w=http://ftp.gnu.org/gnu/wget;     \
+          elif [ $$n = which     ]; then w=http://ftp.gnu.org/gnu/which;    \
           elif [ $$n = xz        ]; then w=http://tukaani.org/xz;           \
           elif [ $$n = zlib      ]; then w=http://www.zlib.net;             \
           else                                                              \
@@ -178,10 +181,9 @@ $(tarballs): $(tdir)/%:
 	    downloader="$(DOWNLOADER)";                                     \
 	  fi;                                                               \
                                                                             \
-	  if ! $$downloader $@ $$tarballurl; then                           \
-	     rm -f $@;                                                      \
-	     echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1;      \
-	  fi;                                                               \
+	  touch $(lockdir)/download;                                        \
+	  $(downloadwrapper) "$$downloader" $(lockdir)/download             \
+	                     $$tarballurl $@;                               \
 	fi
 
 
diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk
index deb9f39..2b3b70b 100644
--- a/reproduce/src/make/dependencies.mk
+++ b/reproduce/src/make/dependencies.mk
@@ -43,7 +43,7 @@ ildir  = $(BDIR)/dependencies/installed/lib
 ilidir = $(BDIR)/dependencies/installed/lib/built
 
 # Define the top-level programs to build (installed in `.local/bin').
-top-level-programs  = astnoisechisel flock metastore unzip zip
+top-level-programs  = astnoisechisel metastore unzip zip
 top-level-libraries = freetype
 all: $(ddir)/texlive-versions.tex                       \
      $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \
@@ -92,7 +92,6 @@ export LDFLAGS           := $(rpath_command) -L$(ildir)
 tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz                  \
                         cmake-$(cmake-version).tar.gz                      \
                         curl-$(curl-version).tar.gz                        \
-                        flock-$(flock-version).tar.xz                      \
                         freetype-$(freetype-version).tar.gz                \
                         ghostscript-$(ghostscript-version).tar.gz          \
                         git-$(git-version).tar.xz                          \
@@ -132,17 +131,16 @@ $(tarballs): $(tdir)/%:
 	    w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz
 	  elif [ $$n = cmake       ]; then w=https://cmake.org/files/v3.12
 	  elif [ $$n = curl        ]; then w=https://curl.haxx.se/download
-	  elif [ $$n = flock       ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version)
 	  elif [ $$n = freetype    ]; then w=https://download.savannah.gnu.org/releases/freetype
 	  elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926
 	  elif [ $$n = git         ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git
-	  elif [ $$n = gnuastro    ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro
-	  elif [ $$n = gsl         ]; then w=http://ftpmirror.gnu.org/gnu/gsl
+	  elif [ $$n = gnuastro    ]; then w=http://ftp.gnu.org/gnu/gnuastro
+	  elif [ $$n = gsl         ]; then w=http://ftp.gnu.org/gnu/gsl
 	  elif [ $$n = install     ]; then w=http://mirror.ctan.org/systems/texlive/tlnet
 	  elif [ $$n = jpegsrc     ]; then w=http://ijg.org/files
 	  elif [ $$n = libbsd      ]; then w=http://libbsd.freedesktop.org/releases
 	  elif [ $$n = libpng      ]; then w=https://download.sourceforge.net/libpng
-	  elif [ $$n = libtool     ]; then w=http://ftpmirror.gnu.org/gnu/libtool
+	  elif [ $$n = libtool     ]; then w=http://ftp.gnu.org/gnu/libtool
 	  elif [ $$n = libgit      ]; then
 	    mergenames=0
 	    w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz
@@ -376,9 +374,6 @@ $(ibdir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz
 $(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz
 	$(call gbuild, $<, ghostscript-$(ghostscript-version))
 
-$(ibdir)/flock: $(tdir)/flock-$(flock-version).tar.xz
-	$(call gbuild, $<, flock-$(flock-version), static)
-
 $(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \
               $(ibdir)/curl
 	$(call gbuild, $<, git-$(git-version), static,             \
diff --git a/reproduce/src/make/download.mk b/reproduce/src/make/download.mk
index a1efde7..036332b 100644
--- a/reproduce/src/make/download.mk
+++ b/reproduce/src/make/download.mk
@@ -69,7 +69,8 @@ $(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir)
 	  ln -s $(INDIR)/$$origname $@
 	else
 	  touch $(lockdir)/download
-	  $(downloadwrapper) wget $(lockdir)/download $$url/$$origname $@
+	  $(downloadwrapper) "wget --no-use-server-timestamps -O" \
+	                     $(lockdir)/download $$url/$$origname $@
 	fi
 
         # Check the md5 sum to see if this is the proper dataset.