From 98d31767a965ec75f4920b666f236cbb6baa91ab Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 27 Mar 2019 19:53:18 +0000 Subject: flock is now built in configure, to allow serial downloads Until now, we were using `flock' (file-lock) for downloading the input datasets in series. But we couldn't do this when downloading the software tarballs because `flock' wasn't yet available. Generally, unlike processing, downloading is much better done in series than in parallel. To enable serial downloads of the software also, with this commit we are installing `flock' in the configure script (not in a Makefile). As a result, besides `flock', we can also benefit from the other good features of the `reproduce/src/bash/download-multi-try' script *(for example attempting download again after some time). Some GNU mirrors may have problems at the time of download, so with this commit, we are using the main GNU FTP server for GNU programs. --- .file-metadata | Bin 3956 -> 4158 bytes configure | 63 ++++++++++++++++++++++++++++++- reproduce/src/bash/download-multi-try | 15 +++++--- reproduce/src/make/dependencies-basic.mk | 52 +++++++++++++------------ reproduce/src/make/dependencies.mk | 13 ++----- reproduce/src/make/download.mk | 3 +- 6 files changed, 104 insertions(+), 42 deletions(-) diff --git a/.file-metadata b/.file-metadata index 0fec14c..ac0a6cb 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/configure b/configure index c8e2b63..c34a68a 100755 --- a/configure +++ b/configure @@ -418,8 +418,12 @@ if [ $rewritepconfig = yes ]; then $pconf.in >> $pconf else # Read the values from existing configuration file. - inbdir=$(awk '$1=="BDIR" {print $3}' $pconf) - downloader=$(awk '$1=="DOWNLOADER" {print $3}' $pconf) + inbdir=$(awk '$1=="BDIR" {print $3}' $pconf) + + # The downloader command may contain multiple elements, so we'll just + # change the (in memory) first and second tokens to empty space and + # write the full line (the original file is unchanged). + downloader=$(awk '$1=="DOWNLOADER" {$1=""; $2=""; print $0}' $pconf) # Make sure all necessary variables have a value err=0 @@ -676,6 +680,61 @@ fi +# Build `flock' as first program +# ------------------------------ +# +# Flock (or file-lock) is a unique program in the pipeline that is +# necessary to serialize the (generally parallel) processing of make when +# necessary. GNU/Linux machines have it as part of their `util-linux' +# programs. But to be consistent, we will be using our own build. +# +# The reason its sepecial is that we need it to serialize the download +# process of the dependency tarballs. +flockversion=$(awk '/flock-version/{print $3}' \ + reproduce/config/pipeline/dependency-versions.mk) +flocktar=flock-$flockversion.tar.gz +flockurl=http://github.com/discoteq/flock/releases/download/v$flockversion/ + +# Prepare/download the tarball. +if ! [ -f $tardir/$flocktar ]; then + if [ -f $ddir/$flocktar ]; then + cp $ddir/$flocktar $tardir/$flocktar + else + if ! $downloader $tardir/$flocktar $flockurl/$flocktar; then + rm -f $tardir/$flocktar; + echo + echo "DOWNLOAD ERROR: Couldn't download the 'flock' tarball:" + echo " $flockurl" + echo + echo "You can manually place it in '$ddir' to avoid downloading." + exit 1 + fi + fi +fi + +# If the tarball is newer than the (possibly existing) program, then delete +# the program. +if [ -f .local/bin/flock ]; then + if [ $tardir/$flocktar -nt .local/bin/flock ]; then + rm .local/bin/flock + fi +fi + +# Build `flock' if necessary. +if ! [ -f .local/bin/flock ]; then + cd $depdir + tar xf $tardir/$flocktar + cd flock-$flockversion + ./configure --prefix=$instdir + make; make install + cd $topdir + rm -rf $depdir/flock-$flockversion +fi + + + + + # Build Basic dependencies # ------------------------ # diff --git a/reproduce/src/bash/download-multi-try b/reproduce/src/bash/download-multi-try index 1be2ffa..cdeca8e 100755 --- a/reproduce/src/bash/download-multi-try +++ b/reproduce/src/bash/download-multi-try @@ -1,11 +1,13 @@ -#!.local/bin/bash -# # Attempt downloading multiple times before crashing whole pipeline. From # the top project directory (for the shebang above), this script must be # run like this: # # $ /path/to/download-multi-try downloader lockfile input-url downloaded-name # +# NOTE: The `downloader' must contain the option to specify the output name +# in its end. For example "wget -O". Any other option can also be placed in +# the middle. +# # Due to temporary network problems, a download may fail suddenly, but # succeed in a second try a few seconds later. Without this script that # temporary glitch in the network will permanently crash the pipeline and @@ -70,7 +72,7 @@ fi -# Try downloading multiple times before crashing +# Try downloading multiple times before crashing. counter=0 maxcounter=10 while [ ! -f "$outname" ]; do @@ -97,9 +99,12 @@ while [ ! -f "$outname" ]; do sleep $tstep fi - # Attempt downloading the file (one-at-a-time). + # Attempt downloading the file (one-at-a-time). Note that the + # `downloader' ends with the respective option to specify the output + # name. For example "wget -O" (so `outname', that comes after it) will + # be the name of the downloaded file. flock "$lockfile" bash -c \ - "if ! $downloader -O$outname $inurl; then rm -f $outname; fi" + "if ! $downloader $outname $inurl; then rm -f $outname; fi" done diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 2e6caf4..cdb4a8c 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -43,12 +43,13 @@ include reproduce/config/pipeline/LOCAL.mk include reproduce/src/make/dependencies-build-rules.mk include reproduce/config/pipeline/dependency-versions.mk -ddir = $(BDIR)/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib -ilidir = $(BDIR)/dependencies/installed/lib/built +lockdir = $(BDIR)/locks +ddir = $(BDIR)/dependencies +tdir = $(BDIR)/dependencies/tarballs +idir = $(BDIR)/dependencies/installed +ibdir = $(BDIR)/dependencies/installed/bin +ildir = $(BDIR)/dependencies/installed/lib +ilidir = $(BDIR)/dependencies/installed/lib/built # We'll need the system's PATH for making links to low-level programs we # won't be building ourselves. @@ -94,6 +95,8 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # # However, downloading from this link is slow (because its just a link). So # its easier to just keep a with the others. +$(lockdir): | $(BDIR); mkdir $@ +downloadwrapper = ./reproduce/src/bash/download-multi-try tarballs = $(foreach t, bash-$(bash-version).tar.gz \ binutils-$(binutils-version).tar.lz \ bzip2-$(bzip2-version).tar.gz \ @@ -123,7 +126,7 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \ xz-$(xz-version).tar.gz \ zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) -$(tarballs): $(tdir)/%: +$(tarballs): $(tdir)/%: $(lockdir) if [ -f $(DEPENDENCIES-DIR)/$* ]; then \ cp $(DEPENDENCIES-DIR)/$* $@; \ else \ @@ -132,32 +135,32 @@ $(tarballs): $(tdir)/%: | awk '{print $$1}' ); \ \ mergenames=1; \ - if [ $$n = bash ]; then w=http://ftpmirror.gnu.org/gnu/bash; \ - elif [ $$n = binutils ]; then w=http://ftpmirror.gnu.org/gnu/binutils; \ + if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash; \ + elif [ $$n = binutils ]; then w=http://ftp.gnu.org/gnu/binutils; \ elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \ elif [ $$n = cert ]; then w=http://akhlaghi.org/src; \ - elif [ $$n = coreutils ]; then w=http://ftpmirror.gnu.org/gnu/coreutils;\ - elif [ $$n = diffutils ]; then w=http://ftpmirror.gnu.org/gnu/diffutils;\ + elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils;\ + elif [ $$n = diffutils ]; then w=http://ftp.gnu.org/gnu/diffutils;\ elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \ - elif [ $$n = gawk ]; then w=http://ftpmirror.gnu.org/gnu/gawk; \ + elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk; \ elif [ $$n = gcc ]; then w=http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version); \ elif [ $$n = gmp ]; then w=https://gmplib.org/download/gmp; \ - elif [ $$n = grep ]; then w=http://ftpmirror.gnu.org/gnu/grep; \ - elif [ $$n = gzip ]; then w=http://ftpmirror.gnu.org/gnu/gzip; \ + elif [ $$n = grep ]; then w=http://ftp.gnu.org/gnu/grep; \ + elif [ $$n = gzip ]; then w=http://ftp.gnu.org/gnu/gzip; \ elif [ $$n = isl ]; then w=ftp://gcc.gnu.org/pub/gcc/infrastructure; \ elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \ elif [ $$n = make ]; then w=http://akhlaghi.org/src; \ elif [ $$n = mpfr ]; then w=http://www.mpfr.org/mpfr-current;\ - elif [ $$n = mpc ]; then w=http://ftpmirror.gnu.org/gnu/mpc;\ - elif [ $$n = ncurses ]; then w=http://ftpmirror.gnu.org/gnu/ncurses;\ + elif [ $$n = mpc ]; then w=http://ftp.gnu.org/gnu/mpc; \ + elif [ $$n = ncurses ]; then w=http://ftp.gnu.org/gnu/ncurses; \ elif [ $$n = openssl ]; then w=http://www.openssl.org/source; \ elif [ $$n = patchelf ]; then w=http://nixos.org/releases/patchelf/patchelf-$(patchelf-version); \ elif [ $$n = pkg ]; then w=http://pkg-config.freedesktop.org/releases; \ - elif [ $$n = readline ]; then w=http://ftpmirror.gnu.org/gnu/readline; \ - elif [ $$n = sed ]; then w=http://ftpmirror.gnu.org/gnu/sed;\ - elif [ $$n = tar ]; then w=http://ftpmirror.gnu.org/gnu/tar;\ - elif [ $$n = wget ]; then w=http://ftpmirror.gnu.org/gnu/wget;\ - elif [ $$n = which ]; then w=http://ftpmirror.gnu.org/gnu/which;\ + elif [ $$n = readline ]; then w=http://ftp.gnu.org/gnu/readline; \ + elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed; \ + elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \ + elif [ $$n = wget ]; then w=http://ftp.gnu.org/gnu/wget; \ + elif [ $$n = which ]; then w=http://ftp.gnu.org/gnu/which; \ elif [ $$n = xz ]; then w=http://tukaani.org/xz; \ elif [ $$n = zlib ]; then w=http://www.zlib.net; \ else \ @@ -178,10 +181,9 @@ $(tarballs): $(tdir)/%: downloader="$(DOWNLOADER)"; \ fi; \ \ - if ! $$downloader $@ $$tarballurl; then \ - rm -f $@; \ - echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1; \ - fi; \ + touch $(lockdir)/download; \ + $(downloadwrapper) "$$downloader" $(lockdir)/download \ + $$tarballurl $@; \ fi diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index deb9f39..2b3b70b 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,7 +43,7 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). -top-level-programs = astnoisechisel flock metastore unzip zip +top-level-programs = astnoisechisel metastore unzip zip top-level-libraries = freetype all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ @@ -92,7 +92,6 @@ export LDFLAGS := $(rpath_command) -L$(ildir) tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ - flock-$(flock-version).tar.xz \ freetype-$(freetype-version).tar.gz \ ghostscript-$(ghostscript-version).tar.gz \ git-$(git-version).tar.xz \ @@ -132,17 +131,16 @@ $(tarballs): $(tdir)/%: w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download - elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git - elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro - elif [ $$n = gsl ]; then w=http://ftpmirror.gnu.org/gnu/gsl + elif [ $$n = gnuastro ]; then w=http://ftp.gnu.org/gnu/gnuastro + elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng - elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool + elif [ $$n = libtool ]; then w=http://ftp.gnu.org/gnu/libtool elif [ $$n = libgit ]; then mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz @@ -376,9 +374,6 @@ $(ibdir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz $(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz $(call gbuild, $<, ghostscript-$(ghostscript-version)) -$(ibdir)/flock: $(tdir)/flock-$(flock-version).tar.xz - $(call gbuild, $<, flock-$(flock-version), static) - $(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \ $(ibdir)/curl $(call gbuild, $<, git-$(git-version), static, \ diff --git a/reproduce/src/make/download.mk b/reproduce/src/make/download.mk index a1efde7..036332b 100644 --- a/reproduce/src/make/download.mk +++ b/reproduce/src/make/download.mk @@ -69,7 +69,8 @@ $(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir) ln -s $(INDIR)/$$origname $@ else touch $(lockdir)/download - $(downloadwrapper) wget $(lockdir)/download $$url/$$origname $@ + $(downloadwrapper) "wget --no-use-server-timestamps -O" \ + $(lockdir)/download $$url/$$origname $@ fi # Check the md5 sum to see if this is the proper dataset. -- cgit v1.2.1