From 98d31767a965ec75f4920b666f236cbb6baa91ab Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 27 Mar 2019 19:53:18 +0000 Subject: flock is now built in configure, to allow serial downloads Until now, we were using `flock' (file-lock) for downloading the input datasets in series. But we couldn't do this when downloading the software tarballs because `flock' wasn't yet available. Generally, unlike processing, downloading is much better done in series than in parallel. To enable serial downloads of the software also, with this commit we are installing `flock' in the configure script (not in a Makefile). As a result, besides `flock', we can also benefit from the other good features of the `reproduce/src/bash/download-multi-try' script *(for example attempting download again after some time). Some GNU mirrors may have problems at the time of download, so with this commit, we are using the main GNU FTP server for GNU programs. --- reproduce/src/make/dependencies-basic.mk | 52 +++++++++++++++++--------------- reproduce/src/make/dependencies.mk | 13 +++----- reproduce/src/make/download.mk | 3 +- 3 files changed, 33 insertions(+), 35 deletions(-) (limited to 'reproduce/src/make') diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 2e6caf4..cdb4a8c 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -43,12 +43,13 @@ include reproduce/config/pipeline/LOCAL.mk include reproduce/src/make/dependencies-build-rules.mk include reproduce/config/pipeline/dependency-versions.mk -ddir = $(BDIR)/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib -ilidir = $(BDIR)/dependencies/installed/lib/built +lockdir = $(BDIR)/locks +ddir = $(BDIR)/dependencies +tdir = $(BDIR)/dependencies/tarballs +idir = $(BDIR)/dependencies/installed +ibdir = $(BDIR)/dependencies/installed/bin +ildir = $(BDIR)/dependencies/installed/lib +ilidir = $(BDIR)/dependencies/installed/lib/built # We'll need the system's PATH for making links to low-level programs we # won't be building ourselves. @@ -94,6 +95,8 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # # However, downloading from this link is slow (because its just a link). So # its easier to just keep a with the others. +$(lockdir): | $(BDIR); mkdir $@ +downloadwrapper = ./reproduce/src/bash/download-multi-try tarballs = $(foreach t, bash-$(bash-version).tar.gz \ binutils-$(binutils-version).tar.lz \ bzip2-$(bzip2-version).tar.gz \ @@ -123,7 +126,7 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \ xz-$(xz-version).tar.gz \ zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) -$(tarballs): $(tdir)/%: +$(tarballs): $(tdir)/%: $(lockdir) if [ -f $(DEPENDENCIES-DIR)/$* ]; then \ cp $(DEPENDENCIES-DIR)/$* $@; \ else \ @@ -132,32 +135,32 @@ $(tarballs): $(tdir)/%: | awk '{print $$1}' ); \ \ mergenames=1; \ - if [ $$n = bash ]; then w=http://ftpmirror.gnu.org/gnu/bash; \ - elif [ $$n = binutils ]; then w=http://ftpmirror.gnu.org/gnu/binutils; \ + if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash; \ + elif [ $$n = binutils ]; then w=http://ftp.gnu.org/gnu/binutils; \ elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \ elif [ $$n = cert ]; then w=http://akhlaghi.org/src; \ - elif [ $$n = coreutils ]; then w=http://ftpmirror.gnu.org/gnu/coreutils;\ - elif [ $$n = diffutils ]; then w=http://ftpmirror.gnu.org/gnu/diffutils;\ + elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils;\ + elif [ $$n = diffutils ]; then w=http://ftp.gnu.org/gnu/diffutils;\ elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \ - elif [ $$n = gawk ]; then w=http://ftpmirror.gnu.org/gnu/gawk; \ + elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk; \ elif [ $$n = gcc ]; then w=http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version); \ elif [ $$n = gmp ]; then w=https://gmplib.org/download/gmp; \ - elif [ $$n = grep ]; then w=http://ftpmirror.gnu.org/gnu/grep; \ - elif [ $$n = gzip ]; then w=http://ftpmirror.gnu.org/gnu/gzip; \ + elif [ $$n = grep ]; then w=http://ftp.gnu.org/gnu/grep; \ + elif [ $$n = gzip ]; then w=http://ftp.gnu.org/gnu/gzip; \ elif [ $$n = isl ]; then w=ftp://gcc.gnu.org/pub/gcc/infrastructure; \ elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \ elif [ $$n = make ]; then w=http://akhlaghi.org/src; \ elif [ $$n = mpfr ]; then w=http://www.mpfr.org/mpfr-current;\ - elif [ $$n = mpc ]; then w=http://ftpmirror.gnu.org/gnu/mpc;\ - elif [ $$n = ncurses ]; then w=http://ftpmirror.gnu.org/gnu/ncurses;\ + elif [ $$n = mpc ]; then w=http://ftp.gnu.org/gnu/mpc; \ + elif [ $$n = ncurses ]; then w=http://ftp.gnu.org/gnu/ncurses; \ elif [ $$n = openssl ]; then w=http://www.openssl.org/source; \ elif [ $$n = patchelf ]; then w=http://nixos.org/releases/patchelf/patchelf-$(patchelf-version); \ elif [ $$n = pkg ]; then w=http://pkg-config.freedesktop.org/releases; \ - elif [ $$n = readline ]; then w=http://ftpmirror.gnu.org/gnu/readline; \ - elif [ $$n = sed ]; then w=http://ftpmirror.gnu.org/gnu/sed;\ - elif [ $$n = tar ]; then w=http://ftpmirror.gnu.org/gnu/tar;\ - elif [ $$n = wget ]; then w=http://ftpmirror.gnu.org/gnu/wget;\ - elif [ $$n = which ]; then w=http://ftpmirror.gnu.org/gnu/which;\ + elif [ $$n = readline ]; then w=http://ftp.gnu.org/gnu/readline; \ + elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed; \ + elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \ + elif [ $$n = wget ]; then w=http://ftp.gnu.org/gnu/wget; \ + elif [ $$n = which ]; then w=http://ftp.gnu.org/gnu/which; \ elif [ $$n = xz ]; then w=http://tukaani.org/xz; \ elif [ $$n = zlib ]; then w=http://www.zlib.net; \ else \ @@ -178,10 +181,9 @@ $(tarballs): $(tdir)/%: downloader="$(DOWNLOADER)"; \ fi; \ \ - if ! $$downloader $@ $$tarballurl; then \ - rm -f $@; \ - echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1; \ - fi; \ + touch $(lockdir)/download; \ + $(downloadwrapper) "$$downloader" $(lockdir)/download \ + $$tarballurl $@; \ fi diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index deb9f39..2b3b70b 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -43,7 +43,7 @@ ildir = $(BDIR)/dependencies/installed/lib ilidir = $(BDIR)/dependencies/installed/lib/built # Define the top-level programs to build (installed in `.local/bin'). -top-level-programs = astnoisechisel flock metastore unzip zip +top-level-programs = astnoisechisel metastore unzip zip top-level-libraries = freetype all: $(ddir)/texlive-versions.tex \ $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ @@ -92,7 +92,6 @@ export LDFLAGS := $(rpath_command) -L$(ildir) tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ - flock-$(flock-version).tar.xz \ freetype-$(freetype-version).tar.gz \ ghostscript-$(ghostscript-version).tar.gz \ git-$(git-version).tar.xz \ @@ -132,17 +131,16 @@ $(tarballs): $(tdir)/%: w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download - elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git - elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro - elif [ $$n = gsl ]; then w=http://ftpmirror.gnu.org/gnu/gsl + elif [ $$n = gnuastro ]; then w=http://ftp.gnu.org/gnu/gnuastro + elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng - elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool + elif [ $$n = libtool ]; then w=http://ftp.gnu.org/gnu/libtool elif [ $$n = libgit ]; then mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz @@ -376,9 +374,6 @@ $(ibdir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz $(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz $(call gbuild, $<, ghostscript-$(ghostscript-version)) -$(ibdir)/flock: $(tdir)/flock-$(flock-version).tar.xz - $(call gbuild, $<, flock-$(flock-version), static) - $(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \ $(ibdir)/curl $(call gbuild, $<, git-$(git-version), static, \ diff --git a/reproduce/src/make/download.mk b/reproduce/src/make/download.mk index a1efde7..036332b 100644 --- a/reproduce/src/make/download.mk +++ b/reproduce/src/make/download.mk @@ -69,7 +69,8 @@ $(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir) ln -s $(INDIR)/$$origname $@ else touch $(lockdir)/download - $(downloadwrapper) wget $(lockdir)/download $$url/$$origname $@ + $(downloadwrapper) "wget --no-use-server-timestamps -O" \ + $(lockdir)/download $$url/$$origname $@ fi # Check the md5 sum to see if this is the proper dataset. -- cgit v1.2.1