diff options
Diffstat (limited to 'reproduce/src')
| -rwxr-xr-x | reproduce/src/bash/download-multi-try | 15 | ||||
| -rw-r--r-- | reproduce/src/bash/git-post-checkout | 8 | ||||
| -rw-r--r-- | reproduce/src/bash/git-pre-commit | 8 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies-basic.mk | 52 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies.mk | 11 | ||||
| -rw-r--r-- | reproduce/src/make/download.mk | 3 |
6 files changed, 50 insertions, 47 deletions
diff --git a/reproduce/src/bash/download-multi-try b/reproduce/src/bash/download-multi-try index 1be2ffa..cdeca8e 100755 --- a/reproduce/src/bash/download-multi-try +++ b/reproduce/src/bash/download-multi-try @@ -1,11 +1,13 @@ -#!.local/bin/bash -# # Attempt downloading multiple times before crashing whole pipeline. From # the top project directory (for the shebang above), this script must be # run like this: # # $ /path/to/download-multi-try downloader lockfile input-url downloaded-name # +# NOTE: The `downloader' must contain the option to specify the output name +# in its end. For example "wget -O". Any other option can also be placed in +# the middle. +# # Due to temporary network problems, a download may fail suddenly, but # succeed in a second try a few seconds later. Without this script that # temporary glitch in the network will permanently crash the pipeline and @@ -70,7 +72,7 @@ fi -# Try downloading multiple times before crashing +# Try downloading multiple times before crashing. counter=0 maxcounter=10 while [ ! -f "$outname" ]; do @@ -97,9 +99,12 @@ while [ ! -f "$outname" ]; do sleep $tstep fi - # Attempt downloading the file (one-at-a-time). + # Attempt downloading the file (one-at-a-time). Note that the + # `downloader' ends with the respective option to specify the output + # name. For example "wget -O" (so `outname', that comes after it) will + # be the name of the downloaded file. flock "$lockfile" bash -c \ - "if ! $downloader -O$outname $inurl; then rm -f $outname; fi" + "if ! $downloader $outname $inurl; then rm -f $outname; fi" done diff --git a/reproduce/src/bash/git-post-checkout b/reproduce/src/bash/git-post-checkout index 153b277..a56c21f 100644 --- a/reproduce/src/bash/git-post-checkout +++ b/reproduce/src/bash/git-post-checkout @@ -4,10 +4,10 @@ # controlled files (with each commit) using the `metastore' program. # # This script is taken from the `examples/hooks/pre-commit' file of the -# `metastore' package (installed within the pipeline, with an MIT -# license). We have just changed the name of the `MSFILE' and also set -# special characters for the installation location of meta-store so our own -# installation is found by Git. +# `metastore' package (installed within the pipeline, with an MIT license +# for copyright). We have just changed the name of the `MSFILE' and also +# set special characters for the installation location of meta-store so our +# own installation is found by Git. # File containig the metadata and metastore executable. MSFILE=".file-metadata" diff --git a/reproduce/src/bash/git-pre-commit b/reproduce/src/bash/git-pre-commit index 0fa6c52..cd06c25 100644 --- a/reproduce/src/bash/git-pre-commit +++ b/reproduce/src/bash/git-pre-commit @@ -4,10 +4,10 @@ # controlled files (with each commit) using the `metastore' program. # # This script is taken from the `examples/hooks/pre-commit' file of the -# `metastore' package (installed within the pipeline, with an MIT -# license). We have just changed the name of the `MSFILE' and also set -# special characters for the installation location of meta-store so our own -# installation is found by Git. +# `metastore' package (installed within the pipeline, with an MIT license +# for copyright). We have just changed the name of the `MSFILE' and also +# set special characters for the installation location of meta-store so our +# own installation is found by Git. # # WARNING: # diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 74a04ed..52a41d9 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -43,12 +43,13 @@ include reproduce/config/pipeline/LOCAL.mk include reproduce/src/make/dependencies-build-rules.mk include reproduce/config/pipeline/dependency-versions.mk -ddir = $(BDIR)/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib -ilidir = $(BDIR)/dependencies/installed/lib/built +lockdir = $(BDIR)/locks +ddir = $(BDIR)/dependencies +tdir = $(BDIR)/dependencies/tarballs +idir = $(BDIR)/dependencies/installed +ibdir = $(BDIR)/dependencies/installed/bin +ildir = $(BDIR)/dependencies/installed/lib +ilidir = $(BDIR)/dependencies/installed/lib/built # We'll need the system's PATH for making links to low-level programs we # won't be building ourselves. @@ -94,6 +95,8 @@ all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) # # However, downloading from this link is slow (because its just a link). So # its easier to just keep a with the others. +$(lockdir): | $(BDIR); mkdir $@ +downloadwrapper = ./reproduce/src/bash/download-multi-try tarballs = $(foreach t, bash-$(bash-version).tar.gz \ binutils-$(binutils-version).tar.lz \ bzip2-$(bzip2-version).tar.gz \ @@ -123,7 +126,7 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \ xz-$(xz-version).tar.gz \ zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) -$(tarballs): $(tdir)/%: +$(tarballs): $(tdir)/%: $(lockdir) if [ -f $(DEPENDENCIES-DIR)/$* ]; then \ cp $(DEPENDENCIES-DIR)/$* $@; \ else \ @@ -132,32 +135,32 @@ $(tarballs): $(tdir)/%: | awk '{print $$1}' ); \ \ mergenames=1; \ - if [ $$n = bash ]; then w=http://ftpmirror.gnu.org/gnu/bash; \ - elif [ $$n = binutils ]; then w=http://ftpmirror.gnu.org/gnu/binutils; \ + if [ $$n = bash ]; then w=http://ftp.gnu.org/gnu/bash; \ + elif [ $$n = binutils ]; then w=http://ftp.gnu.org/gnu/binutils; \ elif [ $$n = bzip ]; then w=http://akhlaghi.org/src; \ elif [ $$n = cert ]; then w=http://akhlaghi.org/src; \ - elif [ $$n = coreutils ]; then w=http://ftpmirror.gnu.org/gnu/coreutils;\ - elif [ $$n = diffutils ]; then w=http://ftpmirror.gnu.org/gnu/diffutils;\ + elif [ $$n = coreutils ]; then w=http://ftp.gnu.org/gnu/coreutils;\ + elif [ $$n = diffutils ]; then w=http://ftp.gnu.org/gnu/diffutils;\ elif [ $$n = findutils ]; then w=http://akhlaghi.org/src; \ - elif [ $$n = gawk ]; then w=http://ftpmirror.gnu.org/gnu/gawk; \ + elif [ $$n = gawk ]; then w=http://ftp.gnu.org/gnu/gawk; \ elif [ $$n = gcc ]; then w=http://ftp.gnu.org/gnu/gcc/gcc-$(gcc-version); \ elif [ $$n = gmp ]; then w=https://gmplib.org/download/gmp; \ - elif [ $$n = grep ]; then w=http://ftpmirror.gnu.org/gnu/grep; \ - elif [ $$n = gzip ]; then w=http://ftpmirror.gnu.org/gnu/gzip; \ + elif [ $$n = grep ]; then w=http://ftp.gnu.org/gnu/grep; \ + elif [ $$n = gzip ]; then w=http://ftp.gnu.org/gnu/gzip; \ elif [ $$n = isl ]; then w=ftp://gcc.gnu.org/pub/gcc/infrastructure; \ elif [ $$n = lzip ]; then w=http://download.savannah.gnu.org/releases/lzip; \ elif [ $$n = make ]; then w=http://akhlaghi.org/src; \ elif [ $$n = mpfr ]; then w=http://www.mpfr.org/mpfr-current;\ - elif [ $$n = mpc ]; then w=http://ftpmirror.gnu.org/gnu/mpc;\ - elif [ $$n = ncurses ]; then w=http://ftpmirror.gnu.org/gnu/ncurses;\ + elif [ $$n = mpc ]; then w=http://ftp.gnu.org/gnu/mpc; \ + elif [ $$n = ncurses ]; then w=http://ftp.gnu.org/gnu/ncurses; \ elif [ $$n = openssl ]; then w=http://www.openssl.org/source; \ elif [ $$n = patchelf ]; then w=http://nixos.org/releases/patchelf/patchelf-$(patchelf-version); \ elif [ $$n = pkg ]; then w=http://pkg-config.freedesktop.org/releases; \ - elif [ $$n = readline ]; then w=http://ftpmirror.gnu.org/gnu/readline; \ - elif [ $$n = sed ]; then w=http://ftpmirror.gnu.org/gnu/sed;\ - elif [ $$n = tar ]; then w=http://ftpmirror.gnu.org/gnu/tar;\ - elif [ $$n = wget ]; then w=http://ftpmirror.gnu.org/gnu/wget;\ - elif [ $$n = which ]; then w=http://ftpmirror.gnu.org/gnu/which;\ + elif [ $$n = readline ]; then w=http://ftp.gnu.org/gnu/readline; \ + elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed; \ + elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \ + elif [ $$n = wget ]; then w=http://ftp.gnu.org/gnu/wget; \ + elif [ $$n = which ]; then w=http://ftp.gnu.org/gnu/which; \ elif [ $$n = xz ]; then w=http://tukaani.org/xz; \ elif [ $$n = zlib ]; then w=http://www.zlib.net; \ else \ @@ -178,10 +181,9 @@ $(tarballs): $(tdir)/%: downloader="$(DOWNLOADER)"; \ fi; \ \ - if ! $$downloader $@ $$tarballurl; then \ - rm -f $@; \ - echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1; \ - fi; \ + touch $(lockdir)/download; \ + $(downloadwrapper) "$$downloader" $(lockdir)/download \ + $$tarballurl $@; \ fi diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index fe29d1b..fdbfd7f 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -103,7 +103,6 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ atlas-$(atlas-version).tar.bz2 \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ - flock-$(flock-version).tar.xz \ freetype-$(freetype-version).tar.gz \ fftw-$(fftw-version).tar.gz \ ghostscript-$(ghostscript-version).tar.gz \ @@ -151,7 +150,6 @@ $(tarballs): $(tdir)/%: w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 elif [ $$n = curl ]; then w=https://curl.haxx.se/download - elif [ $$n = flock ]; then w=https://github.com/discoteq/flock/releases/download/v$(flock-version) elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype elif [ $$n = hdf ]; then @@ -160,14 +158,14 @@ $(tarballs): $(tdir)/%: w=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-$$majorver/hdf5-$(hdf5-version)/src/$* elif [ $$n = ghostscript ]; then w=https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs926 elif [ $$n = git ]; then w=http://mirrors.edge.kernel.org/pub/software/scm/git - elif [ $$n = gnuastro ]; then w=http://ftpmirror.gnu.org/gnu/gnuastro - elif [ $$n = gsl ]; then w=http://ftpmirror.gnu.org/gnu/gsl + elif [ $$n = gnuastro ]; then w=http://ftp.gnu.org/gnu/gnuastro + elif [ $$n = gsl ]; then w=http://ftp.gnu.org/gnu/gsl elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files elif [ $$n = lapack ]; then w=http://www.netlib.org/lapack elif [ $$n = libbsd ]; then w=http://libbsd.freedesktop.org/releases elif [ $$n = libpng ]; then w=https://download.sourceforge.net/libpng - elif [ $$n = libtool ]; then w=http://ftpmirror.gnu.org/gnu/libtool + elif [ $$n = libtool ]; then w=http://ftp.gnu.org/gnu/libtool elif [ $$n = libgit ]; then mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz @@ -511,9 +509,6 @@ $(ibdir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz $(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz $(call gbuild, $<, ghostscript-$(ghostscript-version)) -$(ibdir)/flock: $(tdir)/flock-$(flock-version).tar.xz - $(call gbuild, $<, flock-$(flock-version), static) - $(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \ $(ibdir)/curl $(call gbuild, $<, git-$(git-version), static, \ diff --git a/reproduce/src/make/download.mk b/reproduce/src/make/download.mk index a1efde7..036332b 100644 --- a/reproduce/src/make/download.mk +++ b/reproduce/src/make/download.mk @@ -69,7 +69,8 @@ $(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir) ln -s $(INDIR)/$$origname $@ else touch $(lockdir)/download - $(downloadwrapper) wget $(lockdir)/download $$url/$$origname $@ + $(downloadwrapper) "wget --no-use-server-timestamps -O" \ + $(lockdir)/download $$url/$$origname $@ fi # Check the md5 sum to see if this is the proper dataset. |