diff options
Diffstat (limited to 'reproduce/analysis')
| -rwxr-xr-x | reproduce/analysis/bash/download-multi-try.sh | 59 | ||||
| -rw-r--r-- | reproduce/analysis/make/initialize.mk | 41 | ||||
| -rw-r--r-- | reproduce/analysis/make/top-prepare.mk | 6 |
3 files changed, 71 insertions, 35 deletions
diff --git a/reproduce/analysis/bash/download-multi-try.sh b/reproduce/analysis/bash/download-multi-try.sh index 16950b5..5cc5407 100755 --- a/reproduce/analysis/bash/download-multi-try.sh +++ b/reproduce/analysis/bash/download-multi-try.sh @@ -90,6 +90,37 @@ urlfile=$(echo "$inurl" | awk -F "/" '{print $NF}') +# Function for downloading +download_func () { + + # Set the arguments. + inurl="$1" + + # Attempt downloading the file. Note that the 'downloader' ends with + # the respective option to specify the output name. For example "wget + # -O" (so 'outname', that comes after it) will be the name of the + # downloaded file. + if [ x"$lockfile" = xnolock ]; then + if ! $downloader $outname $inurl; then rm -f $outname; fi + else + flock "$lockfile" sh -c \ + "if ! $downloader $outname \"$inurl\"; then rm -f $outname; fi" + fi + + # Some servers return HTTP 4xx/5xx errors as an HTML page with a 200 + # status, so the downloader exits 0 and saves the HTML body to disk. + # Detect this by checking whether the response starts with an HTML tag + # and treat it as a failed download so backup servers will be tried. + if [ -f "$outname" ] \ + && head -c 500 "$outname" 2>/dev/null | grep -qi '<html'; then + rm -f "$outname" + fi +} + + + + + # Try downloading multiple times before crashing. counter=0 maxcounter=10 @@ -119,30 +150,22 @@ while [ ! -f "$outname" ]; do sleep $tstep fi - # Attempt downloading the file. Note that the 'downloader' ends with - # the respective option to specify the output name. For example "wget - # -O" (so 'outname', that comes after it) will be the name of the - # downloaded file. - if [ x"$lockfile" = xnolock ]; then - if ! $downloader $outname $inurl; then rm -f $outname; fi - else - # Try downloading from the requested URL. - flock "$lockfile" sh -c \ - "if ! $downloader $outname \"$inurl\"; then rm -f $outname; fi" - fi + # First attempt at the download. + download_func "$inurl" # If the download failed, try the backup server(s). if [ ! -f "$outname" ]; then if [ x"$backupservers" != x ]; then for bs in $backupservers; do - # Use this backup server. - if [ x"$lockfile" = xnolock ]; then - if ! $downloader $outname $bs/$urlfile; then rm -f $outname; fi - else - flock "$lockfile" sh -c \ - "if ! $downloader $outname $bs/$urlfile; then rm -f $outname; fi" - fi + # For Zenodo backup servers, append '?download=1' as well. + bsurl="$bs/$urlfile" + case "$bsurl" in + *zenodo.org*) bsurl="${bsurl}?download=1" ;; + esac + + # Download the file. + download_func "$bsurl" # If the file was downloaded, break out of the loop that # parses over the backup servers. diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 7aefd3f..2752f47 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -685,27 +685,40 @@ $(mtexdir)/initialize.tex: # Calculate the latest Maneage commit used to build this project: # - The project may not have the 'maneage' branch (for example # after cloning from a fork that didn't include it!). In this -# case, we'll print a descriptive warning, telling the user what -# should be done (reporting the last merged commit and its date -# is very useful for the future). +# case, if a PDF is to be created, print a descriptive warning, +# telling the user what should be done (reporting the last merged +# commit and its date is very useful for the future). +# - This is not relevant when no PDF is requested because we only +# use this to print the latest Maneage commit as a LaTeX macro. # - The '--dirty' option (used in 'project-commit-hash') isn't # applicable to "commit-ishes" (direct quote from Git's error -# message!). +# message!), so no need to include it here. if git log maneage -1 &> /dev/null; then c=$$(git merge-base HEAD maneage) v=$$(git describe --always --long $$c) d=$$(git show -s --format=%aD $$v | awk '{print $$2, $$3, $$4}') + +# No 'maneage' branch found else - echo - echo "WARNING: no 'maneage' branch found! Without it, the latest merge of " - echo "this project with Maneage can't be reported in the paper (which is bad " - echo "for your readers; that includes yourself in a few years). Please run " - echo "the commands below to fetch the 'maneage' branch from its own server " - echo "and remove this warning (these commands will not affect your project):" - echo " $ git remote add origin-maneage http://git.maneage.org/project.git" - echo " $ git fetch origin-maneage" - echo " $ git branch maneage --track origin-maneage/maneage" - echo + +# Only print a warning if a PDF is to be created. + if [ x$(pdf-build-final) = xyes ]; then + echo + printf "WARNING: no 'maneage' branch found! Without it, " + printf "the latest merge of this project with Maneage can't " + printf "be reported in the paper (which is bad for your " + printf "readers; that includes yourself in a few years). " + printf "Please run the commands below to fetch the " + printf "'maneage' branch from its own server and remove " + printf "this warning (these commands will not affect your " + printf "project):\n" + printf " $ git remote add origin-maneage " + printf "http://git.maneage.org/project.git\n" + printf " $ git fetch origin-maneage\n" + printf " $ git branch maneage --track " + printf "origin-maneage/maneage\n" + echo + fi v="\textcolor{red}{NO-MANEAGE-BRANCH (see printed warning to fix this)}" d="\textcolor{red}{NO-MANEAGE-DATE}" fi diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk index 8d1c2e0..913a062 100644 --- a/reproduce/analysis/make/top-prepare.mk +++ b/reproduce/analysis/make/top-prepare.mk @@ -42,10 +42,10 @@ else all: @if [ "x$(GROUP-NAME)" = x ]; then \ echo "Project is NOT configured for groups, please run"; \ - echo " $$ ./project prepare"; \ + echo " $$ ./project configure"; \ else \ echo "Project is configured for groups, please run"; \ - echo " $$ ./project prepare --group=$(GROUP-NAME) -j8"; \ + echo " $$ ./project configure --group=$(GROUP-NAME) -j8"; \ fi exit 1 endif @@ -61,7 +61,7 @@ endif # # To ensure that 'prepare' and 'make' have the same basic definitions and # environment and that all 'downloads' are managed in one place, both -# './project prepare' and './project make' will first read 'initialize.mk' +# './project configure' and './project make' will first read 'initialize.mk' # and 'downloads.mk'. makesrc = initialize \ prepare |