diff options
Diffstat (limited to 'reproduce/analysis')
| -rwxr-xr-x | reproduce/analysis/bash/download-multi-try | 22 | ||||
| -rw-r--r-- | reproduce/analysis/config/verify-outputs.conf | 1 | ||||
| -rw-r--r-- | reproduce/analysis/make/download.mk | 7 | ||||
| -rw-r--r-- | reproduce/analysis/make/initialize.mk | 19 | ||||
| -rw-r--r-- | reproduce/analysis/make/paper.mk | 4 | ||||
| -rw-r--r-- | reproduce/analysis/make/prepare.mk | 4 | ||||
| -rw-r--r-- | reproduce/analysis/make/top-make.mk | 6 | ||||
| -rw-r--r-- | reproduce/analysis/make/top-prepare.mk | 19 | ||||
| -rw-r--r-- | reproduce/analysis/make/verify.mk | 2 |
9 files changed, 51 insertions, 33 deletions
diff --git a/reproduce/analysis/bash/download-multi-try b/reproduce/analysis/bash/download-multi-try index dec2539..7a5f8f2 100755 --- a/reproduce/analysis/bash/download-multi-try +++ b/reproduce/analysis/bash/download-multi-try @@ -51,11 +51,13 @@ set -e -# Input arguments and necessary sanity checks. +# Input arguments and necessary sanity checks. Note that the 5th argument +# (backup servers) isn't mandatory. inurl="$3" outname="$4" lockfile="$2" downloader="$1" +backupservers="$5" if [ "x$downloader" = x ]; then echo "$0: downloader (first argument) not given."; exit 1; fi @@ -73,6 +75,13 @@ fi +# Separate the actual filename, to possibly use backup server. +urlfile=$(echo "$inurl" | awk -F "/" '{print $NF}') + + + + + # Try downloading multiple times before crashing. counter=0 maxcounter=10 @@ -107,8 +116,19 @@ while [ ! -f "$outname" ]; do if [ x"$lockfile" = xnolock ]; then if ! $downloader $outname $inurl; then rm -f $outname; fi else + # Try downloading from the requested URL. flock "$lockfile" bash -c \ "if ! $downloader $outname $inurl; then rm -f $outname; fi" + + # If it failed, try the backup server(s). + if [ ! -f "$outname" ]; then + if [ x"$backupservers" != x ]; then + for bs in "$backupservers"; do + flock "$lockfile" bash -c \ + "if ! $downloader $outname $bs/$urlfile; then rm -f $outname; fi" + done + fi + fi fi done diff --git a/reproduce/analysis/config/verify-outputs.conf b/reproduce/analysis/config/verify-outputs.conf index e580e04..0542c53 100644 --- a/reproduce/analysis/config/verify-outputs.conf +++ b/reproduce/analysis/config/verify-outputs.conf @@ -1,3 +1,2 @@ # To disable verification of output datasets set this variable to yes - verify-outputs = diff --git a/reproduce/analysis/make/download.mk b/reproduce/analysis/make/download.mk index e4f2ccd..cf5bfa4 100644 --- a/reproduce/analysis/make/download.mk +++ b/reproduce/analysis/make/download.mk @@ -24,9 +24,10 @@ # Download input data # -------------------- # -# The input dataset properties are defined in `$(pconfdir)/INPUTS.mk'. For -# this template we only have one dataset to enable easy processing, so all -# the extra checks in this rule may seem redundant. +# The input dataset properties are defined in +# `$(pconfdir)/INPUTS.conf'. For this template we only have one dataset to +# enable easy processing, so all the extra checks in this rule may seem +# redundant. # # In a real project, you will need more than one dataset. In that case, # just add them to the target list and add an `elif' statement to define it diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index a4acff7..ce4e488 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -141,6 +141,12 @@ export SHELL := $(installdir)/bin/bash export CPPFLAGS := -I$(installdir)/include export LD_LIBRARY_PATH := $(installdir)/lib +# Until we build our own C library, without this, the project's GCC won't +# be able to compile anything if the host C library isn't in a standard +# place: in particular Debian-based operatings sytems. On other systems, it +# will be empty. +export CPATH := $(SYS_CPATH) + # RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is # ultimately redundant. But on some systems, even having a single value # causes crashs (see bug #56682). So we'll just give it no value at all. @@ -151,9 +157,9 @@ export DYLD_LIBRARY_PATH := # remote-shell accesss through this environment variable. export OMPI_MCA_plm_rsh_agent=/bin/false -# Recipe startup script, see `reproduce/software/bash/bashrc.sh'. +# Recipe startup script. export PROJECT_STATUS := make -export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh @@ -242,7 +248,7 @@ distclean: clean # `rm', not our own. $$sys_rm -rf $(BDIR) $$sys_rm -f Makefile .gnuastro .local .build - $$sys_rm -f $(pconfdir)/LOCAL.mk $(gconfdir)/gnuastro-local.conf + $$sys_rm -f $(pconfdir)/LOCAL.conf $(gconfdir)/gnuastro-local.conf @@ -299,10 +305,11 @@ $(packagecontents): paper.pdf | $(texdir) # directories (when building in a group structure, there will be # `build-user1', `build-user2' and etc), are just temporary LaTeX # build files and don't have any relevant/hand-written files in - # them. 2) The `LOCAL.mk' and `gnuastro-local.conf' files just have - # this machine's local settings and are irrelevant for anyone else. + # them. 2) The `LOCAL.conf' and `gnuastro-local.conf' files just + # have this machine's local settings and are irrelevant for anyone + # else. rm -rf $$dir/tex/build/build* - rm $$dir/reproduce/software/config/installation/LOCAL.mk + rm $$dir/reproduce/software/config/installation/LOCAL.conf rm $$dir/reproduce/software/config/gnuastro/gnuastro-local.conf # When submitting to places like arXiv, they will just run LaTeX diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk index af6bdc5..67db364 100644 --- a/reproduce/analysis/make/paper.mk +++ b/reproduce/analysis/make/paper.mk @@ -38,7 +38,7 @@ # # Note that if you don't want the final PDF and just want the processing # and file outputs, you can remove the value of `pdf-build-final' in -# `reproduce/analysis/config/pdf-build.mk'. +# `reproduce/analysis/config/pdf-build.conf'. $(mtexdir)/project.tex: $(mtexdir)/verify.tex # If no PDF is requested, or if LaTeX isn't available, don't @@ -62,7 +62,7 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex echo "can check by running './.local/bin/latex --version'), _AND_" echo "make sure that the 'pdf-build-final' variable has a value." echo "'pdf-build-final' is defined in: " - echo "'reproduce/analysis/config/pdf-build.mk'." + echo "'reproduce/analysis/config/pdf-build.conf'." echo echo "If you don't have LaTeX within the project, please re-run" echo "'./project configure -e' when you have internet access." diff --git a/reproduce/analysis/make/prepare.mk b/reproduce/analysis/make/prepare.mk index 1f0613c..36f25e7 100644 --- a/reproduce/analysis/make/prepare.mk +++ b/reproduce/analysis/make/prepare.mk @@ -19,7 +19,9 @@ # Final-target # # Without this file, `./project make' won't work. -$(BDIR)/software/preparation-done.mk: +prepare-dep = $(subst prepare, ,$(makesrc)) +$(BDIR)/software/preparation-done.mk: \ + $(foreach s, $(prepare-dep), $(mtexdir)/$(s).tex) # If you need to add preparations define targets above to do the # preparations, then set the value below to `yes'. Recall that just diff --git a/reproduce/analysis/make/top-make.mk b/reproduce/analysis/make/top-make.mk index 000c1fd..30d537a 100644 --- a/reproduce/analysis/make/top-make.mk +++ b/reproduce/analysis/make/top-make.mk @@ -21,7 +21,7 @@ # Load the local configuration (created after running # `./project configure'). -include reproduce/software/config/installation/LOCAL.mk +include reproduce/software/config/installation/LOCAL.conf @@ -50,7 +50,7 @@ include reproduce/software/config/installation/LOCAL.mk # # Controlling this requires two variables that are available at this stage: # -# - `GROUP-NAME': from `LOCAL.mk' (which was built by `./project configure'). +# - `GROUP-NAME': from `LOCAL.conf' (which was built by `./project configure'). # - `reproducible_paper_group_name': value to the `--group' option. # # The analysis is only done when both have the same group name. Note that @@ -63,7 +63,7 @@ include reproduce/software/config/installation/LOCAL.mk # # If you are just interested in the processing and don't want to build the # PDF, you can skip the creatation of the final PDF by removing the value -# of `pdf-build-final' in `reproduce/analysis/config/pdf-build.mk'. +# of `pdf-build-final' in `reproduce/analysis/config/pdf-build.conf'. ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) all: paper.pdf else diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk index 5ce120d..1778c6b 100644 --- a/reproduce/analysis/make/top-prepare.mk +++ b/reproduce/analysis/make/top-prepare.mk @@ -25,7 +25,7 @@ # Load the local configuration (created after running # `./project configure'). -include reproduce/software/config/installation/LOCAL.mk +include reproduce/software/config/installation/LOCAL.conf @@ -37,19 +37,7 @@ include reproduce/software/config/installation/LOCAL.mk # See `top-make.mk' for complete explanation. ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) all: $(BDIR)/software/preparation-done.mk - @echo ""; - echo "----------------" - echo "Project preparation has been completed without any errors." - echo "" - echo "Please run the following command to start building the project." - echo "(Replace '8' with the number of CPU threads on your system)" - echo "" - if [ "x$(GROUP-NAME)" = x ]; then \ - echo " $$ ./project make"; \ - else \ - echo " $$ ./project make --group=$(GROUP-NAME) -j8"; \ - fi - echo "" + @echo "Project preparation is complete."; else all: @if [ "x$(GROUP-NAME)" = x ]; then \ @@ -59,6 +47,7 @@ all: echo "Project is configured for groups, please run"; \ echo " $$ ./project prepare --group=$(GROUP-NAME) -j8"; \ fi + exit 1 endif @@ -87,5 +76,5 @@ makesrc = initialize \ # # See `top-make.mk' for complete explanation. project-phase = prepare -include reproduce/analysis/config/*.mk +include reproduce/analysis/config/*.conf include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk) diff --git a/reproduce/analysis/make/verify.mk b/reproduce/analysis/make/verify.mk index a04bce4..f0bcf10 100644 --- a/reproduce/analysis/make/verify.mk +++ b/reproduce/analysis/make/verify.mk @@ -33,7 +33,7 @@ verify-print-tips = \ echo "If you are still developing your project, you can disable"; \ echo "verification by removing the value of the variable in the"; \ echo "following file (from the top project source directory):"; \ - echo " reproduce/analysis/config/verify-outputs.mk"; \ + echo " reproduce/analysis/config/verify-outputs.conf"; \ echo; \ echo "If this is the final version of the file, you can just copy"; \ echo "and paste the calculated checksum (above) for the file in"; \ |