aboutsummaryrefslogtreecommitdiff
path: root/reproduce/analysis
diff options
context:
space:
mode:
Diffstat (limited to 'reproduce/analysis')
-rwxr-xr-xreproduce/analysis/bash/download-multi-try22
-rw-r--r--reproduce/analysis/config/verify-outputs.conf1
-rw-r--r--reproduce/analysis/make/download.mk7
-rw-r--r--reproduce/analysis/make/initialize.mk19
-rw-r--r--reproduce/analysis/make/paper.mk4
-rw-r--r--reproduce/analysis/make/prepare.mk4
-rw-r--r--reproduce/analysis/make/top-make.mk6
-rw-r--r--reproduce/analysis/make/top-prepare.mk19
-rw-r--r--reproduce/analysis/make/verify.mk2
9 files changed, 51 insertions, 33 deletions
diff --git a/reproduce/analysis/bash/download-multi-try b/reproduce/analysis/bash/download-multi-try
index dec2539..7a5f8f2 100755
--- a/reproduce/analysis/bash/download-multi-try
+++ b/reproduce/analysis/bash/download-multi-try
@@ -51,11 +51,13 @@ set -e
-# Input arguments and necessary sanity checks.
+# Input arguments and necessary sanity checks. Note that the 5th argument
+# (backup servers) isn't mandatory.
inurl="$3"
outname="$4"
lockfile="$2"
downloader="$1"
+backupservers="$5"
if [ "x$downloader" = x ]; then
echo "$0: downloader (first argument) not given."; exit 1;
fi
@@ -73,6 +75,13 @@ fi
+# Separate the actual filename, to possibly use backup server.
+urlfile=$(echo "$inurl" | awk -F "/" '{print $NF}')
+
+
+
+
+
# Try downloading multiple times before crashing.
counter=0
maxcounter=10
@@ -107,8 +116,19 @@ while [ ! -f "$outname" ]; do
if [ x"$lockfile" = xnolock ]; then
if ! $downloader $outname $inurl; then rm -f $outname; fi
else
+ # Try downloading from the requested URL.
flock "$lockfile" bash -c \
"if ! $downloader $outname $inurl; then rm -f $outname; fi"
+
+ # If it failed, try the backup server(s).
+ if [ ! -f "$outname" ]; then
+ if [ x"$backupservers" != x ]; then
+ for bs in "$backupservers"; do
+ flock "$lockfile" bash -c \
+ "if ! $downloader $outname $bs/$urlfile; then rm -f $outname; fi"
+ done
+ fi
+ fi
fi
done
diff --git a/reproduce/analysis/config/verify-outputs.conf b/reproduce/analysis/config/verify-outputs.conf
index e580e04..0542c53 100644
--- a/reproduce/analysis/config/verify-outputs.conf
+++ b/reproduce/analysis/config/verify-outputs.conf
@@ -1,3 +1,2 @@
# To disable verification of output datasets set this variable to yes
-
verify-outputs =
diff --git a/reproduce/analysis/make/download.mk b/reproduce/analysis/make/download.mk
index e4f2ccd..cf5bfa4 100644
--- a/reproduce/analysis/make/download.mk
+++ b/reproduce/analysis/make/download.mk
@@ -24,9 +24,10 @@
# Download input data
# --------------------
#
-# The input dataset properties are defined in `$(pconfdir)/INPUTS.mk'. For
-# this template we only have one dataset to enable easy processing, so all
-# the extra checks in this rule may seem redundant.
+# The input dataset properties are defined in
+# `$(pconfdir)/INPUTS.conf'. For this template we only have one dataset to
+# enable easy processing, so all the extra checks in this rule may seem
+# redundant.
#
# In a real project, you will need more than one dataset. In that case,
# just add them to the target list and add an `elif' statement to define it
diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk
index a4acff7..ce4e488 100644
--- a/reproduce/analysis/make/initialize.mk
+++ b/reproduce/analysis/make/initialize.mk
@@ -141,6 +141,12 @@ export SHELL := $(installdir)/bin/bash
export CPPFLAGS := -I$(installdir)/include
export LD_LIBRARY_PATH := $(installdir)/lib
+# Until we build our own C library, without this, the project's GCC won't
+# be able to compile anything if the host C library isn't in a standard
+# place: in particular Debian-based operatings sytems. On other systems, it
+# will be empty.
+export CPATH := $(SYS_CPATH)
+
# RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is
# ultimately redundant. But on some systems, even having a single value
# causes crashs (see bug #56682). So we'll just give it no value at all.
@@ -151,9 +157,9 @@ export DYLD_LIBRARY_PATH :=
# remote-shell accesss through this environment variable.
export OMPI_MCA_plm_rsh_agent=/bin/false
-# Recipe startup script, see `reproduce/software/bash/bashrc.sh'.
+# Recipe startup script.
export PROJECT_STATUS := make
-export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh
+export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh
@@ -242,7 +248,7 @@ distclean: clean
# `rm', not our own.
$$sys_rm -rf $(BDIR)
$$sys_rm -f Makefile .gnuastro .local .build
- $$sys_rm -f $(pconfdir)/LOCAL.mk $(gconfdir)/gnuastro-local.conf
+ $$sys_rm -f $(pconfdir)/LOCAL.conf $(gconfdir)/gnuastro-local.conf
@@ -299,10 +305,11 @@ $(packagecontents): paper.pdf | $(texdir)
# directories (when building in a group structure, there will be
# `build-user1', `build-user2' and etc), are just temporary LaTeX
# build files and don't have any relevant/hand-written files in
- # them. 2) The `LOCAL.mk' and `gnuastro-local.conf' files just have
- # this machine's local settings and are irrelevant for anyone else.
+ # them. 2) The `LOCAL.conf' and `gnuastro-local.conf' files just
+ # have this machine's local settings and are irrelevant for anyone
+ # else.
rm -rf $$dir/tex/build/build*
- rm $$dir/reproduce/software/config/installation/LOCAL.mk
+ rm $$dir/reproduce/software/config/installation/LOCAL.conf
rm $$dir/reproduce/software/config/gnuastro/gnuastro-local.conf
# When submitting to places like arXiv, they will just run LaTeX
diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk
index af6bdc5..67db364 100644
--- a/reproduce/analysis/make/paper.mk
+++ b/reproduce/analysis/make/paper.mk
@@ -38,7 +38,7 @@
#
# Note that if you don't want the final PDF and just want the processing
# and file outputs, you can remove the value of `pdf-build-final' in
-# `reproduce/analysis/config/pdf-build.mk'.
+# `reproduce/analysis/config/pdf-build.conf'.
$(mtexdir)/project.tex: $(mtexdir)/verify.tex
# If no PDF is requested, or if LaTeX isn't available, don't
@@ -62,7 +62,7 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex
echo "can check by running './.local/bin/latex --version'), _AND_"
echo "make sure that the 'pdf-build-final' variable has a value."
echo "'pdf-build-final' is defined in: "
- echo "'reproduce/analysis/config/pdf-build.mk'."
+ echo "'reproduce/analysis/config/pdf-build.conf'."
echo
echo "If you don't have LaTeX within the project, please re-run"
echo "'./project configure -e' when you have internet access."
diff --git a/reproduce/analysis/make/prepare.mk b/reproduce/analysis/make/prepare.mk
index 1f0613c..36f25e7 100644
--- a/reproduce/analysis/make/prepare.mk
+++ b/reproduce/analysis/make/prepare.mk
@@ -19,7 +19,9 @@
# Final-target
#
# Without this file, `./project make' won't work.
-$(BDIR)/software/preparation-done.mk:
+prepare-dep = $(subst prepare, ,$(makesrc))
+$(BDIR)/software/preparation-done.mk: \
+ $(foreach s, $(prepare-dep), $(mtexdir)/$(s).tex)
# If you need to add preparations define targets above to do the
# preparations, then set the value below to `yes'. Recall that just
diff --git a/reproduce/analysis/make/top-make.mk b/reproduce/analysis/make/top-make.mk
index 000c1fd..30d537a 100644
--- a/reproduce/analysis/make/top-make.mk
+++ b/reproduce/analysis/make/top-make.mk
@@ -21,7 +21,7 @@
# Load the local configuration (created after running
# `./project configure').
-include reproduce/software/config/installation/LOCAL.mk
+include reproduce/software/config/installation/LOCAL.conf
@@ -50,7 +50,7 @@ include reproduce/software/config/installation/LOCAL.mk
#
# Controlling this requires two variables that are available at this stage:
#
-# - `GROUP-NAME': from `LOCAL.mk' (which was built by `./project configure').
+# - `GROUP-NAME': from `LOCAL.conf' (which was built by `./project configure').
# - `reproducible_paper_group_name': value to the `--group' option.
#
# The analysis is only done when both have the same group name. Note that
@@ -63,7 +63,7 @@ include reproduce/software/config/installation/LOCAL.mk
#
# If you are just interested in the processing and don't want to build the
# PDF, you can skip the creatation of the final PDF by removing the value
-# of `pdf-build-final' in `reproduce/analysis/config/pdf-build.mk'.
+# of `pdf-build-final' in `reproduce/analysis/config/pdf-build.conf'.
ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME))
all: paper.pdf
else
diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk
index 5ce120d..1778c6b 100644
--- a/reproduce/analysis/make/top-prepare.mk
+++ b/reproduce/analysis/make/top-prepare.mk
@@ -25,7 +25,7 @@
# Load the local configuration (created after running
# `./project configure').
-include reproduce/software/config/installation/LOCAL.mk
+include reproduce/software/config/installation/LOCAL.conf
@@ -37,19 +37,7 @@ include reproduce/software/config/installation/LOCAL.mk
# See `top-make.mk' for complete explanation.
ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME))
all: $(BDIR)/software/preparation-done.mk
- @echo "";
- echo "----------------"
- echo "Project preparation has been completed without any errors."
- echo ""
- echo "Please run the following command to start building the project."
- echo "(Replace '8' with the number of CPU threads on your system)"
- echo ""
- if [ "x$(GROUP-NAME)" = x ]; then \
- echo " $$ ./project make"; \
- else \
- echo " $$ ./project make --group=$(GROUP-NAME) -j8"; \
- fi
- echo ""
+ @echo "Project preparation is complete.";
else
all:
@if [ "x$(GROUP-NAME)" = x ]; then \
@@ -59,6 +47,7 @@ all:
echo "Project is configured for groups, please run"; \
echo " $$ ./project prepare --group=$(GROUP-NAME) -j8"; \
fi
+ exit 1
endif
@@ -87,5 +76,5 @@ makesrc = initialize \
#
# See `top-make.mk' for complete explanation.
project-phase = prepare
-include reproduce/analysis/config/*.mk
+include reproduce/analysis/config/*.conf
include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk)
diff --git a/reproduce/analysis/make/verify.mk b/reproduce/analysis/make/verify.mk
index a04bce4..f0bcf10 100644
--- a/reproduce/analysis/make/verify.mk
+++ b/reproduce/analysis/make/verify.mk
@@ -33,7 +33,7 @@ verify-print-tips = \
echo "If you are still developing your project, you can disable"; \
echo "verification by removing the value of the variable in the"; \
echo "following file (from the top project source directory):"; \
- echo " reproduce/analysis/config/verify-outputs.mk"; \
+ echo " reproduce/analysis/config/verify-outputs.conf"; \
echo; \
echo "If this is the final version of the file, you can just copy"; \
echo "and paste the calculated checksum (above) for the file in"; \