From 0774aac6c562b37800e49767ec28826ca8330d8c Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sat, 18 Jan 2020 00:44:05 +0000 Subject: README-hacking.md: edits and corrections for easier customization The checklist descriptions were slightly edited to be more clear. Also, while following them, I noticed that while removing the "delete-me" parts on `verify.mk', would cause an error: the `if [ $$m == delete-me ];' statement we were saying to delete cause an error because `elif' was the first statement Bash would see. So with this commit, the `download' conditional (which isn't instructed to be deleted) was set to be the top (with an `if') and the `delete-me' conditional now has an `elif'. --- reproduce/analysis/make/verify.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'reproduce/analysis/make') diff --git a/reproduce/analysis/make/verify.mk b/reproduce/analysis/make/verify.mk index 1173f2c..a4afd45 100644 --- a/reproduce/analysis/make/verify.mk +++ b/reproduce/analysis/make/verify.mk @@ -114,8 +114,8 @@ $(mtexdir)/verify.tex: $(foreach s, $(verify-dep), $(mtexdir)/$(s).tex) # Verify TeX macros (the values that go into the PDF text). for m in $(verify-check); do file=$(mtexdir)/$$m.tex - if [ $$m == delete-me ]; then s=711e2f7fa1f16ecbeeb3df6bcb4ec705 - elif [ $$m == download ]; then s=6749e17ce606d57d30cebdbc1a5d23ad + if [ $$m == download ]; then s=6749e17ce606d57d30cebdbc1a5d23ad + elif [ $$m == delete-me ]; then s=711e2f7fa1f16ecbeeb3df6bcb4ec705 else echo; echo "'$$m' not recognized."; exit 1 fi $(call verify-txt-no-comments-leading-space, $$file, $$s) -- cgit v1.2.1 From 7ac86df891798fd1f0ef4d7a40aff43ec9621854 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 20 Jan 2020 00:05:41 +0000 Subject: IMPORTANT!!! Configuration Makefiles now have a .conf suffix Until now, the configuration Makefiles (in `reproduce/software/config/installation' and `reproduce/analysis/config') had a `.mk' suffix, similar to the workhorse Makefiles. Although they are indeed Makefiles, but given their nature (to only keep configuration parameters), it is confusing (especially to early users) for them to also have a `.mk' (similar to the analysis or software building Makefiles). To address this issue, with this commit, all the configuration Makefiles (in those directories) are now given a `.conf' suffix. This is also assumed for all the files that are loaded. The configuration (software building) and running of the template have been checked with this change from scratch, but please report any error that may not have been noticed. THIS IS AN IMPORTANT CHANGE AND WILL CAUSE CRASHES OR UNEXPECTED BEHAVIORS FOR PROJECTS THAT HAVE BRANCHED FROM THIS TEMPLATE. PLEASE CORRECT THE SUFFIX OF ALL YOUR PROJECT'S CONFIGURATION MAKEFILES (IN THE DIRECTORIES ABOVE), OTHERWISE THEY AREN'T AUTOMATICALLY LOADED ANYMORE. --- reproduce/analysis/make/delete-me.mk | 2 +- reproduce/analysis/make/download.mk | 9 +++++---- reproduce/analysis/make/initialize.mk | 9 +++++---- reproduce/analysis/make/paper.mk | 4 ++-- reproduce/analysis/make/top-make.mk | 8 ++++---- reproduce/analysis/make/top-prepare.mk | 4 ++-- reproduce/analysis/make/verify.mk | 2 +- 7 files changed, 20 insertions(+), 18 deletions(-) (limited to 'reproduce/analysis/make') diff --git a/reproduce/analysis/make/delete-me.mk b/reproduce/analysis/make/delete-me.mk index 3ba4909..8b97673 100644 --- a/reproduce/analysis/make/delete-me.mk +++ b/reproduce/analysis/make/delete-me.mk @@ -26,7 +26,7 @@ delete-numdir = $(texdir)/delete-me-num delete-num = $(delete-numdir)/data.txt $(delete-numdir): | $(texdir); mkdir $@ -$(delete-num): $(pconfdir)/delete-me-num.mk | $(delete-numdir) +$(delete-num): $(pconfdir)/delete-me-num.conf | $(delete-numdir) # When the plotted values are re-made, it is necessary to also # delete the TiKZ externalized files so the plot is also re-made. diff --git a/reproduce/analysis/make/download.mk b/reproduce/analysis/make/download.mk index 7c290f4..07e9f27 100644 --- a/reproduce/analysis/make/download.mk +++ b/reproduce/analysis/make/download.mk @@ -24,9 +24,10 @@ # Download input data # -------------------- # -# The input dataset properties are defined in `$(pconfdir)/INPUTS.mk'. For -# this template we only have one dataset to enable easy processing, so all -# the extra checks in this rule may seem redundant. +# The input dataset properties are defined in +# `$(pconfdir)/INPUTS.conf'. For this template we only have one dataset to +# enable easy processing, so all the extra checks in this rule may seem +# redundant. # # In a real project, you will need more than one dataset. In that case, # just add them to the target list and add an `elif' statement to define it @@ -93,5 +94,5 @@ $(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir) # # It is very important to mention the address where the data were # downloaded in the final report. -$(mtexdir)/download.tex: $(pconfdir)/INPUTS.mk | $(mtexdir) +$(mtexdir)/download.tex: $(pconfdir)/INPUTS.conf | $(mtexdir) echo "\\newcommand{\\wfpctwourl}{$(WFPC2URL)}" > $@ diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index cdf2129..fa39699 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -241,7 +241,7 @@ distclean: clean # `rm', not our own. $$sys_rm -rf $(BDIR) $$sys_rm -f Makefile .gnuastro .local .build - $$sys_rm -f $(pconfdir)/LOCAL.mk $(gconfdir)/gnuastro-local.conf + $$sys_rm -f $(pconfdir)/LOCAL.conf $(gconfdir)/gnuastro-local.conf @@ -298,10 +298,11 @@ $(packagecontents): paper.pdf | $(texdir) # directories (when building in a group structure, there will be # `build-user1', `build-user2' and etc), are just temporary LaTeX # build files and don't have any relevant/hand-written files in - # them. 2) The `LOCAL.mk' and `gnuastro-local.conf' files just have - # this machine's local settings and are irrelevant for anyone else. + # them. 2) The `LOCAL.conf' and `gnuastro-local.conf' files just + # have this machine's local settings and are irrelevant for anyone + # else. rm -rf $$dir/tex/build/build* - rm $$dir/reproduce/software/config/installation/LOCAL.mk + rm $$dir/reproduce/software/config/installation/LOCAL.conf rm $$dir/reproduce/software/config/gnuastro/gnuastro-local.conf # When submitting to places like arXiv, they will just run LaTeX diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk index a4eeb2e..8a14573 100644 --- a/reproduce/analysis/make/paper.mk +++ b/reproduce/analysis/make/paper.mk @@ -38,7 +38,7 @@ # # Note that if you don't want the final PDF and just want the processing # and file outputs, you can remove the value of `pdf-build-final' in -# `reproduce/analysis/config/pdf-build.mk'. +# `reproduce/analysis/config/pdf-build.conf'. $(mtexdir)/project.tex: $(mtexdir)/verify.tex # If no PDF is requested, or if LaTeX isn't available, don't @@ -62,7 +62,7 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex echo "can check by running './.local/bin/latex --version'), _AND_" echo "make sure that the 'pdf-build-final' variable has a value." echo "'pdf-build-final' is defined in: " - echo "'reproduce/analysis/config/pdf-build.mk'." + echo "'reproduce/analysis/config/pdf-build.conf'." echo echo "If you don't have LaTeX within the project, please re-run" echo "'./project configure -e' when you have internet access." diff --git a/reproduce/analysis/make/top-make.mk b/reproduce/analysis/make/top-make.mk index 0292d3c..6c940b8 100644 --- a/reproduce/analysis/make/top-make.mk +++ b/reproduce/analysis/make/top-make.mk @@ -21,7 +21,7 @@ # Load the local configuration (created after running # `./project configure'). -include reproduce/software/config/installation/LOCAL.mk +include reproduce/software/config/installation/LOCAL.conf @@ -50,7 +50,7 @@ include reproduce/software/config/installation/LOCAL.mk # # Controlling this requires two variables that are available at this stage: # -# - `GROUP-NAME': from `LOCAL.mk' (which was built by `./project configure'). +# - `GROUP-NAME': from `LOCAL.conf' (which was built by `./project configure'). # - `reproducible_paper_group_name': value to the `--group' option. # # The analysis is only done when both have the same group name. Note that @@ -63,7 +63,7 @@ include reproduce/software/config/installation/LOCAL.mk # # If you are just interested in the processing and don't want to build the # PDF, you can skip the creatation of the final PDF by removing the value -# of `pdf-build-final' in `reproduce/analysis/config/pdf-build.mk'. +# of `pdf-build-final' in `reproduce/analysis/config/pdf-build.conf'. ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) all: paper.pdf else @@ -132,5 +132,5 @@ makesrc = initialize \ # But before that, we need to identify the phase for the Makefiles that are # run both in `./project prepare' and `./project make'. project-phase = make -include reproduce/analysis/config/*.mk +include reproduce/analysis/config/*.conf include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk) diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk index 5ce120d..cefbc6b 100644 --- a/reproduce/analysis/make/top-prepare.mk +++ b/reproduce/analysis/make/top-prepare.mk @@ -25,7 +25,7 @@ # Load the local configuration (created after running # `./project configure'). -include reproduce/software/config/installation/LOCAL.mk +include reproduce/software/config/installation/LOCAL.conf @@ -87,5 +87,5 @@ makesrc = initialize \ # # See `top-make.mk' for complete explanation. project-phase = prepare -include reproduce/analysis/config/*.mk +include reproduce/analysis/config/*.conf include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk) diff --git a/reproduce/analysis/make/verify.mk b/reproduce/analysis/make/verify.mk index a4afd45..d11dcbf 100644 --- a/reproduce/analysis/make/verify.mk +++ b/reproduce/analysis/make/verify.mk @@ -33,7 +33,7 @@ verify-print-tips = \ echo "If you are still developing your project, you can disable"; \ echo "verification by removing the value of the variable in the"; \ echo "following file (from the top project source directory):"; \ - echo " reproduce/analysis/config/verify-outputs.mk"; \ + echo " reproduce/analysis/config/verify-outputs.conf"; \ echo; \ echo "If this is the final version of the file, you can just copy"; \ echo "and paste the calculated checksum (above) for the file in"; \ -- cgit v1.2.1 From 334af0e29a950873a19f958b0e47d847808e8910 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 31 Jan 2020 22:43:11 +0100 Subject: Configure step: compiler checks done before basic settings Until now, the project would first ask for the basic directories, then it would start testing the compiler. But that was problematic because the build directory can come from a previous setting (with `./project configure -e'). Also, it could confuse users to first ask for details, then suddently tell them that you don't have a working C library! We also need to store the CPATH variable in the `LOCAL.conf' because in some cases, the compiler won't work without it. With this commit, the compiler checking has been moved at the start of the configure script. Instead of putting the test program in the build directory, we now make a temporary hidden directory in the source directory and delete that directory as soon as the tests are done. In the process, I also noticed that the copyright year of the two hidden files weren't updated and corrected them. --- reproduce/analysis/make/initialize.mk | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'reproduce/analysis/make') diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index fa39699..1137f28 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -140,6 +140,12 @@ export SHELL := $(installdir)/bin/bash export CPPFLAGS := -I$(installdir)/include export LD_LIBRARY_PATH := $(installdir)/lib +# Until we build our own C library, without this, the project's GCC won't +# be able to compile anything if the host C library isn't in a standard +# place: in particular Debian-based operatings sytems. On other systems, it +# will be empty. +export CPATH := $(SYS_CPATH) + # RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is # ultimately redundant. But on some systems, even having a single value # causes crashs (see bug #56682). So we'll just give it no value at all. -- cgit v1.2.1 From 35ed6cf0df743175688b49a4559793cb7f6e9d66 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sat, 1 Feb 2020 21:24:00 +0100 Subject: IMPORTANT: reproduce/software/bash renamed to reproduce/software/shell Until now the shell scripts in the software building phase were in the `reproduce/software/bash' directory. But given our recent change to a POSIX-only start, the `configure.sh' shell script (which is the main component of this directory) is no longer written with Bash. With this commit, to fix that problem, that directory's name has been changed to `reproduce/software/shell'. --- reproduce/analysis/make/initialize.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'reproduce/analysis/make') diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 1137f28..79f9266 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -156,9 +156,9 @@ export DYLD_LIBRARY_PATH := # remote-shell accesss through this environment variable. export OMPI_MCA_plm_rsh_agent=/bin/false -# Recipe startup script, see `reproduce/software/bash/bashrc.sh'. +# Recipe startup script. export PROJECT_STATUS := make -export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh -- cgit v1.2.1 From c54642a9d2cb63181814a0bfa76b55897cbc4801 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Thu, 20 Feb 2020 18:01:06 +0000 Subject: Preparation phase: prepare.tex not needed to finish preparation Until now, the final preparation target of the preparation phase depended on all the `$(makesrc)' files. This caused a problem because we were telling it to also depend on `prepare.tex' (which is the same file that is being built). With this commit, we are applying the same solution we have already done in `paper.mk' (for `paper.tex'): we are removing `prepare' from the list of prerequisites. This bug was found by Zahra Sharbaf. --- reproduce/analysis/make/prepare.mk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'reproduce/analysis/make') diff --git a/reproduce/analysis/make/prepare.mk b/reproduce/analysis/make/prepare.mk index 1f0613c..36f25e7 100644 --- a/reproduce/analysis/make/prepare.mk +++ b/reproduce/analysis/make/prepare.mk @@ -19,7 +19,9 @@ # Final-target # # Without this file, `./project make' won't work. -$(BDIR)/software/preparation-done.mk: +prepare-dep = $(subst prepare, ,$(makesrc)) +$(BDIR)/software/preparation-done.mk: \ + $(foreach s, $(prepare-dep), $(mtexdir)/$(s).tex) # If you need to add preparations define targets above to do the # preparations, then set the value below to `yes'. Recall that just -- cgit v1.2.1 From 7d0c5ef77395a44a316bb822170543b533db239c Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sat, 29 Feb 2020 21:26:54 +0000 Subject: IMPORTANT: re-preparation can only be done with --prepare-redo Until now, the preparation phase was always executed before the final build phase when running `./project make'. But when it becomes necessary, project preparation can be slow and will un-necessarily slow down the project while the project is growing (focus is on the analysis that is done after preparation). With this commit, preparation will be done automatically the first time that the project is run (`.build/software/preparation-done.mk' doesn't exist). However, after preperation is complete once, future runs of `./project make' won't do preparation any more (by calling `top-prepare.mk'). They will directly call `top-make.mk' for the analysis. To manually invoke preparation after the first attempt, the `./project make' script should be run with the new `--prepare-redo' option. Also, since the preparation phase is now automatically done before the analysis phase, the long notice that describes running `./project make' at the end of the preparation phase has been removed in `top-prepare.mk'. It now just prints a short line, saying the preparation has been complete. Finally, when the project has not been run with the proper group configuration, it ends with an `exit 1' so the main `./project' script doesn't proceed any further. --- reproduce/analysis/make/top-prepare.mk | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) (limited to 'reproduce/analysis/make') diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk index cefbc6b..1778c6b 100644 --- a/reproduce/analysis/make/top-prepare.mk +++ b/reproduce/analysis/make/top-prepare.mk @@ -37,19 +37,7 @@ include reproduce/software/config/installation/LOCAL.conf # See `top-make.mk' for complete explanation. ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) all: $(BDIR)/software/preparation-done.mk - @echo ""; - echo "----------------" - echo "Project preparation has been completed without any errors." - echo "" - echo "Please run the following command to start building the project." - echo "(Replace '8' with the number of CPU threads on your system)" - echo "" - if [ "x$(GROUP-NAME)" = x ]; then \ - echo " $$ ./project make"; \ - else \ - echo " $$ ./project make --group=$(GROUP-NAME) -j8"; \ - fi - echo "" + @echo "Project preparation is complete."; else all: @if [ "x$(GROUP-NAME)" = x ]; then \ @@ -59,6 +47,7 @@ all: echo "Project is configured for groups, please run"; \ echo " $$ ./project prepare --group=$(GROUP-NAME) -j8"; \ fi + exit 1 endif -- cgit v1.2.1