From 7caa2845304c40540a336f840b3ca468bf6c8697 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Tue, 1 Oct 2019 16:17:59 +0100 Subject: Preparation phase added before final building In many real-world scenarios, `./project make' can really benefit from having some basic information about the data before being run. For example when quering a server. If we know how many datasets were downloaded and their general properties, it can greatly optmize the process when we are designing the solution to be run in `./project make'. Therefore with this commit, a new phase has been added to the template's design: `./project prepare'. In the raw template this is empty, because the simple analysis done in the template doesn't warrant it. But everything is ready for projects using the template to add preparation phases prior to the analysis. --- reproduce/analysis/make/prepare.mk | 35 +++++++++ reproduce/analysis/make/top-make.mk | 136 +++++++++++++++++++++++++++++++++ reproduce/analysis/make/top-prepare.mk | 91 ++++++++++++++++++++++ reproduce/analysis/make/top.mk | 136 --------------------------------- reproduce/software/bash/configure.sh | 6 +- 5 files changed, 265 insertions(+), 139 deletions(-) create mode 100644 reproduce/analysis/make/prepare.mk create mode 100644 reproduce/analysis/make/top-make.mk create mode 100644 reproduce/analysis/make/top-prepare.mk delete mode 100644 reproduce/analysis/make/top.mk (limited to 'reproduce') diff --git a/reproduce/analysis/make/prepare.mk b/reproduce/analysis/make/prepare.mk new file mode 100644 index 0000000..3e41fa9 --- /dev/null +++ b/reproduce/analysis/make/prepare.mk @@ -0,0 +1,35 @@ +# Basic preparations, called by `./project prepare'. +# +# Copyright (C) 2019 Mohammad Akhlaghi +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. See . + + + + + +# Final-target +# +# Without this file, `./project make' won't work. +$(BDIR)/software/preparation-done.txt: + + # If you need to add preparations define targets above to do the + # preparations. Recall that before this file, `top-prepare.mk' + # loads `initialize.mk' and `download.mk', so you can safely assume + # everything that is defined there in this Makefile. + # + # TIP: the targets can actually be automatically generated + # Makefiles that are used by `./project make'. They can include + # variables, or actual rules. Just make sure that those Makefiles + # aren't written in the source directory! Even though they are + # Makefiles, they are automatically built, so they should be + # somewhere under $(BDIR). + @touch $@ diff --git a/reproduce/analysis/make/top-make.mk b/reproduce/analysis/make/top-make.mk new file mode 100644 index 0000000..7d20800 --- /dev/null +++ b/reproduce/analysis/make/top-make.mk @@ -0,0 +1,136 @@ +# Top-level Makefile (first to be loaded). +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# . + + + + + +# Load the local configuration (created after running +# `./project configure'). +include reproduce/software/config/installation/LOCAL.mk + + + + + +# Ultimate target of this project +# ------------------------------- +# +# The final paper/report (`paper.pdf') is the main target of this +# project. As defined in the Make paradigm, it must be the first target +# that Make encounters (immediately after loading the local configuration +# settings, necessary for a group building scenario mentioned next). +# +# +# Group build +# ----------- +# +# This project can also be configured to have a shared build directory +# between multiple users. In this scenario, many users (on a server) can +# have their own/separate version controlled project source, but share the +# same build outputs (in a common directory). This will allow a group to +# work separately, on parallel parts of the analysis that don't +# interfere. It is thus very useful in cases were special storage +# requirements or CPU power is necessary and its not possible/efficient for +# each user to have a fully separate copy of the build directory. +# +# Controlling this requires two variables that are available at this stage: +# +# - `GROUP-NAME': from `LOCAL.mk' (which was built by `./project configure'). +# - `reproducible_paper_group_name': value to the `--group' option. +# +# The analysis is only done when both have the same group name. Note that +# when the project isn't being built for a group, both variables will be an +# empty string. +# +# +# Only processing, no LaTeX PDF +# ----------------------------- +# +# If you are just interested in the processing and don't want to build the +# PDF, you can skip the creatation of the final PDF by removing the value +# of `pdf-build-final' in `reproduce/analysis/config/pdf-build.mk'. +ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) +all: paper.pdf +else +all: + @if [ "x$(GROUP-NAME)" = x ]; then \ + echo "Project is NOT configured for groups, please run"; \ + echo " $$ ./project make"; \ + else \ + echo "Project is configured for groups, please run"; \ + echo " $$ ./project make --group=$(GROUP-NAME) -j8"; \ + fi +endif + + + + + +# Define source Makefiles +# ----------------------- +# +# To keep things clean, managable and readable, each set of operations +# is (and must be) classified (modularized) by context into separate +# Makefiles: the more the better. These modular steps are then +# included in this top-level Makefile through the `include' command of +# the next step. Each Makefile should also produce a LaTeX macro file +# with the same fixed name (used to keep all the parameters and +# relevant outputs of the steps in it for the final paper). +# +# In the rare case that no special LaTeX macros are necessary in a +# workhorse Makefile, you can simply make an empty file with `touch +# $@'. This will not add any lines to the final combined LaTeX macros +# file, but will create the file that is a prerequisite to the final +# paper generation. +# +# To (significantly) help in readability, this top-level Makefile should be +# the only one in charge of including Makefiles. So if you care about easy +# maintainence and understandability (even for your self, in one year! It +# is VERY IMPORTANT and as a scientist, you MUST care about it!), do not +# include Makefiles from any other Makefile. +# +# IMPORTANT NOTE: order matters in the inclusion of the processing +# Makefiles. As the project grows, some Makefiles will define +# variables/dependencies that later Makefiles need. Therefore we are using +# a `foreach' loop in the next step to explicitly request loading them in +# the same order that they are defined here (we aren't just using a +# wild-card like the configuration Makefiles). +makesrc = initialize \ + download \ + delete-me \ + paper + + + + + +# Include all analysis Makefiles +# ------------------------------ +# +# 1) All the analysis configuration-Makefiles (Makefiles that only define +# variables with no rules or order). +# +# 2) From the software configuration-Makefiles, we only include the one +# containing software versions, just incase its necessary to +# use/report outside of the acknowledgments section of the paper. +# +# 3) Finally, we'll import all the analysis workhorse-Makefiles which +# contain rules to actually do this project's processing. +include reproduce/analysis/config/*.mk +include reproduce/software/config/installation/versions.mk +include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk) diff --git a/reproduce/analysis/make/top-prepare.mk b/reproduce/analysis/make/top-prepare.mk new file mode 100644 index 0000000..3353638 --- /dev/null +++ b/reproduce/analysis/make/top-prepare.mk @@ -0,0 +1,91 @@ +# Do basic preparations to optimize the project's running. +# +# NOTE: This file is very similar to `top-make.mk', so the large comments +# are not included here. Please see that file for thorough comments on each +# step. +# +# Copyright (C) 2019 Mohammad Akhlaghi +# +# This Makefile is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This Makefile is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# . + + + + + +# Load the local configuration (created after running +# `./project configure'). +include reproduce/software/config/installation/LOCAL.mk + + + + + +# Ultimate target of this project +# ------------------------------- +# +# See `top-make.mk' for complete explanation. +ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) +all: $(BDIR)/software/preparation-done.txt + @echo ""; + echo "----------------" + echo "Project preparation has been completed without any errors." + echo "" + echo "Please run the following command to start building the project." + echo "(Replace '8' with the number of CPU threads on your system)" + echo "" + if [ "x$(GROUP-NAME)" = x ]; then + echo " $$ ./project make" + else + echo " $$ ./project make --group=$(GROUP-NAME) -j8" + fi + echo "" +else +all: + @if [ "x$(GROUP-NAME)" = x ]; then + echo "Project is NOT configured for groups, please run" + echo " $$ ./project prepare" + else + echo "Project is configured for groups, please run" + echo " $$ ./project prepare --group=$(GROUP-NAME) -j8" + fi +endif + + + + + +# Define source Makefiles +# ----------------------- +# +# See `top-make.mk' for complete explanation. +# +# To ensure that `prepare' and `make' have the same basic definitions and +# environment and that all `downloads' are managed in one place, both +# `./project prepare' and `./project make' will first read `initialize.mk' +# and `downloads.mk'. +makesrc = initialize \ + download \ + prepare + + + + + +# Include all analysis Makefiles +# ------------------------------ +# +# See `top-make.mk' for complete explanation. +include reproduce/analysis/config/*.mk +include reproduce/software/config/installation/versions.mk +include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk) diff --git a/reproduce/analysis/make/top.mk b/reproduce/analysis/make/top.mk deleted file mode 100644 index 7d20800..0000000 --- a/reproduce/analysis/make/top.mk +++ /dev/null @@ -1,136 +0,0 @@ -# Top-level Makefile (first to be loaded). -# -# Copyright (C) 2018-2019 Mohammad Akhlaghi -# -# This Makefile is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This Makefile is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# A copy of the GNU General Public License is available at -# . - - - - - -# Load the local configuration (created after running -# `./project configure'). -include reproduce/software/config/installation/LOCAL.mk - - - - - -# Ultimate target of this project -# ------------------------------- -# -# The final paper/report (`paper.pdf') is the main target of this -# project. As defined in the Make paradigm, it must be the first target -# that Make encounters (immediately after loading the local configuration -# settings, necessary for a group building scenario mentioned next). -# -# -# Group build -# ----------- -# -# This project can also be configured to have a shared build directory -# between multiple users. In this scenario, many users (on a server) can -# have their own/separate version controlled project source, but share the -# same build outputs (in a common directory). This will allow a group to -# work separately, on parallel parts of the analysis that don't -# interfere. It is thus very useful in cases were special storage -# requirements or CPU power is necessary and its not possible/efficient for -# each user to have a fully separate copy of the build directory. -# -# Controlling this requires two variables that are available at this stage: -# -# - `GROUP-NAME': from `LOCAL.mk' (which was built by `./project configure'). -# - `reproducible_paper_group_name': value to the `--group' option. -# -# The analysis is only done when both have the same group name. Note that -# when the project isn't being built for a group, both variables will be an -# empty string. -# -# -# Only processing, no LaTeX PDF -# ----------------------------- -# -# If you are just interested in the processing and don't want to build the -# PDF, you can skip the creatation of the final PDF by removing the value -# of `pdf-build-final' in `reproduce/analysis/config/pdf-build.mk'. -ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) -all: paper.pdf -else -all: - @if [ "x$(GROUP-NAME)" = x ]; then \ - echo "Project is NOT configured for groups, please run"; \ - echo " $$ ./project make"; \ - else \ - echo "Project is configured for groups, please run"; \ - echo " $$ ./project make --group=$(GROUP-NAME) -j8"; \ - fi -endif - - - - - -# Define source Makefiles -# ----------------------- -# -# To keep things clean, managable and readable, each set of operations -# is (and must be) classified (modularized) by context into separate -# Makefiles: the more the better. These modular steps are then -# included in this top-level Makefile through the `include' command of -# the next step. Each Makefile should also produce a LaTeX macro file -# with the same fixed name (used to keep all the parameters and -# relevant outputs of the steps in it for the final paper). -# -# In the rare case that no special LaTeX macros are necessary in a -# workhorse Makefile, you can simply make an empty file with `touch -# $@'. This will not add any lines to the final combined LaTeX macros -# file, but will create the file that is a prerequisite to the final -# paper generation. -# -# To (significantly) help in readability, this top-level Makefile should be -# the only one in charge of including Makefiles. So if you care about easy -# maintainence and understandability (even for your self, in one year! It -# is VERY IMPORTANT and as a scientist, you MUST care about it!), do not -# include Makefiles from any other Makefile. -# -# IMPORTANT NOTE: order matters in the inclusion of the processing -# Makefiles. As the project grows, some Makefiles will define -# variables/dependencies that later Makefiles need. Therefore we are using -# a `foreach' loop in the next step to explicitly request loading them in -# the same order that they are defined here (we aren't just using a -# wild-card like the configuration Makefiles). -makesrc = initialize \ - download \ - delete-me \ - paper - - - - - -# Include all analysis Makefiles -# ------------------------------ -# -# 1) All the analysis configuration-Makefiles (Makefiles that only define -# variables with no rules or order). -# -# 2) From the software configuration-Makefiles, we only include the one -# containing software versions, just incase its necessary to -# use/report outside of the acknowledgments section of the paper. -# -# 3) Finally, we'll import all the analysis workhorse-Makefiles which -# contain rules to actually do this project's processing. -include reproduce/analysis/config/*.mk -include reproduce/software/config/installation/versions.mk -include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk) diff --git a/reproduce/software/bash/configure.sh b/reproduce/software/bash/configure.sh index 5c46496..7ef576a 100755 --- a/reproduce/software/bash/configure.sh +++ b/reproduce/software/bash/configure.sh @@ -1387,9 +1387,9 @@ echo `.local/bin/date` > $finaltarget # The configuration is now complete, we can inform the user on the next # step(s) to take. if [ x$reproducible_paper_group_name = x ]; then - buildcommand="./project make -j8" + buildcommand="./project prepare -j8" else - buildcommand="./project make --group=$reproducible_paper_group_name -j8" + buildcommand="./project prepare --group=$reproducible_paper_group_name -j8" fi cat <