diff options
| -rw-r--r-- | img/make-demo-1.pdf | bin | 5173 -> 5503 bytes | |||
| -rw-r--r-- | img/make-demo-10.pdf | bin | 9378 -> 8868 bytes | |||
| -rw-r--r-- | img/make-demo-11.pdf | bin | 9632 -> 9199 bytes | |||
| -rw-r--r-- | img/make-demo-12.pdf | bin | 9760 -> 9387 bytes | |||
| -rw-r--r-- | img/make-demo-13.pdf | bin | 9886 -> 9494 bytes | |||
| -rw-r--r-- | img/make-demo-14.pdf | bin | 10150 -> 9665 bytes | |||
| -rw-r--r-- | img/make-demo-15.pdf | bin | 10297 -> 9821 bytes | |||
| -rw-r--r-- | img/make-demo-16.pdf | bin | 10727 -> 9867 bytes | |||
| -rw-r--r-- | img/make-demo-17.pdf | bin | 10968 -> 10022 bytes | |||
| -rw-r--r-- | img/make-demo-18.pdf | bin | 0 -> 10415 bytes | |||
| -rw-r--r-- | img/make-demo-19.pdf | bin | 0 -> 10580 bytes | |||
| -rw-r--r-- | img/make-demo-2.pdf | bin | 7137 -> 6209 bytes | |||
| -rw-r--r-- | img/make-demo-20.pdf | bin | 0 -> 10832 bytes | |||
| -rw-r--r-- | img/make-demo-21.pdf | bin | 0 -> 11005 bytes | |||
| -rw-r--r-- | img/make-demo-3.pdf | bin | 7388 -> 6727 bytes | |||
| -rw-r--r-- | img/make-demo-4.pdf | bin | 7803 -> 7094 bytes | |||
| -rw-r--r-- | img/make-demo-5.pdf | bin | 8268 -> 7429 bytes | |||
| -rw-r--r-- | img/make-demo-6.pdf | bin | 8458 -> 7611 bytes | |||
| -rw-r--r-- | img/make-demo-7.pdf | bin | 8655 -> 7860 bytes | |||
| -rw-r--r-- | img/make-demo-8.pdf | bin | 8956 -> 8547 bytes | |||
| -rw-r--r-- | img/make-demo-9.pdf | bin | 9138 -> 8687 bytes | |||
| -rw-r--r-- | reproducible-paper.tex | 146 |
22 files changed, 79 insertions, 67 deletions
diff --git a/img/make-demo-1.pdf b/img/make-demo-1.pdf Binary files differindex b0cfd86..f647634 100644 --- a/img/make-demo-1.pdf +++ b/img/make-demo-1.pdf diff --git a/img/make-demo-10.pdf b/img/make-demo-10.pdf Binary files differindex 17f5b06..a296b7f 100644 --- a/img/make-demo-10.pdf +++ b/img/make-demo-10.pdf diff --git a/img/make-demo-11.pdf b/img/make-demo-11.pdf Binary files differindex 40ebda3..ec359c5 100644 --- a/img/make-demo-11.pdf +++ b/img/make-demo-11.pdf diff --git a/img/make-demo-12.pdf b/img/make-demo-12.pdf Binary files differindex a203d6f..9112299 100644 --- a/img/make-demo-12.pdf +++ b/img/make-demo-12.pdf diff --git a/img/make-demo-13.pdf b/img/make-demo-13.pdf Binary files differindex 402e39f..f29e92e 100644 --- a/img/make-demo-13.pdf +++ b/img/make-demo-13.pdf diff --git a/img/make-demo-14.pdf b/img/make-demo-14.pdf Binary files differindex bdb23be..8043daf 100644 --- a/img/make-demo-14.pdf +++ b/img/make-demo-14.pdf diff --git a/img/make-demo-15.pdf b/img/make-demo-15.pdf Binary files differindex 208ed3a..406150d 100644 --- a/img/make-demo-15.pdf +++ b/img/make-demo-15.pdf diff --git a/img/make-demo-16.pdf b/img/make-demo-16.pdf Binary files differindex c94ae94..0a4e127 100644 --- a/img/make-demo-16.pdf +++ b/img/make-demo-16.pdf diff --git a/img/make-demo-17.pdf b/img/make-demo-17.pdf Binary files differindex 58532c0..1e33e00 100644 --- a/img/make-demo-17.pdf +++ b/img/make-demo-17.pdf diff --git a/img/make-demo-18.pdf b/img/make-demo-18.pdf Binary files differnew file mode 100644 index 0000000..b3420ad --- /dev/null +++ b/img/make-demo-18.pdf diff --git a/img/make-demo-19.pdf b/img/make-demo-19.pdf Binary files differnew file mode 100644 index 0000000..09051db --- /dev/null +++ b/img/make-demo-19.pdf diff --git a/img/make-demo-2.pdf b/img/make-demo-2.pdf Binary files differindex 0a4ac54..a82d6e7 100644 --- a/img/make-demo-2.pdf +++ b/img/make-demo-2.pdf diff --git a/img/make-demo-20.pdf b/img/make-demo-20.pdf Binary files differnew file mode 100644 index 0000000..e9ffdc5 --- /dev/null +++ b/img/make-demo-20.pdf diff --git a/img/make-demo-21.pdf b/img/make-demo-21.pdf Binary files differnew file mode 100644 index 0000000..4bc8de0 --- /dev/null +++ b/img/make-demo-21.pdf diff --git a/img/make-demo-3.pdf b/img/make-demo-3.pdf Binary files differindex 504d021..7960445 100644 --- a/img/make-demo-3.pdf +++ b/img/make-demo-3.pdf diff --git a/img/make-demo-4.pdf b/img/make-demo-4.pdf Binary files differindex c67bfc9..9f4d16b 100644 --- a/img/make-demo-4.pdf +++ b/img/make-demo-4.pdf diff --git a/img/make-demo-5.pdf b/img/make-demo-5.pdf Binary files differindex 1b547dd..9dc4bac 100644 --- a/img/make-demo-5.pdf +++ b/img/make-demo-5.pdf diff --git a/img/make-demo-6.pdf b/img/make-demo-6.pdf Binary files differindex 20a104a..a8a6458 100644 --- a/img/make-demo-6.pdf +++ b/img/make-demo-6.pdf diff --git a/img/make-demo-7.pdf b/img/make-demo-7.pdf Binary files differindex ad7e38a..c93e6b1 100644 --- a/img/make-demo-7.pdf +++ b/img/make-demo-7.pdf diff --git a/img/make-demo-8.pdf b/img/make-demo-8.pdf Binary files differindex 825b3b1..d3e2d9b 100644 --- a/img/make-demo-8.pdf +++ b/img/make-demo-8.pdf diff --git a/img/make-demo-9.pdf b/img/make-demo-9.pdf Binary files differindex 9e4e7a1..51dc392 100644 --- a/img/make-demo-9.pdf +++ b/img/make-demo-9.pdf diff --git a/reproducible-paper.tex b/reproducible-paper.tex index 6c8cd71..7cee761 100644 --- a/reproducible-paper.tex +++ b/reproducible-paper.tex @@ -846,55 +846,7 @@ - %% Make demo. - \makedemoslide{img/make-demo-1.pdf} - {Makefiles (\texttt{\*.mk}) keep contextually separate parts of the project, all imported into \texttt{top-make.mk}} - \makedemoslide{img/make-demo-2.pdf} - {First built/output file (downloaded dataset), managed in \texttt{download.mk}.} - \makedemoslide{img/make-demo-3.pdf} - {Downloaded file is used to build analysis-1's first output.} - \makedemoslide{img/make-demo-4.pdf} - {Third built file is for analysis-3, also depends on - a value in a configuration file.} - \makedemoslide{img/make-demo-5.pdf} - {First input is also used to build another file (using - two configuration parameters).} - \makedemoslide{img/make-demo-6.pdf} - {\texttt{out-3b.dat} is automatically built after (depends on) \texttt{out-2a.dat}.} - \makedemoslide{img/make-demo-7.pdf} - {We now need a second input file, which is downloaded.} - \makedemoslide{img/make-demo-8.pdf} - {\texttt{out-1b.dat} depends on \texttt{out-1a.dat}, - \texttt{input2.dat} and \texttt{param-1.conf}.} - \makedemoslide{img/make-demo-9.pdf} - {Final hypothetical output file (of the analysis phase) is - also built.} - \begin{frame}{Benefits of using Make for storing data lineage} - \Large - \begin{itemize} - \setlength\itemsep{5mm} - \item Every output file's position in the analysis is formally defined.\\ - {\normalsize (edges between the nodes of the graph in the previous slide)} - \vspace{3mm} - \begin{itemize} - \Large - \setlength\itemsep{3mm} - \item Make can \alert{parallelize} the analysis: \\Make knows - which steps are indepenent and will run them at the same time.\\ - \item Make can \alert{automatically detect a change} and will - re-do \emph{only} the affected steps.\\ {\normalsize (for - example to change the multiple of sigma in a configuration - file to see its effect)} - \item Easily \alert{backtrace} any step (without needing to remember!).\\ - {\normalsize (very useful to find problems/improvements)} - \end{itemize} - \vspace{3mm} All will speed up your work, and encourage - experimentation for a robust result. - \item Make is \alert{available} on any system: many people are \alert{already familiar} with it. - \item And again: its \alert{all in plain text}!\\{\normalsize (doesn't take much space, easy to read, distribute, parse automatically, or archive)} - \end{itemize} - \end{frame} @@ -979,33 +931,91 @@ \end{center} \end{frame} + + %% Make demo. + \begin{frame} + \LARGE + \vspace{1cm} + \hfill Let's see how all this is managed in a hypothetical project... + \end{frame} + \makedemoslide{img/make-demo-1.pdf} + {Makefiles (\texttt{\*.mk}) keep contextually separate parts of the project, all imported into \texttt{top-make.mk}} + \makedemoslide{img/make-demo-2.pdf} + {The ultimate purpose of the project is to produce a paper/report (in PDF).} + \makedemoslide{img/make-demo-3.pdf} + {The narrative description of the PDF and \LaTeX{} typography is kept in \texttt{paper.tex}.} + \makedemoslide{img/make-demo-4.pdf} + {The numbers of the PDF (blended into the narrative) come from \texttt{project.tex}.} + \makedemoslide{img/make-demo-5.pdf} + {Basic project info comes from \texttt{initialize.tex}.} + \makedemoslide{img/make-demo-6.pdf} + {Reported values about the downloaded inputs come from \texttt{download.tex}.} + \makedemoslide{img/make-demo-7.pdf} + {... for example the number of rows in the second input (a catalog) of the project.} + \makedemoslide{img/make-demo-8.pdf} + {The URL to download \texttt{input2.dat}, and a checksum to validate it, are stored in \texttt{INPUTS.conf}.} \makedemoslide{img/make-demo-9.pdf} - {Let's start from the end of the analysis phase (same graph as before).} + {Reported values from first analysis steps stored in \texttt{analysis1.tex}.} \makedemoslide{img/make-demo-10.pdf} - {Makefiles are also used to connect the analysis with with paper.} + {... for example the average of the numbers in \texttt{out-1b.dat}.} \makedemoslide{img/make-demo-11.pdf} - {Information from the raw inputs is stored in \texttt{download.tex}.} + {But \texttt{out-1b.dat} itself depends on other files and a paramter (for example a multiple of sigma).} \makedemoslide{img/make-demo-12.pdf} - {Values to report from the first analysis step are kept in - \texttt{analysis1.tex}.} + {\texttt{out-1a.dat} is built from a downloaded dataset.} \makedemoslide{img/make-demo-13.pdf} - {Values to report from the second analysis step are kept in - \texttt{analysis2.tex}.} + {Download URL and checksum of \texttt{input1.dat} also stored in \texttt{INPUTS.conf}.} \makedemoslide{img/make-demo-14.pdf} - {Values to report from the third analysis step are kept in - \texttt{analysis3.tex}.} + {Reported values from second analysis steps stored in \texttt{analysis2.tex}.} \makedemoslide{img/make-demo-15.pdf} - {General project information are stored in \texttt{initialize.tex}.} + {... for example the number of selected rows in \texttt{out-2b.dat}.} \makedemoslide{img/make-demo-16.pdf} - {All the \LaTeX{} macros are loaded into the single - \texttt{project.tex} (after the analysis is complete).} - \makedemoslide{img/make-demo-17.pdf} - {With the paper's \LaTeX{} source, \texttt{project.tex} is - used to build the final PDF paper.} + {\texttt{out-2b.dat} is derived from \texttt{out-1b.dat} (for example, rejected some of \texttt{out-1b.dat}'s rows).} \makedemoslide{img/make-demo-17.pdf} - {Full data lineage (raw input $\leftrightarrow$ every - number/paragraph and plot in paper's PDF) is recorded.} + {Reported values from third analysis steps stored in \texttt{analysis3.tex}.} + \makedemoslide{img/make-demo-18.pdf} + {... for example measurements from both \texttt{out-3a.dat} and \texttt{out-3b.dat}.} + \makedemoslide{img/make-demo-19.pdf} + {\texttt{out-3b.dat} is generated from an analysis on \texttt{out-2a.dat}.} + \makedemoslide{img/make-demo-20.pdf} + {But \texttt{out-2a.dat} itself is generated from \texttt{input1.dat} and an analysis which has two settings.} + \makedemoslide{img/make-demo-21.pdf} + {\texttt{out-3a.dat} also depends on \texttt{out-1a.dat} and an analysis with needs one parameter.} + \begin{frame}{\LARGE The whole project is a directed graph (codifying the data's lineage).} + \LARGE + \begin{itemize} + \setlength\itemsep{1cm} + \item Every \alert{file} (source or built) is a \alert{node} in the graph (connected to others).\\ + {\large (The links/connections/dependencies between the nodes, defined by the Makefiles: \texttt{*.mk})} + \item There are two types of nodes/files: + \begin{itemize} + \LARGE + \setlength\itemsep{4mm} + \item \alert{Source} nodes (\texttt{*.conf} and \texttt{paper.tex}) only have an \alert{outward} link. + \item \alert{Built} files always have \alert{inward} \emph{and} {\normalsize (except \texttt{paper.pdf})} \alert{outward} link(s). + \end{itemize} + \item All built files ultimately originate from a \texttt{*.conf} file,\\ + ... and ultimately conclude in \texttt{paper.pdf}. + \end{itemize} + \end{frame} + \begin{frame}{Benefits of using Make} + \Large + \begin{itemize} + \setlength\itemsep{4mm} + \item Make can \alert{parallelize} the analysis: \\Make knows + which steps are indepenent and will run them at the same time.\\ + \item Make can \alert{automatically detect a change} and will + re-do \emph{only} the affected steps.\\ {\normalsize (for + example to change the multiple of sigma in a configuration + file to see its effect)} + \item Easily \alert{backtrace} any step (without needing to remember!).\\ + {\normalsize (very useful to find problems/improvements)} + \item The above will speed up your work, and \alert{encourage experimentation} on methods. + \item Make is \alert{available} on any system: many people are \alert{already familiar} with it. + \item And again: its \alert{all in plain text}!\\{\normalsize (doesn't take much space, easy to read, distribute, parse automatically, or archive)} + \item Recall that the project's \alert{software installation} was also managed in Make. + \end{itemize} + \end{frame} @@ -1026,10 +1036,12 @@ \newcommand{\allopacity}{1} - \begin{frame}{Everything in plain text (machine and human readable)} + \begin{frame}{All questions have an answer now (in + \alert{plain text}: human \& computer readable/archivable).} \include{tex/project-graph} \end{frame} \newcommand{\gitlogo}{} - \begin{frame}{Everything in plain text (machine and human readable)} + \begin{frame}{All questions have an answer now (in + \alert{plain text}: so we can use Git to keep its history).} \include{tex/project-graph} \end{frame} |