From 48e61df320e7571e527049cd3eef5cf96b8cb491 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Mon, 23 Mar 2020 03:16:06 +0000 Subject: Analysis and configuration file sections complete With this commit a description of these two important parts have been added to the project, along with several figures showing various parts of the files that are discussed. I also done some other restructuring of the figures and files to make things fit better into the the description of the paper. --- tex/src/figure-data-lineage.tex | 46 ++++++++++++++-------------- tex/src/figure-download.tex | 8 ----- tex/src/figure-file-architecture.tex | 2 +- tex/src/figure-inputconf.tex | 8 +++++ tex/src/figure-mk20tab3.tex | 59 ------------------------------------ tex/src/figure-src-demoplot.tex | 32 +++++++++++++++++++ tex/src/figure-src-download.tex | 8 +++++ tex/src/figure-src-format.tex | 59 ++++++++++++++++++++++++++++++++++++ tex/src/figure-src-topmake.tex | 24 +++++++++++++++ tex/src/figure-tools-per-year.tex | 34 +++++++++++++++++++++ 10 files changed, 189 insertions(+), 91 deletions(-) delete mode 100644 tex/src/figure-download.tex create mode 100644 tex/src/figure-inputconf.tex delete mode 100644 tex/src/figure-mk20tab3.tex create mode 100644 tex/src/figure-src-demoplot.tex create mode 100644 tex/src/figure-src-download.tex create mode 100644 tex/src/figure-src-format.tex create mode 100644 tex/src/figure-src-topmake.tex create mode 100644 tex/src/figure-tools-per-year.tex (limited to 'tex') diff --git a/tex/src/figure-data-lineage.tex b/tex/src/figure-data-lineage.tex index 010a0be..7379b2f 100644 --- a/tex/src/figure-data-lineage.tex +++ b/tex/src/figure-data-lineage.tex @@ -58,11 +58,11 @@ \node (downloadmk) [node-makefile, at={(-2.93cm,-1.3cm)}, label={[shift={(0,-5mm)}]\texttt{download.mk}}] {}; \node (analysis1mk) [node-makefile, at={(-0.13cm,-1.3cm)}, - label={[shift={(0,-5mm)}]\texttt{analysis1.mk}}] {}; + label={[shift={(0,-5mm)}]\texttt{format.mk}}] {}; \node (analysis2mk) [node-makefile, at={(2.67cm,-1.3cm)}, - label={[shift={(0,-5mm)}]\texttt{analysis2.mk}}] {}; - \node (analysis2mk) [node-makefile, at={(5.47cm,-1.3cm)}, - label={[shift={(0,-5mm)}]\texttt{analysis3.mk}}] {}; + label={[shift={(0,-5mm)}]\texttt{demo-plot.mk}}] {}; + \node (analysis3mk) [node-makefile, at={(5.47cm,-1.3cm)}, + label={[shift={(0,-5mm)}]\texttt{another-step.mk}}] {}; %% verify.mk \node [at={(-5.3cm,-2.8cm)}, @@ -149,7 +149,7 @@ %% analysis1.tex \ifdefined\analysisonetex - \node (a1tex) [node-terminal, at={(-0.13cm,-0.8cm)}] {analysis1.tex}; + \node (a1tex) [node-terminal, at={(-0.13cm,-0.8cm)}] {format.tex}; \draw [rounded corners, -] (a1tex) |- (initialize-south); \fi @@ -167,14 +167,17 @@ %% analysis2.tex \ifdefined\analysistwotex - \node (a2tex) [node-terminal, at={(2.67cm,-0.8cm)}] {analysis2.tex}; + \node (a2tex) [node-terminal, at={(2.67cm,-0.8cm)}] {demo-plot.tex}; \draw [rounded corners, -] (a2tex) |- (initialize-south); \fi %% out-2b.dat \ifdefined\outtwob - \node (out2b) [node-terminal, at={(2.67cm,0.3cm)}] {out-2b.dat}; + \node (menkedemoyear) [node-nonterminal, at={(2.67cm,4.6cm)}] {menke-demo-year.conf}; + \node (a2tex-west) [node-point, at={(1.27cm,-0.8cm)}] {}; + \node (out2b) [node-terminal, at={(2.67cm,0.3cm)}] {tools-per-year.txt}; \draw [->] (out2b) -- (a2tex); + \draw [->,rounded corners] (menkedemoyear.west) -| (a2tex-west) |- (a2tex); \fi %% out-2b dependencies @@ -184,39 +187,36 @@ %% analysis3.tex \ifdefined\analysisthreetex - \node (a3tex) [node-terminal, at={(5.47cm,-0.8cm)}] {analysis3.tex}; - \draw [rounded corners, -] (a3tex) |- (initialize-south); + \node [opacity=0.6] (a3tex) [node-terminal, at={(5.47cm,-0.8cm)}] {another-step.tex}; + \draw [opacity=0.6, rounded corners, -, dashed] (a3tex) |- (initialize-south); \fi %% Outputs of analysis3 \ifdefined\analysisthreeouts - \node (out3a) [node-terminal, at={(5.47cm,2.7cm)}] {out-3a.dat}; - \node (out3b) [node-terminal, at={(5.47cm,1.1cm)}] {out-3b.dat}; + \node [opacity=0.6] (out3a) [node-terminal, at={(5.47cm,2.7cm)}] {out-3a.dat}; + \node [opacity=0.6] (out3b) [node-terminal, at={(5.47cm,1.1cm)}] {out-3b.dat}; \node (a3tex-east) [node-point, at={(6.87cm,-0.8cm)}] {}; - \draw [->,rounded corners] (out3a.east) -| (a3tex-east) |- (a3tex); - \draw [->] (out3b) -- (a3tex); + \draw [opacity=0.6, ->,rounded corners, dashed] (out3a.east) -| (a3tex-east) |- (a3tex); + \draw [opacity=0.6, ->, dashed] (out3b) -- (a3tex); \fi %% out-2a.dat \ifdefined\outtwoa - \node (out2a) [node-terminal, at={(2.67cm,1.9cm)}] {out-2a.dat}; - \draw [->, rounded corners] (out2a.south) |- (out3b); + \node [opacity=0.6] (out2a) [node-terminal, at={(2.67cm,1.9cm)}] {demo-out.dat}; + \draw [opacity=0.6, ->, rounded corners, dashed] (out2a.south) |- (out3b); \fi %% Dependencies of out-2a \ifdefined\outtwoadep - \node (a2conf1) [node-nonterminal, at={(2.67cm,5.3cm)}] {param-2a.conf}; - \node (a2conf2) [node-nonterminal, at={(2.67cm,4.6cm)}] {param-2b.conf}; \node (out2a-west) [node-point, at={(1.27cm,1.9cm)}] {}; - \draw [->,rounded corners] (a2conf1.west) -| (out2a-west) |- (out2a); - \draw [->,rounded corners] (a2conf2.west) -| (out2a-west) |- (out2a); - %\draw [->] (input1) -- (out2a); + \draw [opacity=0.6, ->, dashed] (input2) -- (out2a); \fi %% Dependencies of out-3a \ifdefined\outthreeadep - \node (out3a-west) [node-point, at={(4.07cm,2.7cm)}] {}; - \node (a3conf1) [node-nonterminal, at={(5.47cm,4.6cm)}] {param-3.conf}; - \draw [rounded corners] (a3conf1.west) -| (out3a-west) |- (out3a); + \node [opacity=0.6] (out3a-west) [node-point, at={(4.07cm,2.7cm)}] {}; + \draw [opacity=0.6, ->,rounded corners, dashed] (input2) |- (out3a); + \node [opacity=0.6] (a3conf1) [node-nonterminal, at={(5.47cm,4.6cm)}] {param-3.conf}; + \draw [opacity=0.6, rounded corners, dashed] (a3conf1.west) -| (out3a-west) |- (out3a); \fi \end{tikzpicture} diff --git a/tex/src/figure-download.tex b/tex/src/figure-download.tex deleted file mode 100644 index b9da02f..0000000 --- a/tex/src/figure-download.tex +++ /dev/null @@ -1,8 +0,0 @@ -\begin{tcolorbox} - \footnotesize - \texttt{\mkcomment{Write download URL into the paper (through a LaTeX macro).}} - - \texttt{\mktarget{\$(mtexdir)/download.tex}: \$(\mkvar{indir})/menke20.xlsx} - - \texttt{\mktab{}\mkprog{echo} "\textbackslash{}newcommand{\textbackslash{}menketwentyurl}\{\mktarget{\$(MK20URL)}\}" > \$@} -\end{tcolorbox} diff --git a/tex/src/figure-file-architecture.tex b/tex/src/figure-file-architecture.tex index 8fb1a6d..1fc26c5 100644 --- a/tex/src/figure-file-architecture.tex +++ b/tex/src/figure-file-architecture.tex @@ -88,7 +88,7 @@ \node [node-nonterminal-thin, at={(2.95cm,0.8cm)}] {top-prepare.mk}; \node [node-nonterminal-thin, at={(2.95cm,0.3cm)}] {top-make.mk}; \node [node-nonterminal-thin, at={(2.95cm,-0.2cm)}] {initialize.mk}; - \node [node-nonterminal-thin, at={(2.95cm,-0.7cm)}] {analysis1.mk}; + \node [node-nonterminal-thin, at={(2.95cm,-0.7cm)}] {format.mk}; %% reproduce/analysis/bash/ \node [dirbox, at={(0.15cm,-1.3cm)}, minimum width=2.6cm, minimum height=1.1cm, diff --git a/tex/src/figure-inputconf.tex b/tex/src/figure-inputconf.tex new file mode 100644 index 0000000..f09bebd --- /dev/null +++ b/tex/src/figure-inputconf.tex @@ -0,0 +1,8 @@ +\begin{tcolorbox} + \footnotesize + \texttt{\mkvar{MK20DATA} = menke20.xlsx}\\ + \texttt{\mkvar{MK20MD5}{ } = 8e4eee64791f351fec58680126d558a0}\\ + \texttt{\mkvar{MK20SIZE} = 1.9MB}\\ + \texttt{\mkvar{MK20URL}{ } = https://the.full.url/is/too/large/for/here/media-1.xlsx}\\ + \vspace{-3mm} +\end{tcolorbox} diff --git a/tex/src/figure-mk20tab3.tex b/tex/src/figure-mk20tab3.tex deleted file mode 100644 index 3cc0cd3..0000000 --- a/tex/src/figure-mk20tab3.tex +++ /dev/null @@ -1,59 +0,0 @@ -\begin{tcolorbox} - \footnotesize - \texttt{\mkcomment{1ST MAKE RULE: build the directory hosting the converted table.}} - - \texttt{\mkvar{a1dir} = \$(\mkvar{BDIR})/analysis-1} - - \texttt{\mktarget{\$(a1dir)}:} - - \texttt{\mktab{}\mkprog{mkdir} \$@} - - \vspace{2em} - \texttt{\mkcomment{2ND MAKE RULE: Convert the XLSX table to a simple plain-text table.}} - - \texttt{\mkvar{mk20tab3} = \$(\mkvar{a1dir})/menke20-table-3.txt} - - \texttt{\mktarget{\$(mk20tab3)}: \$(\mkvar{indir})/menke20.xlsx | \$(\mkvar{a1dir})} - - \texttt{\recipecomment{Call XLSX I/O to convert all the spreadsheets into different CSV files.}} - - \texttt{\recipecomment{We only want the `table-3' spreadsheet, but XLSX I/O doesn't allow setting its}} - - \texttt{\recipecomment{output filename. For simplicity, let's assume its written in `table-3.csv'.}} - - \texttt{\mktab{}\mkprog{xlsxio\_xlsx2csv} \$<} - - \vspace{0.5em} - \texttt{\recipecomment{Use GNU AWK to keep the desired columns in space-separated, fixed-width format.}} - - \texttt{\recipecomment{With `FPAT' commas within double quotes are not counted as columns.}} - - \texttt{\mktab{}\mkprog{awk} 'NR>1\{printf("\%-10d\%-10d\%-10d \%s\textbackslash{}n", \$\$2, \$\$3, \$\$(NF-1)*\$\$NF, \$\$1)\}' \textbackslash} - - \texttt{\mktab{}{ }{ }{ }{ }FPAT='([\^{},]+)|("[\^{}"]+")' table-3.csv > \$@} - - \vspace{0.5em} - \texttt{\recipecomment{Delete the temporary CSV file.}} - - \texttt{\mktab{}\mkprog{rm} table-3.csv} - - \vspace{2em} - \texttt{\mkcomment{3RD MAKE RULE: Main LaTeX macro file for reported values.}} - - \texttt{\mktarget{\$(mtexdir)/analysis1.tex}: \$(\mkvar{mk20tab3)}} - - \texttt{\recipecomment{Count the total number of papers in their study to report in this paper.}} - - \texttt{\mktab{}v=\$\$(\mkprog{awk} '\!/\^{}\#/\{c+=\$\$2\} END\{print c\}' \$(\mkvar{mk20tab3)})} - - \texttt{\mktab{}\mkprog{echo} "\textbackslash{}newcommand\{\textbackslash{}menkenumpapers\}\{\$\$v\}" > \$@} - - \vspace{0.5em} - \texttt{\recipecomment{Count total number of journals in that study.}} - - \texttt{\mktab{}v=\$\$(awk 'BEGIN{FIELDWIDTHS="31 10000"} !/\^\#/\{print \$\$2\}' \$(mk20tab3) \textbackslash} - - \texttt{\mktab{ }{ }{ }{ }{ }{ }{ }{ }{ }{ }| uniq | wc -l)} - - \texttt{\mktab{}\mkprog{echo} "\textbackslash{}newcommand\{\textbackslash{}menkenumjournals\}\{\$\$v\}" >> \$@} -\end{tcolorbox} diff --git a/tex/src/figure-src-demoplot.tex b/tex/src/figure-src-demoplot.tex new file mode 100644 index 0000000..6d788f5 --- /dev/null +++ b/tex/src/figure-src-demoplot.tex @@ -0,0 +1,32 @@ +\begin{tcolorbox}[title=\inlinecode{\textcolor{white}{demo-plot.mk}}\hfill(Simplified contents)] + \footnotesize + \texttt{\mkcomment{1ST MAKE RULE: build the directory hosting the converted table.}}\\ + \texttt{\mkvar{a2dir} = \$(\mkvar{texdir})/tools-per-year}\\ + \texttt{\mktarget{\$(a2dir)}:; \mkprog{mkdir} \$@} + + \vspace{2em} + \texttt{\mkcomment{2ND MAKE RULE: extract necessary info from raw table.}}\\ + \texttt{\mkvar{a2mk20f1c} = \$(\mkvar{a2dir})/tools-per-year.txt}\\ + \texttt{\mktarget{\$(a2mk20f1c)}: \$(\mkvar{mk20tab3}) | \$(\mkvar{a2dir})}\\ + \texttt{\mktab{}\mkprog{awk} '!/\^{}\#/ \{all[\$\$1]+=\$\$2; id[\$\$1]+=\$\$3;\}} \textbackslash\\ + \texttt{\mktab{}{ }{ }{ }{ }{ }END\{ for(year in all) print year, 100*id[year]/all[year], all[year] \}}' \textbackslash\\ + \texttt{\mktab{}{ }{ }{ }{ }> \$@} + + \vspace{2em} + \texttt{\mkcomment{3RD MAKE RULE: Main LaTeX macro file for reported values in text.}}\\ + \texttt{\mkvar{pconfdir} = reproduce/analysis/config}\\ + \texttt{\mktarget{\$(mtexdir)/demo-plot.tex}: \$(\mkvar{a2mk20f1c}) \$(\mkvar{pconfdir})/menke-demo-year.conf} + + %% We need an empty line here for the extra space to work. + \texttt{\recipecomment{First year data were taken (first column of first row).}}\\ + \texttt{\mktab{}v=\$\$(awk 'NR==1\{print \$\$1\}' \$(\mkvar{a2mk20f1c}))}\\ + \texttt{\mktab{}\mkprog{echo} "\textbackslash{}newcommand\{\textbackslash{}menkefirstyear\}\{\$\$v\}" > \$@} + + %% We need an empty line here for the extra space to work. + \texttt{\recipecomment{Number of papers in the demonstration year. The year is defined in}} + + \texttt{\recipecomment{`\$(pconfdir)/menke-demo-year.conf' as `menke-demo-year' and also passed onto LaTeX.}}\\ + \texttt{\mktab{}v=\$\$(awk '\$\$1==\$(\mkvar{menke-demo-year})\{print \$\$3\}' \$(\mkvar{a2mk20f1c}))}\\ + \texttt{\mktab{}\mkprog{echo} "\textbackslash{}newcommand\{\textbackslash{}menkenumpapersdemocount\}\{\$\$v\}"{ }>> \$@} \\ + \texttt{\mktab{}\mkprog{echo} "\textbackslash{}newcommand\{\textbackslash{}menkenumpapersdemoyear\}\{\$(\mkvar{menke-demo-year})\}"{ }>> \$@} +\end{tcolorbox} diff --git a/tex/src/figure-src-download.tex b/tex/src/figure-src-download.tex new file mode 100644 index 0000000..74026b8 --- /dev/null +++ b/tex/src/figure-src-download.tex @@ -0,0 +1,8 @@ +\begin{tcolorbox}[title=\inlinecode{\textcolor{white}{download.mk}} \textcolor{white}{(only \LaTeX{} macro's rule.}] + \footnotesize + \texttt{\mkcomment{Write download URL into the paper (through a LaTeX macro).}} + + \texttt{\mktarget{\$(mtexdir)/download.tex}: \$(\mkvar{indir})/menke20.xlsx} + + \texttt{\mktab{}\mkprog{echo} "\textbackslash{}newcommand{\textbackslash{}menketwentyurl}\{\mktarget{\$(MK20URL)}\}" > \$@} +\end{tcolorbox} diff --git a/tex/src/figure-src-format.tex b/tex/src/figure-src-format.tex new file mode 100644 index 0000000..ba4458e --- /dev/null +++ b/tex/src/figure-src-format.tex @@ -0,0 +1,59 @@ +\begin{tcolorbox}[title=\inlinecode{\textcolor{white}{format.mk}}\hfill(Simplified contents)] + \footnotesize + \texttt{\mkcomment{1ST MAKE RULE: build the directory hosting the converted table.}} + + \texttt{\mkvar{a1dir} = \$(\mkvar{BDIR})/format} + + \texttt{\mktarget{\$(a1dir)}:} + + \texttt{\mktab{}\mkprog{mkdir} \$@} + + \vspace{2em} + \texttt{\mkcomment{2ND MAKE RULE: Convert the XLSX table to a simple plain-text table.}} + + \texttt{\mkvar{mk20tab3} = \$(\mkvar{a1dir})/menke20-table-3.txt} + + \texttt{\mktarget{\$(mk20tab3)}: \$(\mkvar{indir})/menke20.xlsx | \$(\mkvar{a1dir})} + + \texttt{\recipecomment{Call XLSX I/O to convert all the spreadsheets into different CSV files.}} + + \texttt{\recipecomment{We only want the `table-3' spreadsheet, but XLSX I/O doesn't allow setting its}} + + \texttt{\recipecomment{output filename. For simplicity, let's assume its written in `table-3.csv'.}} + + \texttt{\mktab{}\mkprog{xlsxio\_xlsx2csv} \$<} + + \vspace{0.5em} + \texttt{\recipecomment{Use GNU AWK to keep the desired columns in space-separated, fixed-width format.}} + + \texttt{\recipecomment{With `FPAT' commas within double quotes are not counted as columns.}} + + \texttt{\mktab{}\mkprog{awk} 'NR>1\{printf("\%-10d\%-10d\%-10d \%s\textbackslash{}n", \$\$2, \$\$3, \$\$(NF-1)*\$\$NF, \$\$1)\}' \textbackslash} + + \texttt{\mktab{}{ }{ }{ }{ }FPAT='([\^{},]+)|("[\^{}"]+")' table-3.csv > \$@} + + \vspace{0.5em} + \texttt{\recipecomment{Delete the temporary CSV file.}} + + \texttt{\mktab{}\mkprog{rm} table-3.csv} + + \vspace{2em} + \texttt{\mkcomment{3RD MAKE RULE: Main LaTeX macro file for reported values.}} + + \texttt{\mktarget{\$(mtexdir)/format.tex}: \$(\mkvar{mk20tab3)}} + + \texttt{\recipecomment{Count the total number of papers in their study to report in this paper.}} + + \texttt{\mktab{}v=\$\$(\mkprog{awk} '\!/\^{}\#/\{c+=\$\$2\} END\{print c\}' \$(\mkvar{mk20tab3)})} + + \texttt{\mktab{}\mkprog{echo} "\textbackslash{}newcommand\{\textbackslash{}menkenumpapers\}\{\$\$v\}" > \$@} + + \vspace{0.5em} + \texttt{\recipecomment{Count total number of journals in that study.}} + + \texttt{\mktab{}v=\$\$(awk 'BEGIN{FIELDWIDTHS="31 10000"} !/\^\#/\{print \$\$2\}' \$(mk20tab3) \textbackslash} + + \texttt{\mktab{ }{ }{ }{ }{ }{ }{ }{ }{ }{ }| uniq | wc -l)} + + \texttt{\mktab{}\mkprog{echo} "\textbackslash{}newcommand\{\textbackslash{}menkenumjournals\}\{\$\$v\}" >> \$@} +\end{tcolorbox} diff --git a/tex/src/figure-src-topmake.tex b/tex/src/figure-src-topmake.tex new file mode 100644 index 0000000..bd4b67d --- /dev/null +++ b/tex/src/figure-src-topmake.tex @@ -0,0 +1,24 @@ +\begin{tcolorbox}[title=\inlinecode{\textcolor{white}{top-make.mk}}\hfill\textcolor{white}{(simplified)}] + \footnotesize + + \texttt{\mkcomment{Ultimate target/purpose of project (`paper.pdf' is the final target of the final subMakefile}}\par + \texttt{\mkcomment{that is loaded/included below)}}\par + \texttt{\mktarget{all}: paper.pdf} + + \vspace{1em} + \texttt{\mkcomment{List of subMakefiles to be loaded in order.}}\par + \texttt{\mkvar{makesrc} = initialize \textbackslash}\par + \texttt{{ }{ }{ }{ }{ }{ }{ }{ }{ }{ }download \textbackslash}\par + \texttt{{ }{ }{ }{ }{ }{ }{ }{ }{ }{ }format \textbackslash}\par + \texttt{{ }{ }{ }{ }{ }{ }{ }{ }{ }{ }demo-plot \textbackslash}\par + \texttt{{ }{ }{ }{ }{ }{ }{ }{ }{ }{ }verify \textbackslash}\par + \texttt{{ }{ }{ }{ }{ }{ }{ }{ }{ }{ }paper}\par + + \vspace{1em} + \texttt{\mkcomment{Load all the configuration files.}}\par + \texttt{\textcolor{purple}{include} reproduce/analysis/config/*.conf} + + \vspace{1em} + \texttt{\mkcomment{Load/include the subMakefiles in the specified order.}}\par + \texttt{\textcolor{purple}{include} \$(\textcolor{blue}{foreach} s, \$(\mkvar{makesrc}), reproduce/analysis/make/\$(\mkvar{s}).mk)} +\end{tcolorbox} diff --git a/tex/src/figure-tools-per-year.tex b/tex/src/figure-tools-per-year.tex new file mode 100644 index 0000000..75557ac --- /dev/null +++ b/tex/src/figure-tools-per-year.tex @@ -0,0 +1,34 @@ +\begin{tikzpicture} + \begin{axis}[ + ymin=0, + ymax=100, + width=\linewidth, + height=0.3\linewidth, + xlabel={Year}, + ylabel={Frac. papers with tools}, + axis y line*=left, + enlarge x limits = false, + yticklabel=\pgfmathprintnumber{\tick}\,\%, + x tick label style={/pgf/number format/1000 sep=}, + ] + + %% Linear plot, showing the number of papers mentioning tools. + \addplot+ [mark=none, very thick, green!60!black] + table {tex/build/tools-per-year/tools-per-year.txt}; + \end{axis} + + %% Add the right-side Y axis. + \begin{axis}[ + ymode=log, + width=\linewidth, + height=0.3\linewidth, + axis x line=none, + axis y line*=right, + enlarge x limits = false, + ylabel=Num. papers (log-scale), + max space between ticks=20, + ] + \addplot+ [ybar, mark=none, fill=red!50!white, red, opacity=0.25] + table [x index=0, y index=2] {tex/build/tools-per-year/tools-per-year.txt}; + \end{axis} +\end{tikzpicture} -- cgit v1.2.1