From 7bdbd6e61f132ac8f6851637d40fd06f8d6b7182 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Tue, 9 Jun 2020 03:58:19 +0100 Subject: Minor edit printing arXiv URL in plain text metadata Until now, in the 'print-copyright' function of 'initialize.mk' (that prints a fixed set of common meta necessary in plain-text files), we were simply printing this line: # Pre-print server: arXiv:1234.56789 But given that all the other elements are click-able URLs, it now prints: # Pre-print server: https://arxiv.org/abs/1234.56789 --- reproduce/analysis/make/initialize.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 19447a6..acc527d 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -442,7 +442,7 @@ print-copyright = \ echo "\# Git commit (that produced this dataset): $(project-commit-hash)" >> $(1); \ echo "\# Project's Git repository: $(metadata-git-repository)" >> $(1); \ if [ x$(metadata-arxiv) != x ]; then \ - echo "\# Pre-print server: arXiv:$(metadata-arxiv)" >> $(1); fi; \ + echo "\# Pre-print server: https://arxiv.org/abs/$(metadata-arxiv)" >> $(1); fi; \ if [ x$(metadata-doi-journal) != x ]; then \ echo "\# DOI (Journal): $(metadata-doi-journal)" >> $(1); fi; \ if [ x$(metadata-doi-zenodo) != x ]; then \ -- cgit v1.2.1 From 2bd2e2f1833300d5102339e2aa417a3099c13960 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Tue, 9 Jun 2020 20:09:09 +0100 Subject: IMPORTANT: bug fix in default data download script of download.mk Summary of possible semantic conflicts 1. The recipe to download input datasets has been modified. You have to re-set the old 'origname' variable to 'localname' (to avoid confusion) and the default dataset URL should now be complete (including the actual filename). See the newly added descriptions in 'INPUTS.conf' for more on this. Until now, when the dataset was already present on the host system, a link couldn't be made to it, causing the project to crash in the checksum phase. This has been fixed with properly naming the main variable as 'localname' to avoid the confusion that caused it. Some other problems have been fixed in this recipe in the meantime: - When the checksum is different, the expected and calculated checksums are printed. - In the default paper, we now print the full URL of the dataset, not just the server, so the checksum of the 'download.tex' step has been updated. --- reproduce/analysis/config/INPUTS.conf | 42 ++++++++++++++++++++++++++++++++--- reproduce/analysis/make/download.mk | 24 +++++++++++--------- reproduce/analysis/make/verify.mk | 2 +- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/reproduce/analysis/config/INPUTS.conf b/reproduce/analysis/config/INPUTS.conf index 6ddaec7..5e6c425 100644 --- a/reproduce/analysis/config/INPUTS.conf +++ b/reproduce/analysis/config/INPUTS.conf @@ -1,6 +1,36 @@ -# Input files necessary for this project. +# Input files necessary for this project, the variables defined in this +# file are primarily used in 'reproduce/analysis/make/download.mk'. See +# there for precise usage of the variables. But comments are also provided +# here. # -# This file is read by the configure script and running Makefiles. +# Necessary variables for each input dataset are listed below. Its good +# that all the variables of each file have the same base-name (in the +# example below 'WFPC2') with descriptive suffixes, also put a short +# comment above each group of variables for each dataset, shortly +# explaining what it is. +# +# 1) Local file name ('WFPC2IMAGE' below): this is the name of the dataset +# on the local system (in 'INDIR', given at configuration time). It is +# recommended that it be the same name as the online version of the +# file like the case here (note how this variable is used in 'WFPC2URL' +# for the dataset's full URL). However, this is not always possible, so +# the local and server filenames may be different. Ultimately, the file +# name is irrelevant, we check the integrity with the checksum. +# +# 2) The MD5 checksum of the file ('WFPC2MD5' below): this is very +# important for an automatic verification of the file. You can +# calculate it by running 'md5sum' on your desired file. +# +# 3) The human-readable size of the file ('WFPC2SIZE' below): this is an +# optional feature which you can use for in the script that is loaded +# at configure time ('reproduce/software/shell/configure.sh'). When +# asking for the input-data directory, you can print some basic +# information of the files for users to get a better feeling of the +# volume. See that script for an example using this demo dataset. +# +# 4) The full dataset URL ('WFPC2URL' below): this is the full URL +# (including the file-name) that can be used to download the dataset +# when necessary. Also, see the description above on local filename. # # Copyright (C) 2018-2020 Mohammad Akhlaghi # @@ -9,7 +39,13 @@ # this notice are preserved. This file is offered as-is, without any # warranty. + + + + +# Demonstration image used in the histogram plot (remove this when +# customizing). WFPC2IMAGE = WFPC2ASSNu5780205bx.fits WFPC2MD5 = a4791e42cd1045892f9c41f11b50bad8 WFPC2SIZE = 62kb -WFPC2URL = https://fits.gsfc.nasa.gov/samples +WFPC2URL = https://fits.gsfc.nasa.gov/samples/$(WFPC2IMAGE) diff --git a/reproduce/analysis/make/download.mk b/reproduce/analysis/make/download.mk index 71ee7d3..bc8b8ce 100644 --- a/reproduce/analysis/make/download.mk +++ b/reproduce/analysis/make/download.mk @@ -58,7 +58,7 @@ $(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir) # Set the necessary parameters for this input file. if [ $* = wfpc2 ]; then - origname=$(WFPC2IMAGE); url=$(WFPC2URL); mdf=$(WFPC2MD5); + localname=$(WFPC2IMAGE); url=$(WFPC2URL); mdf=$(WFPC2MD5); else echo; echo; echo "Not recognized input dataset: '$*.fits'." echo; echo; exit 1 @@ -71,21 +71,25 @@ $(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir) # here points to the final file directly (note that `readlink' is # part of GNU Coreutils). If its not a link, the `readlink' part # has no effect. - if [ -f $(INDIR)/$$origname ]; then - ln -fs $$(readlink -f $(INDIR)/$$origname) $$out + unchecked=$@.unchecked + if [ -f $(INDIR)/$$localname ]; then + ln -fs $$(readlink -f $(INDIR)/$$localname) $$unchecked else touch $(lockdir)/download $(downloadwrapper) "wget --no-use-server-timestamps -O" \ - $(lockdir)/download $$url/$$origname $@ + $(lockdir)/download $$url $$unchecked fi # Check the md5 sum to see if this is the proper dataset. - sum=$$(md5sum $@ | awk '{print $$1}') - if [ $$sum != $$mdf ]; then - wrongname=$(dir $@)/wrong-$(notdir $@) - mv $@ $$wrongname - echo; echo; echo "Wrong MD5 checksum for '$$origname' in $$wrongname" - echo; echo; exit 1 + sum=$$(md5sum $$unchecked | awk '{print $$1}') + if [ $$sum = $$mdf ]; then + mv $$unchecked $@ + else + echo; echo; + echo "Wrong MD5 checksum for input file '$$localname':" + echo " Expected MD5 checksum: $$mdf"; \ + echo " Calculated MD5 checksum: $$sum"; \ + echo; exit 1 fi diff --git a/reproduce/analysis/make/verify.mk b/reproduce/analysis/make/verify.mk index 67b3fea..69711d5 100644 --- a/reproduce/analysis/make/verify.mk +++ b/reproduce/analysis/make/verify.mk @@ -135,7 +135,7 @@ $(mtexdir)/verify.tex: $(foreach s, $(verify-dep), $(mtexdir)/$(s).tex) # Verify TeX macros (the values that go into the PDF text). for m in $(verify-check); do file=$(mtexdir)/$$m.tex - if [ $$m == download ]; then s=6749e17ce606d57d30cebdbc1a5d23ad + if [ $$m == download ]; then s=49e4e9f049aa9da0453a67203d798587 elif [ $$m == delete-me ]; then s=711e2f7fa1f16ecbeeb3df6bcb4ec705 else echo; echo "'$$m' not recognized."; exit 1 fi -- cgit v1.2.1 From db2bd887f76e8860d728fd84a5eb84f3e41f640e Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 10 Jun 2020 02:28:01 +0100 Subject: Updated text of default paper.tex, putting more recent examples The text of the default paper hadn't been changed for a very long time! In this time, three papers using Maneage have been published (which can be very good as an example), Maneage also now has a webpage! With these commit these examples and the webpage have been added and generally it was also polished a little to hopefully be more useful. --- paper.tex | 165 ++++++++++++++++++++++++++----------------------- tex/src/references.tex | 100 +++++++++++++++++++++++------- 2 files changed, 165 insertions(+), 100 deletions(-) diff --git a/paper.tex b/paper.tex index 0993e73..f1ca48f 100644 --- a/paper.tex +++ b/paper.tex @@ -63,25 +63,32 @@ %% Project abstract and keywords. \includeabstract{ - You have completed the reproducible paper template and are ready to - configure and implement it for your own research. This template contains - almost all the elements that you will need in a research project - containing the downloading of raw data and necessary software, building - the software, and processing the data with the software in a - highly-controlled environment. It then allows including the results in - plots and producing the final report, including this abstract, figures - and bibliography. If you design your project with this template's - infra-structure in your work, don't forget to add a notice and clearly - let the readers know that your work is reproducible. If this template - proves useful in your research, please cite \citet{gnuastro}. + Welcome to Maneage (\emph{Man}aging data lin\emph{eage}) and reproducible + papers/projects, for a review of the basics of this system, please see + \citet{akhlaghi20}. You are now ready to configure Maneage and implement + your own research in this framework. Maneage contains almost all the + elements that you will need in a research project, and adding any missing + parts is very easy once you become familiar with it. For example it + already has steps to downloading of raw data and necessary software + (while verifying them with their checksums), building the software, and + processing the data with the software in a highly-controlled + environment. But Maneage is not just for the analysis of your project, + you will also write your paper in it (by replacing this text in + \texttt{paper.tex}): including this abstract, figures and + bibliography. If you design your project with Maneage's infra-structure, + don't forget to add a notice and clearly let the readers know that your + work is reproducible, we should spread the word and show the world how + useful reproducible research is for the sciences, also don't forget to + cite and acknowledge it so we can continue developing it. This PDF was + made with Maneage, commit \projectversion{}. \vspace{0.25cm} \textsl{Keywords}: Add some keywords for your research here. \textsl{Reproducible paper}: All quantitave results (numbers and plots) - in this paper are exactly reproducible with Maneage (version \projectversion{}, - \url{https://maneage.org}).} + in this paper are exactly reproducible with Maneage + (\url{https://maneage.org}). } %% To add the first page's headers. \thispagestyle{firststyle} @@ -93,23 +100,45 @@ %% Start of main body. \section{Congratulations!} Congratulations on running the raw template project! You can now follow the -checklist in the \texttt{README.md} file to customize this template to your -exciting research project. - -Just don't forget to \emph{never} use numbers or fixed strings (for example -database urls like \url{\wfpctwourl}) directly within your \LaTeX{} -source. Read them directly from your configuration files, or processing -outputs, and import them into \LaTeX{} as macros through the -\texttt{tex/build/macros/project.tex} file (created after running the -project). See the several existing examples within the template for a -demonstration. For some recent real-world examples, the reproducible -project sources for Sections 4 and 7.3 of \citet{bacon17} are available at -\href{https://doi.org/10.5281/zenodo.1164774}{zenodo.1164774}\footnote{\url{https://gitlab.com/makhlaghi/muse-udf-origin-only-hst-magnitudes}}, -or -\href{https://doi.org/10.5281/zenodo.1163746}{zenodo.1163746}\footnote{\url{https://gitlab.com/makhlaghi/muse-udf-photometry-astrometry}}. Working +``Customization checklist'' in the \texttt{README-hacking.md} file, +customize this template and start your exciting research project over +it. You can always merge Maneage back into your project to improve its +infra-structure and leaving your own project intact. If you haven't already +read \citet{akhlaghi20}, please do so before continuing, it isn't long +(just 7 pages). + +While you are writing your paper, just don't forget to \emph{not} use +numbers or fixed strings (for example database urls like \url{\wfpctwourl}) +directly within your \LaTeX{} source. Put them in configuration files and +after using them in the analysis, pass them into the \LaTeX{} source +through macros in the same subMakefile that used them. For some already +published examples, please see +\citet{akhlaghi20}\footnote{\url{https://gitlab.com/makhlaghi/maneage-paper}}, +\citet{infantesainz20}\footnote{\url{https://gitlab.com/infantesainz/sdss-extended-psfs-paper}} +and +\citet{akhlaghi19}\footnote{\url{https://gitlab.com/makhlaghi/iau-symposium-355}}. Working in this way, will let you focus clearly on your science and not have to worry about fixing this or that number/name in the text. +Once your project is ready for publication, there is also a ``Publication +checklist'' in \texttt{README-hacking.md} that will guide you in the steps +to do for making your project as FAIR as possible (Findable, Accessibile, +Interoperable, and Reusable). + +The default \LaTeX{} structure within Maneage also has two \LaTeX{} macros +for easy marking of text within your document as \emph{new} and +\emph{notes}. For example, \new{this text has been marked as \texttt{new}.} +\tonote{While this one is marked as \texttt{tonote}.} Please try commenting +the line that defines \texttt{highlightchanges} in \texttt{paper.tex} (by +adding a `\texttt{\%}' at the start of the line or simply deleting the +line). You will then notice that the line that was marked as \texttt{new} +will become black (totally blend in with the rest of the text) and the one +marked \texttt{tonote} will not be in the final PDF. You can thus use +\texttt{highlightchanges} to easily make copies of your research for +existing coauthors (who are just interested in the new parts or notes) and +new co-authors (who don't want to be distracted by these issues in their +first time reading). + Figure \ref{squared} shows a simple plot as a demonstration of creating plots within \LaTeX{} (using the {\small PGFP}lots package). The minimum value in this distribution is $\deletememin$, and $\deletememax$ is the @@ -134,50 +163,31 @@ at the top of this \LaTeX{} source file. Figure \ref{image-histogram} is another demonstration of showing images (datasets) using PGFPlots. It shows a small crop of an image from the -Wide-Field Planetary Camera 2, on board the Hubble Space Telescope from -1993 to 2009. This cropped image is one of the sample FITS files from the -FITS file standard -webpage\footnote{\url{https://fits.gsfc.nasa.gov/fits_samples.html}}. Just -as another basic reporting of measurements on this dataset within the paper -without using numbers in the \LaTeX{} source, the mean is -$\deletemewfpctwomean$ and the median is $\deletemewfpctwomedian$. The -skewness in the histogram of Figure \ref{image-histogram}(b) explains this -difference between the mean and median. The dataset was prepared for -demonstration here with Gnuastro's \textsf{Convert\-Type} program and the +Wide-Field Planetary Camera 2 (that was installed on the Hubble Space +Telescope from 1993 to 2009). As another more realistic demonstration of +reporting results with Maneage, here we report that the mean pixel value in +that image is $\deletemewfpctwomean$ and the median is +$\deletemewfpctwomedian$. The skewness in the histogram of Figure +\ref{image-histogram}(b) explains this difference between the mean and +median. The dataset is visualized here as a black and white image using the +\textsf{Convert\-Type} program of GNU Astronomy Utilities (Gnuastro). The histogram and basic statstics were generated with Gnuastro's \textsf{Statistics} program. {\small PGFP}lots\footnote{\url{https://ctan.org/pkg/pgfplots}} is a great tool to build the plots within \LaTeX{} and removes the necessity to add -further dependencies (to create the plots) to your project. There are -high-level language libraries like Matplotlib which also generate -plots. However, the problem is that they require many dependencies (Python, -Numpy and etc). Installing these dependencies from source, is not easy and -will harm the reproducibility of your paper. Note that after several years, -the binary files of these high-level libraries, that you easily install -today, will no longer be available in common repositories. Therefore -building the libraries from source is the only option to reproduce your -results. - -Furthermore, since {\small PGFP}lots is built by \LaTeX{} it respects all -the properties of your text (for example line width and fonts and -etc). Therefore the final plot blends in your paper much more nicely. It -also has a wonderful +further dependencies, just to create the plots. There are high-level +libraries like Matplotlib which also generate plots. However, the problem +is that they require \emph{many} dependencies, for example see Figure 1 of +\citet{alliez19}. Installing these dependencies from source, is not easy +and will harm the reproducibility of your paper in the future. + +Furthermore, since {\small PGFP}lots builds the plots within \LaTeX{}, it +respects all the properties of your text (for example line width and fonts +and etc). Therefore the final plot blends in your paper much more +nicely. It also has a wonderful manual\footnote{\url{http://mirrors.ctan.org/graphics/pgf/contrib/pgfplots/doc/pgfplots.pdf}}. -This template also defines two \LaTeX{} macros that allow you to mark text -within your document as \emph{new} and \emph{notes}. For example, \new{this - text has been marked as \texttt{new}.} \tonote{While this one is marked - as \texttt{tonote}.} If you comment the line (by adding a `\texttt{\%}' -at the start of the line or simply deleting the line) that defines -\texttt{highlightchanges}, then the one that was marked \texttt{new} will -become black (totally blend in with the rest of the text) and the one -marked \texttt{tonote} will not be in the final PDF. You can thus use -\texttt{highlightchanges} to easily make copies of your research for -existing coauthors (who are just interested in the new parts or notes) and -new co-authors (who don't want to be distracted by these issues in their -first time reading). - \begin{figure}[t] \includetikz{delete-me-image-histogram}{width=\linewidth} @@ -193,15 +203,15 @@ first time reading). \section{Notice and citations} To encourage other scientists to publish similarly reproducible papers, please add a notice close to the start of your paper or in the end of the -abstract clearly mentioning that your work is fully reproducible. - -For the time being, we haven't written a specific paper only for this -template. Until then, we would be grateful if you could cite the first -paper that used the early versions of this template: \citet{gnuastro}. - -After publication, don't forget to upload all the necessary data, software -source code and the project's source to a long-lasting host like Zenodo -(\url{https://zenodo.org}). +abstract clearly mentioning that your work is fully reproducible. One +convention we have adopted until now is to put the Git checkum of the +project as the last word of the abstract, for example see +\citet{akhlaghi19}, \citet{infantesainz20} and \citet{akhlaghi20} + +Finally, don't forget to cite \citet{akhlaghi20} and acknowledge the +funders mentioned below. Otherwise we won't be able to continue working on +Maneage. Also, just as another reminder, before publication, don't forget +to follow the ``Publication checklist'' of \texttt{README-hacking.md}. %% End of main body. @@ -210,11 +220,10 @@ source code and the project's source to a long-lasting host like Zenodo \section{Acknowledgements} \new{Please include the following two paragraphs in the Acknowledgement - section of your paper. This reproducible paper template was developed in - parallel with Gnuastro, so it benefited from the same grants. If you - don't use Gnuastro in your final/customized project, please remove it - from the paragraph below, only mentioning the reproducible paper - template.} + section of your paper. Maneage was developed in parallel with Gnuastro, + so it benefited from the same grants. If you don't use Gnuastro in your + final/customized project, please remove it from the paragraph below, only + mentioning the reproducible paper template.} This research was partly done using GNU Astronomy Utilities (Gnuastro, ascl.net/1801.009), and the reproducible paper template diff --git a/tex/src/references.tex b/tex/src/references.tex index 02a7d50..2610874 100644 --- a/tex/src/references.tex +++ b/tex/src/references.tex @@ -8,26 +8,82 @@ %% notice and this notice are preserved. This file is offered as-is, %% without any warranty. -@ARTICLE{bacon17, - author = {{Bacon}, R. and {Conseil}, S. and {Mary}, D. and {Brinchmann}, J. and - {Shepherd}, M. and {Akhlaghi}, M. and {Weilbacher}, P.~M. and - {Piqueras}, L. and {Wisotzki}, L. and {Lagattuta}, D. and {Epinat}, B. and - {Guerou}, A. and {Inami}, H. and {Cantalupo}, S. and {Courbot}, J.~B. and - {Contini}, T. and {Richard}, J. and {Maseda}, M. and {Bouwens}, R. and - {Bouch{\'e}}, N. and {Kollatschny}, W. and {Schaye}, J. and - {Marino}, R.~A. and {Pello}, R. and {Herenz}, C. and {Guiderdoni}, B. and - {Carollo}, M.}, - title = "{The MUSE Hubble Ultra Deep Field Survey. I. Survey description, data reduction, and source detection}", - journal = {A\&A}, -archivePrefix = "arXiv", - eprint = {1710.03002}, - keywords = {galaxies: distances and redshifts, galaxies: high-redshift, cosmology: observations, methods: data analysis, techniques: imaging spectroscopy, galaxies: formation}, - year = 2017, - month = nov, - volume = 608, - eid = {A1}, - pages = {A1}, - doi = {10.1051/0004-6361/201730833}, - adsurl = {http://adsabs.harvard.edu/abs/2017A\%26A...608A...1B}, - adsnote = {Provided by the SAO/NASA Astrophysics Data System} +@ARTICLE{akhlaghi20, + author = {{Akhlaghi}, Mohammad and {Infante-Sainz}, Ra{\'u}l and + {Roukema}, Boudewijn F. and {Valls-Gabaud}, David and + {Baena-Gall{\'e}}, Roberto}, + title = "{Towards Long-term and Archivable Reproducibility}", + journal = {arXiv e-prints}, + year = 2020, + month = jun, + eid = {arXiv:2006.03018}, + pages = {arXiv:2006.03018}, +archivePrefix = {arXiv}, + eprint = {2006.03018}, + primaryClass = {cs.DL}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2020arXiv200603018A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + + + + + +@ARTICLE{alliez19, + author = {{Alliez}, Pierre and {Di Cosmo}, Roberto and {Guedj}, Benjamin and + {Girault}, Alain and {Hacid}, Mohand-Said and {Legrand}, Arnaud and + {Rougier}, Nicolas P.}, + title = "{Attributing and Referencing (Research) Software: Best Practices and Outlook from Inria}", + journal = {CiSE}, + volume = {22}, + year = "2020", + month = "Jan", + pages = {39-52}, +archivePrefix = {arXiv}, + eprint = {1905.11123}, + primaryClass = {cs.DL}, + doi = {10.1109/MCSE.2019.2949413}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2019arXiv190511123A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + + + + + +@ARTICLE{infantesainz20, + author = {{Infante-Sainz}, Ra{\'u}l and {Trujillo}, Ignacio and + {Rom{\'a}n}, Javier}, + title = "{The Sloan Digital Sky Survey extended point spread functions}", + journal = {MNRAS}, + year = 2020, + month = feb, + volume = {491}, + number = {4}, + pages = {5317-5329}, + doi = {10.1093/mnras/stz3111}, +archivePrefix = {arXiv}, + eprint = {1911.01430}, + primaryClass = {astro-ph.IM}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2020MNRAS.491.5317I}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + + + + + +@ARTICLE{akhlaghi19, + author = {{Akhlaghi}, Mohammad}, + title = "{Carving out the low surface brightness universe with NoiseChisel}", + journal = {IAU Symposium 335}, + year = 2019, + month = sep, + eid = {arXiv:1909.11230}, + pages = {arXiv:1909.11230}, +archivePrefix = {arXiv}, + eprint = {1909.11230}, + primaryClass = {astro-ph.IM}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2019arXiv190911230A}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} } -- cgit v1.2.1 From 202fd39ec98cdbb1cf14f50b1f96da50936b3db8 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 14 Jun 2020 01:31:56 +0100 Subject: Better comments for the top macros of paper.tex The default 'paper.tex' starts by defining some macros and comments describing them. Until now, the text was not too clear and could be confusing for someone that is not at all familiar with Maneage. With this commit, the comments have been edited to be more clear for a first-time reader. For example they all start with FULL CAPS summaries. Two other small things were corrected in 'tex/src/preamble-necessary.tex': - Until now 'project.tex' was included in this preamble. However, because of its importance in Maneage, and prominent place in the demonstration plot of the paper introducing Maneage, it is now included directly in 'paper.tex'. This also allows users to safely ignore/delete this preamble file if their LaTeX style is different. - I noticed that some macros for some astronomical software names from the very first commits in Maneage were still present here! They are no longer used, so they have been removed. --- paper.tex | 76 +++++++++++++++++++++++++++--------------- tex/src/preamble-necessary.tex | 20 ----------- 2 files changed, 50 insertions(+), 46 deletions(-) diff --git a/paper.tex b/paper.tex index f1ca48f..24cadf0 100644 --- a/paper.tex +++ b/paper.tex @@ -2,43 +2,67 @@ %% See the end of the file for license conditions. \documentclass[10pt, twocolumn]{article} -%% This is a convenience variable if you are using PGFPlots to build plots -%% within LaTeX. If you want to import PDF files for figures directly, you -%% can use the standard `\includegraphics' command. See the definition of -%% `\includetikz' in `tex/preamble-pgfplots.tex' for where the files are -%% assumed to be if you use `\includetikz' when `\makepdf' is not defined. +%% (OPTIONAL) CONVENIENCE VARIABLE: Only relevant when you use Maneage's +%% '\includetikz' macro to build plots/figures within LaTeX using TikZ or +%% PGFPlots. If so, when the Figure files (PDFs) are already built, you can +%% avoid TikZ or PGFPlots completely by commenting/removing the definition +%% of '\makepdf' below. This is useful when you don't want to slow-down a +%% LaTeX-only build of the project (for example this happens when you run +%% './project make dist'). See the definition of '\includetikz' in +%% `tex/preamble-pgfplots.tex' for more. \newcommand{\makepdf}{} -%% When defined (value is irrelevant), `\highlightchanges' will cause text -%% in `\tonote' and `\new' to become colored. This is useful in cases that -%% you need to distribute drafts that is undergoing revision and you want -%% to hightlight to your colleagues which parts are new and which parts are -%% only for discussion. +%% (OPTIONAL) CONVENIENCE VARIABLE: Only relevant when +%% 'tex/src/preamble-necessary.tex' is included (in particular the small +%% patch relating to '\highlightchanges'). In there, Maneage defines two +%% macros: `\tonote' and `\new'. When '\highlightchanges' is defined (value +%% is irrelevant), the text in those two macros becomes colored (in the +%% former, the text becomes dark red, in the latter it becomes dark +%% green). When not defined, text in the former isn't printed in the output +%% at all, and text in the latter becomes the same color as the rest of the +%% text. This is useful in cases that you need to distribute drafts and you +%% want to hightlight the new parts and add notes in the middle of the text +%% only for discussion, and build a clean PDF without any such highlights +%% without modifying the text. \newcommand{\highlightchanges}{} -%% Necessary LaTeX preambles to include for relevant functionality. We want -%% to start this file as fast as possible with the actual body of the -%% paper, while keeping modularity in the preambles. +%% VALUES FROM ANALYSIS (NUMBERS AND STRINGS): these are automatically +%% generated by the analysis phase of the project. The files loaded by +%% 'project.tex' only contain macro definitions (with '\newcommand') and +%% nothing else. So they won't interfere with any LaTeX style and can be +%% safely used in any pre-defined style. +\input{tex/build/macros/project.tex} + +%% CUSTOM PREAMBLES FOR DEMO: You can remove them if you are using a +%% specific journal style, or don't need features like BibLaTeX (advanced +%% bibliography management) or PGFPlots (for drawing plots within LaTeX +%% directly from tables of data). If you don't need them, you can also +%% delete their files from your branch to keep the 'tex/src' directory on +%% your branch clean. \input{tex/src/preamble-style.tex} \input{tex/src/preamble-header.tex} \input{tex/src/preamble-biblatex.tex} \input{tex/src/preamble-pgfplots.tex} \input{tex/src/preamble-necessary.tex} -%% Title and author information. For a more fine-grained control of the -%% headers including author name, or paper info, see -%% `tex/src/preamble-header.tex'. Note that if you plan to use a journal's -%% LaTeX style file, you will probably not need to set them, and can also -%% replace this "Title and author information" section with the journal's -%% preferred format. -% -%% NOTE ON TITLE: The title of the project should also be printed as -%% metadata in all output files. So it is defined with other core project -%% metadata in 'reproduce/analysis/config/metadata.conf'. That value is -%% then written in the '\projectitle' LaTeX macro and directly used -%% here. So please set your project's title in that Makefile with other -%% basic information. +%% PROJECT TITLE: The project title should also be printed as metadata in +%% all output files. To avoid inconsistancy caused by manually typing it, +%% the title is defined with other core project metadata in +%% 'reproduce/analysis/config/metadata.conf'. That value is then written in +%% the '\projectitle' LaTeX macro by +%% 'reproduce/analysis/make/initialize.mk' and is directly used here. So +%% please set your project's title in 'metadata.conf' (ideally with other +%% basic project information) and re-run the project to have your new +%% title. If you later use a different LaTeX style, please use the same +%% '\projectitle' in it (after importing 'tex/build/macros/project.tex' +%% like above), don't type it by hand. \title{\large \uppercase{\projecttitle}} + +%% AUTHOR INFORMATION: For a more fine-grained control of the headers +%% including author name, or paper info, see +%% `tex/src/preamble-header.tex'. Note that if you plan to use a journal's +%% LaTeX style file, you will probably set the authors in a different way, +%% feel free to change them here, this part is not related to the analysis. \author[1]{Your name} \author[2]{Coauthor one} \author[1,3]{Coauthor two} diff --git a/tex/src/preamble-necessary.tex b/tex/src/preamble-necessary.tex index 47a178e..bf74c8b 100644 --- a/tex/src/preamble-necessary.tex +++ b/tex/src/preamble-necessary.tex @@ -23,26 +23,6 @@ -%% Values from the analysis. -\input{tex/build/macros/project.tex} - - - - - -% Macros for to help in typing, remove them if you don't need them, but -% this can help as a demo on how you can simply writing of commonly used -% words that need special formatting (like software names). -\newcommand{\snsign}{{\small S}/{\small N}} -\newcommand{\originsoft}{\textsf{ORIGIN}} -\newcommand{\sextractor}{\textsf{SE\-xtractor}} -\newcommand{\noisechisel}{\textsf{Noise\-Chisel}} -\newcommand{\makecatalog}{\textsf{Make\-Catalog}} - - - - - %% For highlighting updates. When this is set, text marked as \new %% will be colored in dark green and text that is marked wtih \tonote %% will be marked in dark red. -- cgit v1.2.1 From 3c63f0febef7c3c8b41ffdd6b7033b1f77f6e21d Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 14 Jun 2020 01:55:33 +0100 Subject: Better explanation in the start of project configuration When './project configure' is run, after the basic checks of the compiler, a small statement is printed telling the user that some configuration questions will now be asked to start building Maneage on the system. Until now this description was confusing: it lead the reader to think that the local configuration (which was recommended to read before continuing) is in another file. With this commit, the text has been edited to explictly mention that the description of the steps following this notice should be read carefully. Thus avoiding that confusion. This issue was mentioned by Michael R. Crusoe. --- reproduce/software/shell/configure.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 789ddd5..e4885d6 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -569,9 +569,13 @@ cat < Date: Sun, 14 Jun 2020 22:09:36 +0100 Subject: Better description for input data directory, pointing to INPUTS.conf Until now, the description of the input-data directory at configure time included a description of the input data (created by reading the values of 'INPUTS.conf'). Maintaining this is easy for a single dataset, but it becomes hard for a general project which may need many input datasets. To avoid extra complexity (for maintaining this list), the description now points a user of the project to the 'INPUTS.conf' file and asks them to look inside of it for seeing the necessary data. This infact helps with the users becoming familiar with the internal structure of Maneage and will allow the authors to focus on not having to worry about updating the low-level 'configure.sh' script. --- reproduce/software/shell/configure.sh | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index e4885d6..d9509ca 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -782,10 +782,6 @@ if [ x"$input_dir" = x ]; then else indir=$input_dir fi -wfpc2name=$(awk '!/^#/ && $1=="WFPC2IMAGE" {print $3}' $adir/INPUTS.conf) -wfpc2md5=$(awk '!/^#/ && $1=="WFPC2MD5" {print $3}' $adir/INPUTS.conf) -wfpc2size=$(awk '!/^#/ && $1=="WFPC2SIZE" {print $3}' $adir/INPUTS.conf) -wfpc2url=$(awk '!/^#/ && $1=="WFPC2URL" {print $3}' $adir/INPUTS.conf) if [ $rewritepconfig = yes ] && [ x"$input_dir" = x ]; then cat < Date: Mon, 15 Jun 2020 01:42:10 +0100 Subject: Configure script now accounts for non-interactive shells The project configuration requires a build-directory at configuration time, two other directories can optionally be given to avoid downloading the project's necessary data and software. It is possible to give these three directories as command-line options, or by interactively giving them after running the configure script. Until now, when these directories weren't given as command-line options, and the running shell was non-interactive, the configure script would crash on the line trying to interactively read the user's given directories (the 'read' command). With this commit, all the 'read' commands for these three directories are now put within an 'if' statement. Therefore, when 'read' fails (the shell is non-interactive), instead of a quiet crash, a descriptive message is printed, telling the user that cause of the problem, and suggesting a fix. This bug was found by Michael R. Crusoe. --- reproduce/software/shell/configure.sh | 39 +++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index d9509ca..b71ea96 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -712,7 +712,14 @@ EOF do # Ask the user (if not already set on the command-line). if [ x"$build_dir" = x ]; then - read -p"Please enter the top build directory: " build_dir + if read -p"Please enter the top build directory: " build_dir; then + just_a_place_holder_to_avoid_not_equal_test=1; + else + echo "ERROR: shell is in non-interactive-mode and no build directory specified." + echo "The build directory (described above) is mandatory, configuration can't continue." + echo "Please use '--build-dir' to specify a build directory non-interactively." + exit 1 + fi fi # If it exists, see if we can write in it. If not, try making it. @@ -782,6 +789,7 @@ if [ x"$input_dir" = x ]; then else indir=$input_dir fi +noninteractive_sleep=2 if [ $rewritepconfig = yes ] && [ x"$input_dir" = x ]; then cat < Date: Mon, 15 Jun 2020 03:26:29 +0100 Subject: OpenSSL now built after Perl After trying a clean build of Maneage in a Docker image (with a minimal debian:stable-20200607-slim OS), I noticed that the building of OpenSSL is failing because it doesn't find the proper Perl functionality. To fix it, with this commit, Perl is set as a prerequisite of OpenSSL and this fixed the problem. --- reproduce/software/make/basic.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index b4745e2..82bc42d 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -805,7 +805,7 @@ $(idir)/etc:; mkdir $@ # Note: cert.pm has to be AFTER the tarball, otherwise the build script # will try to unpack cert.pm and crash (it unpacks the first dependency # under `tdir'). -$(ibidir)/openssl: $(ibidir)/tar \ +$(ibidir)/openssl: $(ibidir)/perl \ $(tdir)/openssl-$(openssl-version).tar.gz \ $(tdir)/cert.pem \ | $(idir)/etc -- cgit v1.2.1 From 6a52c4ee9d2ee8b5723fc9fac4ebc97b77357613 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Tue, 16 Jun 2020 21:45:04 +0100 Subject: XLSX I/O properly accounts for local build Until now, when adding the necessary library flags to the build of XLSX I/O, we were effectively over-writing the 'LDFLAGS' variables. So the compiler was effectively not being told where to look for the necessary libraries. With this commit, to fix the problem, we now append the new linking flags to LDFLAGS in XLSX I/O's build, not over-write it. --- reproduce/software/make/high-level.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 7cc2d51..75fa8ac 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -1224,9 +1224,9 @@ $(ibidir)/xlsxio: $(ibidir)/cmake \ if [ x$(on_mac_os) = xyes ]; then \ export CC=clang; \ export CXX=clang++; \ - export LDFLAGS="-lbz2"; \ + export LDFLAGS="$$LDFLAGS -lbz2"; \ else \ - export LDFLAGS="-lbz2 -lbsd"; \ + export LDFLAGS="$$LDFLAGS -lbz2 -lbsd"; \ fi; \ $(call cbuild, xlsxio-$(xlsxio-version), static, \ -DMINIZIP_DIR:PATH=$(idir) \ -- cgit v1.2.1