diff options
| -rw-r--r-- | .file-metadata | bin | 5337 -> 5532 bytes | |||
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | README-hacking.md | 258 | ||||
| -rwxr-xr-x | configure | 147 | ||||
| -rw-r--r-- | paper.tex | 8 | ||||
| -rwxr-xr-x | reproduce/analysis/bash/download-multi-try (renamed from reproduce/src/bash/download-multi-try) | 0 | ||||
| -rw-r--r-- | reproduce/analysis/config/INPUTS.mk (renamed from reproduce/config/pipeline/INPUTS.mk) | 0 | ||||
| -rw-r--r-- | reproduce/analysis/config/delete-me-num.mk (renamed from reproduce/config/pipeline/delete-me-num.mk) | 0 | ||||
| -rw-r--r-- | reproduce/analysis/config/pdf-build.mk (renamed from reproduce/config/pipeline/pdf-build.mk) | 0 | ||||
| -rw-r--r-- | reproduce/analysis/make/delete-me.mk (renamed from reproduce/src/make/delete-me.mk) | 2 | ||||
| -rw-r--r-- | reproduce/analysis/make/download.mk (renamed from reproduce/src/make/download.mk) | 2 | ||||
| -rw-r--r-- | reproduce/analysis/make/initialize.mk (renamed from reproduce/src/make/initialize.mk) | 26 | ||||
| -rw-r--r-- | reproduce/analysis/make/paper.mk (renamed from reproduce/src/make/paper.mk) | 39 | ||||
| -rw-r--r-- | reproduce/analysis/make/top.mk (renamed from reproduce/src/make/top.mk) | 27 | ||||
| -rw-r--r-- | reproduce/software/bash/git-post-checkout (renamed from reproduce/src/bash/git-post-checkout) | 0 | ||||
| -rw-r--r-- | reproduce/software/bash/git-pre-commit (renamed from reproduce/src/bash/git-pre-commit) | 0 | ||||
| -rw-r--r-- | reproduce/software/bibtex/astropy.tex (renamed from tex/dependencies/astropy.tex) | 0 | ||||
| -rw-r--r-- | reproduce/software/bibtex/cython.tex (renamed from tex/dependencies/cython.tex) | 0 | ||||
| -rw-r--r-- | reproduce/software/bibtex/fftw.tex (renamed from tex/dependencies/fftw.tex) | 0 | ||||
| -rw-r--r-- | reproduce/software/bibtex/gnuastro.tex (renamed from tex/dependencies/gnuastro.tex) | 0 | ||||
| -rw-r--r-- | reproduce/software/bibtex/matplotlib.tex (renamed from tex/dependencies/matplotlib.tex) | 0 | ||||
| -rw-r--r-- | reproduce/software/bibtex/mpi4py.tex (renamed from tex/dependencies/mpi4py.tex) | 0 | ||||
| -rw-r--r-- | reproduce/software/bibtex/numpy.tex (renamed from tex/dependencies/numpy.tex) | 0 | ||||
| -rw-r--r-- | reproduce/software/bibtex/scipy.tex (renamed from tex/dependencies/scipy.tex) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/gnuastro/astbuildprog.conf (renamed from reproduce/config/gnuastro/astbuildprog.conf) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/gnuastro/astconvertt.conf (renamed from reproduce/config/gnuastro/astconvertt.conf) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/gnuastro/aststatistics.conf (renamed from reproduce/config/gnuastro/aststatistics.conf) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/gnuastro/gnuastro.conf (renamed from reproduce/config/gnuastro/gnuastro.conf) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/installation/LOCAL.mk.in (renamed from reproduce/config/pipeline/LOCAL.mk.in) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/installation/numpy-scipy.cfg (renamed from reproduce/config/pipeline/dependency-numpy-scipy.cfg) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/installation/texlive.conf (renamed from reproduce/config/pipeline/texlive.conf) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/installation/texlive.mk (renamed from reproduce/config/pipeline/dependency-texlive.mk) | 0 | ||||
| -rw-r--r-- | reproduce/software/config/installation/versions.mk (renamed from reproduce/config/pipeline/dependency-versions.mk) | 14 | ||||
| -rwxr-xr-x | reproduce/software/make/atlas-multiple.mk (renamed from reproduce/src/make/dependencies-atlas-multiple.mk) | 0 | ||||
| -rwxr-xr-x | reproduce/software/make/atlas-single.mk (renamed from reproduce/src/make/dependencies-atlas-single.mk) | 0 | ||||
| -rw-r--r-- | reproduce/software/make/basic.mk (renamed from reproduce/src/make/dependencies-basic.mk) | 124 | ||||
| -rw-r--r-- | reproduce/software/make/build-rules.mk (renamed from reproduce/src/make/dependencies-build-rules.mk) | 0 | ||||
| -rw-r--r-- | reproduce/software/make/high-level.mk (renamed from reproduce/src/make/dependencies.mk) | 126 | ||||
| -rw-r--r-- | reproduce/software/make/python.mk (renamed from reproduce/src/make/dependencies-python.mk) | 4 | ||||
| -rw-r--r-- | tex/src/delete-me-demo.tex | 4 | ||||
| -rw-r--r-- | tex/src/delete-me.tex | 2 | ||||
| -rw-r--r-- | tex/src/preamble-biblatex.tex | 2 | ||||
| -rw-r--r-- | tex/src/preamble-necessary.tex | 2 |
43 files changed, 429 insertions, 360 deletions
diff --git a/.file-metadata b/.file-metadata Binary files differindex 7e8d8dd..e4c87cf 100644 --- a/.file-metadata +++ b/.file-metadata @@ -33,6 +33,7 @@ mmap_* build .local .build +LOCAL.mk Makefile tex/tikz .gnuastro @@ -42,7 +43,6 @@ tex/pipeline LOCAL_tmp.mk LOCAL_old.mk gnuastro-local.conf -reproduce/config/pipeline/LOCAL.mk reproduce/BDIR/ reproduce/SURVEY/ diff --git a/README-hacking.md b/README-hacking.md index e663ee1..94eec8c 100644 --- a/README-hacking.md +++ b/README-hacking.md @@ -238,38 +238,38 @@ Project architecture In order to customize this template to your research, it is important to first understand its architecture so you can navigate your way in the directories and understand how to implement your research project within -its framework. But before reading this theoretical discussion, please run -the template (described in `README.md`: first run `./configure`, then -`.local/bin/make -j8`) without any change, just to see how it works. - -In order to obtain a reproducible result it is important to have an -identical environment (for example same versions of the programs that it -will use). Therefore, the projects builds its own dependencies during the -`./configure` step. Building of the dependencies is managed by -`reproduce/src/make/dependencies-basic.mk` and -`reproduce/src/make/dependencies.mk`. These Makefiles are called by the -`./configure` script and not used afterwards. The first is intended for -downloading and building the most basic tools like GNU Tar, GNU Bash, GNU -Make, and GNU Compiler Collection (GCC). Therefore it must only contain -very basic and portable Make and shell features. The second is called after -the first, thus enabling usage of the modern and advanced features of GNU -Bash, GNU Make and other low-level GNU tools. Later, if you add a new -program/library for your research, you will need to include a rule on how -to download and build it (mostly in `reproduce/src/make/dependencies.mk`). +its framework: where to add new files and which existing files to modify +for what purpose. But before reading this theoretical discussion, please +run the template (described in `README.md`: first run `./configure`, then +`.local/bin/make -j8`) without any change, just to see how it works (note +that the configure step builds all necessary software, so it can take long, +but you can read along while its working). + +The project has two top-level directories: `reproduce` and +`tex`. `reproduce` hosts all the software building and analysis +steps. `tex` contains all the final paper's components to be compiled into +a PDF using LaTeX. + +The `reproduce` directory has two sub-directories: `software` and +`analysis`. As the name says, the former contains all the instructions to +download, build and install (independent of the host operating system) the +necessary software (these are called by the `./configure` script). The +latter contains instructions on how to use those software to do your +project's analysis. After it finishes, `./configure` will create the following symbolic links in the project's top source directory: 1) `Makefile` in the top directory -which points to `reproduce/src/make/top.mk`. 2) `.build' which points to -the top build directory, and 3) `.local` for easy access to the custom -built software packages installation directory. The first is for practical -necessity (so you can run `make' from the top source directory), but the -latter are just for convenience (fast access to the built products and -software). - -Therefore, by running `.local/bin/make` we will build the project with the -project's custom version of GNU Make, not the host system's Make. The first -file that is read by Make (the template's starting point) is the top-level -`Makefile` (also created by `./configure`). Therefore, we'll start +(which points to `reproduce/analysis/make/top.mk`). 2) `.build` which +points to the top build directory.And 3) `.local` for easy access to the +custom built software packages installation directory. The first is for +practical necessity (so you can run `make` from the top source directory), +but the latter is just for convenience (fast access to the built outputs +and software). + +Therefore, by running `.local/bin/make` we are doing the project's analysis +with its own custom version of GNU Make, not the host system's Make. The +first file that is read by Make (the template's starting point) is the +top-level `Makefile` (created by `./configure`). Therefore, we'll start describing the template's architecture with this file. This file is relatively short and heavily commented so hopefully the descriptions in each comment will be enough to understand the general details. As you read @@ -286,7 +286,7 @@ strategy to deal with large/huge files). To keep the source and (intermediate) built files separate, you _must_ define a top-level build directory variable (or `$(BDIR)`) to host all the -intermediate files (it was defined in `./configure`). This directory +intermediate files (you defined it during `./configure`). This directory doesn't need to be version controlled or even synchronized, or backed-up in other servers: its contents are all products, and can be easily re-created any time. As you define targets for your new rules, it is thus important to @@ -299,30 +299,32 @@ the top `Makefile`: _configuration-Makefiles_ (only independent variables/configurations) and _workhorse-Makefiles_ (Makefiles that actually contain analysis/processing rules). -The configuration-Makefiles are those that satisfy this wildcard: -`reproduce/config/pipeline/*.mk`. These Makefiles don't actually have any -rules, they just have values for various free parameters throughout the -analysis/processing. Open a few of them to see for yourself. These -Makefiles must only contain raw Make variables (project configurations). By -"raw" we mean that the Make variables in these files must not depend on -variables in any other configuration-Makefile. This is because we don't -want to assume any order in reading them. It is also very important to -*not* define any rule, or other Make construct, in these -configuration-Makefiles. - -This enables you to set these configure-Makefiles as a prerequisite to any -target that depends on their variable values. Therefore, if you change any -of their values, all targets that depend on those values will be -re-built. This is very convenient as your project scales up and gets more -complex. +The configuration-Makefiles are those that satisfy these two wildcards: +`reproduce/software/config/installation/*.mk` (for building the necessary +software when you run `./configure`) and `reproduce/analysis/config/*.mk` +(for the high-level analysis, when you run `.local/bin/make`). These +Makefiles don't actually have any rules, they just have values for various +free parameters throughout the configuration or analysis. Open a few of +them to see for yourself. These Makefiles must only contain raw Make +variables (project configurations). By "raw" we mean that the Make +variables in these files must not depend on variables in any other +configuration-Makefile. This is because we don't want to assume any order +in reading them. It is also very important to *not* define any rule, or +other Make construct, in these configuration-Makefiles. + +Following this rule-of-thumb enables you to set these configure-Makefiles +as a prerequisite to any target that depends on their variable +values. Therefore, if you change any of their values, all targets that +depend on those values will be re-built. This is very convenient as your +project scales up and gets more complex. The workhorse-Makefiles are those satisfying this wildcard -`reproduce/src/make/*.mk`. They contain the details of the processing steps -(Makefiles containing rules). Therefore, in this phase *order is -important*, because the prerequisites of most rules will be the targets of -other rules that will be defined prior to them (not a fixed name like -`paper.pdf`). The lower-level rules must be imported into Make before the -higher-level ones. +`reproduce/software/make/*.mk` and `reproduce/analysis/make/*.mk`. They +contain the details of the processing steps (Makefiles containing +rules). Therefore, in this phase *order is important*, because the +prerequisites of most rules will be the targets of other rules that will be +defined prior to them (not a fixed name like `paper.pdf`). The lower-level +rules must be imported into Make before the higher-level ones. All processing steps are assumed to ultimately (usually after many rules) end up in some number, image, figure, or table that will be included in the @@ -345,34 +347,36 @@ other users access to the contents. Therefore the `./configure` and Make steps must be called with special conditions which are managed in the `for-group` script. -Let's see how this design is implemented. When `./configure` finishes: By -creating a `Makefile` in the top directory, it allows us to start "making" -the project. Please open and inspect it as we go along here. The first step -(un-commented line) is to import the local configuration (answers to the -questions `./configure` asked you). They are defined in the -configuration-Makefile `reproduce/config/pipeline/LOCAL.mk` which was also -built by `./configure` (based on the `LOCAL.mk.in` template). - -The next non-commented set of lines define the ultimate target of the whole -project (`paper.pdf`). But to avoid mistakes, a sanity check is necessary -to see if Make is being run with the same group settings as the configure -script (for example when the project is configured for group access using -the `./for-group` script, but Make isn't). Therefore we use a Make -conditional to define the `all` target based on the group permissions being -consistent between the initial configuration and the current run. - -Having defined the top target, our next step is to include all the other -necessary Makefiles. However, order matters in the importing of +Let's see how this design is implemented. The `./configure` script's final +step is to put a `Makefile` in the top directory. This allows us to start +"making" the project. Please open and inspect it as we go along here. The +first step (un-commented line) is to import the local configuration +(answers to the questions `./configure` asked you). They are defined in the +configuration-Makefile `reproduce/software/config/installation/LOCAL.mk` +which was also built by `./configure` (based on the `LOCAL.mk.in` template +of the same directory). + +The next non-commented set of the top `Makefile` defines the ultimate +target of the whole project (`paper.pdf`). But to avoid mistakes, a sanity +check is necessary to see if Make is being run with the same group settings +as the configure script (for example when the project is configured for +group access using the `./for-group` script, but Make isn't). Therefore we +use a Make conditional to define the `all` target based on the group +permissions. + +Having defined the top/ultimate target, our next step is to include all the +other necessary Makefiles. However, order matters in the importing of workhorse-Makefiles and each must also have a TeX macro file with the same base name (without a suffix). Therefore, the next step in the top-level -Makefile is to define a `makesrc` variable to keep the base names (without -a `.mk` suffix) of the workhorse-Makefiles that must be imported, in the -proper order. +Makefile is to define the `makesrc` variable to keep the base names +(without a `.mk` suffix) of the workhorse-Makefiles that must be imported, +in the proper order. -Finally, we'll just import all the configuration-Makefiles with a wildcard -(while ignoring `LOCAL.mk` that was imported before). Also, all -workhorse-Makefiles are imported in the proper order using a Make `foreach` -loop. This finishes the general view of the template's implementation. +Finally, we import all the necessary remaining Makefiles: 1) All the +analysis configuration-Makefiles with a wildcard. 2) The software +configuration-Makefile that contains their versiosn (just incase its +necessary). 3) All workhorse-Makefiles in the proper order using a Make +`foreach` loop. In short, to keep things modular, readable and manageable, follow these recommendations: 1) Set clear-to-understand names for the @@ -385,14 +389,14 @@ manage/understand (even for yourself). As a general rule of thumb, break your rules into as many logically-similar but independent steps as possible. -The `reproduce/src/make/paper.mk` Makefile must be the final Makefile that -is included. This workhorse Makefile ends with the rule to build +The `reproduce/analysis/make/paper.mk` Makefile must be the final Makefile +that is included. This workhorse Makefile ends with the rule to build `paper.pdf` (final target of the whole project). If you look in it, you -will notice that it starts with a rule to create `$(mtexdir)/pipeline.tex` +will notice that it starts with a rule to create `$(mtexdir)/project.tex` (`mtexdir` is just a shorthand name for `$(BDIR)/tex/macros` mentioned -before). `$(mtexdir)/pipeline.tex` is the connection between the +before). `$(mtexdir)/project.tex` is the connection between the processing/analysis steps of the project, and the steps to build the final -PDF. As you see, `$(mtexdir)/pipeline.tex` only instructs LaTeX to import +PDF. As you see, `$(mtexdir)/project.tex` only instructs LaTeX to import the LaTeX macros of each high-level processing step during the analysis (the separate work-horse Makefiles that you defined and included). @@ -415,8 +419,8 @@ Summary Based on the explanation above, some major design points you should have in mind are listed below. - - Define new `reproduce/src/make/XXXXXX.mk` workhorse-Makefile(s) with - good and human-friendly name(s) replacing `XXXXXX`. + - Define new `reproduce/analysis/make/XXXXXX.mk` workhorse-Makefile(s) + with good and human-friendly name(s) replacing `XXXXXX`. - Add `XXXXXX`, as a new line, to the values in `makesrc` of the top-level `Makefile`. @@ -424,13 +428,13 @@ mind are listed below. - Do not use any constant numbers (or important names like filter names) in the workhorse-Makefiles or paper's LaTeX source. Define such constants as logically-grouped, separate configuration-Makefiles in - `reproduce/config/pipeline`. Then set the respective + `reproduce/analysis/config/XXXXX.mk`. Then set this configuration-Makefiles file as a pre-requisite to any rule that uses the variable defined in it. - Through any number of intermediate prerequisites, all processing steps - should end in (be a prerequisite of) `$(mtexdir)/pipeline.tex` (defined - in `reproduce/src/make/paper.mk`). `$(mtexdir)/pipeline.tex` is the + should end in (be a prerequisite of) `$(mtexdir)/project.tex` (defined + in `reproduce/analysis/make/paper.mk`). `$(mtexdir)/project.tex` is the bridge between the processing steps and PDF-building steps. @@ -511,12 +515,12 @@ get more advanced in later stages of your work. - **Title**, **short description** and **author** in source files: In this raw skeleton, the title or short description of your project should be - added in the following two files: `reproduce/src/make/top.mk` (the - first line), and `tex/preamble-header.tex`. In both cases, the texts - you should replace are all in capital letters to make them easier to - identify. Of course, if you use a different LaTeX method of managing - the title and authors, please feel free to use your own methods after - finishing this checklist and doing your first commit. + added in the following two files: `reproduce/analysis/make/top.mk` + (the first line), and `tex/src/preamble-header.tex`. In both cases, + the texts you should replace are all in capital letters to make them + easier to identify. Of course, if you use a different LaTeX method of + managing the title and authors, please feel free to use your own + methods after finishing this checklist and doing your first commit. - **Gnuastro**: GNU Astronomy Utilities (Gnuastro) is currently a dependency of the template which will be built and used. The main @@ -529,28 +533,29 @@ get more advanced in later stages of your work. them. - Delete marked part(s) in `configure`. - - Delete the `reproduce/config/gnuastro` directory. - - Delete `astnoisechisel` from the value of `top-level-programs` in `reproduce/src/make/dependencies.mk`. You can keep the rule to build `astnoisechisel`, since its not in the `top-level-programs` list, it (and all the dependencies that are only needed by Gnuastro) will be ignored. - - Delete marked parts in `reproduce/src/make/initialize.mk`. + - Delete the `reproduce/software/config/gnuastro` directory. + - Delete `gnuastro` from the value of `top-level-programs` in `reproduce/software/make/high-level.mk`. You can keep the rule to build `gnuastro`, since its not in the `top-level-programs` list, it (and all the dependencies that are only needed by Gnuastro) will be ignored. + - Delete marked parts in `reproduce/analysis/make/initialize.mk`. - - **Other dependencies**: If there are any more of the dependencies that - you don't use (or others that you need), then remove (or add) them in - the respective parts of `reproduce/src/make/dependencies.mk`. It is - commented thoroughly and reading over the comments should guide you on - what to add/remove and where. + - **Other dependencies**: If there are any more software that you don't + use (or others that you need), then remove (or add) them in the + respective parts of `top-level-programs` of + `reproduce/software/make/high-level`. It is commented thoroughly and + reading over the comments should guide you on what to add/remove and + where. - **Input dataset (can be done later)**: The input datasets are managed - through the `reproduce/config/pipeline/INPUTS.mk` file. It is best to + through the `reproduce/analysis/config/INPUTS.mk` file. It is best to gather all the information regarding all the input datasets into this one central file. To ensure that the proper dataset is being downloaded and used by the project, it is also recommended get an [MD5 checksum](https://en.wikipedia.org/wiki/MD5) of the file and include that in `INPUTS.mk` so the project can check it automatically. The preparation/downloading of the input datasets is done in - `reproduce/src/make/download.mk`. Have a look there to see how these - values are to be used. This information about the input datasets is - also used in the initial `configure` script (to inform the users), so - also modify that file. You can find all occurrences of the template + `reproduce/analysis/make/download.mk`. Have a look there to see how + these values are to be used. This information about the input datasets + is also used in the initial `configure` script (to inform the users), + so also modify that file. You can find all occurrences of the template dataset with the command below and replace it with your input's dataset. @@ -575,9 +580,9 @@ get more advanced in later stages of your work. - Delete all `delete-me*` files in the following directories: ```shell - $ rm tex/delete-me* - $ rm reproduce/src/make/delete-me* - $ rm reproduce/config/pipeline/delete-me* + $ rm tex/src/delete-me* + $ rm reproduce/analysis/make/delete-me* + $ rm reproduce/analysis/config/delete-me* ``` - **`README.md`**: Correct all the `XXXXX` place holders (name of your @@ -715,10 +720,12 @@ for the benefit of others. a good sign that you should break up the rule into its main components. Try to only have one major processing step per rule. - - *Context-based (many) Makefiles*: This design allows easy inclusion of - many Makefiles (in `reproduce/src/make/*.mk`) for maximal - modularity. So keep the rules for closely related parts of the - processing in separate Makefiles. + - *Context-based (many) Makefiles*: For maximum modularity, this design + allows easy inclusion of many Makefiles: in + `reproduce/analysis/make/*.mk` for analysis steps, and + `reproduce/software/make/*.mk` for building software. So keep the + rules for closely related parts of the processing in separate + Makefiles. - *Descriptive names*: Be very clear and descriptive with the naming of the files and the variables because a few months after the @@ -734,23 +741,24 @@ for the benefit of others. creating a catalog and another two for processing it under models A and B, you can name them like this: `catalog-create.mk`, `catalog-model-a.mk` and `catalog-model-b.mk`. In this way, when - listing the contents of `reproduce/src/make` to see all the + listing the contents of `reproduce/analysis/make` to see all the Makefiles, those related to the catalog will all be close to each other and thus easily found. This also helps in auto-completions by the shell or text editors like Emacs. - *Source directories*: If you need to add files in other languages for - example in shell, Python, AWK or C, keep them in a separate directory - under `reproduce/src`, with the appropriate name. + example in shell, Python, AWK or C, keep the files in the same + language in a separate directory under `reproduce/analysis`, with the + appropriate name. - *Configuration files*: If your research uses special programs as part of the processing, put all their configuration files in a devoted directory (with the program's name) within - `reproduce/config`. Similar to the `reproduce/config/gnuastro` - directory (which is put in the template as a demo in case you use GNU - Astronomy Utilities). It is much cleaner and readable (thus less - buggy) to avoid mixing the configuration files, even if there is no - technical necessity. + `reproduce/software/config`. Similar to the + `reproduce/software/config/gnuastro` directory (which is put in the + template as a demo in case you use GNU Astronomy Utilities). It is + much cleaner and readable (thus less buggy) to avoid mixing the + configuration files, even if there is no technical necessity. - **Contents**: It is good practice to follow the following @@ -837,7 +845,7 @@ for the benefit of others. ``` A more advanced Make programmer will use Make's [call function](https://www.gnu.org/software/make/manual/html_node/Call-Function.html) - to define a wrapper in `reproduce/src/make/initialize.mk`. This + to define a wrapper in `reproduce/analysis/make/initialize.mk`. This wrapper will replace `$(subst .txt,,XXXXX)`. Therefore, it will be possible to greatly simplify this repetitive statement and make the code even more readable throughout the whole project. @@ -848,9 +856,9 @@ for the benefit of others. input data): - *Keep the source tarball of dependencies*: After configuration - finishes, the `.build/dependencies/tarballs` directory will contain - all the software tarballs that were necessary for your project. You - can mirror the contents of this directory to keep a backup of all the + finishes, the `.build/software/tarballs` directory will contain all + the software tarballs that were necessary for your project. You can + mirror the contents of this directory to keep a backup of all the software tarballs used in your project (possibly as another version controlled repository) that is also published with your project. Note that software webpages are not written in stone and can suddenly go @@ -4,15 +4,15 @@ # # Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> # -# This script is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. +# This script is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. # # This script is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. # # A copy of the GNU General Public License is available at # <http://www.gnu.org/licenses/>. @@ -178,13 +178,16 @@ done topdir=$(pwd) lbdir=.build installedlink=.local -cdir=reproduce/config optionaldir="/optional/path" +adir=reproduce/analysis/config +cdir=reproduce/software/config -pdir=$cdir/pipeline -pconf=$pdir/LOCAL.mk -ptconf=$pdir/LOCAL_tmp.mk -poconf=$pdir/LOCAL_old.mk +sbdir=$cdir/installation + +pconf=$sbdir/LOCAL.mk +ptconf=$sbdir/LOCAL_tmp.mk +poconf=$sbdir/LOCAL_old.mk +depverfile=$cdir/installation/versions.mk # --------- Delete for no Gnuastro --------- glconf=$cdir/gnuastro/gnuastro-local.conf # ------------------------------------------ @@ -430,10 +433,10 @@ if [ x"$input_dir" = x ]; then else indir=$input_dir fi -wfpc2name=$(awk '!/^#/ && $1=="WFPC2IMAGE" {print $3}' $pdir/INPUTS.mk) -wfpc2md5=$(awk '!/^#/ && $1=="WFPC2MD5" {print $3}' $pdir/INPUTS.mk) -wfpc2size=$(awk '!/^#/ && $1=="WFPC2SIZE" {print $3}' $pdir/INPUTS.mk) -wfpc2url=$(awk '!/^#/ && $1=="WFPC2URL" {print $3}' $pdir/INPUTS.mk) +wfpc2name=$(awk '!/^#/ && $1=="WFPC2IMAGE" {print $3}' $adir/INPUTS.mk) +wfpc2md5=$(awk '!/^#/ && $1=="WFPC2MD5" {print $3}' $adir/INPUTS.mk) +wfpc2size=$(awk '!/^#/ && $1=="WFPC2SIZE" {print $3}' $adir/INPUTS.mk) +wfpc2url=$(awk '!/^#/ && $1=="WFPC2URL" {print $3}' $adir/INPUTS.mk) if [ $rewritepconfig = yes ] && [ x"$input_dir" = x ]; then cat <<EOF @@ -584,7 +587,6 @@ fi # --------- Delete for no Gnuastro --------- # Get the version of Gnuastro that must be used. -depverfile=reproduce/config/pipeline/dependency-versions.mk gversion=$(awk '$1=="gnuastro-version" {print $NF}' $depverfile) # Gnuastro's local configuration settings @@ -626,15 +628,18 @@ fi rm -f $lbdir ln -s $bdir $lbdir -depdir=$bdir/dependencies -if ! [ -d $depdir ]; then mkdir $depdir; fi +sdir=$bdir/software +if ! [ -d $sdir ]; then mkdir $sdir; fi -tardir=$depdir/tarballs +tardir=$sdir/tarballs if ! [ -d $tardir ]; then mkdir $tardir; fi -instdir=$depdir/installed +instdir=$sdir/installed if ! [ -d $instdir ]; then mkdir $instdir; fi +tmpblddir=$sdir/build-tmp +if ! [ -d $tmpblddir ]; then mkdir $tmpblddir; fi + verdir=$instdir/version-info if ! [ -d $verdir ]; then mkdir $verdir; fi @@ -654,7 +659,7 @@ itidir=$verdir/tex if ! [ -d $itidir ]; then mkdir $itidir; fi texdir=$bdir/tex -if ! [ -d $texdir ]; then mkdir $texdir; ln -s $texdir tex/pipeline; fi +if ! [ -d $texdir ]; then mkdir $texdir; ln -s $texdir tex/build; fi mtexdir=$texdir/macros if ! [ -d $mtexdir ]; then mkdir $mtexdir; fi @@ -663,7 +668,7 @@ rm -f $installedlink ln -s $instdir $installedlink # --------- Delete for no Gnuastro --------- rm -f .gnuastro -ln -s $(pwd)/reproduce/config/gnuastro .gnuastro +ln -s $topdir/reproduce/software/config/gnuastro .gnuastro # ------------------------------------------ @@ -685,8 +690,8 @@ ln -s $(pwd)/reproduce/config/gnuastro .gnuastro # the library came from the system or our build. static_build=no -#oprog=$depdir/static-test -#cprog=$depdir/static-test.c +#oprog=$sdir/static-test +#cprog=$sdir/static-test.c #echo "#include <stdio.h>" > $cprog #echo "int main(void) {return 0;}" >> $cprog #if [ x$CC = x ]; then CC=gcc; fi; @@ -733,8 +738,8 @@ static_build=no # `-rpath-link' is used to write the information of the linked shared # library into the shared object (library or program). But some versions of # LLVM's linker don't accept it an can cause problems. -oprog=$depdir/rpath-test -cprog=$depdir/rpath-test.c +oprog=$sdir/rpath-test +cprog=$sdir/rpath-test.c echo "#include <stdio.h>" > $cprog echo "int main(void) {return 0;}" >> $cprog if [ x$CC = x ]; then CC=gcc; fi; @@ -754,10 +759,10 @@ rm -f $oprog $cprog # # Some programs (like Wget) need dynamic loading (using `libdl'). On # GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But -# Mac OS doesn't need any explicit calling. So we'll check here to use in -# the building of programs. -oprog=$depdir/ldl-test -cprog=$depdir/ldl-test.c +# Mac OS doesn't need any explicit linking. So we'll check here to see if +# it is present (thus necessary) or not. +oprog=$sdir/ldl-test +cprog=$sdir/ldl-test.c cat > $cprog <<EOF #include <stdio.h> #include <dlfcn.h> @@ -790,7 +795,7 @@ NOTE: the built software will NOT BE INSTALLED on your system (no root access is required). They are only for local usage by this project. They will be installed in: - $depdir/installed + $sdir/installed EOF sleep $tsec @@ -821,17 +826,17 @@ fi -# Build `flock' as first program -# ------------------------------ +# Build `flock' +# ------------- # # Flock (or file-lock) is a unique program that is necessary to serialize # the (generally parallel) processing of make when necessary. GNU/Linux # machines have it as part of their `util-linux' programs. But to be # consistent in non-GNU/Linux systems, we will be using our own build. # -# The reason its sepecial is that we need it to serialize the download -# process of the dependency tarballs. -flockversion=$(awk '/flock-version/{print $3}' $pdir/dependency-versions.mk) +# The reason that `flock' is sepecial is that we need it to serialize the +# download process of the software tarballs. +flockversion=$(awk '/flock-version/{print $3}' $depverfile) flocktar=flock-$flockversion.tar.gz flockurl=http://github.com/discoteq/flock/releases/download/v$flockversion/ @@ -862,13 +867,14 @@ fi # Build `flock' if necessary. if ! [ -f $ibidir/flock ]; then - cd $depdir + cd $tmpblddir tar xf $tardir/$flocktar cd flock-$flockversion ./configure --prefix=$instdir - make; make install + make + make install cd $topdir - rm -rf $depdir/flock-$flockversion + rm -rf $tmpblddir/flock-$flockversion echo "Discoteq flock $flockversion" > $ibidir/flock fi @@ -886,11 +892,11 @@ fi # explicitly testing a small C program here to see if the host's C compiler # won't have any problems in building GCC. if [ $host_cc = 0 ]; then - tfile=$depdir/gcc-cdefs-test.c; + tfile=$sdir/gcc-cdefs-test.c; echo "#include <sys/cdefs.h>" > $tfile; echo "int main(void){return 0;}" >> $tfile; - if gcc $tfile -o $depdir/gcc-cdefs-test &> /dev/null; then - rm $depdir/gcc-cdefs-test*; + if gcc $tfile -o $sdir/gcc-cdefs-test &> /dev/null; then + rm $sdir/gcc-cdefs-test*; else host_cc=1 fi; @@ -925,8 +931,8 @@ fi -# Build Basic dependencies -# ------------------------ +# Number of threads for basic builds +# ---------------------------------- # # Since the system might not have GNU Make at this stage, and other Make # implementations can't deal with parallel build properly, we'll just @@ -935,7 +941,7 @@ fi # I found out its because of too many threads. GNU Make will be present on # GNU systems (that have `nproc', part of GNU Coreutils). So to simplify # the test for GNU Make, we'll just try running `nproc'. -if which nproc > /dev/null 2>/dev/null; then +if which nproc &> /dev/null; then if [ $jobs = 0 ]; then numthreads=$(nproc --all); else @@ -944,28 +950,43 @@ if which nproc > /dev/null 2>/dev/null; then else numthreads=1; fi -make -f reproduce/src/make/dependencies-basic.mk \ - rpath_command=$rpath_command \ - static_build=$static_build \ - needs_ldl=$needs_ldl \ - on_mac_os=$on_mac_os \ - numthreads=$numthreads \ - host_cc=$host_cc \ - -j$numthreads -# Rest of dependencies +# Build basic software # -------------------- # +# When building these software we don't have our own un-packing software, +# Bash, Make, or AWK. In this step, we'll install such low-level basic +# tools, but we have to be very portable (and use minimal features in all). +make -f reproduce/software/make/basic.mk \ + rpath_command=$rpath_command \ + static_build=$static_build \ + needs_ldl=$needs_ldl \ + on_mac_os=$on_mac_os \ + numthreads=$numthreads \ + host_cc=$host_cc \ + -j$numthreads + + + + + +# All other software +# ------------------ +# # We will be making all the dependencies before running the top-level # Makefile. To make the job easier, we'll do it in a Makefile, not a # script. Bash and Make were the tools we need to run Makefiles, so we had # to build them in this script. But after this, we can rely on Makefiles. -numthreads=$($instdir/bin/nproc) -./.local/bin/make -f reproduce/src/make/dependencies.mk \ +if [ $jobs = 0 ]; then + numthreads=$($instdir/bin/nproc --all) +else + numthreads=$jobs +fi +./.local/bin/make -f reproduce/software/make/high-level.mk \ rpath_command=$rpath_command \ static_build=$static_build \ on_mac_os=$on_mac_os \ @@ -1101,13 +1122,25 @@ fi + +# Clean the temporary build directory +# --------------------------------- +# +# By the time the script reaches here the temporary software build +# directory should be empty, so just delete it. +rm -rf $tmpblddir + + + + + # Final step: available Makefile # ------------------------------ # # We only want `make' to work after the configuration is complete. So we # will only put in the top-level Makefile after all the steps above are # done. -.local/bin/ln -s $(pwd)/reproduce/src/make/top.mk Makefile +.local/bin/ln -s $topdir/reproduce/analysis/make/top.mk Makefile @@ -57,7 +57,7 @@ \textsl{Keywords}: Add some keywords for your research here. \textsl{Reproducible paper}: All quantitave results (numbers and plots) - in this paper are exactly reproducible (version \pipelineversion{}, + in this paper are exactly reproducible (version \projectversion{}, \url{https://gitlab.com/makhlaghi/reproducible-paper}).} %% To add the first page's headers. @@ -77,7 +77,7 @@ Just don't forget to \emph{never} use numbers or fixed strings (for example database urls like \url{\wfpctwourl}) directly within your \LaTeX{} source. Read them directly from your configuration files, or processing outputs, and import them into \LaTeX{} as macros through the -\texttt{tex/pipeline/macros/pipeline.tex} file (created after running the +\texttt{tex/build/macros/project.tex} file (created after running the project). See the several existing examples within the template for a demonstration. For some recent real-world examples, the reproducible project sources for Sections 4 and 7.3 of \citet{bacon17} are available at @@ -194,7 +194,7 @@ source code and the project's source to a long-lasting host like Zenodo This research was partly done using GNU Astronomy Utilities (Gnuastro, ascl.net/1801.009), and the reproducible paper template -\pipelineversion. Work on Gnuastro and the reproducible paper template has +\projectversion. Work on Gnuastro and the reproducible paper template has been funded by the Japanese Ministry of Education, Culture, Sports, Science, and Technology (MEXT) scholarship and its Grant-in-Aid for Scientific Research (21244012, 24253003), the European Research Council @@ -203,7 +203,7 @@ and innovation programme under Marie Sklodowska-Curie grant agreement No 721463 to the SUNDIAL ITN, and from the Spanish Ministry of Economy and Competitiveness (MINECO) under grant number AYA2016-76219-P. -\input{tex/pipeline/macros/dependencies.tex} +\input{tex/build/macros/dependencies.tex} %% Tell BibLaTeX to put the bibliography list here. \printbibliography diff --git a/reproduce/src/bash/download-multi-try b/reproduce/analysis/bash/download-multi-try index 1fd7497..1fd7497 100755 --- a/reproduce/src/bash/download-multi-try +++ b/reproduce/analysis/bash/download-multi-try diff --git a/reproduce/config/pipeline/INPUTS.mk b/reproduce/analysis/config/INPUTS.mk index eb38295..eb38295 100644 --- a/reproduce/config/pipeline/INPUTS.mk +++ b/reproduce/analysis/config/INPUTS.mk diff --git a/reproduce/config/pipeline/delete-me-num.mk b/reproduce/analysis/config/delete-me-num.mk index 17f608c..17f608c 100644 --- a/reproduce/config/pipeline/delete-me-num.mk +++ b/reproduce/analysis/config/delete-me-num.mk diff --git a/reproduce/config/pipeline/pdf-build.mk b/reproduce/analysis/config/pdf-build.mk index 3a86ff3..3a86ff3 100644 --- a/reproduce/config/pipeline/pdf-build.mk +++ b/reproduce/analysis/config/pdf-build.mk diff --git a/reproduce/src/make/delete-me.mk b/reproduce/analysis/make/delete-me.mk index 701a316..c4cfffe 100644 --- a/reproduce/src/make/delete-me.mk +++ b/reproduce/analysis/make/delete-me.mk @@ -82,7 +82,7 @@ $(histogram): $(dddemodir)/%-hist.txt: $(indir)/%.fits | $(dddemodir) # ---------------- # # This is just as a demonstration on how to get analysic configuration -# parameters from variables defined in `reproduce/config/pipeline'. +# parameters from variables defined in `reproduce/analysis/config/'. stats = $(dddemodir)/wfpc2-stats.txt $(stats): $(dddemodir)/%-stats.txt: $(indir)/%.fits | $(dddemodir) aststatistics $< -h0 --mean --median > $@ diff --git a/reproduce/src/make/download.mk b/reproduce/analysis/make/download.mk index dfc49da..a721863 100644 --- a/reproduce/src/make/download.mk +++ b/reproduce/analysis/make/download.mk @@ -48,7 +48,7 @@ # process with a file and make sure that only one downloading event is in # progress at every moment. $(indir):; mkdir $@ -downloadwrapper = $(srcdir)/bash/download-multi-try +downloadwrapper = $(bashdir)/download-multi-try inputdatasets = $(foreach i, wfpc2, $(indir)/$(i).fits) $(inputdatasets): $(indir)/%.fits: | $(indir) $(lockdir) diff --git a/reproduce/src/make/initialize.mk b/reproduce/analysis/make/initialize.mk index cd533f2..a034494 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -31,14 +31,14 @@ # be necessary to put a lock on them. This project uses the `flock' program # to achieve this. texdir = $(BDIR)/tex -srcdir = reproduce/src lockdir = $(BDIR)/locks indir = $(BDIR)/inputs mtexdir = $(texdir)/macros -pconfdir = reproduce/config/pipeline -installdir = $(BDIR)/dependencies/installed +bashdir = reproduce/analysis/bash +pconfdir = reproduce/analysis/config +installdir = $(BDIR)/software/installed # --------- Delete for no Gnuastro --------- -gconfdir = reproduce/config/gnuastro +gconfdir = reproduce/software/config/gnuastro # ------------------------------------------ @@ -233,16 +233,14 @@ $(packagecontents): | $(texdir) cp configure COPYING for-group README.md README-hacking.md $$dir/ # Build the top-level directories. - mkdir $$dir/reproduce $$dir/tex $$dir/tex/tikz $$dir/tex/pipeline + mkdir $$dir/reproduce $$dir/tex $$dir/tex/tikz $$dir/tex/build - # Copy all the `reproduce' contents except for the `build' symbolic - # link. + # Copy all the necessary `reproduce' and `tex' contents. shopt -s extglob cp -r tex/src $$dir/tex/src cp tex/tikz/*.pdf $$dir/tex/tikz - cp -r reproduce/!(build) $$dir/reproduce - cp -r tex/pipeline/!($(packagebasename)) $$dir/tex/pipeline - cp -r tex/dependencies $$dir/tex/dependencies + cp -r reproduce/ $$dir/reproduce + cp -r tex/build/!($(packagebasename)) $$dir/tex/build # Clean up un-necessary/local files: 1) the $(texdir)/build* # directories (when building in a group structure, there will be @@ -250,9 +248,9 @@ $(packagecontents): | $(texdir) # build files and don't have any relevant/hand-written files in # them. 2) The `LOCAL.mk' and `gnuastro-local.conf' files just have # this machine's local settings and are irrelevant for anyone else. - rm -rf $$dir/tex/pipeline/build* - rm $$dir/reproduce/config/pipeline/LOCAL.mk - rm $$dir/reproduce/config/gnuastro/gnuastro-local.conf + rm -rf $$dir/tex/build/build* + rm $$dir/reproduce/software/config/installation/LOCAL.mk + rm $$dir/reproduce/software/config/gnuastro/gnuastro-local.conf # PROJECT SPECIFIC: under this comment, copy any other file for # packaging, or remove any of the copied files above to suite your @@ -338,4 +336,4 @@ $(mtexdir)/initialize.tex: | $(mtexdir) # Version of the project. @v=$$(git describe --dirty --always); - echo "\newcommand{\pipelineversion}{$$v}" > $@ + echo "\newcommand{\projectversion}{$$v}" > $@ diff --git a/reproduce/src/make/paper.mk b/reproduce/analysis/make/paper.mk index 0c42bee..5378ee3 100644 --- a/reproduce/src/make/paper.mk +++ b/reproduce/analysis/make/paper.mk @@ -23,35 +23,32 @@ # # To report the input settings and results, the final report's PDF (final # target of this project) uses macros generated from various steps of the -# project. All these macros are defined in `$(mtexdir)/pipeline.tex'. +# project. All these macros are defined through `$(mtexdir)/project.tex'. # -# `$(mtexdir)/pipeline.tex' is actually just a combination of separate -# files that keep the LaTeX macros related to each workhorse Makefile (in +# `$(mtexdir)/project.tex' is actually just a combination of separate files +# that keep the LaTeX macros related to each workhorse Makefile (in # `reproduce/src/make/*.mk'). Those individual macros are pre-requisites to -# `$(mtexdir)/pipeline.tex'. The only workhorse Makefile that doesn't need +# `$(mtexdir)/project.tex'. The only workhorse Makefile that doesn't need # to produce LaTeX macros is this Makefile (`reproduce/src/make/paper.mk'). # # This file is thus the interface between the processing scripts and the # final PDF: when we get to this point, all the processing has been # completed. # -# Note that if you don't want the final PDF and just want the -# processing and file outputs, you can remove the value of -# `pdf-build-final' in `reproduce/config/pipeline/pdf-build.mk'. -$(mtexdir)/pipeline.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s).tex) +# Note that if you don't want the final PDF and just want the processing +# and file outputs, you can remove the value of `pdf-build-final' in +# `reproduce/analysis/config/pdf-build.mk'. +$(mtexdir)/project.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s).tex) # If no PDF is requested, or if LaTeX isn't available, don't # continue to building the final PDF. Otherwise, merge all the TeX # macros into one for building the PDF. @if [ -f .local/bin/pdflatex ] && [ x"$(pdf-build-final)" != x ]; then - # First make sure the `tex/pipeline' symbolic link exists. - if [ ! -e tex/pipeline ]; then ln -s $(texdir) tex/pipeline; fi - # Put a LaTeX input command for all the necessary macro files. - rm -f $(mtexdir)/pipeline.tex + rm -f $(mtexdir)/project.tex for t in $(subst paper,,$(makesrc)); do - echo "\input{tex/pipeline/macros/$$t.tex}" >> $(mtexdir)/pipeline.tex + echo "\input{tex/build/macros/$$t.tex}" >> $(mtexdir)/project.tex done else echo @@ -64,7 +61,7 @@ $(mtexdir)/pipeline.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s echo "can check by running './.local/bin/latex --version'), _AND_" echo "make sure that the 'pdf-build-final' variable has a value." echo "'pdf-build-final' is defined in: " - echo "'reproduce/config/pipeline/pdf-build.mk'." + echo "'reproduce/analysis/config/pdf-build.mk'." echo echo "If you don't have LaTeX within the project, please re-run" echo "'./configure' when you have internet access. To speed it up," @@ -88,15 +85,15 @@ $(mtexdir)/pipeline.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s # necessary bibliography before making the final paper. So we'll first have # one run of LaTeX (similar to the `paper.pdf' recipe), then `biber'. # -# NOTE: `$(mtexdir)/pipeline.tex' is an order-only-prerequisite for +# NOTE: `$(mtexdir)/project.tex' is an order-only-prerequisite for # `paper.bbl'. This is because we need to run LaTeX in both the `paper.bbl' # recipe and the `paper.pdf' recipe. But if `tex/src/references.tex' hasn't # been modified, we don't want to re-build the bibliography, only the final # PDF. $(texbdir)/paper.bbl: tex/src/references.tex \ - | $(tikzdir) $(texbdir) $(mtexdir)/pipeline.tex - # If `$(mtexdir)/pipeline.tex' is empty, don't build PDF. - @macros=$$(cat $(mtexdir)/pipeline.tex) + | $(tikzdir) $(texbdir) $(mtexdir)/project.tex + # If `$(mtexdir)/project.tex' is empty, don't build PDF. + @macros=$$(cat $(mtexdir)/project.tex) if [ x"$$macros" != x ]; then # We'll run LaTeX first to generate the `.bcf' file (necessary @@ -121,11 +118,11 @@ $(texbdir)/paper.bbl: tex/src/references.tex \ # to run everything cleanly from there, it is necessary to add the current # directory (top project directory) to the `TEXINPUTS' environment # variable. -paper.pdf: $(mtexdir)/pipeline.tex paper.tex $(texbdir)/paper.bbl \ +paper.pdf: $(mtexdir)/project.tex paper.tex $(texbdir)/paper.bbl \ | $(tikzdir) $(texbdir) - # If `$(mtexdir)/pipeline.tex' is empty, don't build the PDF. - @macros=$$(cat $(mtexdir)/pipeline.tex) + # If `$(mtexdir)/project.tex' is empty, don't build the PDF. + @macros=$$(cat $(mtexdir)/project.tex) if [ x"$$macros" != x ]; then # Go into the top TeX build directory and make the paper. diff --git a/reproduce/src/make/top.mk b/reproduce/analysis/make/top.mk index 763dbd7..a85c530 100644 --- a/reproduce/src/make/top.mk +++ b/reproduce/analysis/make/top.mk @@ -20,7 +20,7 @@ # Load the local configuration (created after running `./configure'). -include reproduce/config/pipeline/LOCAL.mk +include reproduce/software/config/installation/LOCAL.mk @@ -63,7 +63,7 @@ include reproduce/config/pipeline/LOCAL.mk # # If you are just interested in the processing and don't want to build the # PDF, you can skip the creatation of the final PDF by removing the value -# of `pdf-build-final' in `reproduce/config/pipeline/pdf-build.mk'. +# of `pdf-build-final' in `reproduce/analysis/config/pdf-build.mk'. ifeq (x$(reproducible_paper_group_name),x$(GROUP-NAME)) all: paper.pdf else @@ -119,17 +119,18 @@ makesrc = initialize \ -# Include all Makefiles -# --------------------- +# Include all analysis Makefiles +# ------------------------------ # -# We have two classes of Makefiles, separated by context and their location: +# 1) All the analysis configuration-Makefiles (Makefiles that only define +# variables with no rules or order). # -# 1) First, we'll include all the configuration-Makefiles. These -# Makefiles only define variables with no rules or order. We just -# won't include `LOCAL.mk' because it has already been included -# above. +# 2) From the software configuration-Makefiles, we only include the one +# containing software versions, just incase its necessary to +# use/report outside of the acknowledgments section of the paper. # -# 2) Then, we'll import the workhorse-Makefiles which contain rules to -# actually do this project's processing. -include $(filter-out %LOCAL.mk, reproduce/config/pipeline/*.mk) -include $(foreach s,$(makesrc), reproduce/src/make/$(s).mk) +# 3) Finally, we'll import all the analysis workhorse-Makefiles which +# contain rules to actually do this project's processing. +include reproduce/analysis/config/*.mk +include reproduce/software/config/installation/versions.mk +include $(foreach s,$(makesrc), reproduce/analysis/make/$(s).mk) diff --git a/reproduce/src/bash/git-post-checkout b/reproduce/software/bash/git-post-checkout index 9552f01..9552f01 100644 --- a/reproduce/src/bash/git-post-checkout +++ b/reproduce/software/bash/git-post-checkout diff --git a/reproduce/src/bash/git-pre-commit b/reproduce/software/bash/git-pre-commit index dbe0ecc..dbe0ecc 100644 --- a/reproduce/src/bash/git-pre-commit +++ b/reproduce/software/bash/git-pre-commit diff --git a/tex/dependencies/astropy.tex b/reproduce/software/bibtex/astropy.tex index 450212d..450212d 100644 --- a/tex/dependencies/astropy.tex +++ b/reproduce/software/bibtex/astropy.tex diff --git a/tex/dependencies/cython.tex b/reproduce/software/bibtex/cython.tex index 110cb19..110cb19 100644 --- a/tex/dependencies/cython.tex +++ b/reproduce/software/bibtex/cython.tex diff --git a/tex/dependencies/fftw.tex b/reproduce/software/bibtex/fftw.tex index 60c24a8..60c24a8 100644 --- a/tex/dependencies/fftw.tex +++ b/reproduce/software/bibtex/fftw.tex diff --git a/tex/dependencies/gnuastro.tex b/reproduce/software/bibtex/gnuastro.tex index 91788c9..91788c9 100644 --- a/tex/dependencies/gnuastro.tex +++ b/reproduce/software/bibtex/gnuastro.tex diff --git a/tex/dependencies/matplotlib.tex b/reproduce/software/bibtex/matplotlib.tex index 8578899..8578899 100644 --- a/tex/dependencies/matplotlib.tex +++ b/reproduce/software/bibtex/matplotlib.tex diff --git a/tex/dependencies/mpi4py.tex b/reproduce/software/bibtex/mpi4py.tex index 4e0045f..4e0045f 100644 --- a/tex/dependencies/mpi4py.tex +++ b/reproduce/software/bibtex/mpi4py.tex diff --git a/tex/dependencies/numpy.tex b/reproduce/software/bibtex/numpy.tex index 0d9028f..0d9028f 100644 --- a/tex/dependencies/numpy.tex +++ b/reproduce/software/bibtex/numpy.tex diff --git a/tex/dependencies/scipy.tex b/reproduce/software/bibtex/scipy.tex index 60500d2..60500d2 100644 --- a/tex/dependencies/scipy.tex +++ b/reproduce/software/bibtex/scipy.tex diff --git a/reproduce/config/gnuastro/astbuildprog.conf b/reproduce/software/config/gnuastro/astbuildprog.conf index c15baf1..c15baf1 100644 --- a/reproduce/config/gnuastro/astbuildprog.conf +++ b/reproduce/software/config/gnuastro/astbuildprog.conf diff --git a/reproduce/config/gnuastro/astconvertt.conf b/reproduce/software/config/gnuastro/astconvertt.conf index 6466455..6466455 100644 --- a/reproduce/config/gnuastro/astconvertt.conf +++ b/reproduce/software/config/gnuastro/astconvertt.conf diff --git a/reproduce/config/gnuastro/aststatistics.conf b/reproduce/software/config/gnuastro/aststatistics.conf index 1d13e3f..1d13e3f 100644 --- a/reproduce/config/gnuastro/aststatistics.conf +++ b/reproduce/software/config/gnuastro/aststatistics.conf diff --git a/reproduce/config/gnuastro/gnuastro.conf b/reproduce/software/config/gnuastro/gnuastro.conf index 57fcadc..57fcadc 100644 --- a/reproduce/config/gnuastro/gnuastro.conf +++ b/reproduce/software/config/gnuastro/gnuastro.conf diff --git a/reproduce/config/pipeline/LOCAL.mk.in b/reproduce/software/config/installation/LOCAL.mk.in index 785bb6a..785bb6a 100644 --- a/reproduce/config/pipeline/LOCAL.mk.in +++ b/reproduce/software/config/installation/LOCAL.mk.in diff --git a/reproduce/config/pipeline/dependency-numpy-scipy.cfg b/reproduce/software/config/installation/numpy-scipy.cfg index 4b7a7b0..4b7a7b0 100644 --- a/reproduce/config/pipeline/dependency-numpy-scipy.cfg +++ b/reproduce/software/config/installation/numpy-scipy.cfg diff --git a/reproduce/config/pipeline/texlive.conf b/reproduce/software/config/installation/texlive.conf index 53054e1..53054e1 100644 --- a/reproduce/config/pipeline/texlive.conf +++ b/reproduce/software/config/installation/texlive.conf diff --git a/reproduce/config/pipeline/dependency-texlive.mk b/reproduce/software/config/installation/texlive.mk index 0acf5ab..0acf5ab 100644 --- a/reproduce/config/pipeline/dependency-texlive.mk +++ b/reproduce/software/config/installation/texlive.mk diff --git a/reproduce/config/pipeline/dependency-versions.mk b/reproduce/software/config/installation/versions.mk index c6c8043..149f9ae 100644 --- a/reproduce/config/pipeline/dependency-versions.mk +++ b/reproduce/software/config/installation/versions.mk @@ -21,8 +21,8 @@ atlas-version = 3.10.3 bash-version = 5.0 binutils-version = 2.31.1 cfitsio-version = 3.45 -cmake-version = 3.12.4 -coreutils-version = 8.30 +cmake-version = 3.14.2 +coreutils-version = 8.31 curl-version = 7.63.0 diffutils-version = 3.7 fftw-version = 3.3.8 @@ -30,10 +30,10 @@ file-version = 5.36 findutils-version = 4.6.0.199-e3fc flock-version = 0.2.3 freetype-version = 2.9 -gawk-version = 4.2.1 +gawk-version = 5.0.0 gcc-version = 8.3.0 ghostscript-version = 9.26 -git-version = 2.20.1 +git-version = 2.21.0 gmp-version = 6.1.2 gnuastro-version = 0.8 grep-version = 3.3 @@ -59,12 +59,12 @@ openmpi-version = 4.0.1 openssl-version = 1.1.1a patchelf-version = 0.9 pkgconfig-version = 0.29.2 -python-version = 3.6.8 +python-version = 3.7.3 readline-version = 8.0 sed-version = 4.7 -tar-version = 1.31 +tar-version = 1.32 unzip-version = 6.0 -wget-version = 1.20.1 +wget-version = 1.20.3 which-version = 2.21 xz-version = 5.2.4 zip-version = 3.0 diff --git a/reproduce/src/make/dependencies-atlas-multiple.mk b/reproduce/software/make/atlas-multiple.mk index fef25c7..fef25c7 100755 --- a/reproduce/src/make/dependencies-atlas-multiple.mk +++ b/reproduce/software/make/atlas-multiple.mk diff --git a/reproduce/src/make/dependencies-atlas-single.mk b/reproduce/software/make/atlas-single.mk index dde2926..dde2926 100755 --- a/reproduce/src/make/dependencies-atlas-single.mk +++ b/reproduce/software/make/atlas-single.mk diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/software/make/basic.mk index b56d01d..cfa05d2 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/software/make/basic.mk @@ -1,20 +1,17 @@ -# Build the VERY BASIC project dependencies before everything else assuming +# Build the VERY BASIC project software before higher-level ones. Assuming # minimal/generic Make and Shell. # # ------------------------------------------------------------------------ # !!!!! IMPORTANT NOTES !!!!! # # This Makefile will be run by the initial `./configure' script. It is not -# included into the reproduction pipe after that. +# included into the project after that. # -# This Makefile builds very low-level and basic tools like GNU Tar, and -# various compression programs, GNU Bash, and GNU Make. Therefore this is -# the only Makefile in the reproduction pipeline where you MUST NOT assume -# that modern GNU Bash or GNU Make are used. After this Makefile, other -# Makefiles can safely assume the fixed version of all these software. -# -# This Makefile is a very simplified version of `dependencies.mk' in the -# same directory. See there for more comments. +# This Makefile builds very low-level and basic tools like GNU Tar, GNU +# Bash, GNU Make, GCC and etc. Therefore this is the only Makefile in the +# project where you CANNOT assume that GNU Bash or GNU Make are used. After +# this Makefile (where GNU Bash and GNU Make are built), other Makefiles +# can safely assume the fixed version of all these software. # # ------------------------------------------------------------------------ # @@ -35,18 +32,18 @@ # Top level environment -include reproduce/config/pipeline/LOCAL.mk -include reproduce/src/make/dependencies-build-rules.mk -include reproduce/config/pipeline/dependency-versions.mk +include reproduce/software/make/build-rules.mk +include reproduce/software/config/installation/LOCAL.mk +include reproduce/software/config/installation/versions.mk lockdir = $(BDIR)/locks -ddir = $(BDIR)/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib -ibidir = $(BDIR)/dependencies/installed/version-info/bin -ilidir = $(BDIR)/dependencies/installed/version-info/lib +tdir = $(BDIR)/software/tarballs +ddir = $(BDIR)/software/build-tmp +idir = $(BDIR)/software/installed +ibdir = $(BDIR)/software/installed/bin +ildir = $(BDIR)/software/installed/lib +ibidir = $(BDIR)/software/installed/version-info/bin +ilidir = $(BDIR)/software/installed/version-info/lib # We'll need the system's PATH for making links to low-level programs we # won't be building ourselves. @@ -58,11 +55,11 @@ export CCACHE_DISABLE := 1 export PATH := $(ibdir):$(PATH) export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) -export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) +export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) +export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) -# Define the programs that don't depend on any other. +# Define the top-level programs (that don't depend on any other). top-level-programs = low-level-links wget gcc all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) @@ -73,10 +70,10 @@ all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) # Tarballs # -------- # -# Prepare tarballs. Difference with that in `dependencies.mk': `.ONESHELL' -# is not recognized by some versions of Make (even older GNU Makes). So -# we'll have to make sure the recipe doesn't break into multiple shell -# calls (so we can preserve the variables). +# Prepare tarballs. Difference with that in `high-level.mk': `.ONESHELL' is +# not recognized by some versions of Make (even older GNU Makes). So we'll +# have to make sure the recipe doesn't break into multiple shell calls (so +# we can preserve the variables). # # Software hosted at akhlaghi.org/src: As of our latest check (November # 2018) their major release tarballs either crash or don't build on some @@ -93,7 +90,7 @@ all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) # However, downloading from this link is slow (because its just a link). So # its easier to just keep a with the others. $(lockdir): | $(BDIR); mkdir $@ -downloadwrapper = ./reproduce/src/bash/download-multi-try +downloadwrapper = ./reproduce/analysis/bash/download-multi-try tarballs = $(foreach t, bash-$(bash-version).tar.gz \ binutils-$(binutils-version).tar.lz \ bzip2-$(bzip2-version).tar.gz \ @@ -121,9 +118,11 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \ readline-$(readline-version).tar.gz \ sed-$(sed-version).tar.xz \ tar-$(tar-version).tar.gz \ + unzip-$(unzip-version).tar.gz \ wget-$(wget-version).tar.lz \ which-$(which-version).tar.gz \ xz-$(xz-version).tar.gz \ + zip-$(zip-version).tar.gz \ zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: | $(lockdir) @@ -164,9 +163,15 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = readline ]; then w=http://ftp.gnu.org/gnu/readline; \ elif [ $$n = sed ]; then w=http://ftp.gnu.org/gnu/sed; \ elif [ $$n = tar ]; then w=http://ftp.gnu.org/gnu/tar; \ + elif [ $$n = unzip ]; then \ + mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//'); \ + w=ftp://ftp.info-zip.org/pub/infozip/src/unzip$$v.tgz; \ elif [ $$n = wget ]; then w=http://ftp.gnu.org/gnu/wget; \ elif [ $$n = which ]; then w=http://ftp.gnu.org/gnu/which; \ elif [ $$n = xz ]; then w=http://tukaani.org/xz; \ + elif [ $$n = zip ]; then \ + mergenames=0; v=$$(echo $(zip-version) | sed -e's/\.//'); \ + w=ftp://ftp.info-zip.org/pub/infozip/src/zip$$v.tgz; \ elif [ $$n = zlib ]; then w=http://www.zlib.net; \ else \ echo; echo; echo; \ @@ -178,7 +183,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) if [ $$mergenames = 1 ]; then tarballurl=$$w/"$*"; \ else tarballurl=$$w; \ fi; \ - \ + \ echo "Downloading $$tarballurl"; \ if [ -f $(ibdir)/wget ]; then \ downloader="wget --no-use-server-timestamps -O"; \ @@ -344,18 +349,51 @@ $(ibidir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz && ln -fs libbz2.so.1.0 libbz2.so \ && echo "Bzip2 $(bzip2-version)" > $@ +$(ibidir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz + v=$$(echo $(unzip-version) | sed -e's/\.//'); \ + $(call gbuild, $<, unzip$$v, static,, \ + -f unix/Makefile generic_gcc \ + CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ + -f unix/Makefile \ + BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ + && echo "Unzip $(unzip-version)" > $@ + +$(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz + v=$$(echo $(zip-version) | sed -e's/\.//'); \ + $(call gbuild, $<, zip$$v, static,, \ + -f unix/Makefile generic_gcc \ + CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ + -f unix/Makefile \ + BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ + && echo "Zip $(zip-version)" > $@ + +# Some programs (like Wget and CMake) that use zlib need it to be dynamic +# so they use our custom build. So we won't force a static-only build. +# +# Note for a static-only build: Zlib's `./configure' doesn't use Autoconf's +# configure script, it just accepts a direct `--static' option. +$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz + $(call gbuild, $<, zlib-$(zlib-version)) \ + && echo "Zlib $(zlib-version)" > $@ + # GNU Tar: When built statically, tar gives a segmentation fault on -# unpacking Bash. So we'll build it dynamically. +# unpacking Bash. So we'll build it dynamically. Note that technically, zip +# and unzip aren't dependencies of Tar, but for a clean build, we'll set +# Tar to be the last compression-related software (the first-set of +# software to be built). $(ibidir)/tar: $(tdir)/tar-$(tar-version).tar.gz \ $(ibidir)/bzip2 \ - $(ibidir)/lzip \ + $(ibidir)/unzip \ $(ibidir)/gzip \ + $(ibidir)/lzip \ + $(ilidir)/zlib \ + $(ibidir)/zip \ $(ibidir)/xz # Since all later programs depend on Tar, the configuration will be # stuck here, only making Tar. So its more efficient to built it on # multiple threads (when the user's Make doesn't pass down the # number of threads). - $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads)) \ + $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads) V=1) \ && echo "GNU Tar $(tar-version)" > $@ @@ -555,22 +593,11 @@ $(ibidir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ # Downloader # ---------- # -# Some programs (like Wget and CMake) that use zlib need it to be dynamic -# so they use our custom build. So we won't force a static-only build. -# -# Note for a static-only build: Zlib's `./configure' doesn't use Autoconf's -# configure script, it just accepts a direct `--static' option. -$(idir)/etc:; mkdir $@ -$(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \ - $(ibidir)/bash - $(call gbuild, $<, zlib-$(zlib-version)) \ - && echo "Zlib $(zlib-version)" > $@ - # OpenSSL: Some programs/libraries later need dynamic linking. So we'll # build libssl (and libcrypto) dynamically also. # # Until we find a nice and generic way to create an updated CA file in the -# project, the certificates will be available in a file for this pipeline +# project, the certificates will be available in a file for this project # along with the other tarballs. # # In case you do want a static OpenSSL and libcrypto, then uncomment the @@ -579,9 +606,10 @@ $(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \ #ifeq ($(static_build),yes) #openssl-static = no-dso no-dynamic-engine no-shared #endif +$(idir)/etc:; mkdir $@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ $(tdir)/cert.pem \ - $(ilidir)/zlib | $(idir)/etc + $(ibidir)/bash | $(idir)/etc # According to OpenSSL's Wiki (link bellow), it can't automatically # detect Mac OS's structure. It will need some help. So we'll use # the `on_mac_os' Make variable that we defined in the configure @@ -763,13 +791,15 @@ $(ibidir)/which: $(tdir)/which-$(which-version).tar.gz \ # Binutils' linker `ld' is apparently only good for GNU/Linux systems and # other OSs have their own. So for now we aren't actually building # Binutils (`ld' isn't a prerequisite of GCC). -$(ibidir)/binutils: $(tdir)/binutils-$(binutils-version).tar.lz +$(ibidir)/binutils: $(tdir)/binutils-$(binutils-version).tar.lz \ + $(ibidir)/bash $(call gbuild, $<, binutils-$(binutils-version), static) \ && echo "GNU Binutils $(binutils-version)" > $@ # `file' is not a prerequisite of GCC. However, since it is low level, it is # set as a prerequisite of GCC to have it installed. -$(ibidir)/file: $(tdir)/file-$(file-version).tar.gz +$(ibidir)/file: $(tdir)/file-$(file-version).tar.gz \ + $(ibidir)/bash $(call gbuild, $<, file-$(file-version), static) \ && echo "File $(file-version)" > $@ diff --git a/reproduce/src/make/dependencies-build-rules.mk b/reproduce/software/make/build-rules.mk index a8c8731..a8c8731 100644 --- a/reproduce/src/make/dependencies-build-rules.mk +++ b/reproduce/software/make/build-rules.mk diff --git a/reproduce/src/make/dependencies.mk b/reproduce/software/make/high-level.mk index fd9bffa..f7b1f4f 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/software/make/high-level.mk @@ -26,23 +26,23 @@ # Top level environment -include reproduce/config/pipeline/LOCAL.mk -include reproduce/src/make/dependencies-build-rules.mk -include reproduce/config/pipeline/dependency-texlive.mk -include reproduce/config/pipeline/dependency-versions.mk +include reproduce/software/make/build-rules.mk +include reproduce/software/config/installation/LOCAL.mk +include reproduce/software/config/installation/texlive.mk +include reproduce/software/config/installation/versions.mk lockdir = $(BDIR)/locks -ddir = $(BDIR)/dependencies -dtexdir = $(shell pwd)/tex/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib -ibidir = $(BDIR)/dependencies/installed/version-info/bin -ilidir = $(BDIR)/dependencies/installed/version-info/lib -itidir = $(BDIR)/dependencies/installed/version-info/tex -ictdir = $(BDIR)/dependencies/installed/version-info/cite -ipydir = $(BDIR)/dependencies/installed/version-info/python +tdir = $(BDIR)/software/tarballs +ddir = $(BDIR)/software/build-tmp +idir = $(BDIR)/software/installed +ibdir = $(BDIR)/software/installed/bin +ildir = $(BDIR)/software/installed/lib +dtexdir = $(shell pwd)/reproduce/software/bibtex +ibidir = $(BDIR)/software/installed/version-info/bin +ilidir = $(BDIR)/software/installed/version-info/lib +itidir = $(BDIR)/software/installed/version-info/tex +ictdir = $(BDIR)/software/installed/version-info/cite +ipydir = $(BDIR)/software/installed/version-info/python # Define the top-level programs to build (installed in `.local/bin'). # @@ -51,8 +51,8 @@ ipydir = $(BDIR)/dependencies/installed/version-info/python # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. top-level-libraries = # atlas -top-level-programs = gnuastro metastore unzip zip top-level-python = astroquery matplotlib +top-level-programs = gnuastro metastore all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ $(foreach p, $(top-level-python), $(ipydir)/$(p)) \ @@ -87,15 +87,21 @@ export LDFLAGS := $(rpath_command) -L$(ildir) # We want the download to happen on a single thread. So we need to define a # lock, and call a special script we have written for this job. These are -# placed here because we want them both in the `dependencies.mk' and -# `dependencies-python.mk'. +# placed here because we want them both in the `high-level.mk' and +# `python.mk'. $(lockdir): | $(BDIR); mkdir $@ downloader="wget --no-use-server-timestamps -O"; -downloadwrapper = ./reproduce/src/bash/download-multi-try +downloadwrapper = ./reproduce/analysis/bash/download-multi-try + + + # Python packages -include reproduce/src/make/dependencies-python.mk +include reproduce/software/make/python.mk + + + # Tarballs @@ -127,11 +133,9 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libgit2-$(libgit2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ - unzip-$(unzip-version).tar.gz \ openblas-$(openblas-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ - zip-$(zip-version).tar.gz \ , $(tdir)/$(t) ) $(tarballs): $(tdir)/%: | $(lockdir) if [ -f $(DEPENDENCIES-DIR)/$* ]; then @@ -156,7 +160,12 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = atlas ]; then mergenames=0 w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download - elif [ $$n = cmake ]; then w=https://cmake.org/files/v3.12 + elif [ $$n = cmake ]; then + mergenames=0 + majv=$$(echo $(cmake-version) \ + | sed -e's/\./ /' \ + | awk '{printf("%d.%d", $$1, $$2)}') + w=https://cmake.org/files/v$$majv/cmake-$(cmake-version).tar.gz elif [ $$n = curl ]; then w=https://curl.haxx.se/download elif [ $$n = fftw ]; then w=ftp://ftp.fftw.org/pub/fftw elif [ $$n = freetype ]; then w=https://download.savannah.gnu.org/releases/freetype @@ -185,13 +194,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff - elif [ $$n = unzip ]; then w=ftp://ftp.info-zip.org/pub/infozip/src - mergenames=0; v=$$(echo $(unzip-version) | sed -e's/\.//') - w=ftp://ftp.info-zip.org/pub/infozip/src/unzip$$v.tgz elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib - elif [ $$n = zip ]; then - mergenames=0; v=$$(echo $(zip-version) | sed -e's/\.//') - w=ftp://ftp.info-zip.org/pub/infozip/src/zip$$v.tgz else echo; echo; echo; echo "'$$n' not recognized as a dependency name to download." @@ -469,19 +472,20 @@ $(ibidir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ # # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. - if [ x$(on_mac_os) = xyes ]; then \ - export CC=clang; \ - export CXX=clang++; \ - fi; \ - cd $(ddir) \ - && rm -rf cmake-$(cmake-version) \ - && tar xf $< && cd cmake-$(cmake-version) \ - && ./bootstrap --prefix=$(idir) --system-curl --system-zlib\ - --system-bzip2 --system-liblzma --no-qt-gui \ - && make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 \ - && make install \ - && cd .. \ - && rm -rf cmake-$(cmake-version) \ + if [ x$(on_mac_os) = xyes ]; then \ + export CC=clang; \ + export CXX=clang++; \ + fi; \ + cd $(ddir) \ + && rm -rf cmake-$(cmake-version) \ + && tar xf $< \ + && cd cmake-$(cmake-version) \ + && ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ + --system-bzip2 --system-liblzma --no-qt-gui \ + && make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 \ + && make install \ + && cd .. \ + && rm -rf cmake-$(cmake-version) \ && echo "CMake $(cmake-version)" > $@ # cURL (and its library, which is needed by several programs here) can @@ -573,7 +577,7 @@ $(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ -e's|@GROUP[@]|'$$group'|g' \ -e's|@BINDIR[@]|$(ibdir)|g' \ -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ - reproduce/src/bash/git-$$f > .git/hooks/$$f + reproduce/software/bash/git-$$f > .git/hooks/$$f chmod +x .git/hooks/$$f echo "Metastore (forked) $(metastore-version)" > $@ done @@ -609,23 +613,20 @@ endif && cp $(dtexdir)/gnuastro.tex $(ictdir)/ \ && echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ -$(ibidir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz - v=$$(echo $(unzip-version) | sed -e's/\.//') - $(call gbuild, $<, unzip$$v, static,, \ - -f unix/Makefile generic_gcc \ - CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ - -f unix/Makefile \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ - && echo "Unzip $(unzip-version)" > $@ -$(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz - v=$$(echo $(zip-version) | sed -e's/\.//') - $(call gbuild, $<, zip$$v, static,, \ - -f unix/Makefile generic_gcc \ - CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ - -f unix/Makefile \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ - && echo "Zip $(zip-version)" > $@ + + + + + + + + + + + + + @@ -639,7 +640,7 @@ $(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz # `texlive-ready-tlmgr' and use its contents to mark if we can use it or # not. $(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \ - reproduce/config/pipeline/texlive.conf + reproduce/software/config/installation/texlive.conf # Unpack, enter the directory, and install based on the given # configuration (prerequisite of this rule). @@ -649,7 +650,8 @@ $(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \ tar xf $(tdir)/install-tl-unx.tar.gz cd install-tl-* sed -e's|@installdir[@]|$(idir)|g' \ - $$topdir/reproduce/config/pipeline/texlive.conf > texlive.conf + $$topdir/reproduce/software/config/installation/texlive.conf \ + > texlive.conf # TeX Live's installation may fail due to any reason. But TeX Live # is optional (only necessary for building the final PDF). So we @@ -680,7 +682,7 @@ $(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \ # To keep things modular and simple, we'll break up the installation of TeX # Live itself (only very basic TeX and LaTeX) and the installation of its # necessary packages into two packages. -$(itidir)/texlive: reproduce/config/pipeline/dependency-texlive.mk \ +$(itidir)/texlive: reproduce/software/config/installation/texlive.mk \ $(itidir)/texlive-ready-tlmgr # To work with TeX live installation, we'll need the internet. diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/software/make/python.mk index 837b0ad..5ec99f0 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/software/make/python.mk @@ -415,7 +415,7 @@ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ else \ export LDFLAGS="$(LDFLAGS) -shared"; \ fi; \ - conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + conf="$$(pwd)/reproduce/software/config/installation/numpy-scipy.cfg"; \ $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf, \ Numpy $(numpy-version)) \ && cp $(dtexdir)/numpy.tex $(ictdir)/ \ @@ -463,7 +463,7 @@ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ else \ export LDFLAGS="$(LDFLAGS) -shared"; \ fi; \ - conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ + conf="$$(pwd)/reproduce/software/config/installation/numpy-scipy.cfg"; \ $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf) \ && cp $(dtexdir)/scipy.tex $(ictdir)/ \ && echo "Scipy $(scipy-version) \citep{scipy2007,scipy2011}" > $@ diff --git a/tex/src/delete-me-demo.tex b/tex/src/delete-me-demo.tex index abd45f6..67024bb 100644 --- a/tex/src/delete-me-demo.tex +++ b/tex/src/delete-me-demo.tex @@ -20,7 +20,7 @@ %% The displayed WFPC2 image. \node[anchor=south west] (img) at (0,0) {\includegraphics[width=0.5\linewidth] - {tex/pipeline/delete-me-demo/wfpc2.pdf}}; + {tex/build/delete-me-demo/wfpc2.pdf}}; %% Its label \node[anchor=south west] at (0.45\linewidth,0.45\linewidth) @@ -42,7 +42,7 @@ ] \addplot [const plot mark mid, fill=red] table [x index=0, y index=1] - {tex/pipeline/delete-me-demo/wfpc2-hist.txt} + {tex/build/delete-me-demo/wfpc2-hist.txt} \closedcycle; \end{axis} diff --git a/tex/src/delete-me.tex b/tex/src/delete-me.tex index 12fd3b7..552e92a 100644 --- a/tex/src/delete-me.tex +++ b/tex/src/delete-me.tex @@ -26,7 +26,7 @@ %% A particular plot. \addplot+[scatter, only marks] - table {tex/pipeline/delete-me/data.txt}; + table {tex/build/delete-me/data.txt}; \end{axis} \end{tikzpicture} diff --git a/tex/src/preamble-biblatex.tex b/tex/src/preamble-biblatex.tex index b4c02f9..ca54f90 100644 --- a/tex/src/preamble-biblatex.tex +++ b/tex/src/preamble-biblatex.tex @@ -56,7 +56,7 @@ \DeclareFieldFormat[article]{pages}{#1} \DeclareFieldFormat{pages}{\mkfirstpage[{\mkpageprefix[bookpagination]}]{#1}} \addbibresource{tex/src/references.tex} -\addbibresource{tex/pipeline/macros/dependencies-bib.tex} +\addbibresource{tex/build/macros/dependencies-bib.tex} \renewbibmacro{in:}{} \renewcommand*{\bibfont}{\footnotesize} \DefineBibliographyStrings{english}{references = {References}} diff --git a/tex/src/preamble-necessary.tex b/tex/src/preamble-necessary.tex index 2abcb9e..1deb8ec 100644 --- a/tex/src/preamble-necessary.tex +++ b/tex/src/preamble-necessary.tex @@ -24,7 +24,7 @@ %% Values from the analysis. -\input{tex/pipeline/macros/pipeline.tex} +\input{tex/build/macros/project.tex} |