diff options
| -rw-r--r-- | .dir-locals.el | 4 | ||||
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | README-hacking.md | 37 | ||||
| -rw-r--r-- | README.md | 16 | ||||
| -rwxr-xr-x | project | 101 | ||||
| -rw-r--r-- | reproduce/analysis/make/initialize.mk | 10 | ||||
| -rw-r--r-- | reproduce/software/config/installation/LOCAL.conf.in | 1 | ||||
| -rw-r--r-- | reproduce/software/make/README.md | 2 | ||||
| -rw-r--r-- | reproduce/software/make/basic.mk | 153 | ||||
| -rw-r--r-- | reproduce/software/make/high-level.mk | 13 | ||||
| -rwxr-xr-x | reproduce/software/shell/bashrc.sh (renamed from reproduce/software/bash/bashrc.sh) | 0 | ||||
| -rwxr-xr-x | reproduce/software/shell/configure.sh (renamed from reproduce/software/bash/configure.sh) | 842 | ||||
| -rwxr-xr-x | reproduce/software/shell/git-post-checkout (renamed from reproduce/software/bash/git-post-checkout) | 0 | ||||
| -rwxr-xr-x | reproduce/software/shell/git-pre-commit (renamed from reproduce/software/bash/git-pre-commit) | 0 |
14 files changed, 583 insertions, 599 deletions
diff --git a/.dir-locals.el b/.dir-locals.el index 8c3e759..dae2993 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -9,8 +9,8 @@ ;; ;; For more information see (info "(emacs) Directory Variables") ;; -;; Copyright (C) 2017 Mosè Giordano <mose@gnu.org> -;; Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +;; Copyright (C) 2017-2020 Mosè Giordano <mose@gnu.org> +;; Copyright (C) 2018-2020 Mohammad Akhlaghi <mohammad@akhlaghi.org> ;; ;; Copying and distribution of this file, with or without modification, ;; are permitted in any medium without royalty provided the copyright @@ -9,7 +9,7 @@ # To help readability, please add new files based on the length of the # line you wish to add. # -# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2020 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # Copying and distribution of this file, with or without modification, # are permitted in any medium without royalty provided the copyright @@ -43,3 +43,4 @@ tex/pipeline LOCAL_tmp.mk LOCAL_old.mk gnuastro-local.conf +.compiler_test_dir_please_delete
\ No newline at end of file diff --git a/README-hacking.md b/README-hacking.md index 33d129b..8e88547 100644 --- a/README-hacking.md +++ b/README-hacking.md @@ -280,28 +280,20 @@ example if you run `.local/bin/ls` you will be using the `ls` of the template, which is problably different from your system's `ls` (run them both with `--version` to check). -Once the project is configured for your system, `./project prepare` and -`./project make` will do the basic preparations and run the project's -analysis with the custom version of software. The `project` script is just -a wrapper, and with the commands above, it will call `top-prepare.mk` and -`top-make.mk` (both are in the `reproduce/analysis/make` directory). - -In the template, no particular preparation is necessary, so it will -immediately finish and instruct you to run `./project make`. But in some -projects, it can be very useful to do some very basic preparatory steps on -the input data that can greatly optimize running of `./project make`. For -example, you may need to query a server, to find how many input files there -are. Once that number is known in the preparation phase, `./project make` -can parallelize the analysis much more effectively. +Once the project is configured for your system, `./project make` will do +the basic preparations and run the project's analysis with the custom +version of software. The `project` script is just a wrapper, and with the +`make` argument, it will first call `top-prepare.mk` and `top-make.mk` +(both are in the `reproduce/analysis/make` directory). In terms of organization, `top-prepare.mk` and `top-make.mk` have an -identical design, only a minor difference. So, let's continue the -template's architecture with `top-make.mk`. Once you understand that, -you'll clearly understand `top-prepare.mk` also. These very high-level -files are relatively short and heavily commented so hopefully the -descriptions in each comment will be enough to understand the general -details. As you read this section, please also look at the contents of the -mentioned files and directories to fully understand what is going on. +identical design, only minor differences. So, let's continue the template's +architecture with `top-make.mk`. Once you understand that, you'll clearly +understand `top-prepare.mk` also. These very high-level files are +relatively short and heavily commented so hopefully the descriptions in +each comment will be enough to understand the general details. As you read +this section, please also look at the contents of the mentioned files and +directories to fully understand what is going on. Before starting to look into the top `top-make.mk`, it is important to recall that Make defines dependencies by files. Therefore, the @@ -588,7 +580,6 @@ First custom commit ```shell $ ./project configure # Build the project's software environment (can take an hour or so). - $ ./project prepare # Pre-processing preparations (doing nothing in the raw template). $ ./project make # Do the processing and build paper (just a simple demo in the template). # Open 'paper.pdf' and see if everything is ok. @@ -982,8 +973,8 @@ for the benefit of others. - *Environment of each recipe*: If you need to define a special environment (or alises, or scripts to run) for all the recipes in - your Makefiles, you can use the Bash startup file - `reproduce/software/bash/bashrc.sh`. This file is loaded before every + your Makefiles, you can use a Bash startup file + `reproduce/software/shell/bashrc.sh`. This file is loaded before every Make recipe is run, just like the `.bashrc` in your home directory is loaded everytime you start a new interactive, non-login terminal. See the comments in that file for more. @@ -21,7 +21,6 @@ received this source from arXiv, please see the respective section below. $ git clone XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX $ cd XXXXXXXXXXXXXXXXXX $ ./project configure -$ ./project prepare $ ./project make ``` @@ -77,20 +76,7 @@ requiring root/administrator permissions. $ ./project configure ``` -3. In some cases, the project's analysis may need some preparations to - optimize its processing. This is usually mainly related to input data, - and some very basic calculations that can help the management of the - overall lproject in the main/next step. To do the basic preparations, - please run this command to do the preparation on `8` threads. If your - CPU has a different number of threads, change the number (you can see - the number of threads available to your operating system by running - `./.local/bin/nproc`) - - ```shell - $ ./project prepare -j8 - ``` - -4. Run the following command to reproduce all the analysis and build the +3. Run the following command to reproduce all the analysis and build the final `paper.pdf` on `8` threads. If your CPU has a different number of threads, change the number (you can see the number of threads available to your operating system by running `./.local/bin/nproc`) @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # # High-level script to manage the project. # Run `./project --help' for a description of how to use it. @@ -66,14 +66,12 @@ print_help() { # Print the output. cat <<EOF Usage: $scriptname configure [OPTIONS] - $scriptname prepare [OPTIONS] $scriptname make [OPTIONS] Top-level script to manage the reproducible project. The high-level operation is defined by the (mandatory) second argument: configure - Configure project for this machine (e.g., build software). - prepare - Low-level preparations to optimize building with 'make'. make - Run the project (do analysis and build outputs). RECOMMENDATION: If this is the first time you are configuring this @@ -146,12 +144,11 @@ func_operation_set() { fi } -while [[ $# -gt 0 ]] +while [ $# -gt 0 ] do case $1 in # Main operation. configure) func_operation_set $1; shift;; - prepare) func_operation_set $1; shift;; make) func_operation_set $1; shift;; @@ -163,8 +160,8 @@ do -e*|--existing-conf=*) on_off_option_error --existing-conf -e;; --host-cc) host_cc=1; shift;; --host-cc=*) on_off_option_error --host-cc;; - -i|--inputdir) input_dir="$2"; check_v "$1" "$input_dir"; shift;shift;; - -i=*|--inputdir=*) input_dir="${1#*=}"; check_v "$1" "$input_dir"; shift;; + -i|--input-dir) input_dir="$2"; check_v "$1" "$input_dir"; shift;shift;; + -i=*|--input-dir=*) input_dir="${1#*=}"; check_v "$1" "$input_dir"; shift;; -i*) input_dir=$(echo "$1" | sed -e's/-i//'); check_v "$1" "$input_dir"; shift;; -m|--minmapsize) minmapsize="$2"; check_v "$1" "$minmapsize"; shift;shift;; -m=*|--minmapsize=*) minmapsize="${1#*=}"; check_v "$1" "$minmapsize"; shift;; @@ -212,8 +209,8 @@ if ! [ x$check_config = x ]; then # Find the color option to pass to `ls'. Note that `--color' (for GNU # Coreutils `ls') should be checked first because it also has `-G', but # for something else. - if ls --color &> /dev/null; then coloropt="--color=auto" - elif ls -G &> /dev/null; then coloropt="-G" + if ls --color 2> /dev/null > /dev/null; then coloropt="--color=auto" + elif ls -G 2> /dev/null > /dev/null; then coloropt="-G" else coloropt="" fi @@ -294,48 +291,10 @@ controlled_env() { -# Error messages -# -------------- -# -# Having the error messages here helps the over-all process be more -# readable. -print_error_abort() { - case $1 in - prepare) - cat <<EOF - -The project isn't configured for this system, or the configuration wasn't -successful. To configure the project, please use this command: - - $ ./project configure - -(TIP: if you have already ran this command once, run it with '-e' to use -the previous configuration, run with '--help' for more info) - -EOF - exit 1; - ;; - make) - cat <<EOF - -The project preparation hasn't been completed, or it wasn't successful. To -prepare the project prior to building it, please use this command: - - $ ./project prepare - -EOF - exit 1; - ;; - esac -} - - - - # Do requested operation # ---------------------- perms="u+r,u+w,g+r,g+w,o-r,o-w,o-x" -configscript=./reproduce/software/bash/configure.sh +configscript=./reproduce/software/shell/configure.sh case $operation in # Build the project's software. @@ -355,7 +314,7 @@ case $operation in # `reproduce/*/bash' should need executable flags, so we are giving # them executable flags by default. If any other file in your project # needs such flags, add them here. - chmod +x reproduce/software/bash/* reproduce/analysis/bash/* + chmod +x reproduce/software/shell/* reproduce/analysis/bash/* # If the user requested, clean the TeX directory from the extra # (to-be-built) directories that may already be there (and will not @@ -398,32 +357,39 @@ case $operation in - # Run the input management. - prepare) + # Run the project. + make) # Make sure the configure script has been completed properly # (`configuration-done.txt' exists). if ! [ -f .build/software/configuration-done.txt ]; then - print_error_abort $operation - fi + cat <<EOF - # Run input-preparations in control environment - controlled_env reproduce/analysis/make/top-prepare.mk - ;; +The project is either (1) not configured on this system, or (2) the +configuration wasn't successful. +(1) If it hasn't been configured at all, use the command below to configure +it (set a build directory and let it build its necessary software in it). + $ ./project configure +(2) If it has been configured, but the configuration failed in a step, you +can re-configure it using your previous settings with the command +below. All successful steps will be skipped, allowing a fast completion. + $ ./project configure -e - # Run the project - make) +If there was a problem, please let us know by filling this online form: + http://savannah.nongnu.org/support/?func=additem&group=reproduce - # Make sure the configure script has been completed properly - # (`configuration-done.txt' exists). - if ! [ -f .build/software/preparation-done.mk ]; then - print_error_abort $operation +EOF + exit 1 fi + # Run input-preparations (optionally build Makefiles with special + # values for optimizing the main `top-make.mk'). + controlled_env reproduce/analysis/make/top-prepare.mk + # Run the actual project. controlled_env reproduce/analysis/make/top-make.mk ;; @@ -434,9 +400,14 @@ case $operation in # Operation not specified. *) - echo "No operation defined." - echo "Please run with '--help' for more information." - echo "Available operations are: 'configure', 'prepare', or 'make')." + cat <<EOF + +No operation defined! + +Please run with '--help' for more information. +(TIP: available operations are: 'configure' or 'make'). + +EOF exit 1 ;; esac diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index fa39699..79f9266 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -140,6 +140,12 @@ export SHELL := $(installdir)/bin/bash export CPPFLAGS := -I$(installdir)/include export LD_LIBRARY_PATH := $(installdir)/lib +# Until we build our own C library, without this, the project's GCC won't +# be able to compile anything if the host C library isn't in a standard +# place: in particular Debian-based operatings sytems. On other systems, it +# will be empty. +export CPATH := $(SYS_CPATH) + # RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is # ultimately redundant. But on some systems, even having a single value # causes crashs (see bug #56682). So we'll just give it no value at all. @@ -150,9 +156,9 @@ export DYLD_LIBRARY_PATH := # remote-shell accesss through this environment variable. export OMPI_MCA_plm_rsh_agent=/bin/false -# Recipe startup script, see `reproduce/software/bash/bashrc.sh'. +# Recipe startup script. export PROJECT_STATUS := make -export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh diff --git a/reproduce/software/config/installation/LOCAL.conf.in b/reproduce/software/config/installation/LOCAL.conf.in index 1045f72..0579218 100644 --- a/reproduce/software/config/installation/LOCAL.conf.in +++ b/reproduce/software/config/installation/LOCAL.conf.in @@ -12,5 +12,6 @@ BDIR = @bdir@ INDIR = @indir@ DEPENDENCIES-DIR = @ddir@ +SYS_CPATH = @sys_cpath@ DOWNLOADER = @downloader@ GROUP-NAME = @groupname@ diff --git a/reproduce/software/make/README.md b/reproduce/software/make/README.md index fc04dfb..d71a3d5 100644 --- a/reproduce/software/make/README.md +++ b/reproduce/software/make/README.md @@ -2,7 +2,7 @@ Software building instructions ============================== This directory contains Makefiles that are called by the high-level -`reproduce/software/bash/configure.sh` script. The main target for the +`reproduce/software/shell/configure.sh` script. The main target for the installation of each software is a simple plain text file that contains the name of the software and its version (which is put in the paper in the end). Once built, these plain-text files are all put in the proper diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 93dcdc2..d08ba62 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -48,7 +48,7 @@ ibidir = $(BDIR)/software/installed/version-info/proglib # We'll need the system's PATH for making links to low-level programs we # won't be building ourselves. -syspath := $(PATH) +syspath := $(PATH) # As we build more programs, we want to use this project's built programs # and libraries, not the host's. @@ -58,7 +58,15 @@ export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) -export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH) \ + +# This is the "basic" tools where we are relying on the host operating +# system, but are slowly populating our basic software envirnoment. To run +# (system or template) programs, `LD_LIBRARY_PATH' is necessary, so here, +# we'll first tell the programs to look into any possible pre-defined +# `LD_LIBRARY_PATH', then we'll add our own newly installed libraries. We +# will also make sure that there is no "current directory" in it (by +# removing a starting or trailing `:' and any occurance of `::'. +export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH):$(ildir) \ | sed -e's/::/:/g' -e's/^://' -e's/:$$//') # RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is @@ -66,9 +74,9 @@ export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH) \ # causes crashs (see bug #56682). So we'll just give it no value at all. export DYLD_LIBRARY_PATH := -# Recipe startup script, see `reproduce/software/bash/bashrc.sh'. +# Recipe startup script. export PROJECT_STATUS := configure_basic -export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh # Define the top-level basic programs (that don't depend on any other). top-level-programs = low-level-links gcc @@ -658,6 +666,66 @@ $(ibidir)/bash: $(ibidir)/readline \ +# The `-shared' flag will cause problems while building Perl on macOS, so +# we'll only use this configuration option when we are GNU/Linux +# systems. However, since the whole option must be used (which includes `=' +# and empty space), its easier to define the variable as a Make variable +# outside the recipe, not as a shell variable inside it. +ifeq ($(on_mac_os),yes) +perl-conflddlflags = +else +perl-conflddlflags = -Dlddlflags="-shared $$LDFLAGS" +endif +$(ibidir)/perl: | $(ibidir)/bash \ + $(tdir)/perl-$(perl-version).tar.gz + major_version=$$(echo $(perl-version) \ + | sed -e's/\./ /g' \ + | awk '{printf("%d", $$1)}'); \ + base_version=$$(echo $(perl-version) \ + | sed -e's/\./ /g' \ + | awk '{printf("%d.%d", $$1, $$2)}'); \ + cd $(ddir) \ + && rm -rf perl-$(perl-version) \ + && if ! tar xf $(word 1,$(filter $(tdir)/%,$|)); then \ + echo; echo "Tar error"; exit 1; \ + fi \ + && cd perl-$(perl-version) \ + && sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ + -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ + Configure > Configure-tmp \ + && mv -f Configure-tmp Configure \ + && chmod +x Configure \ + && ./Configure -des \ + -Dusethreads \ + -Duseshrplib \ + -Dprefix=$(idir) \ + -Dvendorprefix=$(idir) \ + -Dprivlib=$(idir)/share/perl$$major_version/core_perl \ + -Darchlib=$(idir)/lib/perl$$major_version/$$base_version/core_perl \ + -Dsitelib=$(idir)/share/perl$$major_version/site_perl \ + -Dsitearch=$(idir)/lib/perl$$major_version/$$basever/site_perl \ + -Dvendorlib=$(idir)/share/perl$$major_version/vendor_perl \ + -Dvendorarch=$(idir)/lib/perl$$major_version/$$base_version/vendor_perl \ + -Dscriptdir=$(idir)/bin/core_perl \ + -Dsitescript=$(idir)/bin/site_perl \ + -Dvendorscript=$(idir)/bin/vendor_perl \ + -Dinc_version_list=none \ + -Dman1ext=1perl \ + -Dman3ext=3perl \ + -Dcccdlflags='-fPIC' \ + $(perl-conflddlflags) \ + -Dldflags="$$LDFLAGS" \ + && make SHELL=$(ibdir)/bash -j$(numthreads) \ + && make SHELL=$(ibdir)/bash install \ + && cd .. \ + && rm -rf perl-$(perl-version) \ + && cd $$topdir \ + && echo "Perl $(perl-version)" > $@ + + + + + # Coreutils # --------- # @@ -677,8 +745,10 @@ $(ibidir)/bash: $(ibidir)/readline \ # The echo after the PatchELF loop is to avoid a crash if the last # file that PatchELF encounters is not usable (and it returns with # an error). +# +# Coreutils uses Perl to create man pages! $(ibidir)/coreutils: $(ibidir)/openssl \ - | $(ibidir)/bash \ + | $(ibidir)/perl \ $(tdir)/coreutils-$(coreutils-version).tar.xz cd $(ddir) \ && rm -rf coreutils-$(coreutils-version) \ @@ -847,17 +917,17 @@ $(ibidir)/wget: $(ibidir)/libiconv \ # there is no access to the system's PATH. $(ibidir)/diffutils: | $(ibidir)/coreutils \ $(tdir)/diffutils-$(diffutils-version).tar.xz - $(call gbuild, diffutils-$(diffutils-version), static, , V=1) \ + $(call gbuild, diffutils-$(diffutils-version), static,,V=1) \ && echo "GNU Diffutils $(diffutils-version)" > $@ $(ibidir)/file: | $(ibidir)/coreutils \ $(tdir)/file-$(file-version).tar.gz - $(call gbuild, file-$(file-version), static) \ + $(call gbuild, file-$(file-version), static,,V=1) \ && echo "File $(file-version)" > $@ $(ibidir)/findutils: | $(ibidir)/coreutils \ $(tdir)/findutils-$(findutils-version).tar.xz - $(call gbuild, findutils-$(findutils-version), static, , V=1) \ + $(call gbuild, findutils-$(findutils-version), static,,V=1) \ && echo "GNU Findutils $(findutils-version)" > $@ $(ibidir)/gawk: $(ibidir)/gmp \ @@ -910,13 +980,13 @@ $(ibidir)/gmp: | $(ibidir)/m4 \ $(ibidir)/glibtool: | $(ibidir)/m4 \ $(tdir)/libtool-$(libtool-version).tar.xz $(call gbuild, libtool-$(libtool-version), static, \ - --program-prefix=g) \ + --program-prefix=g, V=1) \ && ln -s $(ibdir)/glibtoolize $(ibdir)/libtoolize \ && echo "GNU Libtool $(libtool-version)" > $@ $(ibidir)/grep: | $(ibidir)/coreutils \ $(tdir)/grep-$(grep-version).tar.xz - $(call gbuild, grep-$(grep-version), static) \ + $(call gbuild, grep-$(grep-version), static,,V=1) \ && echo "GNU Grep $(grep-version)" > $@ $(ibidir)/libbsd: | $(ibidir)/coreutils \ @@ -927,7 +997,7 @@ $(ibidir)/libbsd: | $(ibidir)/coreutils \ $(ibidir)/m4: | $(ibidir)/coreutils \ $(ibidir)/texinfo \ $(tdir)/m4-$(m4-version).tar.gz - $(call gbuild, m4-$(m4-version), static) \ + $(call gbuild, m4-$(m4-version), static,,V=1) \ && echo "GNU M4 $(m4-version)" > $@ # Metastore is used (through a Git hook) to restore the source modification @@ -997,7 +1067,7 @@ $(ibidir)/metastore: $(needlibbsd) \ -e's|@GROUP[@]|'$$group'|g' \ -e's|@BINDIR[@]|$(ibdir)|g' \ -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ - reproduce/software/bash/git-$$f > .git/hooks/$$f \ + reproduce/software/shell/git-$$f > .git/hooks/$$f \ && chmod +x .git/hooks/$$f; \ done; \ fi \ @@ -1020,63 +1090,6 @@ $(ibidir)/mpfr: $(ibidir)/gmp \ $(call gbuild, mpfr-$(mpfr-version), static, , , make check) \ && echo "GNU Multiple Precision Floating-Point Reliably $(mpfr-version)" > $@ -# The `-shared' flag will cause problems while building Perl on macOS, so -# we'll only use this configuration option when we are GNU/Linux -# systems. However, since the whole option must be used (which includes `=' -# and empty space), its easier to define the variable as a Make variable -# outside the recipe, not as a shell variable inside it. -ifeq ($(on_mac_os),yes) -perl-conflddlflags = -else -perl-conflddlflags = -Dlddlflags="-shared $$LDFLAGS" -endif -$(ibidir)/perl: | $(ibidir)/coreutils \ - $(tdir)/perl-$(perl-version).tar.gz - major_version=$$(echo $(perl-version) \ - | sed -e's/\./ /g' \ - | awk '{printf("%d", $$1)}'); \ - base_version=$$(echo $(perl-version) \ - | sed -e's/\./ /g' \ - | awk '{printf("%d.%d", $$1, $$2)}'); \ - cd $(ddir) \ - && rm -rf perl-$(perl-version) \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$|)); then \ - echo; echo "Tar error"; exit 1; \ - fi \ - && cd perl-$(perl-version) \ - && sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ - Configure > Configure-tmp \ - && mv -f Configure-tmp Configure \ - && chmod +x Configure \ - && ./Configure -des \ - -Dusethreads \ - -Duseshrplib \ - -Dprefix=$(idir) \ - -Dvendorprefix=$(idir) \ - -Dprivlib=$(idir)/share/perl$$major_version/core_perl \ - -Darchlib=$(idir)/lib/perl$$major_version/$$base_version/core_perl \ - -Dsitelib=$(idir)/share/perl$$major_version/site_perl \ - -Dsitearch=$(idir)/lib/perl$$major_version/$$basever/site_perl \ - -Dvendorlib=$(idir)/share/perl$$major_version/vendor_perl \ - -Dvendorarch=$(idir)/lib/perl$$major_version/$$base_version/vendor_perl \ - -Dscriptdir=$(idir)/bin/core_perl \ - -Dsitescript=$(idir)/bin/site_perl \ - -Dvendorscript=$(idir)/bin/vendor_perl \ - -Dinc_version_list=none \ - -Dman1ext=1perl \ - -Dman3ext=3perl \ - -Dcccdlflags='-fPIC' \ - $(perl-conflddlflags) \ - -Dldflags="$$LDFLAGS" \ - && make SHELL=$(ibdir)/bash -j$(numthreads) \ - && make SHELL=$(ibdir)/bash install \ - && cd .. \ - && rm -rf perl-$(perl-version) \ - && cd $$topdir \ - && echo "Perl $(perl-version)" > $@ - - $(ibidir)/pkg-config: | $(ibidir)/coreutils \ $(tdir)/pkg-config-$(pkgconfig-version).tar.gz # An existing `libiconv' can cause a conflict with `pkg-config', @@ -1101,7 +1114,7 @@ $(ibidir)/pkg-config: | $(ibidir)/coreutils \ $(ibidir)/sed: | $(ibidir)/coreutils \ $(tdir)/sed-$(sed-version).tar.xz - $(call gbuild, sed-$(sed-version), static) \ + $(call gbuild, sed-$(sed-version), static,,V=1) \ && echo "GNU Sed $(sed-version)" > $@ $(ibidir)/texinfo: | $(ibidir)/perl \ diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index bc77dab..53bef55 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -70,8 +70,8 @@ all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ .SHELLFLAGS := --noprofile --norc -ec export CCACHE_DISABLE := 1 export PATH := $(ibdir) -export CXX := $(ibdir)/g++ export CC := $(ibdir)/gcc +export CXX := $(ibdir)/g++ export SHELL := $(ibdir)/bash export F77 := $(ibdir)/gfortran export LD_RUN_PATH := $(ildir):$(il64dir) @@ -79,6 +79,11 @@ export PKG_CONFIG_PATH := $(ildir)/pkgconfig export LD_LIBRARY_PATH := $(ildir):$(il64dir) export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +# Until we build our own C library, without this, our GCC won't be able to +# compile anything! Note that on most systems (in particular +# non-Debian-based), `sys_cpath' will be empty. +export CPATH := $(sys_cpath) + # RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is # ultimately redundant. But on some systems, even having a single value # causes crashs (see bug #56682). So we'll just give it no value at all. @@ -91,9 +96,9 @@ export DYLD_LIBRARY_PATH := # for `ld'. export LIBRARY_PATH := $(sys_library_path) -# Recipe startup script, see `reproduce/software/bash/bashrc.sh'. +# Recipe startup script, see `reproduce/software/shell/bashrc.sh'. export PROJECT_STATUS := configure_highlevel -export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh # Building flags: # @@ -634,7 +639,7 @@ $(ibidir)/healpix: $(ibidir)/cfitsio \ && echo "HEALPix $(healpix-version) \citep{healpix}" > $@ $(ibidir)/libjpeg: | $(tdir)/jpegsrc.$(libjpeg-version).tar.gz - $(call gbuild, jpeg-9b, static) \ + $(call gbuild, jpeg-9b, static,,V=1) \ && echo "Libjpeg $(libjpeg-version)" > $@ $(ibidir)/libnsl: $(ibidir)/libtirpc \ diff --git a/reproduce/software/bash/bashrc.sh b/reproduce/software/shell/bashrc.sh index b2a151f..b2a151f 100755 --- a/reproduce/software/bash/bashrc.sh +++ b/reproduce/software/shell/bashrc.sh diff --git a/reproduce/software/bash/configure.sh b/reproduce/software/shell/configure.sh index c681298..e8f2736 100755 --- a/reproduce/software/bash/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -1,4 +1,4 @@ -#! /bin/bash +#! /bin/sh # # Necessary preparations/configurations for the reproducible project. # @@ -57,7 +57,8 @@ glconf=$cdir/gnuastro/gnuastro-local.conf # In case someone opens the files output from the configuration scripts in # a text editor and wants to edit them, it is important to let them know # that their changes are not going to be permenant. -function create_file_with_notice() { +create_file_with_notice () +{ if echo "# IMPORTANT: file can be RE-WRITTEN after './project configure'" > "$1" then echo "#" >> "$1" @@ -81,7 +82,8 @@ function create_file_with_notice() { # # Since the build directory will go into a symbolic link, we want it to be # an absolute address. With this function we can make sure of that. -function absolute_dir() { +absolute_dir () +{ if stat "$1" 1> /dev/null; then echo "$(cd "$(dirname "$1")" && pwd )/$(basename "$1")" else @@ -93,6 +95,398 @@ function absolute_dir() { +# Check for C/C++ compilers +# ------------------------- +# +# To build the software, we'll need some basic tools (the compilers in +# particular) to be present. +hascc=0; +if type cc > /dev/null 2>/dev/null; then + if type c++ > /dev/null 2>/dev/null; then export CC=cc; hascc=1; fi +else + if type gcc > /dev/null 2>/dev/null; then + if type g++ > /dev/null 2>/dev/null; then export CC=gcc; hascc=1; fi + else + if type clang > /dev/null 2>/dev/null; then + if type clang++ > /dev/null 2>/dev/null; then export CC=clang; hascc=1; fi + fi + fi +fi +if [ $hascc = 0 ]; then + cat <<EOF +______________________________________________________ +!!!!!!! C/C++ Compiler NOT FOUND !!!!!!! + +To build the project's software, the host system needs to have basic C and +C++ compilers. The executables that were checked are 'cc', 'gcc' and +'clang' for a C compiler, and 'c++', 'g++' and 'clang++' for a C++ +compiler. If you have a relevant compiler that is not checked, please get +in touch with us (with the form below) so we add it: + + https://savannah.nongnu.org/support/?func=additem&group=reproduce +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +EOF + exit 1 +fi + + + + + +# Special directory for compiler testing +# -------------------------------------- +# +# This directory will be deleted when the compiler testing is finished. +compilertestdir=.compiler_test_dir_please_delete +if ! [ -d $compilertestdir ]; then mkdir $compilertestdir; fi + + + + + +# Check C compiler +# ---------------- +gcc_works=0 +testprog=$compilertestdir/test +testsource=$compilertestdir/test.c +echo; echo; echo "Checking host C compiler..."; +cat > $testsource <<EOF +#include <stdio.h> +#include <stdlib.h> +int main(void){printf("...C compiler works.\n"); + return EXIT_SUCCESS;} +EOF +if $CC $testsource -o$testprog && $testprog; then + rm $testsource $testprog +else + rm $testsource + cat <<EOF + +______________________________________________________ +!!!!!!! C compiler doesn't work !!!!!!! + +Host C compiler ('gcc') can't build a simple program. + +A working C compiler is necessary for building the project's software. +Please use the error message above to find a good solution and re-run the +project configuration. + +If you can't find a solution, please send the error message above to the +link below and we'll try to help + +https://savannah.nongnu.org/support/?func=additem&group=reproduce + +TIP: Once you find the solution, you can use the '-e' option to use +existing configuration: + + $ ./project configure -e + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +EOF + exit 1 +fi + + + + + +# See if the linker accepts -Wl,-rpath-link +# ----------------------------------------- +# +# `-rpath-link' is used to write the information of the linked shared +# library into the shared object (library or program). But some versions of +# LLVM's linker don't accept it an can cause problems. +cat > $testsource <<EOF +#include <stdio.h> +#include <stdlib.h> +int main(void) {return EXIT_SUCCESS;} +EOF +if $CC $testsource -o$testprog -Wl,-rpath-link 2>/dev/null > /dev/null; then + export rpath_command="-Wl,-rpath-link=$instdir/lib" +else + export rpath_command="" +fi +rm -f $testprog $testsource + + + + + +# See if we need the dynamic-linker (-ldl) +# ---------------------------------------- +# +# Some programs (like Wget) need dynamic loading (using `libdl'). On +# GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But +# Mac OS doesn't need any explicit linking. So we'll check here to see if +# it is present (thus necessary) or not. +cat > $testsource <<EOF +#include <stdio.h> +#include <dlfcn.h> +int +main(void) { + void *handle=dlopen ("/lib/CEDD_LIB.so.6", RTLD_LAZY); + return 0; +} +EOF +if $CC $testsource -o$testprog 2>/dev/null > /dev/null; then + needs_ldl=no; +else + needs_ldl=yes; +fi +rm -f $testprog $testsource + + + + + +# See if the C compiler can build static libraries +# ------------------------------------------------ + +# We are manually only working with shared libraries: because some +# high-level programs like Wget and cURL need dynamic linking and if we +# build the libraries statically, our own builds will be ignored and these +# programs will go and find their necessary libraries on the host system. +# +# Another good advantage of shared libraries is that we can actually use +# the shared library tool of the system (`ldd' with GNU C Library) and see +# exactly where each linked library comes from. But in static building, +# unless you follow the build closely, its not easy to see if the source of +# the library came from the system or our build. +static_build=no + + + + + +# If we are on a Mac OS system +# ---------------------------- +# +# For the time being, we'll use the existance of `otool' to see if we are +# on a Mac OS system or not. Some tools (for example OpenSSL) need to know +# this. +# +# On Mac OS, the building of GCC crashes sometimes while building libiberty +# with CLang's `g++'. Until we find a solution, we'll just use the host's C +# compiler. +if type otool > /dev/null 2>/dev/null; then + host_cc=1 + on_mac_os=yes +else + host_cc=0 + on_mac_os=no +fi + + + + + +# Necessary C library element positions +# ------------------------------------- +# +# On some systems (in particular Debian-based OSs), the static C library +# and necessary headers in a non-standard place, and we can't build GCC. So +# we need to find them first. The `sys/cdefs.h' header is also in a +# similarly different location. +sys_cpath="" +sys_library_path="" +if [ x"$$on_mac_os" != xyes ]; then + + # Get the GCC target name of the compiler, when its given, special + # C libraries and headers are in a sub-directory of the host. + gcctarget=$(gcc -v 2>&1 \ + | tr ' ' '\n' \ + | awk '/\-\-target/' \ + | sed -e's/\-\-target=//') + if [ x"$gcctarget" != x ]; then + if [ -f /usr/lib/$gcctarget/libc.a ]; then + export sys_library_path=/usr/lib/$gcctarget + export sys_cpath=/usr/include/$gcctarget + fi + fi + + # For a check: + #echo "sys_library_path: $sys_library_path" + #echo "sys_cpath: $sys_cpath" +fi + + + + + +# See if a link-able static C library exists +# ------------------------------------------ +# +# After building GCC, we must use PatchELF to correct its RPATHs. However, +# PatchELF links internally with `libstdc++'. So a dynamicly linked +# PatchELF cannot be used to correct the links to `libstdc++' in general +# (on some systems this causes no problem, but on others it doesn't!). +# +# However, to build a Static PatchELF, we need to be able to link with the +# static C library, which is not always available on some GNU/Linux +# systems. Therefore we need to check this here. If we can't build a static +# PatchELF, we won't build any GCC either. +if [ x"$host_cc" = x0 ]; then + echo; echo; echo "Checking if static C library is available..."; + cat > $testsource <<EOF +#include <stdio.h> +#include <stdlib.h> +#include <sys/cdefs.h> +int main(void){printf("...yes\n"); + return EXIT_SUCCESS;} +EOF + cc_call="$CC $testsource $CPPFLAGS $LDFLAGS -o$testprog -static -lc" + if $cc_call && $testprog; then + gccwarning=0 + good_static_libc=1 + rm $testsource $testprog + else + echo; echo "Compilation command:"; echo "$cc_call" + good_static_libc=0 + rm $testsource + gccwarning=1 + host_cc=1 + cat <<EOF + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!! Warning !!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +The 'sys/cdefs.h' header cannot be included, or a usable static C library +('libc.a', in any directory) cannot be used with the current settings of +this system. SEE THE ERROR MESSAGE ABOVE. + +Because of this, we can't build GCC. You either 1) don't have them, or 2) +the default system environment aren't enough to find them. + +1) If you don't have them, your operating system provides them as separate +packages that you must manually install. Please look into your operating +system documentation or contact someone familiar with it. For example on +some Redhat-based GNU/Linux distributions, the static C library package can +be installed with this command: + + $ sudo yum install glibc-static + +2) If you have 'libc.a' and 'sys/cdefs.h', but in a non-standard location (for +example in '/PATH/TO/STATIC/LIBC/libc.a' and +'/PATH/TO/SYS/CDEFS_H/sys/cdefs.h'), please run the commands below, then +re-configure the project to fix this problem. + + $ export LDFLAGS="-L/PATH/TO/STATIC/LIBC \$LDFLAGS" + $ export CPPFLAGS="-I/PATH/TO/SYS/CDEFS_H \$LDFLAGS" + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +EOF + fi +fi + +# Print a warning if GCC is not meant to be built. +if [ x"$gccwarning" = x1 ]; then + cat <<EOF + +PLEASE SEE THE WARNINGS ABOVE. + +Since GCC is pretty low-level, this configuration script will continue in 5 +seconds and use your system's C compiler (it won't build a custom GCC). But +please consider installing the necessary package(s) to complete your C +compiler, then re-run './project configure'. + +EOF + sleep 5 +fi + + + + + +# Fortran compiler +# ---------------- +# +# If GCC is ultimately build within the project, the user won't need to +# have a fortran compiler, we'll build it internally for high-level +# programs. However, when the host C compiler is to be used, the user needs +# to have a Fortran compiler available. +if [ $host_cc = 1 ]; then + + # See if a Fortran compiler exists. + hasfc=0; + if type gfortran > /dev/null 2>/dev/null; then hasfc=1; fi + if [ $hasfc = 0 ]; then + cat <<EOF +______________________________________________________ +!!!!!!! Fortran Compiler NOT FOUND !!!!!!! + +Because the project won't be building its own GCC (which includes a Fortran +compiler), you need to have a Fortran compiler available. Fortran is +commonly necessary for many lower-level scientific programs. Currently we +search for 'gfortran'. If you have a Fortran compiler that is not checked, +please get in touch with us (with the form below) so we add it: + + https://savannah.nongnu.org/support/?func=additem&group=reproduce + +Note: GCC will not be built because you are either using the '--host-cc' +option, or you are using an operating system that currently has bugs when +building GCC. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +EOF + exit 1 + fi + + # See if the Fortran compiler works + testsource=$compilertestdir/test.f + echo; echo; echo "Checking host Fortran compiler..."; + echo " PRINT *, \"... Fortran Compiler works.\"" > $testsource + echo " END" >> $testsource + if gfortran $testsource -o$testprog && $testprog; then + rm $testsource $testprog + else + rm $testsource + cat <<EOF + +______________________________________________________ +!!!!!!! Fortran compiler doesn't work !!!!!!! + +Host Fortran compiler ('gfortran') can't build a simple program. + +A working Fortran compiler is necessary for building some of the project's +software. Please use the error message above to find a good solution and +re-run the project configuration. + +If you can't find a solution, please send the error message above to the +link below and we'll try to help + +https://savannah.nongnu.org/support/?func=additem&group=reproduce + +TIP: Once you find the solution, you can use the '-e' option to use +existing configuration: + + $ ./project configure -e + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +EOF + exit 1 + fi +fi + + + + + +# Delete the compiler testing directory +# ------------------------------------- +# +# This directory was made above to make sure the necessary compilers can be +# run. +rm -rf $compilertestdir + + + + + # Inform the user # --------------- # @@ -239,7 +633,7 @@ EOF bdir= currentdir=$(pwd) junkname=pure-junk-974adfkj38 - while [ x$bdir == x ] + while [ x$bdir = x ] do # Ask the user (if not already set on the command-line). if [ x"$build_dir" = x ]; then @@ -264,10 +658,12 @@ EOF fi fi - # Make sure the given directory is not a subdirectory of the - # source directory. + # If its given, make sure it isn't a subdirectory of the source + # directory. if ! [ x"$bdir" = x ]; then - if [[ $bdir == $currentdir* ]]; then + echo "Given build directory: $bdir" + if echo "$bdir/" \ + | grep '^'$currentdir 2> /dev/null > /dev/null; then # If it was newly created, it will be empty, so delete it. if ! [ "$(ls -A $bdir)" ]; then rm --dir $bdir; fi @@ -388,6 +784,7 @@ if [ $rewritepconfig = yes ]; then sed -e's|@bdir[@]|'"$bdir"'|' \ -e's|@indir[@]|'"$indir"'|' \ -e's|@ddir[@]|'"$ddir"'|' \ + -e's|@sys_cpath[@]|'"$sys_cpath"'|' \ -e's|@downloader[@]|'"$downloader"'|' \ -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \ $pconf.in >> $pconf @@ -631,401 +1028,6 @@ fi -# Check for C/C++ compilers -# ------------------------- -# -# To build the software, we'll need some basic tools (the compilers in -# particular) to be present. -hascc=0; -if type cc > /dev/null 2>/dev/null; then - if type c++ > /dev/null 2>/dev/null; then hascc=1; fi -else - if type gcc > /dev/null 2>/dev/null; then - if type g++ > /dev/null 2>/dev/null; then hascc=1; fi - else - if type clang > /dev/null 2>/dev/null; then - if type clang++ > /dev/null 2>/dev/null; then hascc=1; fi - fi - fi -fi -if [ $hascc = 0 ]; then - cat <<EOF -______________________________________________________ -!!!!!!! C/C++ Compiler NOT FOUND !!!!!!! - -To build the project's software, the host system needs to have basic C and -C++ compilers. The executables that were checked are 'cc', 'gcc' and -'clang' for a C compiler, and 'c++', 'g++' and 'clang++' for a C++ -compiler. If you have a relevant compiler that is not checked, please get -in touch with us (with the form below) so we add it: - - https://savannah.nongnu.org/support/?func=additem&group=reproduce -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -EOF - exit 1 -fi - - - - - -# Check C compiler -# ---------------- -gcc_works=0 -testprog=$tmpblddir/test-c -testsource=$tmpblddir/test.c -echo; echo; echo "Checking host C compiler..."; -echo "#include <stdio.h>" > $testsource -echo "#include <stdlib.h>" >> $testsource -echo "int main(void){printf(\"...C compiler works.\n\");" >> $testsource -echo " return EXIT_SUCCESS;}" >> $testsource -if gcc $testsource -o$testprog && $testprog; then - rm $testsource $testprog -else - rm $testsource - cat <<EOF - -______________________________________________________ -!!!!!!! C compiler doesn't work !!!!!!! - -Host C compiler ('gcc') can't build a simple program. - -A working C compiler is necessary for building the project's software. -Please use the error message above to find a good solution and re-run the -project configuration. - -If you can't find a solution, please send the error message above to the -link below and we'll try to help - -https://savannah.nongnu.org/support/?func=additem&group=reproduce - -TIP: Once you find the solution, you can use the '-e' option to use -existing configuration: - - $ ./project configure -e - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -EOF - exit 1 -fi - - - - - -# See if the linker accepts -Wl,-rpath-link -# ----------------------------------------- -# -# `-rpath-link' is used to write the information of the linked shared -# library into the shared object (library or program). But some versions of -# LLVM's linker don't accept it an can cause problems. -oprog=$sdir/rpath-test -cprog=$sdir/rpath-test.c -echo "#include <stdio.h>" > $cprog -echo "int main(void) {return 0;}" >> $cprog -if [ x$CC = x ]; then CC=gcc; fi; -if $CC $cprog -o$oprog -Wl,-rpath-link &> /dev/null; then - export rpath_command="-Wl,-rpath-link=$instdir/lib" -else - export rpath_command="" -fi -rm -f $oprog $cprog - - - - - -# See if we need the dynamic-linker (-ldl) -# ---------------------------------------- -# -# Some programs (like Wget) need dynamic loading (using `libdl'). On -# GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But -# Mac OS doesn't need any explicit linking. So we'll check here to see if -# it is present (thus necessary) or not. -oprog=$sdir/ldl-test -cprog=$sdir/ldl-test.c -cat > $cprog <<EOF -#include <stdio.h> -#include <dlfcn.h> -int -main(void) { - void *handle=dlopen ("/lib/CEDD_LIB.so.6", RTLD_LAZY); - return 0; -} -EOF -if gcc $cprog -o$oprog &> /dev/null; then needs_ldl=no; else needs_ldl=yes; fi -rm -f $oprog $cprog - - - - - -# See if the C compiler can build static libraries -# ------------------------------------------------ - -# We are manually only working with shared libraries: because some -# high-level programs like Wget and cURL need dynamic linking and if we -# build the libraries statically, our own builds will be ignored and these -# programs will go and find their necessary libraries on the host system. -# -# Another good advantage of shared libraries is that we can actually use -# the shared library tool of the system (`ldd' with GNU C Library) and see -# exactly where each linked library comes from. But in static building, -# unless you follow the build closely, its not easy to see if the source of -# the library came from the system or our build. -static_build=no - -#oprog=$sdir/static-test -#cprog=$sdir/static-test.c -#echo "#include <stdio.h>" > $cprog -#echo "int main(void) {return 0;}" >> $cprog -#if [ x$CC = x ]; then CC=gcc; fi; -#if $CC $cprog -o$oprog -static &> /dev/null; then -# export static_build="yes" -#else -# export static_build="no" -#fi -#rm -f $oprog $cprog -#if [ $printnotice = yes ] && [ $static_build = "no" ]; then -# cat <<EOF -#_________________________________________________________________________ -#!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!! -# -#Your system's C compiler ('$CC') doesn't support building static -#libraries. Therefore the dependencies will be built dynamically. This means -#that they will depend more strongly on changes/updates in the host -#system. For high-level applications (like most research projects in natural -#sciences), this shouldn't be a significant problem. -# -#But generally, for reproducibility, its better to build static libraries -#and programs. For more on their difference (and generally an introduction -#on linking), please see the link below: -# -#https://www.gnu.org/software/gnuastro/manual/html_node/Linking.html -# -#If you have other compilers on your system, you can select a different -#compiler by setting the 'CC' environment variable before running -#'./project configure'. -# -#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -# -#EOF -# sleep 5 -#fi - - - - - -# If we are on a Mac OS system -# ---------------------------- -# -# For the time being, we'll use the existance of `otool' to see if we are -# on a Mac OS system or not. Some tools (for example OpenSSL) need to know -# this. -# -# On Mac OS, the building of GCC crashes sometimes while building libiberty -# with CLang's `g++'. Until we find a solution, we'll just use the host's C -# compiler. -if type otool > /dev/null 2>/dev/null; then - host_cc=1 - on_mac_os=yes -else - host_cc=0 - on_mac_os=no -fi - - - - - -# Necessary C library element positions -# ------------------------------------- -# -# On some systems (in particular Debian-based OSs), the static C library -# and necessary headers in a non-standard place, and we can't build GCC. So -# we need to find them first. The `sys/cdefs.h' header is also in a -# similarly different location. -sys_cppflags="" -sys_library_path="" -if [ x"$$on_mac_os" != xyes ]; then - - # Get the GCC target name of the compiler, when its given, special - # C libraries and headers are in a sub-directory of the host. - gcctarget=$(gcc -v 2>&1 \ - | tr ' ' '\n' \ - | awk '/\-\-target/' \ - | sed -e's/\-\-target=//') - if [ x"$gcctarget" != x ]; then - if [ -f /usr/lib/$gcctarget/libc.a ]; then - export sys_library_path=/usr/lib/$gcctarget - export sys_cppflags=-I/usr/include/$gcctarget - fi - fi - - # For a check: - #echo "sys_library_path: $sys_library_path" - #echo "sys_cppflags: $sys_cppflags" -fi - - - - - -# See if a link-able static C library exists -# ------------------------------------------ -# -# After building GCC, we must use PatchELF to correct its RPATHs. However, -# PatchELF links internally with `libstdc++'. So a dynamicly linked -# PatchELF cannot be used to correct the links to `libstdc++' in general -# (on some systems this causes no problem, but on others it doesn't!). -# -# However, to build a Static PatchELF, we need to be able to link with the -# static C library, which is not always available on some GNU/Linux -# systems. Therefore we need to check this here. If we can't build a static -# PatchELF, we won't build any GCC either. -if [ x"$host_cc" = x0 ]; then - testprog=$tmpblddir/test-c - testsource=$tmpblddir/test.c - echo; echo; echo "Checking if static C library is available..."; - echo "#include <stdio.h>" > $testsource - echo "#include <stdlib.h>" >> $testsource - echo "#include <sys/cdefs.h>" >> $testsource - echo "int main(void){printf(\"...yes\n\");" >> $testsource - echo " return EXIT_SUCCESS;}" >> $testsource - cc_call="gcc $testsource $CPPFLAGS $LDFLAGS -o$testprog -static -lc" - if $cc_call && $testprog; then - gccwarning=0 - good_static_libc=1 - rm $testsource $testprog - else - echo; echo "Compilation command:"; echo "$cc_call" - good_static_libc=0 - rm $testsource - gccwarning=1 - host_cc=1 - cat <<EOF - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!!!!!!!!!!!!!!!!!!!!!! Warning !!!!!!!!!!!!!!!!!!!!!! -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -A usable static C library ('libc.a', in any directory) cannot be linked, -and 'sys/cdefs.h' cannot be included with the current settings of this -system. Because of this we can't build a static PatchELF, hence we can't -build GCC. - -If you have 'libc.a', but in a non-standard location (for example in -'/PATH/TO/STATIC/LIBC/libc.a' and '/PATH/TO/SYS/CDEFS_H/sys/cdefs.h'), -please run the commands below, then re-configure the project to fix this -problem. - -export LDFLAGS="-L/PATH/TO/STATIC/LIBC \$LDFLAGS" -export CPPFLAGS="-I/PATH/TO/SYS/CDEFS_H \$LDFLAGS" - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -EOF - fi -fi - -# Print a warning if GCC is not meant to be built. -if [ x"$gccwarning" = x1 ]; then - cat <<EOF - -PLEASE SEE THE WARNINGS ABOVE. - -Since GCC is pretty low-level, this configuration script will continue in 5 -seconds and use your system's C compiler (it won't build a custom GCC). But -please consider installing the necessary package(s) to complete your C -compiler, then re-run './project configure'. - -EOF - sleep 5 -fi - - - - - -# Fortran compiler -# ---------------- -# -# If GCC is ultimately build within the project, the user won't need to -# have a fortran compiler, we'll build it internally for high-level -# programs. However, when the host C compiler is to be used, the user needs -# to have a Fortran compiler available. -if [ $host_cc = 1 ]; then - - # See if a Fortran compiler exists. - hasfc=0; - if type gfortran > /dev/null 2>/dev/null; then hasfc=1; fi - if [ $hasfc = 0 ]; then - cat <<EOF -______________________________________________________ -!!!!!!! Fortran Compiler NOT FOUND !!!!!!! - -Because the project won't be building its own GCC (which includes a Fortran -compiler), you need to have a Fortran compiler available. Fortran is -commonly necessary for many lower-level scientific programs. Currently we -search for 'gfortran'. If you have a Fortran compiler that is not checked, -please get in touch with us (with the form below) so we add it: - - https://savannah.nongnu.org/support/?func=additem&group=reproduce - -Note: GCC will not be built because you are either using the '--host-cc' -option, or you are using an operating system that currently has bugs when -building GCC. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -EOF - exit 1 - fi - - # See if the Fortran compiler works - testprog=$tmpblddir/test-f - testsource=$tmpblddir/test.f - echo; echo; echo "Checking host Fortran compiler..."; - echo " PRINT *, \"... Fortran Compiler works.\"" > $testsource - echo " END" >> $testsource - if gfortran $testsource -o$testprog && $testprog; then - rm $testsource $testprog - else - rm $testsource - cat <<EOF - -______________________________________________________ -!!!!!!! Fortran compiler doesn't work !!!!!!! - -Host Fortran compiler ('gfortran') can't build a simple program. - -A working Fortran compiler is necessary for building some of the project's -software. Please use the error message above to find a good solution and -re-run the project configuration. - -If you can't find a solution, please send the error message above to the -link below and we'll try to help - -https://savannah.nongnu.org/support/?func=additem&group=reproduce - -TIP: Once you find the solution, you can use the '-e' option to use -existing configuration: - - $ ./project configure -e - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -EOF - exit 1 - fi -fi - - - - - # inform the user that the build process is starting # ------------------------------------------------- if [ $printnotice = yes ]; then @@ -1047,13 +1049,12 @@ will be installed in: **TIP**: you can see which software is being installed at every moment with the following command. See "Inspecting status" section of 'README-hacking.md' for more. In short, run it while the project is being -configured (in another terminal, but on this same directory: '`pwd`'): +configured (in another terminal, but on this same directory: 'pwd'): - $ while true; do echo; date; ls .build/software/build-tmp; sleep 1; done + $ ./project --check-config ------------------------- - EOF sleep $tsec fi @@ -1154,18 +1155,26 @@ fi -# library_path (ONLY FOR BASIC) -# ----------------------------- +# Paths needed by the host compiler (only for `basic.mk') +# ------------------------------------------------------- # -# During the basic build, we need to include possibly existing special C -# compiler targets (if they exist). -export CPPFLAGS="$CPPFLAGS $sys_cppflags" +# At the end of the basic build, we need to build GCC. But GCC will build +# in multiple phases, making its own simple compiler in order to build +# itself completely. The intermediate/simple compiler doesn't recognize +# some system specific locations like `/usr/lib/ARCHITECTURE' that some +# operating systems use. We thus need to tell the intermediate compiler +# where its necessary libraries and headers are. if [ x"$sys_library_path" != x ]; then if [ x"$LIBRARY_PATH" = x ]; then export LIBRARY_PATH="$sys_library_path" else export LIBRARY_PATH="$LIBRARY_PATH:$sys_library_path" fi + if [ x"$CPATH" = x ]; then + export CPATH="$sys_cpath" + else + export CPATH="$CPATH:$sys_cpath" + fi fi @@ -1179,7 +1188,7 @@ fi # Bash, Make, or AWK. In this step, we'll install such low-level basic # tools, but we have to be very portable (and use minimal features in all). echo; echo "Building necessary software (if necessary)..." -make -f reproduce/software/make/basic.mk \ +make -k -f reproduce/software/make/basic.mk \ good_static_libc=$good_static_libc \ rpath_command=$rpath_command \ static_build=$static_build \ @@ -1206,12 +1215,13 @@ else numthreads=$jobs fi .local/bin/env -i HOME=$bdir \ - .local/bin/make -f reproduce/software/make/high-level.mk \ + .local/bin/make -k -f reproduce/software/make/high-level.mk \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ static_build=$static_build \ numthreads=$numthreads \ on_mac_os=$on_mac_os \ + sys_cpath=$sys_cpath \ host_cc=$host_cc \ -j$numthreads @@ -1271,8 +1281,8 @@ fi # After everything is installed, we'll put all the names and versions in a # human-readable paragraph and also prepare the BibTeX citation for the # software. -function prepare_name_version() { - +prepare_name_version () +{ # First see if the (possible) `*' in the input arguments corresponds to # anything. Note that some of the given directories may be empty (no # software installed). @@ -1377,9 +1387,9 @@ echo `.local/bin/date` > $finaltarget # The configuration is now complete, we can inform the user on the next # step(s) to take. if [ x$reproducible_paper_group_name = x ]; then - buildcommand="./project prepare -j8" + buildcommand="./project make -j8" else - buildcommand="./project prepare --group=$reproducible_paper_group_name -j8" + buildcommand="./project make --group=$reproducible_paper_group_name -j8" fi cat <<EOF diff --git a/reproduce/software/bash/git-post-checkout b/reproduce/software/shell/git-post-checkout index f6366dc..f6366dc 100755 --- a/reproduce/software/bash/git-post-checkout +++ b/reproduce/software/shell/git-post-checkout diff --git a/reproduce/software/bash/git-pre-commit b/reproduce/software/shell/git-pre-commit index 9d550db..9d550db 100755 --- a/reproduce/software/bash/git-pre-commit +++ b/reproduce/software/shell/git-pre-commit |