diff options
Diffstat (limited to 'reproduce/software')
| -rw-r--r-- | reproduce/software/config/installation/LOCAL.conf.in | 1 | ||||
| -rw-r--r-- | reproduce/software/make/README.md | 2 | ||||
| -rw-r--r-- | reproduce/software/make/basic.mk | 153 | ||||
| -rw-r--r-- | reproduce/software/make/high-level.mk | 13 | ||||
| -rwxr-xr-x | reproduce/software/shell/bashrc.sh (renamed from reproduce/software/bash/bashrc.sh) | 0 | ||||
| -rwxr-xr-x | reproduce/software/shell/configure.sh (renamed from reproduce/software/bash/configure.sh) | 842 | ||||
| -rwxr-xr-x | reproduce/software/shell/git-post-checkout (renamed from reproduce/software/bash/git-post-checkout) | 0 | ||||
| -rwxr-xr-x | reproduce/software/shell/git-pre-commit (renamed from reproduce/software/bash/git-pre-commit) | 0 |
8 files changed, 520 insertions, 491 deletions
diff --git a/reproduce/software/config/installation/LOCAL.conf.in b/reproduce/software/config/installation/LOCAL.conf.in index 1045f72..0579218 100644 --- a/reproduce/software/config/installation/LOCAL.conf.in +++ b/reproduce/software/config/installation/LOCAL.conf.in @@ -12,5 +12,6 @@ BDIR = @bdir@ INDIR = @indir@ DEPENDENCIES-DIR = @ddir@ +SYS_CPATH = @sys_cpath@ DOWNLOADER = @downloader@ GROUP-NAME = @groupname@ diff --git a/reproduce/software/make/README.md b/reproduce/software/make/README.md index fc04dfb..d71a3d5 100644 --- a/reproduce/software/make/README.md +++ b/reproduce/software/make/README.md @@ -2,7 +2,7 @@ Software building instructions ============================== This directory contains Makefiles that are called by the high-level -`reproduce/software/bash/configure.sh` script. The main target for the +`reproduce/software/shell/configure.sh` script. The main target for the installation of each software is a simple plain text file that contains the name of the software and its version (which is put in the paper in the end). Once built, these plain-text files are all put in the proper diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 93dcdc2..d08ba62 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -48,7 +48,7 @@ ibidir = $(BDIR)/software/installed/version-info/proglib # We'll need the system's PATH for making links to low-level programs we # won't be building ourselves. -syspath := $(PATH) +syspath := $(PATH) # As we build more programs, we want to use this project's built programs # and libraries, not the host's. @@ -58,7 +58,15 @@ export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) -export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH) \ + +# This is the "basic" tools where we are relying on the host operating +# system, but are slowly populating our basic software envirnoment. To run +# (system or template) programs, `LD_LIBRARY_PATH' is necessary, so here, +# we'll first tell the programs to look into any possible pre-defined +# `LD_LIBRARY_PATH', then we'll add our own newly installed libraries. We +# will also make sure that there is no "current directory" in it (by +# removing a starting or trailing `:' and any occurance of `::'. +export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH):$(ildir) \ | sed -e's/::/:/g' -e's/^://' -e's/:$$//') # RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is @@ -66,9 +74,9 @@ export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH) \ # causes crashs (see bug #56682). So we'll just give it no value at all. export DYLD_LIBRARY_PATH := -# Recipe startup script, see `reproduce/software/bash/bashrc.sh'. +# Recipe startup script. export PROJECT_STATUS := configure_basic -export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh # Define the top-level basic programs (that don't depend on any other). top-level-programs = low-level-links gcc @@ -658,6 +666,66 @@ $(ibidir)/bash: $(ibidir)/readline \ +# The `-shared' flag will cause problems while building Perl on macOS, so +# we'll only use this configuration option when we are GNU/Linux +# systems. However, since the whole option must be used (which includes `=' +# and empty space), its easier to define the variable as a Make variable +# outside the recipe, not as a shell variable inside it. +ifeq ($(on_mac_os),yes) +perl-conflddlflags = +else +perl-conflddlflags = -Dlddlflags="-shared $$LDFLAGS" +endif +$(ibidir)/perl: | $(ibidir)/bash \ + $(tdir)/perl-$(perl-version).tar.gz + major_version=$$(echo $(perl-version) \ + | sed -e's/\./ /g' \ + | awk '{printf("%d", $$1)}'); \ + base_version=$$(echo $(perl-version) \ + | sed -e's/\./ /g' \ + | awk '{printf("%d.%d", $$1, $$2)}'); \ + cd $(ddir) \ + && rm -rf perl-$(perl-version) \ + && if ! tar xf $(word 1,$(filter $(tdir)/%,$|)); then \ + echo; echo "Tar error"; exit 1; \ + fi \ + && cd perl-$(perl-version) \ + && sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ + -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ + Configure > Configure-tmp \ + && mv -f Configure-tmp Configure \ + && chmod +x Configure \ + && ./Configure -des \ + -Dusethreads \ + -Duseshrplib \ + -Dprefix=$(idir) \ + -Dvendorprefix=$(idir) \ + -Dprivlib=$(idir)/share/perl$$major_version/core_perl \ + -Darchlib=$(idir)/lib/perl$$major_version/$$base_version/core_perl \ + -Dsitelib=$(idir)/share/perl$$major_version/site_perl \ + -Dsitearch=$(idir)/lib/perl$$major_version/$$basever/site_perl \ + -Dvendorlib=$(idir)/share/perl$$major_version/vendor_perl \ + -Dvendorarch=$(idir)/lib/perl$$major_version/$$base_version/vendor_perl \ + -Dscriptdir=$(idir)/bin/core_perl \ + -Dsitescript=$(idir)/bin/site_perl \ + -Dvendorscript=$(idir)/bin/vendor_perl \ + -Dinc_version_list=none \ + -Dman1ext=1perl \ + -Dman3ext=3perl \ + -Dcccdlflags='-fPIC' \ + $(perl-conflddlflags) \ + -Dldflags="$$LDFLAGS" \ + && make SHELL=$(ibdir)/bash -j$(numthreads) \ + && make SHELL=$(ibdir)/bash install \ + && cd .. \ + && rm -rf perl-$(perl-version) \ + && cd $$topdir \ + && echo "Perl $(perl-version)" > $@ + + + + + # Coreutils # --------- # @@ -677,8 +745,10 @@ $(ibidir)/bash: $(ibidir)/readline \ # The echo after the PatchELF loop is to avoid a crash if the last # file that PatchELF encounters is not usable (and it returns with # an error). +# +# Coreutils uses Perl to create man pages! $(ibidir)/coreutils: $(ibidir)/openssl \ - | $(ibidir)/bash \ + | $(ibidir)/perl \ $(tdir)/coreutils-$(coreutils-version).tar.xz cd $(ddir) \ && rm -rf coreutils-$(coreutils-version) \ @@ -847,17 +917,17 @@ $(ibidir)/wget: $(ibidir)/libiconv \ # there is no access to the system's PATH. $(ibidir)/diffutils: | $(ibidir)/coreutils \ $(tdir)/diffutils-$(diffutils-version).tar.xz - $(call gbuild, diffutils-$(diffutils-version), static, , V=1) \ + $(call gbuild, diffutils-$(diffutils-version), static,,V=1) \ && echo "GNU Diffutils $(diffutils-version)" > $@ $(ibidir)/file: | $(ibidir)/coreutils \ $(tdir)/file-$(file-version).tar.gz - $(call gbuild, file-$(file-version), static) \ + $(call gbuild, file-$(file-version), static,,V=1) \ && echo "File $(file-version)" > $@ $(ibidir)/findutils: | $(ibidir)/coreutils \ $(tdir)/findutils-$(findutils-version).tar.xz - $(call gbuild, findutils-$(findutils-version), static, , V=1) \ + $(call gbuild, findutils-$(findutils-version), static,,V=1) \ && echo "GNU Findutils $(findutils-version)" > $@ $(ibidir)/gawk: $(ibidir)/gmp \ @@ -910,13 +980,13 @@ $(ibidir)/gmp: | $(ibidir)/m4 \ $(ibidir)/glibtool: | $(ibidir)/m4 \ $(tdir)/libtool-$(libtool-version).tar.xz $(call gbuild, libtool-$(libtool-version), static, \ - --program-prefix=g) \ + --program-prefix=g, V=1) \ && ln -s $(ibdir)/glibtoolize $(ibdir)/libtoolize \ && echo "GNU Libtool $(libtool-version)" > $@ $(ibidir)/grep: | $(ibidir)/coreutils \ $(tdir)/grep-$(grep-version).tar.xz - $(call gbuild, grep-$(grep-version), static) \ + $(call gbuild, grep-$(grep-version), static,,V=1) \ && echo "GNU Grep $(grep-version)" > $@ $(ibidir)/libbsd: | $(ibidir)/coreutils \ @@ -927,7 +997,7 @@ $(ibidir)/libbsd: | $(ibidir)/coreutils \ $(ibidir)/m4: | $(ibidir)/coreutils \ $(ibidir)/texinfo \ $(tdir)/m4-$(m4-version).tar.gz - $(call gbuild, m4-$(m4-version), static) \ + $(call gbuild, m4-$(m4-version), static,,V=1) \ && echo "GNU M4 $(m4-version)" > $@ # Metastore is used (through a Git hook) to restore the source modification @@ -997,7 +1067,7 @@ $(ibidir)/metastore: $(needlibbsd) \ -e's|@GROUP[@]|'$$group'|g' \ -e's|@BINDIR[@]|$(ibdir)|g' \ -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ - reproduce/software/bash/git-$$f > .git/hooks/$$f \ + reproduce/software/shell/git-$$f > .git/hooks/$$f \ && chmod +x .git/hooks/$$f; \ done; \ fi \ @@ -1020,63 +1090,6 @@ $(ibidir)/mpfr: $(ibidir)/gmp \ $(call gbuild, mpfr-$(mpfr-version), static, , , make check) \ && echo "GNU Multiple Precision Floating-Point Reliably $(mpfr-version)" > $@ -# The `-shared' flag will cause problems while building Perl on macOS, so -# we'll only use this configuration option when we are GNU/Linux -# systems. However, since the whole option must be used (which includes `=' -# and empty space), its easier to define the variable as a Make variable -# outside the recipe, not as a shell variable inside it. -ifeq ($(on_mac_os),yes) -perl-conflddlflags = -else -perl-conflddlflags = -Dlddlflags="-shared $$LDFLAGS" -endif -$(ibidir)/perl: | $(ibidir)/coreutils \ - $(tdir)/perl-$(perl-version).tar.gz - major_version=$$(echo $(perl-version) \ - | sed -e's/\./ /g' \ - | awk '{printf("%d", $$1)}'); \ - base_version=$$(echo $(perl-version) \ - | sed -e's/\./ /g' \ - | awk '{printf("%d.%d", $$1, $$2)}'); \ - cd $(ddir) \ - && rm -rf perl-$(perl-version) \ - && if ! tar xf $(word 1,$(filter $(tdir)/%,$|)); then \ - echo; echo "Tar error"; exit 1; \ - fi \ - && cd perl-$(perl-version) \ - && sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ - Configure > Configure-tmp \ - && mv -f Configure-tmp Configure \ - && chmod +x Configure \ - && ./Configure -des \ - -Dusethreads \ - -Duseshrplib \ - -Dprefix=$(idir) \ - -Dvendorprefix=$(idir) \ - -Dprivlib=$(idir)/share/perl$$major_version/core_perl \ - -Darchlib=$(idir)/lib/perl$$major_version/$$base_version/core_perl \ - -Dsitelib=$(idir)/share/perl$$major_version/site_perl \ - -Dsitearch=$(idir)/lib/perl$$major_version/$$basever/site_perl \ - -Dvendorlib=$(idir)/share/perl$$major_version/vendor_perl \ - -Dvendorarch=$(idir)/lib/perl$$major_version/$$base_version/vendor_perl \ - -Dscriptdir=$(idir)/bin/core_perl \ - -Dsitescript=$(idir)/bin/site_perl \ - -Dvendorscript=$(idir)/bin/vendor_perl \ - -Dinc_version_list=none \ - -Dman1ext=1perl \ - -Dman3ext=3perl \ - -Dcccdlflags='-fPIC' \ - $(perl-conflddlflags) \ - -Dldflags="$$LDFLAGS" \ - && make SHELL=$(ibdir)/bash -j$(numthreads) \ - && make SHELL=$(ibdir)/bash install \ - && cd .. \ - && rm -rf perl-$(perl-version) \ - && cd $$topdir \ - && echo "Perl $(perl-version)" > $@ - - $(ibidir)/pkg-config: | $(ibidir)/coreutils \ $(tdir)/pkg-config-$(pkgconfig-version).tar.gz # An existing `libiconv' can cause a conflict with `pkg-config', @@ -1101,7 +1114,7 @@ $(ibidir)/pkg-config: | $(ibidir)/coreutils \ $(ibidir)/sed: | $(ibidir)/coreutils \ $(tdir)/sed-$(sed-version).tar.xz - $(call gbuild, sed-$(sed-version), static) \ + $(call gbuild, sed-$(sed-version), static,,V=1) \ && echo "GNU Sed $(sed-version)" > $@ $(ibidir)/texinfo: | $(ibidir)/perl \ diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index bc77dab..53bef55 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -70,8 +70,8 @@ all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ .SHELLFLAGS := --noprofile --norc -ec export CCACHE_DISABLE := 1 export PATH := $(ibdir) -export CXX := $(ibdir)/g++ export CC := $(ibdir)/gcc +export CXX := $(ibdir)/g++ export SHELL := $(ibdir)/bash export F77 := $(ibdir)/gfortran export LD_RUN_PATH := $(ildir):$(il64dir) @@ -79,6 +79,11 @@ export PKG_CONFIG_PATH := $(ildir)/pkgconfig export LD_LIBRARY_PATH := $(ildir):$(il64dir) export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig +# Until we build our own C library, without this, our GCC won't be able to +# compile anything! Note that on most systems (in particular +# non-Debian-based), `sys_cpath' will be empty. +export CPATH := $(sys_cpath) + # RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is # ultimately redundant. But on some systems, even having a single value # causes crashs (see bug #56682). So we'll just give it no value at all. @@ -91,9 +96,9 @@ export DYLD_LIBRARY_PATH := # for `ld'. export LIBRARY_PATH := $(sys_library_path) -# Recipe startup script, see `reproduce/software/bash/bashrc.sh'. +# Recipe startup script, see `reproduce/software/shell/bashrc.sh'. export PROJECT_STATUS := configure_highlevel -export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh # Building flags: # @@ -634,7 +639,7 @@ $(ibidir)/healpix: $(ibidir)/cfitsio \ && echo "HEALPix $(healpix-version) \citep{healpix}" > $@ $(ibidir)/libjpeg: | $(tdir)/jpegsrc.$(libjpeg-version).tar.gz - $(call gbuild, jpeg-9b, static) \ + $(call gbuild, jpeg-9b, static,,V=1) \ && echo "Libjpeg $(libjpeg-version)" > $@ $(ibidir)/libnsl: $(ibidir)/libtirpc \ diff --git a/reproduce/software/bash/bashrc.sh b/reproduce/software/shell/bashrc.sh index b2a151f..b2a151f 100755 --- a/reproduce/software/bash/bashrc.sh +++ b/reproduce/software/shell/bashrc.sh diff --git a/reproduce/software/bash/configure.sh b/reproduce/software/shell/configure.sh index c681298..e8f2736 100755 --- a/reproduce/software/bash/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -1,4 +1,4 @@ -#! /bin/bash +#! /bin/sh # # Necessary preparations/configurations for the reproducible project. # @@ -57,7 +57,8 @@ glconf=$cdir/gnuastro/gnuastro-local.conf # In case someone opens the files output from the configuration scripts in # a text editor and wants to edit them, it is important to let them know # that their changes are not going to be permenant. -function create_file_with_notice() { +create_file_with_notice () +{ if echo "# IMPORTANT: file can be RE-WRITTEN after './project configure'" > "$1" then echo "#" >> "$1" @@ -81,7 +82,8 @@ function create_file_with_notice() { # # Since the build directory will go into a symbolic link, we want it to be # an absolute address. With this function we can make sure of that. -function absolute_dir() { +absolute_dir () +{ if stat "$1" 1> /dev/null; then echo "$(cd "$(dirname "$1")" && pwd )/$(basename "$1")" else @@ -93,6 +95,398 @@ function absolute_dir() { +# Check for C/C++ compilers +# ------------------------- +# +# To build the software, we'll need some basic tools (the compilers in +# particular) to be present. +hascc=0; +if type cc > /dev/null 2>/dev/null; then + if type c++ > /dev/null 2>/dev/null; then export CC=cc; hascc=1; fi +else + if type gcc > /dev/null 2>/dev/null; then + if type g++ > /dev/null 2>/dev/null; then export CC=gcc; hascc=1; fi + else + if type clang > /dev/null 2>/dev/null; then + if type clang++ > /dev/null 2>/dev/null; then export CC=clang; hascc=1; fi + fi + fi +fi +if [ $hascc = 0 ]; then + cat <<EOF +______________________________________________________ +!!!!!!! C/C++ Compiler NOT FOUND !!!!!!! + +To build the project's software, the host system needs to have basic C and +C++ compilers. The executables that were checked are 'cc', 'gcc' and +'clang' for a C compiler, and 'c++', 'g++' and 'clang++' for a C++ +compiler. If you have a relevant compiler that is not checked, please get +in touch with us (with the form below) so we add it: + + https://savannah.nongnu.org/support/?func=additem&group=reproduce +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +EOF + exit 1 +fi + + + + + +# Special directory for compiler testing +# -------------------------------------- +# +# This directory will be deleted when the compiler testing is finished. +compilertestdir=.compiler_test_dir_please_delete +if ! [ -d $compilertestdir ]; then mkdir $compilertestdir; fi + + + + + +# Check C compiler +# ---------------- +gcc_works=0 +testprog=$compilertestdir/test +testsource=$compilertestdir/test.c +echo; echo; echo "Checking host C compiler..."; +cat > $testsource <<EOF +#include <stdio.h> +#include <stdlib.h> +int main(void){printf("...C compiler works.\n"); + return EXIT_SUCCESS;} +EOF +if $CC $testsource -o$testprog && $testprog; then + rm $testsource $testprog +else + rm $testsource + cat <<EOF + +______________________________________________________ +!!!!!!! C compiler doesn't work !!!!!!! + +Host C compiler ('gcc') can't build a simple program. + +A working C compiler is necessary for building the project's software. +Please use the error message above to find a good solution and re-run the +project configuration. + +If you can't find a solution, please send the error message above to the +link below and we'll try to help + +https://savannah.nongnu.org/support/?func=additem&group=reproduce + +TIP: Once you find the solution, you can use the '-e' option to use +existing configuration: + + $ ./project configure -e + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +EOF + exit 1 +fi + + + + + +# See if the linker accepts -Wl,-rpath-link +# ----------------------------------------- +# +# `-rpath-link' is used to write the information of the linked shared +# library into the shared object (library or program). But some versions of +# LLVM's linker don't accept it an can cause problems. +cat > $testsource <<EOF +#include <stdio.h> +#include <stdlib.h> +int main(void) {return EXIT_SUCCESS;} +EOF +if $CC $testsource -o$testprog -Wl,-rpath-link 2>/dev/null > /dev/null; then + export rpath_command="-Wl,-rpath-link=$instdir/lib" +else + export rpath_command="" +fi +rm -f $testprog $testsource + + + + + +# See if we need the dynamic-linker (-ldl) +# ---------------------------------------- +# +# Some programs (like Wget) need dynamic loading (using `libdl'). On +# GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But +# Mac OS doesn't need any explicit linking. So we'll check here to see if +# it is present (thus necessary) or not. +cat > $testsource <<EOF +#include <stdio.h> +#include <dlfcn.h> +int +main(void) { + void *handle=dlopen ("/lib/CEDD_LIB.so.6", RTLD_LAZY); + return 0; +} +EOF +if $CC $testsource -o$testprog 2>/dev/null > /dev/null; then + needs_ldl=no; +else + needs_ldl=yes; +fi +rm -f $testprog $testsource + + + + + +# See if the C compiler can build static libraries +# ------------------------------------------------ + +# We are manually only working with shared libraries: because some +# high-level programs like Wget and cURL need dynamic linking and if we +# build the libraries statically, our own builds will be ignored and these +# programs will go and find their necessary libraries on the host system. +# +# Another good advantage of shared libraries is that we can actually use +# the shared library tool of the system (`ldd' with GNU C Library) and see +# exactly where each linked library comes from. But in static building, +# unless you follow the build closely, its not easy to see if the source of +# the library came from the system or our build. +static_build=no + + + + + +# If we are on a Mac OS system +# ---------------------------- +# +# For the time being, we'll use the existance of `otool' to see if we are +# on a Mac OS system or not. Some tools (for example OpenSSL) need to know +# this. +# +# On Mac OS, the building of GCC crashes sometimes while building libiberty +# with CLang's `g++'. Until we find a solution, we'll just use the host's C +# compiler. +if type otool > /dev/null 2>/dev/null; then + host_cc=1 + on_mac_os=yes +else + host_cc=0 + on_mac_os=no +fi + + + + + +# Necessary C library element positions +# ------------------------------------- +# +# On some systems (in particular Debian-based OSs), the static C library +# and necessary headers in a non-standard place, and we can't build GCC. So +# we need to find them first. The `sys/cdefs.h' header is also in a +# similarly different location. +sys_cpath="" +sys_library_path="" +if [ x"$$on_mac_os" != xyes ]; then + + # Get the GCC target name of the compiler, when its given, special + # C libraries and headers are in a sub-directory of the host. + gcctarget=$(gcc -v 2>&1 \ + | tr ' ' '\n' \ + | awk '/\-\-target/' \ + | sed -e's/\-\-target=//') + if [ x"$gcctarget" != x ]; then + if [ -f /usr/lib/$gcctarget/libc.a ]; then + export sys_library_path=/usr/lib/$gcctarget + export sys_cpath=/usr/include/$gcctarget + fi + fi + + # For a check: + #echo "sys_library_path: $sys_library_path" + #echo "sys_cpath: $sys_cpath" +fi + + + + + +# See if a link-able static C library exists +# ------------------------------------------ +# +# After building GCC, we must use PatchELF to correct its RPATHs. However, +# PatchELF links internally with `libstdc++'. So a dynamicly linked +# PatchELF cannot be used to correct the links to `libstdc++' in general +# (on some systems this causes no problem, but on others it doesn't!). +# +# However, to build a Static PatchELF, we need to be able to link with the +# static C library, which is not always available on some GNU/Linux +# systems. Therefore we need to check this here. If we can't build a static +# PatchELF, we won't build any GCC either. +if [ x"$host_cc" = x0 ]; then + echo; echo; echo "Checking if static C library is available..."; + cat > $testsource <<EOF +#include <stdio.h> +#include <stdlib.h> +#include <sys/cdefs.h> +int main(void){printf("...yes\n"); + return EXIT_SUCCESS;} +EOF + cc_call="$CC $testsource $CPPFLAGS $LDFLAGS -o$testprog -static -lc" + if $cc_call && $testprog; then + gccwarning=0 + good_static_libc=1 + rm $testsource $testprog + else + echo; echo "Compilation command:"; echo "$cc_call" + good_static_libc=0 + rm $testsource + gccwarning=1 + host_cc=1 + cat <<EOF + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!! Warning !!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +The 'sys/cdefs.h' header cannot be included, or a usable static C library +('libc.a', in any directory) cannot be used with the current settings of +this system. SEE THE ERROR MESSAGE ABOVE. + +Because of this, we can't build GCC. You either 1) don't have them, or 2) +the default system environment aren't enough to find them. + +1) If you don't have them, your operating system provides them as separate +packages that you must manually install. Please look into your operating +system documentation or contact someone familiar with it. For example on +some Redhat-based GNU/Linux distributions, the static C library package can +be installed with this command: + + $ sudo yum install glibc-static + +2) If you have 'libc.a' and 'sys/cdefs.h', but in a non-standard location (for +example in '/PATH/TO/STATIC/LIBC/libc.a' and +'/PATH/TO/SYS/CDEFS_H/sys/cdefs.h'), please run the commands below, then +re-configure the project to fix this problem. + + $ export LDFLAGS="-L/PATH/TO/STATIC/LIBC \$LDFLAGS" + $ export CPPFLAGS="-I/PATH/TO/SYS/CDEFS_H \$LDFLAGS" + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +EOF + fi +fi + +# Print a warning if GCC is not meant to be built. +if [ x"$gccwarning" = x1 ]; then + cat <<EOF + +PLEASE SEE THE WARNINGS ABOVE. + +Since GCC is pretty low-level, this configuration script will continue in 5 +seconds and use your system's C compiler (it won't build a custom GCC). But +please consider installing the necessary package(s) to complete your C +compiler, then re-run './project configure'. + +EOF + sleep 5 +fi + + + + + +# Fortran compiler +# ---------------- +# +# If GCC is ultimately build within the project, the user won't need to +# have a fortran compiler, we'll build it internally for high-level +# programs. However, when the host C compiler is to be used, the user needs +# to have a Fortran compiler available. +if [ $host_cc = 1 ]; then + + # See if a Fortran compiler exists. + hasfc=0; + if type gfortran > /dev/null 2>/dev/null; then hasfc=1; fi + if [ $hasfc = 0 ]; then + cat <<EOF +______________________________________________________ +!!!!!!! Fortran Compiler NOT FOUND !!!!!!! + +Because the project won't be building its own GCC (which includes a Fortran +compiler), you need to have a Fortran compiler available. Fortran is +commonly necessary for many lower-level scientific programs. Currently we +search for 'gfortran'. If you have a Fortran compiler that is not checked, +please get in touch with us (with the form below) so we add it: + + https://savannah.nongnu.org/support/?func=additem&group=reproduce + +Note: GCC will not be built because you are either using the '--host-cc' +option, or you are using an operating system that currently has bugs when +building GCC. +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +EOF + exit 1 + fi + + # See if the Fortran compiler works + testsource=$compilertestdir/test.f + echo; echo; echo "Checking host Fortran compiler..."; + echo " PRINT *, \"... Fortran Compiler works.\"" > $testsource + echo " END" >> $testsource + if gfortran $testsource -o$testprog && $testprog; then + rm $testsource $testprog + else + rm $testsource + cat <<EOF + +______________________________________________________ +!!!!!!! Fortran compiler doesn't work !!!!!!! + +Host Fortran compiler ('gfortran') can't build a simple program. + +A working Fortran compiler is necessary for building some of the project's +software. Please use the error message above to find a good solution and +re-run the project configuration. + +If you can't find a solution, please send the error message above to the +link below and we'll try to help + +https://savannah.nongnu.org/support/?func=additem&group=reproduce + +TIP: Once you find the solution, you can use the '-e' option to use +existing configuration: + + $ ./project configure -e + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +EOF + exit 1 + fi +fi + + + + + +# Delete the compiler testing directory +# ------------------------------------- +# +# This directory was made above to make sure the necessary compilers can be +# run. +rm -rf $compilertestdir + + + + + # Inform the user # --------------- # @@ -239,7 +633,7 @@ EOF bdir= currentdir=$(pwd) junkname=pure-junk-974adfkj38 - while [ x$bdir == x ] + while [ x$bdir = x ] do # Ask the user (if not already set on the command-line). if [ x"$build_dir" = x ]; then @@ -264,10 +658,12 @@ EOF fi fi - # Make sure the given directory is not a subdirectory of the - # source directory. + # If its given, make sure it isn't a subdirectory of the source + # directory. if ! [ x"$bdir" = x ]; then - if [[ $bdir == $currentdir* ]]; then + echo "Given build directory: $bdir" + if echo "$bdir/" \ + | grep '^'$currentdir 2> /dev/null > /dev/null; then # If it was newly created, it will be empty, so delete it. if ! [ "$(ls -A $bdir)" ]; then rm --dir $bdir; fi @@ -388,6 +784,7 @@ if [ $rewritepconfig = yes ]; then sed -e's|@bdir[@]|'"$bdir"'|' \ -e's|@indir[@]|'"$indir"'|' \ -e's|@ddir[@]|'"$ddir"'|' \ + -e's|@sys_cpath[@]|'"$sys_cpath"'|' \ -e's|@downloader[@]|'"$downloader"'|' \ -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \ $pconf.in >> $pconf @@ -631,401 +1028,6 @@ fi -# Check for C/C++ compilers -# ------------------------- -# -# To build the software, we'll need some basic tools (the compilers in -# particular) to be present. -hascc=0; -if type cc > /dev/null 2>/dev/null; then - if type c++ > /dev/null 2>/dev/null; then hascc=1; fi -else - if type gcc > /dev/null 2>/dev/null; then - if type g++ > /dev/null 2>/dev/null; then hascc=1; fi - else - if type clang > /dev/null 2>/dev/null; then - if type clang++ > /dev/null 2>/dev/null; then hascc=1; fi - fi - fi -fi -if [ $hascc = 0 ]; then - cat <<EOF -______________________________________________________ -!!!!!!! C/C++ Compiler NOT FOUND !!!!!!! - -To build the project's software, the host system needs to have basic C and -C++ compilers. The executables that were checked are 'cc', 'gcc' and -'clang' for a C compiler, and 'c++', 'g++' and 'clang++' for a C++ -compiler. If you have a relevant compiler that is not checked, please get -in touch with us (with the form below) so we add it: - - https://savannah.nongnu.org/support/?func=additem&group=reproduce -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -EOF - exit 1 -fi - - - - - -# Check C compiler -# ---------------- -gcc_works=0 -testprog=$tmpblddir/test-c -testsource=$tmpblddir/test.c -echo; echo; echo "Checking host C compiler..."; -echo "#include <stdio.h>" > $testsource -echo "#include <stdlib.h>" >> $testsource -echo "int main(void){printf(\"...C compiler works.\n\");" >> $testsource -echo " return EXIT_SUCCESS;}" >> $testsource -if gcc $testsource -o$testprog && $testprog; then - rm $testsource $testprog -else - rm $testsource - cat <<EOF - -______________________________________________________ -!!!!!!! C compiler doesn't work !!!!!!! - -Host C compiler ('gcc') can't build a simple program. - -A working C compiler is necessary for building the project's software. -Please use the error message above to find a good solution and re-run the -project configuration. - -If you can't find a solution, please send the error message above to the -link below and we'll try to help - -https://savannah.nongnu.org/support/?func=additem&group=reproduce - -TIP: Once you find the solution, you can use the '-e' option to use -existing configuration: - - $ ./project configure -e - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -EOF - exit 1 -fi - - - - - -# See if the linker accepts -Wl,-rpath-link -# ----------------------------------------- -# -# `-rpath-link' is used to write the information of the linked shared -# library into the shared object (library or program). But some versions of -# LLVM's linker don't accept it an can cause problems. -oprog=$sdir/rpath-test -cprog=$sdir/rpath-test.c -echo "#include <stdio.h>" > $cprog -echo "int main(void) {return 0;}" >> $cprog -if [ x$CC = x ]; then CC=gcc; fi; -if $CC $cprog -o$oprog -Wl,-rpath-link &> /dev/null; then - export rpath_command="-Wl,-rpath-link=$instdir/lib" -else - export rpath_command="" -fi -rm -f $oprog $cprog - - - - - -# See if we need the dynamic-linker (-ldl) -# ---------------------------------------- -# -# Some programs (like Wget) need dynamic loading (using `libdl'). On -# GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But -# Mac OS doesn't need any explicit linking. So we'll check here to see if -# it is present (thus necessary) or not. -oprog=$sdir/ldl-test -cprog=$sdir/ldl-test.c -cat > $cprog <<EOF -#include <stdio.h> -#include <dlfcn.h> -int -main(void) { - void *handle=dlopen ("/lib/CEDD_LIB.so.6", RTLD_LAZY); - return 0; -} -EOF -if gcc $cprog -o$oprog &> /dev/null; then needs_ldl=no; else needs_ldl=yes; fi -rm -f $oprog $cprog - - - - - -# See if the C compiler can build static libraries -# ------------------------------------------------ - -# We are manually only working with shared libraries: because some -# high-level programs like Wget and cURL need dynamic linking and if we -# build the libraries statically, our own builds will be ignored and these -# programs will go and find their necessary libraries on the host system. -# -# Another good advantage of shared libraries is that we can actually use -# the shared library tool of the system (`ldd' with GNU C Library) and see -# exactly where each linked library comes from. But in static building, -# unless you follow the build closely, its not easy to see if the source of -# the library came from the system or our build. -static_build=no - -#oprog=$sdir/static-test -#cprog=$sdir/static-test.c -#echo "#include <stdio.h>" > $cprog -#echo "int main(void) {return 0;}" >> $cprog -#if [ x$CC = x ]; then CC=gcc; fi; -#if $CC $cprog -o$oprog -static &> /dev/null; then -# export static_build="yes" -#else -# export static_build="no" -#fi -#rm -f $oprog $cprog -#if [ $printnotice = yes ] && [ $static_build = "no" ]; then -# cat <<EOF -#_________________________________________________________________________ -#!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!! -# -#Your system's C compiler ('$CC') doesn't support building static -#libraries. Therefore the dependencies will be built dynamically. This means -#that they will depend more strongly on changes/updates in the host -#system. For high-level applications (like most research projects in natural -#sciences), this shouldn't be a significant problem. -# -#But generally, for reproducibility, its better to build static libraries -#and programs. For more on their difference (and generally an introduction -#on linking), please see the link below: -# -#https://www.gnu.org/software/gnuastro/manual/html_node/Linking.html -# -#If you have other compilers on your system, you can select a different -#compiler by setting the 'CC' environment variable before running -#'./project configure'. -# -#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -# -#EOF -# sleep 5 -#fi - - - - - -# If we are on a Mac OS system -# ---------------------------- -# -# For the time being, we'll use the existance of `otool' to see if we are -# on a Mac OS system or not. Some tools (for example OpenSSL) need to know -# this. -# -# On Mac OS, the building of GCC crashes sometimes while building libiberty -# with CLang's `g++'. Until we find a solution, we'll just use the host's C -# compiler. -if type otool > /dev/null 2>/dev/null; then - host_cc=1 - on_mac_os=yes -else - host_cc=0 - on_mac_os=no -fi - - - - - -# Necessary C library element positions -# ------------------------------------- -# -# On some systems (in particular Debian-based OSs), the static C library -# and necessary headers in a non-standard place, and we can't build GCC. So -# we need to find them first. The `sys/cdefs.h' header is also in a -# similarly different location. -sys_cppflags="" -sys_library_path="" -if [ x"$$on_mac_os" != xyes ]; then - - # Get the GCC target name of the compiler, when its given, special - # C libraries and headers are in a sub-directory of the host. - gcctarget=$(gcc -v 2>&1 \ - | tr ' ' '\n' \ - | awk '/\-\-target/' \ - | sed -e's/\-\-target=//') - if [ x"$gcctarget" != x ]; then - if [ -f /usr/lib/$gcctarget/libc.a ]; then - export sys_library_path=/usr/lib/$gcctarget - export sys_cppflags=-I/usr/include/$gcctarget - fi - fi - - # For a check: - #echo "sys_library_path: $sys_library_path" - #echo "sys_cppflags: $sys_cppflags" -fi - - - - - -# See if a link-able static C library exists -# ------------------------------------------ -# -# After building GCC, we must use PatchELF to correct its RPATHs. However, -# PatchELF links internally with `libstdc++'. So a dynamicly linked -# PatchELF cannot be used to correct the links to `libstdc++' in general -# (on some systems this causes no problem, but on others it doesn't!). -# -# However, to build a Static PatchELF, we need to be able to link with the -# static C library, which is not always available on some GNU/Linux -# systems. Therefore we need to check this here. If we can't build a static -# PatchELF, we won't build any GCC either. -if [ x"$host_cc" = x0 ]; then - testprog=$tmpblddir/test-c - testsource=$tmpblddir/test.c - echo; echo; echo "Checking if static C library is available..."; - echo "#include <stdio.h>" > $testsource - echo "#include <stdlib.h>" >> $testsource - echo "#include <sys/cdefs.h>" >> $testsource - echo "int main(void){printf(\"...yes\n\");" >> $testsource - echo " return EXIT_SUCCESS;}" >> $testsource - cc_call="gcc $testsource $CPPFLAGS $LDFLAGS -o$testprog -static -lc" - if $cc_call && $testprog; then - gccwarning=0 - good_static_libc=1 - rm $testsource $testprog - else - echo; echo "Compilation command:"; echo "$cc_call" - good_static_libc=0 - rm $testsource - gccwarning=1 - host_cc=1 - cat <<EOF - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!!!!!!!!!!!!!!!!!!!!!! Warning !!!!!!!!!!!!!!!!!!!!!! -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -A usable static C library ('libc.a', in any directory) cannot be linked, -and 'sys/cdefs.h' cannot be included with the current settings of this -system. Because of this we can't build a static PatchELF, hence we can't -build GCC. - -If you have 'libc.a', but in a non-standard location (for example in -'/PATH/TO/STATIC/LIBC/libc.a' and '/PATH/TO/SYS/CDEFS_H/sys/cdefs.h'), -please run the commands below, then re-configure the project to fix this -problem. - -export LDFLAGS="-L/PATH/TO/STATIC/LIBC \$LDFLAGS" -export CPPFLAGS="-I/PATH/TO/SYS/CDEFS_H \$LDFLAGS" - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -EOF - fi -fi - -# Print a warning if GCC is not meant to be built. -if [ x"$gccwarning" = x1 ]; then - cat <<EOF - -PLEASE SEE THE WARNINGS ABOVE. - -Since GCC is pretty low-level, this configuration script will continue in 5 -seconds and use your system's C compiler (it won't build a custom GCC). But -please consider installing the necessary package(s) to complete your C -compiler, then re-run './project configure'. - -EOF - sleep 5 -fi - - - - - -# Fortran compiler -# ---------------- -# -# If GCC is ultimately build within the project, the user won't need to -# have a fortran compiler, we'll build it internally for high-level -# programs. However, when the host C compiler is to be used, the user needs -# to have a Fortran compiler available. -if [ $host_cc = 1 ]; then - - # See if a Fortran compiler exists. - hasfc=0; - if type gfortran > /dev/null 2>/dev/null; then hasfc=1; fi - if [ $hasfc = 0 ]; then - cat <<EOF -______________________________________________________ -!!!!!!! Fortran Compiler NOT FOUND !!!!!!! - -Because the project won't be building its own GCC (which includes a Fortran -compiler), you need to have a Fortran compiler available. Fortran is -commonly necessary for many lower-level scientific programs. Currently we -search for 'gfortran'. If you have a Fortran compiler that is not checked, -please get in touch with us (with the form below) so we add it: - - https://savannah.nongnu.org/support/?func=additem&group=reproduce - -Note: GCC will not be built because you are either using the '--host-cc' -option, or you are using an operating system that currently has bugs when -building GCC. -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -EOF - exit 1 - fi - - # See if the Fortran compiler works - testprog=$tmpblddir/test-f - testsource=$tmpblddir/test.f - echo; echo; echo "Checking host Fortran compiler..."; - echo " PRINT *, \"... Fortran Compiler works.\"" > $testsource - echo " END" >> $testsource - if gfortran $testsource -o$testprog && $testprog; then - rm $testsource $testprog - else - rm $testsource - cat <<EOF - -______________________________________________________ -!!!!!!! Fortran compiler doesn't work !!!!!!! - -Host Fortran compiler ('gfortran') can't build a simple program. - -A working Fortran compiler is necessary for building some of the project's -software. Please use the error message above to find a good solution and -re-run the project configuration. - -If you can't find a solution, please send the error message above to the -link below and we'll try to help - -https://savannah.nongnu.org/support/?func=additem&group=reproduce - -TIP: Once you find the solution, you can use the '-e' option to use -existing configuration: - - $ ./project configure -e - -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -EOF - exit 1 - fi -fi - - - - - # inform the user that the build process is starting # ------------------------------------------------- if [ $printnotice = yes ]; then @@ -1047,13 +1049,12 @@ will be installed in: **TIP**: you can see which software is being installed at every moment with the following command. See "Inspecting status" section of 'README-hacking.md' for more. In short, run it while the project is being -configured (in another terminal, but on this same directory: '`pwd`'): +configured (in another terminal, but on this same directory: 'pwd'): - $ while true; do echo; date; ls .build/software/build-tmp; sleep 1; done + $ ./project --check-config ------------------------- - EOF sleep $tsec fi @@ -1154,18 +1155,26 @@ fi -# library_path (ONLY FOR BASIC) -# ----------------------------- +# Paths needed by the host compiler (only for `basic.mk') +# ------------------------------------------------------- # -# During the basic build, we need to include possibly existing special C -# compiler targets (if they exist). -export CPPFLAGS="$CPPFLAGS $sys_cppflags" +# At the end of the basic build, we need to build GCC. But GCC will build +# in multiple phases, making its own simple compiler in order to build +# itself completely. The intermediate/simple compiler doesn't recognize +# some system specific locations like `/usr/lib/ARCHITECTURE' that some +# operating systems use. We thus need to tell the intermediate compiler +# where its necessary libraries and headers are. if [ x"$sys_library_path" != x ]; then if [ x"$LIBRARY_PATH" = x ]; then export LIBRARY_PATH="$sys_library_path" else export LIBRARY_PATH="$LIBRARY_PATH:$sys_library_path" fi + if [ x"$CPATH" = x ]; then + export CPATH="$sys_cpath" + else + export CPATH="$CPATH:$sys_cpath" + fi fi @@ -1179,7 +1188,7 @@ fi # Bash, Make, or AWK. In this step, we'll install such low-level basic # tools, but we have to be very portable (and use minimal features in all). echo; echo "Building necessary software (if necessary)..." -make -f reproduce/software/make/basic.mk \ +make -k -f reproduce/software/make/basic.mk \ good_static_libc=$good_static_libc \ rpath_command=$rpath_command \ static_build=$static_build \ @@ -1206,12 +1215,13 @@ else numthreads=$jobs fi .local/bin/env -i HOME=$bdir \ - .local/bin/make -f reproduce/software/make/high-level.mk \ + .local/bin/make -k -f reproduce/software/make/high-level.mk \ sys_library_path=$sys_library_path \ rpath_command=$rpath_command \ static_build=$static_build \ numthreads=$numthreads \ on_mac_os=$on_mac_os \ + sys_cpath=$sys_cpath \ host_cc=$host_cc \ -j$numthreads @@ -1271,8 +1281,8 @@ fi # After everything is installed, we'll put all the names and versions in a # human-readable paragraph and also prepare the BibTeX citation for the # software. -function prepare_name_version() { - +prepare_name_version () +{ # First see if the (possible) `*' in the input arguments corresponds to # anything. Note that some of the given directories may be empty (no # software installed). @@ -1377,9 +1387,9 @@ echo `.local/bin/date` > $finaltarget # The configuration is now complete, we can inform the user on the next # step(s) to take. if [ x$reproducible_paper_group_name = x ]; then - buildcommand="./project prepare -j8" + buildcommand="./project make -j8" else - buildcommand="./project prepare --group=$reproducible_paper_group_name -j8" + buildcommand="./project make --group=$reproducible_paper_group_name -j8" fi cat <<EOF diff --git a/reproduce/software/bash/git-post-checkout b/reproduce/software/shell/git-post-checkout index f6366dc..f6366dc 100755 --- a/reproduce/software/bash/git-post-checkout +++ b/reproduce/software/shell/git-post-checkout diff --git a/reproduce/software/bash/git-pre-commit b/reproduce/software/shell/git-pre-commit index 9d550db..9d550db 100755 --- a/reproduce/software/bash/git-pre-commit +++ b/reproduce/software/shell/git-pre-commit |