aboutsummaryrefslogtreecommitdiff
path: root/reproduce/software/bash
diff options
context:
space:
mode:
Diffstat (limited to 'reproduce/software/bash')
-rwxr-xr-xreproduce/software/bash/configure.sh1224
1 files changed, 1224 insertions, 0 deletions
diff --git a/reproduce/software/bash/configure.sh b/reproduce/software/bash/configure.sh
new file mode 100755
index 0000000..65b728e
--- /dev/null
+++ b/reproduce/software/bash/configure.sh
@@ -0,0 +1,1224 @@
+#! /bin/bash
+#
+# Necessary preparations/configurations for the reproducible project.
+#
+# Copyright (C) 2018-2019 Mohammad Akhlaghi <mohammad@akhlaghi.org>
+#
+# This script is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation, either version 3 of the License, or (at your
+# option) any later version.
+#
+# This script is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# A copy of the GNU General Public License is available at
+# <http://www.gnu.org/licenses/>.
+
+
+# Script settings
+# ---------------
+# Stop the script if there are any errors.
+set -e
+
+
+
+
+
+# Internal directories
+# --------------------
+#
+# These are defined to help make this script more readable.
+topdir=$(pwd)
+optionaldir="/optional/path"
+adir=reproduce/analysis/config
+cdir=reproduce/software/config
+
+sbdir=$cdir/installation
+
+pconf=$sbdir/LOCAL.mk
+ptconf=$sbdir/LOCAL_tmp.mk
+poconf=$sbdir/LOCAL_old.mk
+depverfile=$cdir/installation/versions.mk
+# --------- Delete for no Gnuastro ---------
+glconf=$cdir/gnuastro/gnuastro-local.conf
+# ------------------------------------------
+
+
+
+
+
+# Notice for top of generated files
+# ---------------------------------
+#
+# In case someone opens the files output from the configuration scripts in
+# a text editor and wants to edit them, it is important to let them know
+# that their changes are not going to be permenant.
+function create_file_with_notice() {
+ if echo "# IMPORTANT: file can be RE-WRITTEN after './project configure'" > "$1"
+ then
+ echo "#" >> "$1"
+ echo "# This file was created during configuration" >> "$1"
+ echo "# ('./project configure'). Therefore, it is not under" >> "$1"
+ echo "# version control and any manual changes to it will be" >> "$1"
+ echo "# over-written if the project re-configured." >> "$1"
+ echo "#" >> "$1"
+ else
+ echo; echo "Can't write to $1"; echo;
+ exit 1
+ fi
+}
+
+
+
+
+
+# Get absolute address
+# --------------------
+#
+# Since the build directory will go into a symbolic link, we want it to be
+# an absolute address. With this function we can make sure of that.
+function absolute_dir() {
+ if stat "$1" 1> /dev/null; then
+ echo "$(cd "$(dirname "$1")" && pwd )/$(basename "$1")"
+ else
+ exit 1;
+ fi
+}
+
+
+
+
+
+# Inform the user
+# ---------------
+#
+# Print some basic information so the user gets a feeling of what is going
+# on and is prepared on what will happen next.
+cat <<EOF
+
+-----------------------------
+Project's local configuration
+-----------------------------
+
+Local configuration includes things like top-level directories, or
+processing steps. It is STRONGLY recommended to read the comments, and set
+the best values for your system (where necessary).
+
+EOF
+
+
+
+
+
+# What to do with possibly existing configuration file
+# ----------------------------------------------------
+#
+# `LOCAL.mk' is the top-most local configuration for the project. If it
+# already exists when this script is run, we'll make a copy of it as backup
+# (for example the user might have ran `./project configure' by mistake).
+printnotice=yes
+rewritepconfig=yes
+rewritegconfig=yes
+if [ -f $pconf ] || [ -f $glconf ]; then
+ if [ $existing_conf = 1 ]; then
+ printnotice=no
+ if [ -f $pconf ]; then rewritepconfig=no; fi
+ if [ -f $glconf ]; then rewritegconfig=no; fi
+ fi
+fi
+
+
+
+
+# Make sure the group permissions satisfy the previous configuration (if it
+# exists and we don't want to re-write it).
+if [ $rewritepconfig = no ]; then
+ oldgroupname=$(awk '/GROUP-NAME/ {print $3; exit 0}' $pconf)
+ if [ "x$oldgroupname" = "x$reproducible_paper_group_name" ]; then
+ just_a_place_holder_to_avoid_not_equal_test=1;
+ else
+ echo "-----------------------------"
+ echo "!!!!!!!! ERROR !!!!!!!!"
+ echo "-----------------------------"
+ if [ "x$oldgroupname" = x ]; then
+ status="NOT configured for groups"
+ confcommand="./project configure"
+ else
+ status="configured for '$oldgroupname' group"
+ confcommand="./project configure --group=$oldgroupname"
+ fi
+ echo "Project was previously $status!"
+ echo "Either enable re-write of this configuration file,"
+ echo "or re-run this configuration like this:"
+ echo
+ echo " $confcommand"; echo
+ exit 1
+ fi
+fi
+
+
+
+
+
+# Identify the downloader tool
+# ----------------------------
+#
+# After this script finishes, we will have both Wget and cURL for
+# downloading any necessary dataset during the processing. However, to
+# complete the configuration, we may also need to download the source code
+# of some necessary software packages (including the downloaders). So we
+# need to check the host's available tool for downloading at this step.
+if [ $rewritepconfig = yes ]; then
+ if type wget > /dev/null 2>/dev/null; then
+ name=$(which wget)
+
+ # By default Wget keeps the remote file's timestamp, so we'll have
+ # to disable it manually.
+ downloader="$name --no-use-server-timestamps -O";
+ elif type curl > /dev/null 2>/dev/null; then
+ name=$(which curl)
+
+ # - cURL doesn't keep the remote file's timestamp by default.
+ # - With the `-L' option, we tell cURL to follow redirects.
+ downloader="$name -L -o"
+ else
+ cat <<EOF
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!! Warning !!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+Couldn't find GNU Wget, or cURL on this system. These programs are used for
+downloading necessary programs and data if they aren't already present (in
+directories that you can specify with this configure script). Therefore if
+the necessary files are not present, the project will crash.
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+EOF
+ downloader="no-downloader-found"
+ fi;
+fi
+
+
+
+
+
+# Build directory
+# ---------------
+if [ $rewritepconfig = yes ]; then
+ cat <<EOF
+
+===============
+Build directory
+===============
+
+The project's "source" (this directory) and "build" directories are treated
+separately. This greatly helps in managing the many intermediate files that
+are created during the build. The intermediate build files don't need to be
+archived or backed up: you can always re-build them with the contents of
+the source directory. The build directory also needs a relatively large
+amount of free space (atleast serveral Giga-bytes), while the source
+directory (all plain text) will usually be a mega-byte or less.
+
+'.build' (a symbolic link to the build directory) will also be created
+during this configuration. It can help encourage you to set the actual
+build directory in a very different address from this one (one that can be
+deleted and has large volume), while having easy access to it from here.
+
+EOF
+ bdir=
+ junkname=pure-junk-974adfkj38
+ while [ x$bdir == x ]
+ do
+ # Ask the user (if not already set on the command-line).
+ if [ x"$build_dir" = x ]; then
+ read -p"Please enter the top build directory: " build_dir
+ fi
+
+ # If it exists, see if we can write in it. If not, try making it.
+ if [ -d $build_dir ]; then
+ if mkdir $build_dir/$junkname 2> /dev/null; then
+ bdir=$(absolute_dir $build_dir)
+ echo " -- Build directory: '$bdir'"
+ rm -rf $build_dir/$junkname
+ else
+ echo " -- Can't write in '$build_dir'"
+ fi
+ else
+ if mkdir $build_dir 2> /dev/null; then
+ bdir=$(absolute_dir $build_dir)
+ echo " -- Build directory set to (the newly created): '$bdir'"
+ else
+ echo " -- Can't create '$build_dir'"
+ fi
+ fi
+
+ # Reset `build_dir' to blank, so it continues asking when the
+ # previous value wasn't usable.
+ build_dir=
+ done
+fi
+
+
+
+
+
+# Input directory
+# ---------------
+if [ x"$input_dir" = x ]; then
+ indir=$optionaldir
+else
+ indir=$input_dir
+fi
+wfpc2name=$(awk '!/^#/ && $1=="WFPC2IMAGE" {print $3}' $adir/INPUTS.mk)
+wfpc2md5=$(awk '!/^#/ && $1=="WFPC2MD5" {print $3}' $adir/INPUTS.mk)
+wfpc2size=$(awk '!/^#/ && $1=="WFPC2SIZE" {print $3}' $adir/INPUTS.mk)
+wfpc2url=$(awk '!/^#/ && $1=="WFPC2URL" {print $3}' $adir/INPUTS.mk)
+if [ $rewritepconfig = yes ] && [ x"$input_dir" = x ]; then
+ cat <<EOF
+
+----------------------------------
+(OPTIONAL) Input dataset directory
+----------------------------------
+
+This project needs the dataset(s) listed below. If you already have them,
+please specify the directory hosting them on this system. If you don't,
+they will be downloaded automatically. Each file is shown with its total
+volume and its 128-bit MD5 checksum in parenthesis.
+
+ $wfpc2name ($wfpc2size, $wfpc2md5):
+ A 100x100 Hubble Space Telescope WFPC II image used in the FITS
+ standard webpage as a demonstration of this file format.
+ URL: $wfpc2url/$wfpc2name
+
+NOTE I: This directory, or the datasets above, are optional. If it doesn't
+exist, the files will be downloaded in the build directory and used.
+
+NOTE II: This directory (if given) will only be read, nothing will be
+written into it, so no writing permissions are necessary.
+
+TIP: If you have these files in multiple directories on your system and
+don't want to download them or make duplicates, you can create symbolic
+links to them and put those symbolic links in the given top-level
+directory.
+
+EOF
+ read -p"(OPTIONAL) Input datasets directory ($indir): " inindir
+ if [ x$inindir != x ]; then
+ indir=$inindir
+ echo " -- Using '$indir'"
+ fi
+fi
+
+
+
+
+
+# Dependency tarball directory
+# ----------------------------
+if [ x"$software_dir" = x ]; then
+ ddir=$optionaldir
+else
+ ddir=$software_dir
+fi
+if [ $rewritepconfig = yes ] && [ x"$software_dir" = x ]; then
+ cat <<EOF
+
+---------------------------------------
+(OPTIONAL) Software tarball directory
+---------------------------------------
+
+To ensure an identical build environment, the project will use its own
+build of the programs it needs. Therefore the tarball of the relevant
+programs are necessary. If a tarball isn't present in the specified
+directory, *IT WILL BE DOWNLOADED* automatically.
+
+If you don't specify any directory here, or it doesn't contain the tarball
+of a dependency, it is necessary to have an internet connection. The
+project will download the tarballs it needs automatically.
+
+EOF
+ read -p"(OPTIONAL) Directory of dependency tarballs ($ddir): " tmpddir
+ if [ x"$tmpddir" != x ]; then
+ ddir=$tmpddir
+ echo " -- Using '$ddir'"
+ fi
+fi
+
+
+
+
+
+# Write the parameters into the local configuration file.
+if [ $rewritepconfig = yes ]; then
+
+ # Add commented notice.
+ create_file_with_notice $pconf
+
+ # Write the values.
+ sed -e's|@bdir[@]|'"$bdir"'|' \
+ -e's|@indir[@]|'"$indir"'|' \
+ -e's|@ddir[@]|'"$ddir"'|' \
+ -e's|@downloader[@]|'"$downloader"'|' \
+ -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \
+ $pconf.in >> $pconf
+else
+ # Read the values from existing configuration file.
+ inbdir=$(awk '$1=="BDIR" {print $3}' $pconf)
+
+ # Read the software directory.
+ ddir=$(awk '$1=="DEPENDENCIES-DIR" {print $3}' $pconf)
+
+ # The downloader command may contain multiple elements, so we'll just
+ # change the (in memory) first and second tokens to empty space and
+ # write the full line (the original file is unchanged).
+ downloader=$(awk '$1=="DOWNLOADER" {$1=""; $2=""; print $0}' $pconf)
+
+ # Make sure all necessary variables have a value
+ err=0
+ verr=0
+ novalue=""
+ if [ x"$inbdir" = x ]; then novalue="BDIR, "; fi
+ if [ x"$downloader" = x ]; then novalue="$novalue"DOWNLOADER; fi
+ if [ x"$novalue" != x ]; then verr=1; err=1; fi
+
+ # Make sure `bdir' is an absolute path and it exists.
+ berr=0
+ ierr=0
+ bdir=$(absolute_dir $inbdir)
+
+ if ! [ -d $bdir ]; then if ! mkdir $bdir; then berr=1; err=1; fi; fi
+ if [ $err = 1 ]; then
+ cat <<EOF
+
+#################################################################
+######## ERORR reading existing configuration file ############
+#################################################################
+EOF
+ if [ $verr = 1 ]; then
+ cat <<EOF
+
+These variables have no value: $novalue.
+EOF
+ fi
+ if [ $berr = 1 ]; then
+ cat <<EOF
+
+Couldn't create the build directory '$bdir' (value to 'BDIR') in
+'$pconf'.
+EOF
+ fi
+
+ cat <<EOF
+
+Please run the configure script again (accepting to re-write existing
+configuration file) so all the values can be filled and checked.
+#################################################################
+EOF
+ fi
+fi
+
+
+
+
+
+# --------- Delete for no Gnuastro ---------
+# Get the version of Gnuastro that must be used.
+gversion=$(awk '$1=="gnuastro-version" {print $NF}' $depverfile)
+
+# Gnuastro's local configuration settings
+if [ $rewritegconfig = yes ]; then
+ create_file_with_notice $glconf
+ echo "# Minimum number of bytes to use HDD/SSD instead of RAM." >> $glconf
+ echo " minmapsize $minmapsize" >> $glconf
+ echo >> $glconf
+ echo "# Version of Gnuastro that must be used." >> $glconf
+ echo " onlyversion $gversion" >> $glconf
+else
+ ingversion=$(awk '$1=="onlyversion" {print $NF}' $glconf)
+ if [ x$ingversion != x$gversion ]; then
+ cat <<EOF
+______________________________________________________
+!!!!!!!!!!!!!!!!!!CONFIGURATION ERROR!!!!!!!!!!!!!!!!!
+
+Gnuastro's version in '$glconf' ($ingversion) doesn't match the tarball
+version that this project was designed to use in '$depverfile'
+($gversion). Please re-run after removing the former file:
+
+ $ rm $glconf
+ $ ./project configure
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+EOF
+ exit 1
+ fi
+fi
+# ------------------------------------------
+
+
+
+
+
+# Delete final configuration target
+# ---------------------------------
+#
+# We only want to start running the project later if this script has
+# completed successfully. To make sure it hasn't crashed in the middle
+# (without the user noticing), in the end of this script we make a file and
+# we'll delete it here (at the start). Therefore if the script crashed in
+# the middle that file won't exist.
+sdir=$bdir/software
+finaltarget=$sdir/configuration-done.txt
+if ! [ -d $sdir ]; then mkdir $sdir; fi
+rm -f $finaltarget
+
+
+
+
+
+# Project's top-level directories
+# -------------------------------
+#
+# These directories are possibly needed by many steps of process, so to
+# avoid too many directory dependencies throughout the software and
+# analysis Makefiles (thus making them hard to read), we are just building
+# them here
+# Software tarballs
+tardir=$sdir/tarballs
+if ! [ -d $tardir ]; then mkdir $tardir; fi
+
+# Installed software
+instdir=$sdir/installed
+if ! [ -d $instdir ]; then mkdir $instdir; fi
+
+# To record software versions and citation.
+verdir=$instdir/version-info
+if ! [ -d $verdir ]; then mkdir $verdir; fi
+
+# Program and library versions and citation.
+ibidir=$verdir/proglib
+if ! [ -d $ibidir ]; then mkdir $ibidir; fi
+
+# Python module versions and citation.
+ipydir=$verdir/python
+if ! [ -d $ipydir ]; then mkdir $ipydir; fi
+
+# Used software BibTeX entries.
+ictdir=$verdir/cite
+if ! [ -d $ictdir ]; then mkdir $ictdir; fi
+
+# TeXLive versions.
+itidir=$verdir/tex
+if ! [ -d $itidir ]; then mkdir $itidir; fi
+
+# Top-level LaTeX.
+texdir=$bdir/tex
+if ! [ -d $texdir ]; then mkdir $texdir; fi
+
+# LaTeX macros.
+mtexdir=$texdir/macros
+if ! [ -d $mtexdir ]; then mkdir $mtexdir; fi
+
+
+# TeX build directory. If built in a group scenario, the TeX build
+# directory must be separate for each member (so they can work on their
+# relevant parts of the paper without conflicting with each other).
+if [ "x$reproducible_paper_group_name" = x ]; then
+ texbdir=$texdir/build
+else
+ user=$(whoami)
+ texbdir=$texdir/build-$user
+fi
+if ! [ -d $texbdir ]; then mkdir $texbdir; fi
+
+# TiKZ (for building figures within LaTeX).
+tikzdir=$texbdir/tikz
+if ! [ -d $tikzdir ]; then mkdir $tikzdir; fi
+
+
+# Set the symbolic links for easy access to the top project build
+# directories. Note that these are put in each user's source/cloned
+# directory, not in the build directory (which can be shared between many
+# users and thus may already exist).
+#
+# Note: if we don't delete them first, it can happen that an extra link
+# will be created in each directory that points to its parent. So to be
+# safe, we are deleting all the links on each re-configure of the project.
+rm -f .build .local tex/build tex/tikz .gnuastro
+ln -s $bdir .build
+ln -s $instdir .local
+ln -s $texdir tex/build
+ln -s $tikzdir tex/tikz
+# --------- Delete for no Gnuastro ---------
+ln -s $topdir/reproduce/software/config/gnuastro .gnuastro
+# ------------------------------------------
+
+
+# Temporary software un-packing/build directory: if the host has the
+# standard `/dev/shm' mounting-point, we'll do it in shared memory (on the
+# RAM), to avoid harming/over-using the HDDs/SSDs. The RAM of most systems
+# today (>8GB) is large enough for the parallel building of the software.
+#
+# For the name of the directory under `/dev/shm' (for this project), we'll
+# use the names of the two parent directories to the current/running
+# directory, separated by a `-' instead of `/'. We'll then appended that
+# with the user's name (in case multiple users may be working on similar
+# project names). Maybe later, we can use something like `mktemp' to add
+# random characters to this name and make it unique to every run (even for
+# a single user).
+tmpblddir=$sdir/build-tmp
+rm -rf $tmpblddir/* $tmpblddir # If its a link, we need to empty its
+ # contents first, then itself.
+
+# Set the top-level shared memory location.
+if [ -d /dev/shm ]; then shmdir=/dev/shm
+else shmdir=""
+fi
+
+# If a shared memory mounted directory exists and there is enough space
+# there (in RAM), build a temporary directory for this project.
+needed_space=2000000
+if [ x"$shmdir" != x ]; then
+ available_space=$(df $shmdir | awk 'NR==2{print $4}')
+ if [ $available_space -gt $needed_space ]; then
+ dirname=$(pwd | sed -e's/\// /g' \
+ | awk '{l=NF-1; printf("%s-%s",$l, $NF)}')
+ tbshmdir=$shmdir/"$dirname"-$(whoami)
+ if ! [ -d $tbshmdir ]; then mkdir $tbshmdir; fi
+ fi
+else
+ tbshmdir=""
+fi
+
+# If a shared memory directory was created set `build-tmp' to be a
+# symbolic link to it. Otherwise, just build the temporary build
+# directory under the project build directory.
+if [ x$tbshmdir = x ]; then mkdir $tmpblddir;
+else ln -s $tbshmdir $tmpblddir;
+fi
+
+
+
+
+
+# Check for C/C++ compilers
+# -------------------------
+#
+# To build the software, we'll need some basic tools (the compilers in
+# particular) to be present.
+hascc=0;
+if type cc > /dev/null 2>/dev/null; then
+ if type c++ > /dev/null 2>/dev/null; then hascc=1; fi
+else
+ if type gcc > /dev/null 2>/dev/null; then
+ if type g++ > /dev/null 2>/dev/null; then hascc=1; fi
+ else
+ if type clang > /dev/null 2>/dev/null; then
+ if type clang++ > /dev/null 2>/dev/null; then hascc=1; fi
+ fi
+ fi
+fi
+if [ $hascc = 0 ]; then
+ cat <<EOF
+______________________________________________________
+!!!!!!! C/C++ Compiler NOT FOUND !!!!!!!
+
+To build the project's software, the host system needs to have basic C and
+C++ compilers. The executables that were checked are 'cc', 'gcc' and
+'clang' for a C compiler, and 'c++', 'g++' and 'clang++' for a C++
+compiler. If you have a relevant compiler that is not checked, please get
+in touch with us (with the form below) so we add it:
+
+ https://savannah.nongnu.org/support/?func=additem&group=reproduce
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+EOF
+ exit 1
+fi
+
+
+
+
+
+# See if the C compiler can build static libraries
+# ------------------------------------------------
+
+# We are manually only working with shared libraries: because some
+# high-level programs like Wget and cURL need dynamic linking and if we
+# build the libraries statically, our own builds will be ignored and these
+# programs will go and find their necessary libraries on the host system.
+#
+# Another good advantage of shared libraries is that we can actually use
+# the shared library tool of the system (`ldd' with GNU C Library) and see
+# exactly where each linked library comes from. But in static building,
+# unless you follow the build closely, its not easy to see if the source of
+# the library came from the system or our build.
+static_build=no
+
+#oprog=$sdir/static-test
+#cprog=$sdir/static-test.c
+#echo "#include <stdio.h>" > $cprog
+#echo "int main(void) {return 0;}" >> $cprog
+#if [ x$CC = x ]; then CC=gcc; fi;
+#if $CC $cprog -o$oprog -static &> /dev/null; then
+# export static_build="yes"
+#else
+# export static_build="no"
+#fi
+#rm -f $oprog $cprog
+#if [ $printnotice = yes ] && [ $static_build = "no" ]; then
+# cat <<EOF
+#_________________________________________________________________________
+#!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!
+#
+#Your system's C compiler ('$CC') doesn't support building static
+#libraries. Therefore the dependencies will be built dynamically. This means
+#that they will depend more strongly on changes/updates in the host
+#system. For high-level applications (like most research projects in natural
+#sciences), this shouldn't be a significant problem.
+#
+#But generally, for reproducibility, its better to build static libraries
+#and programs. For more on their difference (and generally an introduction
+#on linking), please see the link below:
+#
+#https://www.gnu.org/software/gnuastro/manual/html_node/Linking.html
+#
+#If you have other compilers on your system, you can select a different
+#compiler by setting the 'CC' environment variable before running
+#'./project configure'.
+#
+#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+#
+#EOF
+# sleep 5
+#fi
+
+
+
+
+
+# See if the linker accepts -Wl,-rpath-link
+# -----------------------------------------
+#
+# `-rpath-link' is used to write the information of the linked shared
+# library into the shared object (library or program). But some versions of
+# LLVM's linker don't accept it an can cause problems.
+oprog=$sdir/rpath-test
+cprog=$sdir/rpath-test.c
+echo "#include <stdio.h>" > $cprog
+echo "int main(void) {return 0;}" >> $cprog
+if [ x$CC = x ]; then CC=gcc; fi;
+if $CC $cprog -o$oprog -Wl,-rpath-link &> /dev/null; then
+ export rpath_command="-Wl,-rpath-link=$instdir/lib"
+else
+ export rpath_command=""
+fi
+rm -f $oprog $cprog
+
+
+
+
+
+# See if we need the dynamic-linker (-ldl)
+# ----------------------------------------
+#
+# Some programs (like Wget) need dynamic loading (using `libdl'). On
+# GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But
+# Mac OS doesn't need any explicit linking. So we'll check here to see if
+# it is present (thus necessary) or not.
+oprog=$sdir/ldl-test
+cprog=$sdir/ldl-test.c
+cat > $cprog <<EOF
+#include <stdio.h>
+#include <dlfcn.h>
+int
+main(void) {
+ void *handle=dlopen ("/lib/CEDD_LIB.so.6", RTLD_LAZY);
+ return 0;
+}
+EOF
+if gcc $cprog -o$oprog &> /dev/null; then needs_ldl=no; else needs_ldl=yes; fi
+rm -f $oprog $cprog
+
+
+
+
+
+# inform the user that the build process is starting
+# -------------------------------------------------
+if [ $printnotice = yes ]; then
+ tsec=10
+ cat <<EOF
+
+-------------------------
+Building dependencies ...
+-------------------------
+
+Necessary dependency programs and libraries will be built in $tsec sec.
+
+NOTE: the built software will NOT BE INSTALLED on your system (no root
+access is required). They are only for local usage by this project. They
+will be installed in:
+
+ $sdir/installed
+
+EOF
+ sleep $tsec
+fi
+
+
+
+
+
+# If we are on a Mac OS system
+# ----------------------------
+#
+# For the time being, we'll use the existance of `otool' to see if we are
+# on a Mac OS system or not. Some tools (for example OpenSSL) need to know
+# this.
+#
+# On Mac OS, the building of GCC crashes sometimes while building libiberty
+# with CLang's `g++'. Until we find a solution, we'll just use the host's C
+# compiler.
+if type otool > /dev/null 2>/dev/null; then
+ host_cc=1
+ on_mac_os=yes
+else
+ on_mac_os=no
+fi
+
+
+
+
+
+# Build `flock'
+# -------------
+#
+# Flock (or file-lock) is a unique program that is necessary to serialize
+# the (generally parallel) processing of make when necessary. GNU/Linux
+# machines have it as part of their `util-linux' programs. But to be
+# consistent in non-GNU/Linux systems, we will be using our own build.
+#
+# The reason that `flock' is sepecial is that we need it to serialize the
+# download process of the software tarballs.
+flockversion=$(awk '/flock-version/{print $3}' $depverfile)
+flocktar=flock-$flockversion.tar.gz
+flockurl=http://github.com/discoteq/flock/releases/download/v$flockversion/
+
+# Prepare/download the tarball.
+if ! [ -f $tardir/$flocktar ]; then
+ if [ -f $ddir/$flocktar ]; then
+ cp $ddir/$flocktar $tardir/$flocktar
+ else
+ if ! $downloader $tardir/$flocktar $flockurl/$flocktar; then
+ rm -f $tardir/$flocktar;
+ echo
+ echo "DOWNLOAD ERROR: Couldn't download the 'flock' tarball:"
+ echo " $flockurl"
+ echo
+ echo "You can manually place it in '$ddir' to avoid downloading."
+ exit 1
+ fi
+ fi
+fi
+
+# If the tarball is newer than the (possibly existing) program (the version
+# has changed), then delete the program.
+if [ -f .local/bin/flock ]; then
+ if [ $tardir/$flocktar -nt $ibidir/flock ]; then
+ rm $ibidir/flock
+ fi
+fi
+
+# Build `flock' if necessary.
+if ! [ -f $ibidir/flock ]; then
+ cd $tmpblddir
+ tar xf $tardir/$flocktar
+ cd flock-$flockversion
+ ./configure --prefix=$instdir
+ make
+ make install
+ cd $topdir
+ rm -rf $tmpblddir/flock-$flockversion
+ echo "Discoteq flock $flockversion" > $ibidir/flock
+fi
+
+
+
+
+
+# See if GCC can be built
+# -----------------------
+#
+# On some GNU/Linux distros, the C compiler is broken into `multilib' (for
+# 32-bit and 64-bit support, with their own headers). On these systems,
+# `/usr/include/sys/cdefs.h' and `/usr/lib/libc.a' are not available by
+# default. So GCC will crash with different ugly errors! The only solution
+# is that user manually installs the `multilib' part as root, before
+# running the configure script.
+#
+# Note that `sys/cdefs.h' may be available in other directories (for
+# example `/usr/include/x86_64-linux-gnu/') that are automatically included
+# in an installed GCC. HOWEVER during the build of GCC, all those other
+# directories are ignored. So even if they exist, they are useless.
+warningsleep=0
+if [ $host_cc = 0 ]; then
+ if ! [ -f /usr/include/sys/cdefs.h ]; then
+ host_cc=1
+ warningsleep=1
+ cat <<EOF
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!! Warning !!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+This system doesn't have '/usr/include/sys/cdefs.h'. Because of this, the
+project can't build its custom GCC to ensure better reproducibility. We
+strongly recommend installing the proper package (for your operating
+system) that installs this necessary file. For example on some Debian-based
+GNU/Linux distros, you need these two packages: 'gcc-multilib' and
+'g++-multilib'.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+EOF
+ fi
+
+ if [ -f /usr/lib/libc.a ] || [ -f /usr/lib64/libc.a ]; then
+ # This is just a place holder
+ host_cc=$host_cc
+ else
+ host_cc=1
+ warningsleep=1
+ cat <<EOF
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!! Warning !!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+This system doesn't have '/usr/lib/libc.a' or '/usr/lib64/libc.a'. Because
+of this, the project can't build its custom GCC to ensure better
+reproducibility. We strongly recommend installing the proper package (for
+your operating system) that installs this necessary file.
+
+Some possible solutions:
+ 1. On some Debian-based GNU/Linux distros, these two packages may fix the
+ problem: 'gcc-multilib' and 'g++-multilib'.
+ 2. (BE CAREFUL!) If you have '/usr/lib/x86_64-linux-gnu' but don't have
+ '/usr/lib64', then running the following command might fix this
+ particular problem by making a symbolic link.
+ $ sudo ln -s /usr/lib/x86_64-linux-gnu /usr/lib64
+ After the configure script is finished, delete the link with 'rm
+ /usr/lib64' (you won't need it any more as far as this project is
+ concerned).
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+EOF
+ fi
+
+ if [ $warningsleep = 1 ]; then
+ cat <<EOF
+
+PLEASE SEE THE WARNINGS ABOVE.
+
+Since GCC is pretty low-level, this configuration script will continue in 5
+seconds and use your system's C compiler (it won't build a custom GCC). But
+please consider installing the necessary package(s) to complete your C
+compiler, then re-run './project configure'.
+
+EOF
+ sleep 5
+ fi
+fi
+
+
+
+
+
+# Fortran compiler
+# ----------------
+#
+# If GCC is ultimately build within the project, the user won't need to
+# have a fortran compiler, we'll build it internally for high-level
+# programs. However, when the host C compiler is to be used, the user needs
+# to have a Fortran compiler available.
+if [ $host_cc = 1 ]; then
+ hasfc=0;
+ if type gfortran > /dev/null 2>/dev/null; then hasfc=1; fi
+
+ if [ $hasfc = 0 ]; then
+ cat <<EOF
+______________________________________________________
+!!!!!!! Fortran Compiler NOT FOUND !!!!!!!
+
+Because the project won't be building its own GCC (which includes a Fortran
+compiler), you need to have a Fortran compiler available. Fortran is
+commonly necessary for many lower-level scientific programs. Currently we
+search for 'gfortran'. If you have a Fortran compiler that is not checked,
+please get in touch with us (with the form below) so we add it:
+
+ https://savannah.nongnu.org/support/?func=additem&group=reproduce
+
+Note: GCC will not be built because you are either using the '--host-cc'
+option, or you are using an operating system that currently has bugs when
+building GCC.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+EOF
+ exit 1
+ fi
+fi
+
+
+
+
+
+# Number of threads for basic builds
+# ----------------------------------
+#
+# Since the system might not have GNU Make at this stage, and other Make
+# implementations can't deal with parallel build properly, we'll just
+# default to 1 thread. This is because some versions of Make complain about
+# not having enough 'pipe' (memory) on some systems. After some searching,
+# I found out its because of too many threads. GNU Make will be present on
+# GNU systems (that have `nproc', part of GNU Coreutils). So to simplify
+# the test for GNU Make, we'll just try running `nproc'.
+if which nproc &> /dev/null; then
+ if [ $jobs = 0 ]; then
+ numthreads=$(nproc --all);
+ else
+ numthreads=$jobs
+ fi
+else
+ numthreads=1;
+fi
+
+
+
+
+
+# Build basic software
+# --------------------
+#
+# When building these software we don't have our own un-packing software,
+# Bash, Make, or AWK. In this step, we'll install such low-level basic
+# tools, but we have to be very portable (and use minimal features in all).
+make -f reproduce/software/make/basic.mk \
+ rpath_command=$rpath_command \
+ static_build=$static_build \
+ needs_ldl=$needs_ldl \
+ on_mac_os=$on_mac_os \
+ numthreads=$numthreads \
+ host_cc=$host_cc \
+ -j$numthreads
+
+
+
+
+
+# All other software
+# ------------------
+#
+# We will be making all the dependencies before running the top-level
+# Makefile. To make the job easier, we'll do it in a Makefile, not a
+# script. Bash and Make were the tools we need to run Makefiles, so we had
+# to build them in this script. But after this, we can rely on Makefiles.
+if [ $jobs = 0 ]; then
+ numthreads=$($instdir/bin/nproc --all)
+else
+ numthreads=$jobs
+fi
+.local/bin/env -i HOME=$bdir \
+ .local/bin/make -f reproduce/software/make/high-level.mk \
+ rpath_command=$rpath_command \
+ static_build=$static_build \
+ numthreads=$numthreads \
+ on_mac_os=$on_mac_os \
+ host_cc=$host_cc \
+ -j$numthreads
+
+
+
+
+
+# Make sure TeX Live installed successfully
+# -----------------------------------------
+#
+# TeX Live is managed over the internet, so if there isn't any, or it
+# suddenly gets cut, it can't be built. However, when TeX Live isn't
+# installed, the project can do all its processing independent of it. It
+# will just stop at the stage when all the processing is complete and it is
+# only necessary to build the PDF. So we don't want to stop the project's
+# configuration and building if its not present.
+if [ -f $itidir/texlive-ready-tlmgr ]; then
+ texlive_result=$(cat $itidir/texlive-ready-tlmgr)
+else
+ texlive_result="NOT!"
+fi
+if [ x"$texlive_result" = x"NOT!" ]; then
+ cat <<EOF
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!! Warning !!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+TeX Live couldn't be installed during the configuration (probably because
+there were downloading problems). TeX Live is only necessary in making the
+final PDF (which is only done after all the analysis has been complete). It
+is not used at all during the analysis.
+
+Therefore, if you don't need the final PDF, and just want to do the
+analysis, you can safely ignore this warning and continue.
+
+If you later have internet access and would like to add TeX live to your
+project, please delete the respective files, then re-run configure as shown
+below. Within configure, answer 'n' (for "no") when asked to re-write the
+configuration files.
+
+ rm .local/version-info/tex/texlive-ready-tlmgr
+ ./project configure
+
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+EOF
+fi
+
+
+
+
+
+# Citation of installed software
+#
+# After everything is installed, we'll put all the names and versions in a
+# human-readable paragraph and also prepare the BibTeX citation for the
+# software.
+function prepare_name_version() {
+
+ # First see if the (possible) `*' in the input arguments corresponds to
+ # anything. Note that some of the given directories may be empty (no
+ # software installed).
+ hasfiles=0
+ for f in $@; do
+ if [ -f $f ]; then hasfiles=1; break; fi;
+ done
+
+ # If there are any files, merge all the names in a paragraph.
+ if [ $hasfiles = 1 ]; then
+
+ # Count how many names there are. This is necessary to identify the
+ # last element.
+ num=$(.local/bin/cat $@ \
+ | .local/bin/sed '/^\s*$/d' \
+ | .local/bin/wc -l)
+
+ # Put them all in one paragraph.
+ .local/bin/cat $@ \
+ | .local/bin/sort \
+ | sed 's/_/\\_/' \
+ | .local/bin/awk 'NF>0 { \
+ c++; \
+ if(c==1) \
+ { \
+ if('$num'==1) printf("%s", $0); \
+ else printf("%s", $0); \
+ } \
+ else if(c=='$num') printf(" and %s\n", $0); \
+ else printf(", %s", $0) \
+ }'
+ fi
+}
+
+# Report the different software in separate contexts (separating Python and
+# TeX packages from the C/C++ programs and libraries).
+proglibs=$(prepare_name_version $verdir/proglib/*)
+pymodules=$(prepare_name_version $verdir/python/*)
+texpkg=$(prepare_name_version $verdir/tex/texlive)
+
+# Write them as one paragraph for LaTeX.
+pkgver=$mtexdir/dependencies.tex
+.local/bin/echo "This research was done with the following free" > $pkgver
+.local/bin/echo "software programs and libraries: $proglibs." >> $pkgver
+if [ x"$pymodules" != x ]; then
+ .local/bin/echo "Within Python, the following modules" >> $pkgver
+ echo "were used: $pymodules." >> $pkgver
+fi
+.local/bin/echo "The \LaTeX{} source of the paper was compiled" >> $pkgver
+.local/bin/echo "to make the PDF using the following packages:" >> $pkgver
+.local/bin/echo "$texpkg. We are very grateful to all their" >> $pkgver
+.local/bin/echo "creators for freely providing this necessary" >> $pkgver
+.local/bin/echo "infrastructure. This research (and many " >> $pkgver
+.local/bin/echo "others) would not be possible without them." >> $pkgver
+
+# Prepare the BibTeX entries for the used software (if there are any).
+hasentry=0
+bibfiles="$ictdir/*"
+for f in $bibfiles; do if [ -f $f ]; then hasentry=1; break; fi; done;
+
+# Make sure we start with an empty output file.
+pkgbib=$mtexdir/dependencies-bib.tex
+echo "" > $pkgbib
+
+# Fill it in with all the BibTeX entries in this directory. We'll just
+# avoid writing any comments (usually copyright notices) and also put an
+# empty line after each file's contents to make the output more readable.
+if [ $hasentry = 1 ]; then
+ for f in $bibfiles; do
+ awk '!/^%/{print} END{print ""}' $f >> $pkgbib
+ done
+fi
+
+
+
+
+
+# Clean the temporary build directory
+# ---------------------------------
+#
+# By the time the script reaches here the temporary software build
+# directory should be empty, so just delete it. Note `tmpblddir' may be a
+# symbolic link to shared memory. So, to work in any scenario, first delete
+# the contents of the directory (if it has any), then delete `tmpblddir'.
+.local/bin/rm -rf $tmpblddir/* $tmpblddir
+
+
+
+
+
+# Register successful completion
+# ------------------------------
+echo `.local/bin/date` > $finaltarget
+
+
+
+
+
+
+# Final notice
+# ------------
+#
+# The configuration is now complete, we can inform the user on the next
+# step(s) to take.
+if [ x$reproducible_paper_group_name = x ]; then
+ buildcommand="./project make -j8"
+else
+ buildcommand="./project make --group=$reproducible_paper_group_name -j8"
+fi
+cat <<EOF
+
+----------------
+The project and its environment are configured with no errors.
+
+Please run the following command to start.
+(Replace '8' with the number of CPU threads)
+
+ $buildcommand
+
+To change the configuration later, please re-run './project configure', DO
+NOT manually edit the relevant files.
+
+EOF