From 35ed6cf0df743175688b49a4559793cb7f6e9d66 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sat, 1 Feb 2020 21:24:00 +0100 Subject: IMPORTANT: reproduce/software/bash renamed to reproduce/software/shell Until now the shell scripts in the software building phase were in the `reproduce/software/bash' directory. But given our recent change to a POSIX-only start, the `configure.sh' shell script (which is the main component of this directory) is no longer written with Bash. With this commit, to fix that problem, that directory's name has been changed to `reproduce/software/shell'. --- README-hacking.md | 4 +- project | 4 +- reproduce/analysis/make/initialize.mk | 4 +- reproduce/software/bash/bashrc.sh | 45 - reproduce/software/bash/configure.sh | 1407 ---------------------------- reproduce/software/bash/git-post-checkout | 66 -- reproduce/software/bash/git-pre-commit | 83 -- reproduce/software/make/README.md | 2 +- reproduce/software/make/basic.mk | 6 +- reproduce/software/make/high-level.mk | 4 +- reproduce/software/shell/bashrc.sh | 45 + reproduce/software/shell/configure.sh | 1407 ++++++++++++++++++++++++++++ reproduce/software/shell/git-post-checkout | 66 ++ reproduce/software/shell/git-pre-commit | 83 ++ 14 files changed, 1613 insertions(+), 1613 deletions(-) delete mode 100755 reproduce/software/bash/bashrc.sh delete mode 100755 reproduce/software/bash/configure.sh delete mode 100755 reproduce/software/bash/git-post-checkout delete mode 100755 reproduce/software/bash/git-pre-commit create mode 100755 reproduce/software/shell/bashrc.sh create mode 100755 reproduce/software/shell/configure.sh create mode 100755 reproduce/software/shell/git-post-checkout create mode 100755 reproduce/software/shell/git-pre-commit diff --git a/README-hacking.md b/README-hacking.md index 324ada4..244cb23 100644 --- a/README-hacking.md +++ b/README-hacking.md @@ -959,8 +959,8 @@ for the benefit of others. - *Environment of each recipe*: If you need to define a special environment (or alises, or scripts to run) for all the recipes in - your Makefiles, you can use the Bash startup file - `reproduce/software/bash/bashrc.sh`. This file is loaded before every + your Makefiles, you can use a Bash startup file + `reproduce/software/shell/bashrc.sh`. This file is loaded before every Make recipe is run, just like the `.bashrc` in your home directory is loaded everytime you start a new interactive, non-login terminal. See the comments in that file for more. diff --git a/project b/project index 89361ae..f9a7537 100755 --- a/project +++ b/project @@ -294,7 +294,7 @@ controlled_env() { # Do requested operation # ---------------------- perms="u+r,u+w,g+r,g+w,o-r,o-w,o-x" -configscript=./reproduce/software/bash/configure.sh +configscript=./reproduce/software/shell/configure.sh case $operation in # Build the project's software. @@ -314,7 +314,7 @@ case $operation in # `reproduce/*/bash' should need executable flags, so we are giving # them executable flags by default. If any other file in your project # needs such flags, add them here. - chmod +x reproduce/software/bash/* reproduce/analysis/bash/* + chmod +x reproduce/software/shell/* reproduce/analysis/bash/* # If the user requested, clean the TeX directory from the extra # (to-be-built) directories that may already be there (and will not diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 1137f28..79f9266 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -156,9 +156,9 @@ export DYLD_LIBRARY_PATH := # remote-shell accesss through this environment variable. export OMPI_MCA_plm_rsh_agent=/bin/false -# Recipe startup script, see `reproduce/software/bash/bashrc.sh'. +# Recipe startup script. export PROJECT_STATUS := make -export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh diff --git a/reproduce/software/bash/bashrc.sh b/reproduce/software/bash/bashrc.sh deleted file mode 100755 index b2a151f..0000000 --- a/reproduce/software/bash/bashrc.sh +++ /dev/null @@ -1,45 +0,0 @@ -# Bash startup file for better control of project environment. -# -# To have better control over the environment of each analysis step (Make -# recipe), besides having environment variables (directly included from -# Make), it may also be useful to have a Bash startup file (this file). All -# of the Makefiles set this file as the `BASH_ENV' environment variable, so -# it is loaded into all the Make recipes within the project. -# -# The special `PROJECT_STATUS' environment variable is defined in every -# top-level Makefile of the project. It defines the the state of the Make -# that is calling this script. It can have three values: -# -# configure_basic -# --------------- -# When doing basic configuration, therefore the executed steps cannot -# make any assumptions about the version of Bash (or any other -# program). Therefore it is important for any step in this step to be -# highly portable. -# -# configure_highlevel -# ------------------- -# When building the higher-level programs, so the versions of the -# most basic tools are set and you may safely assume certain -# features. -# -# make -# ---- -# When doing the project's analysis: all software have known -# versions. -# -# -# Copyright (C) 2019-2020 Mohammad Akhlaghi -# -# This script is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This script is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# A copy of the GNU General Public License is available at -# . diff --git a/reproduce/software/bash/configure.sh b/reproduce/software/bash/configure.sh deleted file mode 100755 index a6fbc4e..0000000 --- a/reproduce/software/bash/configure.sh +++ /dev/null @@ -1,1407 +0,0 @@ -#! /bin/sh -# -# Necessary preparations/configurations for the reproducible project. -# -# Copyright (C) 2018-2020 Mohammad Akhlaghi -# -# This script is free software: you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or (at your -# option) any later version. -# -# This script is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -# Public License for more details. -# -# A copy of the GNU General Public License is available at -# . - - -# Script settings -# --------------- -# Stop the script if there are any errors. -set -e - - - - - -# Internal directories -# -------------------- -# -# These are defined to help make this script more readable. -topdir=$(pwd) -optionaldir="/optional/path" -adir=reproduce/analysis/config -cdir=reproduce/software/config - -sbdir=$cdir/installation - -pconf=$sbdir/LOCAL.conf -ptconf=$sbdir/LOCAL_tmp.conf -poconf=$sbdir/LOCAL_old.conf -depverfile=$cdir/installation/versions.conf -depshafile=$cdir/installation/checksums.conf -# --------- Delete for no Gnuastro --------- -glconf=$cdir/gnuastro/gnuastro-local.conf -# ------------------------------------------ - - - - - -# Notice for top of generated files -# --------------------------------- -# -# In case someone opens the files output from the configuration scripts in -# a text editor and wants to edit them, it is important to let them know -# that their changes are not going to be permenant. -create_file_with_notice () -{ - if echo "# IMPORTANT: file can be RE-WRITTEN after './project configure'" > "$1" - then - echo "#" >> "$1" - echo "# This file was created during configuration" >> "$1" - echo "# ('./project configure'). Therefore, it is not under" >> "$1" - echo "# version control and any manual changes to it will be" >> "$1" - echo "# over-written if the project re-configured." >> "$1" - echo "#" >> "$1" - else - echo; echo "Can't write to $1"; echo; - exit 1 - fi -} - - - - - -# Get absolute address -# -------------------- -# -# Since the build directory will go into a symbolic link, we want it to be -# an absolute address. With this function we can make sure of that. -absolute_dir () -{ - if stat "$1" 1> /dev/null; then - echo "$(cd "$(dirname "$1")" && pwd )/$(basename "$1")" - else - exit 1; - fi -} - - - - - -# Check for C/C++ compilers -# ------------------------- -# -# To build the software, we'll need some basic tools (the compilers in -# particular) to be present. -hascc=0; -if type cc > /dev/null 2>/dev/null; then - if type c++ > /dev/null 2>/dev/null; then export CC=cc; hascc=1; fi -else - if type gcc > /dev/null 2>/dev/null; then - if type g++ > /dev/null 2>/dev/null; then export CC=gcc; hascc=1; fi - else - if type clang > /dev/null 2>/dev/null; then - if type clang++ > /dev/null 2>/dev/null; then export CC=clang; hascc=1; fi - fi - fi -fi -if [ $hascc = 0 ]; then - cat < $testsource < -#include -int main(void){printf("...C compiler works.\n"); - return EXIT_SUCCESS;} -EOF -if $CC $testsource -o$testprog && $testprog; then - rm $testsource $testprog -else - rm $testsource - cat < $testsource < -#include -int main(void) {return EXIT_SUCCESS;} -EOF -if $CC $testsource -o$testprog -Wl,-rpath-link 2>/dev/null > /dev/null; then - export rpath_command="-Wl,-rpath-link=$instdir/lib" -else - export rpath_command="" -fi -rm -f $testprog $testsource - - - - - -# See if we need the dynamic-linker (-ldl) -# ---------------------------------------- -# -# Some programs (like Wget) need dynamic loading (using `libdl'). On -# GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But -# Mac OS doesn't need any explicit linking. So we'll check here to see if -# it is present (thus necessary) or not. -cat > $testsource < -#include -int -main(void) { - void *handle=dlopen ("/lib/CEDD_LIB.so.6", RTLD_LAZY); - return 0; -} -EOF -if $CC $testsource -o$testprog 2>/dev/null > /dev/null; then - needs_ldl=no; -else - needs_ldl=yes; -fi -rm -f $testprog $testsource - - - - - -# See if the C compiler can build static libraries -# ------------------------------------------------ - -# We are manually only working with shared libraries: because some -# high-level programs like Wget and cURL need dynamic linking and if we -# build the libraries statically, our own builds will be ignored and these -# programs will go and find their necessary libraries on the host system. -# -# Another good advantage of shared libraries is that we can actually use -# the shared library tool of the system (`ldd' with GNU C Library) and see -# exactly where each linked library comes from. But in static building, -# unless you follow the build closely, its not easy to see if the source of -# the library came from the system or our build. -static_build=no - - - - - -# If we are on a Mac OS system -# ---------------------------- -# -# For the time being, we'll use the existance of `otool' to see if we are -# on a Mac OS system or not. Some tools (for example OpenSSL) need to know -# this. -# -# On Mac OS, the building of GCC crashes sometimes while building libiberty -# with CLang's `g++'. Until we find a solution, we'll just use the host's C -# compiler. -if type otool > /dev/null 2>/dev/null; then - host_cc=1 - on_mac_os=yes -else - host_cc=0 - on_mac_os=no -fi - - - - - -# Necessary C library element positions -# ------------------------------------- -# -# On some systems (in particular Debian-based OSs), the static C library -# and necessary headers in a non-standard place, and we can't build GCC. So -# we need to find them first. The `sys/cdefs.h' header is also in a -# similarly different location. -sys_cpath="" -sys_library_path="" -if [ x"$$on_mac_os" != xyes ]; then - - # Get the GCC target name of the compiler, when its given, special - # C libraries and headers are in a sub-directory of the host. - gcctarget=$(gcc -v 2>&1 \ - | tr ' ' '\n' \ - | awk '/\-\-target/' \ - | sed -e's/\-\-target=//') - if [ x"$gcctarget" != x ]; then - if [ -f /usr/lib/$gcctarget/libc.a ]; then - export sys_library_path=/usr/lib/$gcctarget - export sys_cpath=/usr/include/$gcctarget - fi - fi - - # For a check: - #echo "sys_library_path: $sys_library_path" - #echo "sys_cpath: $sys_cpath" -fi - - - - - -# See if a link-able static C library exists -# ------------------------------------------ -# -# After building GCC, we must use PatchELF to correct its RPATHs. However, -# PatchELF links internally with `libstdc++'. So a dynamicly linked -# PatchELF cannot be used to correct the links to `libstdc++' in general -# (on some systems this causes no problem, but on others it doesn't!). -# -# However, to build a Static PatchELF, we need to be able to link with the -# static C library, which is not always available on some GNU/Linux -# systems. Therefore we need to check this here. If we can't build a static -# PatchELF, we won't build any GCC either. -if [ x"$host_cc" = x0 ]; then - echo; echo; echo "Checking if static C library is available..."; - cat > $testsource < -#include -#include -int main(void){printf("...yes\n"); - return EXIT_SUCCESS;} -EOF - cc_call="$CC $testsource $CPPFLAGS $LDFLAGS -o$testprog -static -lc" - if $cc_call && $testprog; then - gccwarning=0 - good_static_libc=1 - rm $testsource $testprog - else - echo; echo "Compilation command:"; echo "$cc_call" - good_static_libc=0 - rm $testsource - gccwarning=1 - host_cc=1 - cat < /dev/null 2>/dev/null; then hasfc=1; fi - if [ $hasfc = 0 ]; then - cat < $testsource - echo " END" >> $testsource - if gfortran $testsource -o$testprog && $testprog; then - rm $testsource $testprog - else - rm $testsource - cat < /dev/null 2>/dev/null; then - name=$(which wget) - - # By default Wget keeps the remote file's timestamp, so we'll have - # to disable it manually. - downloader="$name --no-use-server-timestamps -O"; - elif type curl > /dev/null 2>/dev/null; then - name=$(which curl) - - # - cURL doesn't keep the remote file's timestamp by default. - # - With the `-L' option, we tell cURL to follow redirects. - downloader="$name -L -o" - else - cat < /dev/null; then - instring="the already existing" - bdir=$(absolute_dir $build_dir) - rm -rf $build_dir/$junkname - else - echo " -- Can't write in '$build_dir'"; echo - fi - else - if mkdir $build_dir 2> /dev/null; then - instring="the newly created" - bdir=$(absolute_dir $build_dir) - else - echo " -- Can't create '$build_dir'"; echo - fi - fi - - # If its given, make sure it isn't a subdirectory of the source - # directory. - if ! [ x"$bdir" = x ]; then - echo "Given build directory: $bdir" - if echo "$bdir/" \ - | grep '^'$currentdir 2> /dev/null > /dev/null; then - - # If it was newly created, it will be empty, so delete it. - if ! [ "$(ls -A $bdir)" ]; then rm --dir $bdir; fi - - # Inform the user that this is not acceptable and reset `bdir'. - bdir= - echo " -- The build-directory cannot be under the source-directory." - echo " Please specify another build-directory that is outside of the source." - echo "" - else - echo " -- Build directory set to ($instring): '$bdir'" - fi - fi - - # Reset `build_dir' to blank, so it continues asking when the - # previous value wasn't usable. - build_dir= - done -fi - - - - - -# Input directory -# --------------- -if [ x"$input_dir" = x ]; then - indir=$optionaldir -else - indir=$input_dir -fi -wfpc2name=$(awk '!/^#/ && $1=="WFPC2IMAGE" {print $3}' $adir/INPUTS.conf) -wfpc2md5=$(awk '!/^#/ && $1=="WFPC2MD5" {print $3}' $adir/INPUTS.conf) -wfpc2size=$(awk '!/^#/ && $1=="WFPC2SIZE" {print $3}' $adir/INPUTS.conf) -wfpc2url=$(awk '!/^#/ && $1=="WFPC2URL" {print $3}' $adir/INPUTS.conf) -if [ $rewritepconfig = yes ] && [ x"$input_dir" = x ]; then - cat <> $pconf -else - # Read the values from existing configuration file. - inbdir=$(awk '$1=="BDIR" {print $3}' $pconf) - - # Read the software directory. - ddir=$(awk '$1=="DEPENDENCIES-DIR" {print $3}' $pconf) - - # The downloader command may contain multiple elements, so we'll just - # change the (in memory) first and second tokens to empty space and - # write the full line (the original file is unchanged). - downloader=$(awk '$1=="DOWNLOADER" {$1=""; $2=""; print $0}' $pconf) - - # Make sure all necessary variables have a value - err=0 - verr=0 - novalue="" - if [ x"$inbdir" = x ]; then novalue="BDIR, "; fi - if [ x"$downloader" = x ]; then novalue="$novalue"DOWNLOADER; fi - if [ x"$novalue" != x ]; then verr=1; err=1; fi - - # Make sure `bdir' is an absolute path and it exists. - berr=0 - ierr=0 - bdir=$(absolute_dir $inbdir) - - if ! [ -d $bdir ]; then if ! mkdir $bdir; then berr=1; err=1; fi; fi - if [ $err = 1 ]; then - cat <> $glconf - echo " minmapsize $minmapsize" >> $glconf - echo >> $glconf - echo "# Version of Gnuastro that must be used." >> $glconf - echo " onlyversion $gversion" >> $glconf -else - ingversion=$(awk '$1=="onlyversion" {print $NF}' $glconf) - if [ x$ingversion != x$gversion ]; then - cat <8GB) is large enough for the parallel building of the software. -# -# For the name of the directory under `/dev/shm' (for this project), we'll -# use the names of the two parent directories to the current/running -# directory, separated by a `-' instead of `/'. We'll then appended that -# with the user's name (in case multiple users may be working on similar -# project names). Maybe later, we can use something like `mktemp' to add -# random characters to this name and make it unique to every run (even for -# a single user). -tmpblddir=$sdir/build-tmp -rm -rf $tmpblddir/* $tmpblddir # If its a link, we need to empty its - # contents first, then itself. - -# Set the top-level shared memory location. -if [ -d /dev/shm ]; then shmdir=/dev/shm -else shmdir="" -fi - -# If a shared memory mounted directory exists and there is enough space -# there (in RAM), build a temporary directory for this project. -needed_space=2000000 -if [ x"$shmdir" != x ]; then - available_space=$(df $shmdir | awk 'NR==2{print $4}') - if [ $available_space -gt $needed_space ]; then - dirname=$(pwd | sed -e's/\// /g' \ - | awk '{l=NF-1; printf("%s-%s",$l, $NF)}') - tbshmdir=$shmdir/"$dirname"-$(whoami) - if ! [ -d $tbshmdir ]; then mkdir $tbshmdir; fi - fi -else - tbshmdir="" -fi - -# If a shared memory directory was created set `build-tmp' to be a -# symbolic link to it. Otherwise, just build the temporary build -# directory under the project build directory. -if [ x$tbshmdir = x ]; then mkdir $tmpblddir; -else ln -s $tbshmdir $tmpblddir; -fi - - - - - -# inform the user that the build process is starting -# ------------------------------------------------- -if [ $printnotice = yes ]; then - tsec=10 - cat < /dev/null; then - if [ $jobs = 0 ]; then - numthreads=$(nproc --all); - else - numthreads=$jobs - fi -else - numthreads=1; -fi - - - - - -# Build `flock' before other program -# ---------------------------------- -# -# Flock (or file-lock) is a unique program that is necessary to serialize -# the (generally parallel) processing of make when necessary. GNU/Linux -# machines have it as part of their `util-linux' programs. But to be -# consistent in non-GNU/Linux systems, we will be using our own build. -# -# The reason that `flock' is sepecial is that we need it to serialize the -# download process of the software tarballs. -flockversion=$(awk '/flock-version/{print $3}' $depverfile) -flockchecksum=$(awk '/flock-checksum/{print $3}' $depshafile) -flocktar=flock-$flockversion.tar.gz -flockurl=http://github.com/discoteq/flock/releases/download/v$flockversion/ - -# Prepare/download the tarball. -if ! [ -f $tardir/$flocktar ]; then - flocktarname=$tardir/$flocktar - ucname=$flocktarname.unchecked - if [ -f $ddir/$flocktar ]; then - cp $ddir/$flocktar $ucname - else - if ! $downloader $ucname $flockurl/$flocktar; then - rm -f $ucname; - echo - echo "DOWNLOAD ERROR: Couldn't download the 'flock' tarball:" - echo " $flockurl" - echo - echo "You can manually place it in '$ddir' to avoid downloading." - exit 1 - fi - fi - - # Make sure this is the correct tarball. - if type sha512sum > /dev/null 2>/dev/null; then - checksum=$(sha512sum "$ucname" | awk '{print $1}') - if [ x$checksum = x$flockchecksum ]; then mv "$ucname" "$flocktarname" - else echo "ERROR: Non-matching checksum for '$flocktar'."; exit 1 - fi; - else mv "$ucname" "$flocktarname" - fi -fi - -# If the tarball is newer than the (possibly existing) program (the version -# has changed), then delete the program. -if [ -f .local/bin/flock ]; then - if [ $tardir/$flocktar -nt $ibidir/flock ]; then - rm $ibidir/flock - fi -fi - -# Build `flock' if necessary. -if ! [ -f $ibidir/flock ]; then - cd $tmpblddir - tar xf $tardir/$flocktar - cd flock-$flockversion - ./configure --prefix=$instdir - make - make install - cd $topdir - rm -rf $tmpblddir/flock-$flockversion - echo "Discoteq flock $flockversion" > $ibidir/flock -fi - - - - - -# Paths needed by the host compiler (only for `basic.mk') -# ------------------------------------------------------- -# -# At the end of the basic build, we need to build GCC. But GCC will build -# in multiple phases, making its own simple compiler in order to build -# itself completely. The intermediate/simple compiler doesn't recognize -# some system specific locations like `/usr/lib/ARCHITECTURE' that some -# operating systems use. We thus need to tell the intermediate compiler -# where its necessary libraries and headers are. -if [ x"$sys_library_path" != x ]; then - if [ x"$LIBRARY_PATH" = x ]; then - export LIBRARY_PATH="$sys_library_path" - else - export LIBRARY_PATH="$LIBRARY_PATH:$sys_library_path" - fi - if [ x"$CPATH" = x ]; then - export CPATH="$sys_cpath" - else - export CPATH="$CPATH:$sys_cpath" - fi -fi - - - - - -# Build basic software -# -------------------- -# -# When building these software we don't have our own un-packing software, -# Bash, Make, or AWK. In this step, we'll install such low-level basic -# tools, but we have to be very portable (and use minimal features in all). -echo; echo "Building necessary software (if necessary)..." -make -f reproduce/software/make/basic.mk \ - good_static_libc=$good_static_libc \ - rpath_command=$rpath_command \ - static_build=$static_build \ - numthreads=$numthreads \ - needs_ldl=$needs_ldl \ - on_mac_os=$on_mac_os \ - host_cc=$host_cc \ - -j$numthreads - - - - - -# All other software -# ------------------ -# -# We will be making all the dependencies before running the top-level -# Makefile. To make the job easier, we'll do it in a Makefile, not a -# script. Bash and Make were the tools we need to run Makefiles, so we had -# to build them in this script. But after this, we can rely on Makefiles. -if [ $jobs = 0 ]; then - numthreads=$($instdir/bin/nproc --all) -else - numthreads=$jobs -fi -.local/bin/env -i HOME=$bdir \ - .local/bin/make -f reproduce/software/make/high-level.mk \ - sys_library_path=$sys_library_path \ - rpath_command=$rpath_command \ - static_build=$static_build \ - numthreads=$numthreads \ - on_mac_os=$on_mac_os \ - sys_cpath=$sys_cpath \ - host_cc=$host_cc \ - -j$numthreads - - - - - -# Make sure TeX Live installed successfully -# ----------------------------------------- -# -# TeX Live is managed over the internet, so if there isn't any, or it -# suddenly gets cut, it can't be built. However, when TeX Live isn't -# installed, the project can do all its processing independent of it. It -# will just stop at the stage when all the processing is complete and it is -# only necessary to build the PDF. So we don't want to stop the project's -# configuration and building if its not present. -if [ -f $itidir/texlive-ready-tlmgr ]; then - texlive_result=$(cat $itidir/texlive-ready-tlmgr) -else - texlive_result="NOT!" -fi -if [ x"$texlive_result" = x"NOT!" ]; then - cat <0 { \ - c++; \ - if(c==1) \ - { \ - if('$num'==1) printf("%s", $0); \ - else printf("%s", $0); \ - } \ - else if(c=='$num') printf(" and %s\n", $0); \ - else printf(", %s", $0) \ - }' - fi -} - -# Report the different software in separate contexts (separating Python and -# TeX packages from the C/C++ programs and libraries). -proglibs=$(prepare_name_version $verdir/proglib/*) -pymodules=$(prepare_name_version $verdir/python/*) -texpkg=$(prepare_name_version $verdir/tex/texlive) - -# Write them as one paragraph for LaTeX. -pkgver=$mtexdir/dependencies.tex -.local/bin/echo "This research was done with the following free" > $pkgver -.local/bin/echo "software programs and libraries: $proglibs." >> $pkgver -if [ x"$pymodules" != x ]; then - .local/bin/echo "Within Python, the following modules" >> $pkgver - echo "were used: $pymodules." >> $pkgver -fi -.local/bin/echo "The \LaTeX{} source of the paper was compiled" >> $pkgver -.local/bin/echo "to make the PDF using the following packages:" >> $pkgver -.local/bin/echo "$texpkg. We are very grateful to all their" >> $pkgver -.local/bin/echo "creators for freely providing this necessary" >> $pkgver -.local/bin/echo "infrastructure. This research (and many " >> $pkgver -.local/bin/echo "others) would not be possible without them." >> $pkgver - -# Prepare the BibTeX entries for the used software (if there are any). -hasentry=0 -bibfiles="$ictdir/*" -for f in $bibfiles; do if [ -f $f ]; then hasentry=1; break; fi; done; - -# Make sure we start with an empty output file. -pkgbib=$mtexdir/dependencies-bib.tex -echo "" > $pkgbib - -# Fill it in with all the BibTeX entries in this directory. We'll just -# avoid writing any comments (usually copyright notices) and also put an -# empty line after each file's contents to make the output more readable. -if [ $hasentry = 1 ]; then - for f in $bibfiles; do - awk '!/^%/{print} END{print ""}' $f >> $pkgbib - done -fi - - - - - -# Clean the temporary build directory -# --------------------------------- -# -# By the time the script reaches here the temporary software build -# directory should be empty, so just delete it. Note `tmpblddir' may be a -# symbolic link to shared memory. So, to work in any scenario, first delete -# the contents of the directory (if it has any), then delete `tmpblddir'. -.local/bin/rm -rf $tmpblddir/* $tmpblddir - - - - - -# Register successful completion -# ------------------------------ -echo `.local/bin/date` > $finaltarget - - - - - - -# Final notice -# ------------ -# -# The configuration is now complete, we can inform the user on the next -# step(s) to take. -if [ x$reproducible_paper_group_name = x ]; then - buildcommand="./project make -j8" -else - buildcommand="./project make --group=$reproducible_paper_group_name -j8" -fi -cat < -# Copyright (C) 2018-2020 Mohammad Akhlaghi -# -# This script is taken from the `examples/hooks/pre-commit' file of the -# `metastore' package (installed within the project, with an MIT license -# for copyright). We have just changed the name of the `MSFILE' and also -# set special characters for the installation location of meta-store so our -# own installation is found by Git. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the -# following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -# USE OR OTHER DEALINGS IN THE SOFTWARE. - - -# File containig the metadata and metastore executable. -MSFILE=".file-metadata" -MSBIN=@BINDIR@/metastore - -# If metastore is not installed, then ignore this script (exit with a -# status of 0). -if [ ! -f $MSBIN ]; then exit 0; fi - -# Delete all temporary files -find @TOP_PROJECT_DIR@/ -name "*~" -type f -delete - -# Function to help in reporting a crash. -exit_on_fail() { - "$@" - if [ $? -ne 0 ]; then - echo "Failed to execute: $@" >&2 - exit 1 - fi -} - -# Check if the metadata file exists. -if [ ! -e "$MSFILE" ]; then - echo "\"$MSFILE\" missing" >&2 - exit 1 -fi - -# Run metastore. -exit_on_fail \ - $MSBIN -a -m -e -E -q -O @USER@ -G @GROUP@ -f "$MSFILE" - -# Return with a success code (0). -exit 0 diff --git a/reproduce/software/bash/git-pre-commit b/reproduce/software/bash/git-pre-commit deleted file mode 100755 index 9d550db..0000000 --- a/reproduce/software/bash/git-pre-commit +++ /dev/null @@ -1,83 +0,0 @@ -#!@BINDIR@/bash -# -# The example hook script to store the metadata information of version -# controlled files (with each commit) using the `metastore' program. -# -# Copyright (C) 2016 Przemyslaw Pawelczyk -# Copyright (C) 2018-2020 Mohammad Akhlaghi -# -# WARNING: -# -# If the commit is aborted (e.g. by not entering any synopsis), -# then updated metastore file (.metadata by default) is not reverted, -# so its new version remains in the index. -# To undo any changes in metastore file written since HEAD commit, -# you may want to reset and checkout HEAD version of the file: -# -# git reset HEAD -- .metadata -# git checkout HEAD -- .metadata -# -# This script is taken from the `examples/hooks/pre-commit' file of the -# `metastore' package (installed within the project, with an MIT license -# for copyright). Here, the name of the `MSFILE' and also set special -# characters for the installation location of meta-store so our own -# installation is found by Git. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, -# distribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the -# following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -# USE OR OTHER DEALINGS IN THE SOFTWARE. - -# File containig the metadata and metastore executable. -MSFILE=".file-metadata" -MSBIN=@BINDIR@/metastore - -# If metastore is not installed, then ignore this script (exit with a -# status of 0). -if [ ! -f $MSBIN ]; then exit 0; fi - -# Function to help in reporting a crash. -exit_on_fail() { - "$@" - if [ $? -ne 0 ]; then - echo "Failed to execute: $@" >&2 - exit 1 - fi -} - -# Run metastore. -exit_on_fail \ - $MSBIN -O @USER@ -G @GROUP@ -s -f "$MSFILE" - -# If it's first metastore commit, store again to include $MSFILE in $MSFILE. -if ! git-ls-tree --name-only HEAD 2>/dev/null | grep -Fqx "$MSFILE"; then - exit_on_fail \ - $MSBIN -O @USER@ -G @GROUP@ -s -f "$MSFILE" -fi - -# Check if the metadata file exists. -if [ ! -e "$MSFILE" ]; then - echo "\"$MSFILE\" missing" >&2 - exit 1 -fi - -# Add the metadata file to the Git repository. -exit_on_fail \ - git-add "$MSFILE" - -# Return with a success code (0). -exit 0 diff --git a/reproduce/software/make/README.md b/reproduce/software/make/README.md index fc04dfb..d71a3d5 100644 --- a/reproduce/software/make/README.md +++ b/reproduce/software/make/README.md @@ -2,7 +2,7 @@ Software building instructions ============================== This directory contains Makefiles that are called by the high-level -`reproduce/software/bash/configure.sh` script. The main target for the +`reproduce/software/shell/configure.sh` script. The main target for the installation of each software is a simple plain text file that contains the name of the software and its version (which is put in the paper in the end). Once built, these plain-text files are all put in the proper diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 7e900b8..d08ba62 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -74,9 +74,9 @@ export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH):$(ildir) \ # causes crashs (see bug #56682). So we'll just give it no value at all. export DYLD_LIBRARY_PATH := -# Recipe startup script, see `reproduce/software/bash/bashrc.sh'. +# Recipe startup script. export PROJECT_STATUS := configure_basic -export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh # Define the top-level basic programs (that don't depend on any other). top-level-programs = low-level-links gcc @@ -1067,7 +1067,7 @@ $(ibidir)/metastore: $(needlibbsd) \ -e's|@GROUP[@]|'$$group'|g' \ -e's|@BINDIR[@]|$(ibdir)|g' \ -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ - reproduce/software/bash/git-$$f > .git/hooks/$$f \ + reproduce/software/shell/git-$$f > .git/hooks/$$f \ && chmod +x .git/hooks/$$f; \ done; \ fi \ diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 02ae1a5..53bef55 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -96,9 +96,9 @@ export DYLD_LIBRARY_PATH := # for `ld'. export LIBRARY_PATH := $(sys_library_path) -# Recipe startup script, see `reproduce/software/bash/bashrc.sh'. +# Recipe startup script, see `reproduce/software/shell/bashrc.sh'. export PROJECT_STATUS := configure_highlevel -export BASH_ENV := $(shell pwd)/reproduce/software/bash/bashrc.sh +export BASH_ENV := $(shell pwd)/reproduce/software/shell/bashrc.sh # Building flags: # diff --git a/reproduce/software/shell/bashrc.sh b/reproduce/software/shell/bashrc.sh new file mode 100755 index 0000000..b2a151f --- /dev/null +++ b/reproduce/software/shell/bashrc.sh @@ -0,0 +1,45 @@ +# Bash startup file for better control of project environment. +# +# To have better control over the environment of each analysis step (Make +# recipe), besides having environment variables (directly included from +# Make), it may also be useful to have a Bash startup file (this file). All +# of the Makefiles set this file as the `BASH_ENV' environment variable, so +# it is loaded into all the Make recipes within the project. +# +# The special `PROJECT_STATUS' environment variable is defined in every +# top-level Makefile of the project. It defines the the state of the Make +# that is calling this script. It can have three values: +# +# configure_basic +# --------------- +# When doing basic configuration, therefore the executed steps cannot +# make any assumptions about the version of Bash (or any other +# program). Therefore it is important for any step in this step to be +# highly portable. +# +# configure_highlevel +# ------------------- +# When building the higher-level programs, so the versions of the +# most basic tools are set and you may safely assume certain +# features. +# +# make +# ---- +# When doing the project's analysis: all software have known +# versions. +# +# +# Copyright (C) 2019-2020 Mohammad Akhlaghi +# +# This script is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This script is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# . diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh new file mode 100755 index 0000000..a6fbc4e --- /dev/null +++ b/reproduce/software/shell/configure.sh @@ -0,0 +1,1407 @@ +#! /bin/sh +# +# Necessary preparations/configurations for the reproducible project. +# +# Copyright (C) 2018-2020 Mohammad Akhlaghi +# +# This script is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This script is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# . + + +# Script settings +# --------------- +# Stop the script if there are any errors. +set -e + + + + + +# Internal directories +# -------------------- +# +# These are defined to help make this script more readable. +topdir=$(pwd) +optionaldir="/optional/path" +adir=reproduce/analysis/config +cdir=reproduce/software/config + +sbdir=$cdir/installation + +pconf=$sbdir/LOCAL.conf +ptconf=$sbdir/LOCAL_tmp.conf +poconf=$sbdir/LOCAL_old.conf +depverfile=$cdir/installation/versions.conf +depshafile=$cdir/installation/checksums.conf +# --------- Delete for no Gnuastro --------- +glconf=$cdir/gnuastro/gnuastro-local.conf +# ------------------------------------------ + + + + + +# Notice for top of generated files +# --------------------------------- +# +# In case someone opens the files output from the configuration scripts in +# a text editor and wants to edit them, it is important to let them know +# that their changes are not going to be permenant. +create_file_with_notice () +{ + if echo "# IMPORTANT: file can be RE-WRITTEN after './project configure'" > "$1" + then + echo "#" >> "$1" + echo "# This file was created during configuration" >> "$1" + echo "# ('./project configure'). Therefore, it is not under" >> "$1" + echo "# version control and any manual changes to it will be" >> "$1" + echo "# over-written if the project re-configured." >> "$1" + echo "#" >> "$1" + else + echo; echo "Can't write to $1"; echo; + exit 1 + fi +} + + + + + +# Get absolute address +# -------------------- +# +# Since the build directory will go into a symbolic link, we want it to be +# an absolute address. With this function we can make sure of that. +absolute_dir () +{ + if stat "$1" 1> /dev/null; then + echo "$(cd "$(dirname "$1")" && pwd )/$(basename "$1")" + else + exit 1; + fi +} + + + + + +# Check for C/C++ compilers +# ------------------------- +# +# To build the software, we'll need some basic tools (the compilers in +# particular) to be present. +hascc=0; +if type cc > /dev/null 2>/dev/null; then + if type c++ > /dev/null 2>/dev/null; then export CC=cc; hascc=1; fi +else + if type gcc > /dev/null 2>/dev/null; then + if type g++ > /dev/null 2>/dev/null; then export CC=gcc; hascc=1; fi + else + if type clang > /dev/null 2>/dev/null; then + if type clang++ > /dev/null 2>/dev/null; then export CC=clang; hascc=1; fi + fi + fi +fi +if [ $hascc = 0 ]; then + cat < $testsource < +#include +int main(void){printf("...C compiler works.\n"); + return EXIT_SUCCESS;} +EOF +if $CC $testsource -o$testprog && $testprog; then + rm $testsource $testprog +else + rm $testsource + cat < $testsource < +#include +int main(void) {return EXIT_SUCCESS;} +EOF +if $CC $testsource -o$testprog -Wl,-rpath-link 2>/dev/null > /dev/null; then + export rpath_command="-Wl,-rpath-link=$instdir/lib" +else + export rpath_command="" +fi +rm -f $testprog $testsource + + + + + +# See if we need the dynamic-linker (-ldl) +# ---------------------------------------- +# +# Some programs (like Wget) need dynamic loading (using `libdl'). On +# GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But +# Mac OS doesn't need any explicit linking. So we'll check here to see if +# it is present (thus necessary) or not. +cat > $testsource < +#include +int +main(void) { + void *handle=dlopen ("/lib/CEDD_LIB.so.6", RTLD_LAZY); + return 0; +} +EOF +if $CC $testsource -o$testprog 2>/dev/null > /dev/null; then + needs_ldl=no; +else + needs_ldl=yes; +fi +rm -f $testprog $testsource + + + + + +# See if the C compiler can build static libraries +# ------------------------------------------------ + +# We are manually only working with shared libraries: because some +# high-level programs like Wget and cURL need dynamic linking and if we +# build the libraries statically, our own builds will be ignored and these +# programs will go and find their necessary libraries on the host system. +# +# Another good advantage of shared libraries is that we can actually use +# the shared library tool of the system (`ldd' with GNU C Library) and see +# exactly where each linked library comes from. But in static building, +# unless you follow the build closely, its not easy to see if the source of +# the library came from the system or our build. +static_build=no + + + + + +# If we are on a Mac OS system +# ---------------------------- +# +# For the time being, we'll use the existance of `otool' to see if we are +# on a Mac OS system or not. Some tools (for example OpenSSL) need to know +# this. +# +# On Mac OS, the building of GCC crashes sometimes while building libiberty +# with CLang's `g++'. Until we find a solution, we'll just use the host's C +# compiler. +if type otool > /dev/null 2>/dev/null; then + host_cc=1 + on_mac_os=yes +else + host_cc=0 + on_mac_os=no +fi + + + + + +# Necessary C library element positions +# ------------------------------------- +# +# On some systems (in particular Debian-based OSs), the static C library +# and necessary headers in a non-standard place, and we can't build GCC. So +# we need to find them first. The `sys/cdefs.h' header is also in a +# similarly different location. +sys_cpath="" +sys_library_path="" +if [ x"$$on_mac_os" != xyes ]; then + + # Get the GCC target name of the compiler, when its given, special + # C libraries and headers are in a sub-directory of the host. + gcctarget=$(gcc -v 2>&1 \ + | tr ' ' '\n' \ + | awk '/\-\-target/' \ + | sed -e's/\-\-target=//') + if [ x"$gcctarget" != x ]; then + if [ -f /usr/lib/$gcctarget/libc.a ]; then + export sys_library_path=/usr/lib/$gcctarget + export sys_cpath=/usr/include/$gcctarget + fi + fi + + # For a check: + #echo "sys_library_path: $sys_library_path" + #echo "sys_cpath: $sys_cpath" +fi + + + + + +# See if a link-able static C library exists +# ------------------------------------------ +# +# After building GCC, we must use PatchELF to correct its RPATHs. However, +# PatchELF links internally with `libstdc++'. So a dynamicly linked +# PatchELF cannot be used to correct the links to `libstdc++' in general +# (on some systems this causes no problem, but on others it doesn't!). +# +# However, to build a Static PatchELF, we need to be able to link with the +# static C library, which is not always available on some GNU/Linux +# systems. Therefore we need to check this here. If we can't build a static +# PatchELF, we won't build any GCC either. +if [ x"$host_cc" = x0 ]; then + echo; echo; echo "Checking if static C library is available..."; + cat > $testsource < +#include +#include +int main(void){printf("...yes\n"); + return EXIT_SUCCESS;} +EOF + cc_call="$CC $testsource $CPPFLAGS $LDFLAGS -o$testprog -static -lc" + if $cc_call && $testprog; then + gccwarning=0 + good_static_libc=1 + rm $testsource $testprog + else + echo; echo "Compilation command:"; echo "$cc_call" + good_static_libc=0 + rm $testsource + gccwarning=1 + host_cc=1 + cat < /dev/null 2>/dev/null; then hasfc=1; fi + if [ $hasfc = 0 ]; then + cat < $testsource + echo " END" >> $testsource + if gfortran $testsource -o$testprog && $testprog; then + rm $testsource $testprog + else + rm $testsource + cat < /dev/null 2>/dev/null; then + name=$(which wget) + + # By default Wget keeps the remote file's timestamp, so we'll have + # to disable it manually. + downloader="$name --no-use-server-timestamps -O"; + elif type curl > /dev/null 2>/dev/null; then + name=$(which curl) + + # - cURL doesn't keep the remote file's timestamp by default. + # - With the `-L' option, we tell cURL to follow redirects. + downloader="$name -L -o" + else + cat < /dev/null; then + instring="the already existing" + bdir=$(absolute_dir $build_dir) + rm -rf $build_dir/$junkname + else + echo " -- Can't write in '$build_dir'"; echo + fi + else + if mkdir $build_dir 2> /dev/null; then + instring="the newly created" + bdir=$(absolute_dir $build_dir) + else + echo " -- Can't create '$build_dir'"; echo + fi + fi + + # If its given, make sure it isn't a subdirectory of the source + # directory. + if ! [ x"$bdir" = x ]; then + echo "Given build directory: $bdir" + if echo "$bdir/" \ + | grep '^'$currentdir 2> /dev/null > /dev/null; then + + # If it was newly created, it will be empty, so delete it. + if ! [ "$(ls -A $bdir)" ]; then rm --dir $bdir; fi + + # Inform the user that this is not acceptable and reset `bdir'. + bdir= + echo " -- The build-directory cannot be under the source-directory." + echo " Please specify another build-directory that is outside of the source." + echo "" + else + echo " -- Build directory set to ($instring): '$bdir'" + fi + fi + + # Reset `build_dir' to blank, so it continues asking when the + # previous value wasn't usable. + build_dir= + done +fi + + + + + +# Input directory +# --------------- +if [ x"$input_dir" = x ]; then + indir=$optionaldir +else + indir=$input_dir +fi +wfpc2name=$(awk '!/^#/ && $1=="WFPC2IMAGE" {print $3}' $adir/INPUTS.conf) +wfpc2md5=$(awk '!/^#/ && $1=="WFPC2MD5" {print $3}' $adir/INPUTS.conf) +wfpc2size=$(awk '!/^#/ && $1=="WFPC2SIZE" {print $3}' $adir/INPUTS.conf) +wfpc2url=$(awk '!/^#/ && $1=="WFPC2URL" {print $3}' $adir/INPUTS.conf) +if [ $rewritepconfig = yes ] && [ x"$input_dir" = x ]; then + cat <> $pconf +else + # Read the values from existing configuration file. + inbdir=$(awk '$1=="BDIR" {print $3}' $pconf) + + # Read the software directory. + ddir=$(awk '$1=="DEPENDENCIES-DIR" {print $3}' $pconf) + + # The downloader command may contain multiple elements, so we'll just + # change the (in memory) first and second tokens to empty space and + # write the full line (the original file is unchanged). + downloader=$(awk '$1=="DOWNLOADER" {$1=""; $2=""; print $0}' $pconf) + + # Make sure all necessary variables have a value + err=0 + verr=0 + novalue="" + if [ x"$inbdir" = x ]; then novalue="BDIR, "; fi + if [ x"$downloader" = x ]; then novalue="$novalue"DOWNLOADER; fi + if [ x"$novalue" != x ]; then verr=1; err=1; fi + + # Make sure `bdir' is an absolute path and it exists. + berr=0 + ierr=0 + bdir=$(absolute_dir $inbdir) + + if ! [ -d $bdir ]; then if ! mkdir $bdir; then berr=1; err=1; fi; fi + if [ $err = 1 ]; then + cat <> $glconf + echo " minmapsize $minmapsize" >> $glconf + echo >> $glconf + echo "# Version of Gnuastro that must be used." >> $glconf + echo " onlyversion $gversion" >> $glconf +else + ingversion=$(awk '$1=="onlyversion" {print $NF}' $glconf) + if [ x$ingversion != x$gversion ]; then + cat <8GB) is large enough for the parallel building of the software. +# +# For the name of the directory under `/dev/shm' (for this project), we'll +# use the names of the two parent directories to the current/running +# directory, separated by a `-' instead of `/'. We'll then appended that +# with the user's name (in case multiple users may be working on similar +# project names). Maybe later, we can use something like `mktemp' to add +# random characters to this name and make it unique to every run (even for +# a single user). +tmpblddir=$sdir/build-tmp +rm -rf $tmpblddir/* $tmpblddir # If its a link, we need to empty its + # contents first, then itself. + +# Set the top-level shared memory location. +if [ -d /dev/shm ]; then shmdir=/dev/shm +else shmdir="" +fi + +# If a shared memory mounted directory exists and there is enough space +# there (in RAM), build a temporary directory for this project. +needed_space=2000000 +if [ x"$shmdir" != x ]; then + available_space=$(df $shmdir | awk 'NR==2{print $4}') + if [ $available_space -gt $needed_space ]; then + dirname=$(pwd | sed -e's/\// /g' \ + | awk '{l=NF-1; printf("%s-%s",$l, $NF)}') + tbshmdir=$shmdir/"$dirname"-$(whoami) + if ! [ -d $tbshmdir ]; then mkdir $tbshmdir; fi + fi +else + tbshmdir="" +fi + +# If a shared memory directory was created set `build-tmp' to be a +# symbolic link to it. Otherwise, just build the temporary build +# directory under the project build directory. +if [ x$tbshmdir = x ]; then mkdir $tmpblddir; +else ln -s $tbshmdir $tmpblddir; +fi + + + + + +# inform the user that the build process is starting +# ------------------------------------------------- +if [ $printnotice = yes ]; then + tsec=10 + cat < /dev/null; then + if [ $jobs = 0 ]; then + numthreads=$(nproc --all); + else + numthreads=$jobs + fi +else + numthreads=1; +fi + + + + + +# Build `flock' before other program +# ---------------------------------- +# +# Flock (or file-lock) is a unique program that is necessary to serialize +# the (generally parallel) processing of make when necessary. GNU/Linux +# machines have it as part of their `util-linux' programs. But to be +# consistent in non-GNU/Linux systems, we will be using our own build. +# +# The reason that `flock' is sepecial is that we need it to serialize the +# download process of the software tarballs. +flockversion=$(awk '/flock-version/{print $3}' $depverfile) +flockchecksum=$(awk '/flock-checksum/{print $3}' $depshafile) +flocktar=flock-$flockversion.tar.gz +flockurl=http://github.com/discoteq/flock/releases/download/v$flockversion/ + +# Prepare/download the tarball. +if ! [ -f $tardir/$flocktar ]; then + flocktarname=$tardir/$flocktar + ucname=$flocktarname.unchecked + if [ -f $ddir/$flocktar ]; then + cp $ddir/$flocktar $ucname + else + if ! $downloader $ucname $flockurl/$flocktar; then + rm -f $ucname; + echo + echo "DOWNLOAD ERROR: Couldn't download the 'flock' tarball:" + echo " $flockurl" + echo + echo "You can manually place it in '$ddir' to avoid downloading." + exit 1 + fi + fi + + # Make sure this is the correct tarball. + if type sha512sum > /dev/null 2>/dev/null; then + checksum=$(sha512sum "$ucname" | awk '{print $1}') + if [ x$checksum = x$flockchecksum ]; then mv "$ucname" "$flocktarname" + else echo "ERROR: Non-matching checksum for '$flocktar'."; exit 1 + fi; + else mv "$ucname" "$flocktarname" + fi +fi + +# If the tarball is newer than the (possibly existing) program (the version +# has changed), then delete the program. +if [ -f .local/bin/flock ]; then + if [ $tardir/$flocktar -nt $ibidir/flock ]; then + rm $ibidir/flock + fi +fi + +# Build `flock' if necessary. +if ! [ -f $ibidir/flock ]; then + cd $tmpblddir + tar xf $tardir/$flocktar + cd flock-$flockversion + ./configure --prefix=$instdir + make + make install + cd $topdir + rm -rf $tmpblddir/flock-$flockversion + echo "Discoteq flock $flockversion" > $ibidir/flock +fi + + + + + +# Paths needed by the host compiler (only for `basic.mk') +# ------------------------------------------------------- +# +# At the end of the basic build, we need to build GCC. But GCC will build +# in multiple phases, making its own simple compiler in order to build +# itself completely. The intermediate/simple compiler doesn't recognize +# some system specific locations like `/usr/lib/ARCHITECTURE' that some +# operating systems use. We thus need to tell the intermediate compiler +# where its necessary libraries and headers are. +if [ x"$sys_library_path" != x ]; then + if [ x"$LIBRARY_PATH" = x ]; then + export LIBRARY_PATH="$sys_library_path" + else + export LIBRARY_PATH="$LIBRARY_PATH:$sys_library_path" + fi + if [ x"$CPATH" = x ]; then + export CPATH="$sys_cpath" + else + export CPATH="$CPATH:$sys_cpath" + fi +fi + + + + + +# Build basic software +# -------------------- +# +# When building these software we don't have our own un-packing software, +# Bash, Make, or AWK. In this step, we'll install such low-level basic +# tools, but we have to be very portable (and use minimal features in all). +echo; echo "Building necessary software (if necessary)..." +make -f reproduce/software/make/basic.mk \ + good_static_libc=$good_static_libc \ + rpath_command=$rpath_command \ + static_build=$static_build \ + numthreads=$numthreads \ + needs_ldl=$needs_ldl \ + on_mac_os=$on_mac_os \ + host_cc=$host_cc \ + -j$numthreads + + + + + +# All other software +# ------------------ +# +# We will be making all the dependencies before running the top-level +# Makefile. To make the job easier, we'll do it in a Makefile, not a +# script. Bash and Make were the tools we need to run Makefiles, so we had +# to build them in this script. But after this, we can rely on Makefiles. +if [ $jobs = 0 ]; then + numthreads=$($instdir/bin/nproc --all) +else + numthreads=$jobs +fi +.local/bin/env -i HOME=$bdir \ + .local/bin/make -f reproduce/software/make/high-level.mk \ + sys_library_path=$sys_library_path \ + rpath_command=$rpath_command \ + static_build=$static_build \ + numthreads=$numthreads \ + on_mac_os=$on_mac_os \ + sys_cpath=$sys_cpath \ + host_cc=$host_cc \ + -j$numthreads + + + + + +# Make sure TeX Live installed successfully +# ----------------------------------------- +# +# TeX Live is managed over the internet, so if there isn't any, or it +# suddenly gets cut, it can't be built. However, when TeX Live isn't +# installed, the project can do all its processing independent of it. It +# will just stop at the stage when all the processing is complete and it is +# only necessary to build the PDF. So we don't want to stop the project's +# configuration and building if its not present. +if [ -f $itidir/texlive-ready-tlmgr ]; then + texlive_result=$(cat $itidir/texlive-ready-tlmgr) +else + texlive_result="NOT!" +fi +if [ x"$texlive_result" = x"NOT!" ]; then + cat <0 { \ + c++; \ + if(c==1) \ + { \ + if('$num'==1) printf("%s", $0); \ + else printf("%s", $0); \ + } \ + else if(c=='$num') printf(" and %s\n", $0); \ + else printf(", %s", $0) \ + }' + fi +} + +# Report the different software in separate contexts (separating Python and +# TeX packages from the C/C++ programs and libraries). +proglibs=$(prepare_name_version $verdir/proglib/*) +pymodules=$(prepare_name_version $verdir/python/*) +texpkg=$(prepare_name_version $verdir/tex/texlive) + +# Write them as one paragraph for LaTeX. +pkgver=$mtexdir/dependencies.tex +.local/bin/echo "This research was done with the following free" > $pkgver +.local/bin/echo "software programs and libraries: $proglibs." >> $pkgver +if [ x"$pymodules" != x ]; then + .local/bin/echo "Within Python, the following modules" >> $pkgver + echo "were used: $pymodules." >> $pkgver +fi +.local/bin/echo "The \LaTeX{} source of the paper was compiled" >> $pkgver +.local/bin/echo "to make the PDF using the following packages:" >> $pkgver +.local/bin/echo "$texpkg. We are very grateful to all their" >> $pkgver +.local/bin/echo "creators for freely providing this necessary" >> $pkgver +.local/bin/echo "infrastructure. This research (and many " >> $pkgver +.local/bin/echo "others) would not be possible without them." >> $pkgver + +# Prepare the BibTeX entries for the used software (if there are any). +hasentry=0 +bibfiles="$ictdir/*" +for f in $bibfiles; do if [ -f $f ]; then hasentry=1; break; fi; done; + +# Make sure we start with an empty output file. +pkgbib=$mtexdir/dependencies-bib.tex +echo "" > $pkgbib + +# Fill it in with all the BibTeX entries in this directory. We'll just +# avoid writing any comments (usually copyright notices) and also put an +# empty line after each file's contents to make the output more readable. +if [ $hasentry = 1 ]; then + for f in $bibfiles; do + awk '!/^%/{print} END{print ""}' $f >> $pkgbib + done +fi + + + + + +# Clean the temporary build directory +# --------------------------------- +# +# By the time the script reaches here the temporary software build +# directory should be empty, so just delete it. Note `tmpblddir' may be a +# symbolic link to shared memory. So, to work in any scenario, first delete +# the contents of the directory (if it has any), then delete `tmpblddir'. +.local/bin/rm -rf $tmpblddir/* $tmpblddir + + + + + +# Register successful completion +# ------------------------------ +echo `.local/bin/date` > $finaltarget + + + + + + +# Final notice +# ------------ +# +# The configuration is now complete, we can inform the user on the next +# step(s) to take. +if [ x$reproducible_paper_group_name = x ]; then + buildcommand="./project make -j8" +else + buildcommand="./project make --group=$reproducible_paper_group_name -j8" +fi +cat < +# Copyright (C) 2018-2020 Mohammad Akhlaghi +# +# This script is taken from the `examples/hooks/pre-commit' file of the +# `metastore' package (installed within the project, with an MIT license +# for copyright). We have just changed the name of the `MSFILE' and also +# set special characters for the installation location of meta-store so our +# own installation is found by Git. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the +# following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +# USE OR OTHER DEALINGS IN THE SOFTWARE. + + +# File containig the metadata and metastore executable. +MSFILE=".file-metadata" +MSBIN=@BINDIR@/metastore + +# If metastore is not installed, then ignore this script (exit with a +# status of 0). +if [ ! -f $MSBIN ]; then exit 0; fi + +# Delete all temporary files +find @TOP_PROJECT_DIR@/ -name "*~" -type f -delete + +# Function to help in reporting a crash. +exit_on_fail() { + "$@" + if [ $? -ne 0 ]; then + echo "Failed to execute: $@" >&2 + exit 1 + fi +} + +# Check if the metadata file exists. +if [ ! -e "$MSFILE" ]; then + echo "\"$MSFILE\" missing" >&2 + exit 1 +fi + +# Run metastore. +exit_on_fail \ + $MSBIN -a -m -e -E -q -O @USER@ -G @GROUP@ -f "$MSFILE" + +# Return with a success code (0). +exit 0 diff --git a/reproduce/software/shell/git-pre-commit b/reproduce/software/shell/git-pre-commit new file mode 100755 index 0000000..9d550db --- /dev/null +++ b/reproduce/software/shell/git-pre-commit @@ -0,0 +1,83 @@ +#!@BINDIR@/bash +# +# The example hook script to store the metadata information of version +# controlled files (with each commit) using the `metastore' program. +# +# Copyright (C) 2016 Przemyslaw Pawelczyk +# Copyright (C) 2018-2020 Mohammad Akhlaghi +# +# WARNING: +# +# If the commit is aborted (e.g. by not entering any synopsis), +# then updated metastore file (.metadata by default) is not reverted, +# so its new version remains in the index. +# To undo any changes in metastore file written since HEAD commit, +# you may want to reset and checkout HEAD version of the file: +# +# git reset HEAD -- .metadata +# git checkout HEAD -- .metadata +# +# This script is taken from the `examples/hooks/pre-commit' file of the +# `metastore' package (installed within the project, with an MIT license +# for copyright). Here, the name of the `MSFILE' and also set special +# characters for the installation location of meta-store so our own +# installation is found by Git. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the +# following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +# NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +# USE OR OTHER DEALINGS IN THE SOFTWARE. + +# File containig the metadata and metastore executable. +MSFILE=".file-metadata" +MSBIN=@BINDIR@/metastore + +# If metastore is not installed, then ignore this script (exit with a +# status of 0). +if [ ! -f $MSBIN ]; then exit 0; fi + +# Function to help in reporting a crash. +exit_on_fail() { + "$@" + if [ $? -ne 0 ]; then + echo "Failed to execute: $@" >&2 + exit 1 + fi +} + +# Run metastore. +exit_on_fail \ + $MSBIN -O @USER@ -G @GROUP@ -s -f "$MSFILE" + +# If it's first metastore commit, store again to include $MSFILE in $MSFILE. +if ! git-ls-tree --name-only HEAD 2>/dev/null | grep -Fqx "$MSFILE"; then + exit_on_fail \ + $MSBIN -O @USER@ -G @GROUP@ -s -f "$MSFILE" +fi + +# Check if the metadata file exists. +if [ ! -e "$MSFILE" ]; then + echo "\"$MSFILE\" missing" >&2 + exit 1 +fi + +# Add the metadata file to the Git repository. +exit_on_fail \ + git-add "$MSFILE" + +# Return with a success code (0). +exit 0 -- cgit v1.2.1