From 6ef4cc854d1df46b719de5d66b45537b0aa11f92 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Sun, 28 Jul 2019 02:32:20 +0100 Subject: Single wrapper instead of old ./configure, Makefile and ./for-group Until now, to work on a project, it was necessary to `./configure' it and build the software. Then we had to run `.local/bin/make' to run the project and do the analysis every time. If the project was a shared project between many users on a large server, it was necessary to call the `./for-group' script. This way of managing the project had a major problem: since the user directly called the lower-level `./configure' or `.local/bin/make' it was not possible to provide high-level control (for example limiting the environment variables). This was especially noticed recently with a bug that was related to environment variables (bug #56682). With this commit, this problem is solved using a single script called `project' in the top directory. To configure and build the project, users can now run these commands: $ ./project configure $ ./project make To work on the project with other users in a group these commands can be used: $ ./project configure --group=GROUPNAME $ ./project make --group=GROUPNAME The old options to both configure and make the project are still valid. Run `./project --help' to see a list. For example: $ ./project configure -e --host-cc $ ./project make -j8 The old `configure' script has been moved to `reproduce/software/bash/configure.sh' and is called by the new `./project' script. The `./project' script now just manages the options, then passes control to the `configure.sh' script. For the "make" step, it also reads the options, then calls Make. So in the lower-level nothing has changed. Only the `./project' script is now the single/direct user interface of the project. On a parallel note: as part of bug #56682, we also found out that on some macOS systems, the `DYLD_LIBRARY_PATH' environment variable has to be set to blank. This is no problem because RPATH is automatically set in macOS and the executables and libraries contain the absolute address of the libraries they should link with. But having `DYLD_LIBRARY_PATH' can conflict with some low-level system libraries and cause very hard to debug linking errors (like that reported in the bug report). This fixes bug #56682. --- reproduce/software/bash/configure.sh | 1224 ++++++++++++++++++++ reproduce/software/config/installation/LOCAL.mk.in | 4 +- reproduce/software/make/atlas-multiple.mk | 4 +- reproduce/software/make/atlas-single.mk | 4 +- reproduce/software/make/basic.mk | 10 +- reproduce/software/make/high-level.mk | 13 +- reproduce/software/make/python.mk | 4 +- 7 files changed, 1249 insertions(+), 14 deletions(-) create mode 100755 reproduce/software/bash/configure.sh (limited to 'reproduce/software') diff --git a/reproduce/software/bash/configure.sh b/reproduce/software/bash/configure.sh new file mode 100755 index 0000000..65b728e --- /dev/null +++ b/reproduce/software/bash/configure.sh @@ -0,0 +1,1224 @@ +#! /bin/bash +# +# Necessary preparations/configurations for the reproducible project. +# +# Copyright (C) 2018-2019 Mohammad Akhlaghi +# +# This script is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This script is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# A copy of the GNU General Public License is available at +# . + + +# Script settings +# --------------- +# Stop the script if there are any errors. +set -e + + + + + +# Internal directories +# -------------------- +# +# These are defined to help make this script more readable. +topdir=$(pwd) +optionaldir="/optional/path" +adir=reproduce/analysis/config +cdir=reproduce/software/config + +sbdir=$cdir/installation + +pconf=$sbdir/LOCAL.mk +ptconf=$sbdir/LOCAL_tmp.mk +poconf=$sbdir/LOCAL_old.mk +depverfile=$cdir/installation/versions.mk +# --------- Delete for no Gnuastro --------- +glconf=$cdir/gnuastro/gnuastro-local.conf +# ------------------------------------------ + + + + + +# Notice for top of generated files +# --------------------------------- +# +# In case someone opens the files output from the configuration scripts in +# a text editor and wants to edit them, it is important to let them know +# that their changes are not going to be permenant. +function create_file_with_notice() { + if echo "# IMPORTANT: file can be RE-WRITTEN after './project configure'" > "$1" + then + echo "#" >> "$1" + echo "# This file was created during configuration" >> "$1" + echo "# ('./project configure'). Therefore, it is not under" >> "$1" + echo "# version control and any manual changes to it will be" >> "$1" + echo "# over-written if the project re-configured." >> "$1" + echo "#" >> "$1" + else + echo; echo "Can't write to $1"; echo; + exit 1 + fi +} + + + + + +# Get absolute address +# -------------------- +# +# Since the build directory will go into a symbolic link, we want it to be +# an absolute address. With this function we can make sure of that. +function absolute_dir() { + if stat "$1" 1> /dev/null; then + echo "$(cd "$(dirname "$1")" && pwd )/$(basename "$1")" + else + exit 1; + fi +} + + + + + +# Inform the user +# --------------- +# +# Print some basic information so the user gets a feeling of what is going +# on and is prepared on what will happen next. +cat < /dev/null 2>/dev/null; then + name=$(which wget) + + # By default Wget keeps the remote file's timestamp, so we'll have + # to disable it manually. + downloader="$name --no-use-server-timestamps -O"; + elif type curl > /dev/null 2>/dev/null; then + name=$(which curl) + + # - cURL doesn't keep the remote file's timestamp by default. + # - With the `-L' option, we tell cURL to follow redirects. + downloader="$name -L -o" + else + cat < /dev/null; then + bdir=$(absolute_dir $build_dir) + echo " -- Build directory: '$bdir'" + rm -rf $build_dir/$junkname + else + echo " -- Can't write in '$build_dir'" + fi + else + if mkdir $build_dir 2> /dev/null; then + bdir=$(absolute_dir $build_dir) + echo " -- Build directory set to (the newly created): '$bdir'" + else + echo " -- Can't create '$build_dir'" + fi + fi + + # Reset `build_dir' to blank, so it continues asking when the + # previous value wasn't usable. + build_dir= + done +fi + + + + + +# Input directory +# --------------- +if [ x"$input_dir" = x ]; then + indir=$optionaldir +else + indir=$input_dir +fi +wfpc2name=$(awk '!/^#/ && $1=="WFPC2IMAGE" {print $3}' $adir/INPUTS.mk) +wfpc2md5=$(awk '!/^#/ && $1=="WFPC2MD5" {print $3}' $adir/INPUTS.mk) +wfpc2size=$(awk '!/^#/ && $1=="WFPC2SIZE" {print $3}' $adir/INPUTS.mk) +wfpc2url=$(awk '!/^#/ && $1=="WFPC2URL" {print $3}' $adir/INPUTS.mk) +if [ $rewritepconfig = yes ] && [ x"$input_dir" = x ]; then + cat <> $pconf +else + # Read the values from existing configuration file. + inbdir=$(awk '$1=="BDIR" {print $3}' $pconf) + + # Read the software directory. + ddir=$(awk '$1=="DEPENDENCIES-DIR" {print $3}' $pconf) + + # The downloader command may contain multiple elements, so we'll just + # change the (in memory) first and second tokens to empty space and + # write the full line (the original file is unchanged). + downloader=$(awk '$1=="DOWNLOADER" {$1=""; $2=""; print $0}' $pconf) + + # Make sure all necessary variables have a value + err=0 + verr=0 + novalue="" + if [ x"$inbdir" = x ]; then novalue="BDIR, "; fi + if [ x"$downloader" = x ]; then novalue="$novalue"DOWNLOADER; fi + if [ x"$novalue" != x ]; then verr=1; err=1; fi + + # Make sure `bdir' is an absolute path and it exists. + berr=0 + ierr=0 + bdir=$(absolute_dir $inbdir) + + if ! [ -d $bdir ]; then if ! mkdir $bdir; then berr=1; err=1; fi; fi + if [ $err = 1 ]; then + cat <> $glconf + echo " minmapsize $minmapsize" >> $glconf + echo >> $glconf + echo "# Version of Gnuastro that must be used." >> $glconf + echo " onlyversion $gversion" >> $glconf +else + ingversion=$(awk '$1=="onlyversion" {print $NF}' $glconf) + if [ x$ingversion != x$gversion ]; then + cat <8GB) is large enough for the parallel building of the software. +# +# For the name of the directory under `/dev/shm' (for this project), we'll +# use the names of the two parent directories to the current/running +# directory, separated by a `-' instead of `/'. We'll then appended that +# with the user's name (in case multiple users may be working on similar +# project names). Maybe later, we can use something like `mktemp' to add +# random characters to this name and make it unique to every run (even for +# a single user). +tmpblddir=$sdir/build-tmp +rm -rf $tmpblddir/* $tmpblddir # If its a link, we need to empty its + # contents first, then itself. + +# Set the top-level shared memory location. +if [ -d /dev/shm ]; then shmdir=/dev/shm +else shmdir="" +fi + +# If a shared memory mounted directory exists and there is enough space +# there (in RAM), build a temporary directory for this project. +needed_space=2000000 +if [ x"$shmdir" != x ]; then + available_space=$(df $shmdir | awk 'NR==2{print $4}') + if [ $available_space -gt $needed_space ]; then + dirname=$(pwd | sed -e's/\// /g' \ + | awk '{l=NF-1; printf("%s-%s",$l, $NF)}') + tbshmdir=$shmdir/"$dirname"-$(whoami) + if ! [ -d $tbshmdir ]; then mkdir $tbshmdir; fi + fi +else + tbshmdir="" +fi + +# If a shared memory directory was created set `build-tmp' to be a +# symbolic link to it. Otherwise, just build the temporary build +# directory under the project build directory. +if [ x$tbshmdir = x ]; then mkdir $tmpblddir; +else ln -s $tbshmdir $tmpblddir; +fi + + + + + +# Check for C/C++ compilers +# ------------------------- +# +# To build the software, we'll need some basic tools (the compilers in +# particular) to be present. +hascc=0; +if type cc > /dev/null 2>/dev/null; then + if type c++ > /dev/null 2>/dev/null; then hascc=1; fi +else + if type gcc > /dev/null 2>/dev/null; then + if type g++ > /dev/null 2>/dev/null; then hascc=1; fi + else + if type clang > /dev/null 2>/dev/null; then + if type clang++ > /dev/null 2>/dev/null; then hascc=1; fi + fi + fi +fi +if [ $hascc = 0 ]; then + cat <" > $cprog +#echo "int main(void) {return 0;}" >> $cprog +#if [ x$CC = x ]; then CC=gcc; fi; +#if $CC $cprog -o$oprog -static &> /dev/null; then +# export static_build="yes" +#else +# export static_build="no" +#fi +#rm -f $oprog $cprog +#if [ $printnotice = yes ] && [ $static_build = "no" ]; then +# cat <" > $cprog +echo "int main(void) {return 0;}" >> $cprog +if [ x$CC = x ]; then CC=gcc; fi; +if $CC $cprog -o$oprog -Wl,-rpath-link &> /dev/null; then + export rpath_command="-Wl,-rpath-link=$instdir/lib" +else + export rpath_command="" +fi +rm -f $oprog $cprog + + + + + +# See if we need the dynamic-linker (-ldl) +# ---------------------------------------- +# +# Some programs (like Wget) need dynamic loading (using `libdl'). On +# GNU/Linux systems, we'll need the `-ldl' flag to link such programs. But +# Mac OS doesn't need any explicit linking. So we'll check here to see if +# it is present (thus necessary) or not. +oprog=$sdir/ldl-test +cprog=$sdir/ldl-test.c +cat > $cprog < +#include +int +main(void) { + void *handle=dlopen ("/lib/CEDD_LIB.so.6", RTLD_LAZY); + return 0; +} +EOF +if gcc $cprog -o$oprog &> /dev/null; then needs_ldl=no; else needs_ldl=yes; fi +rm -f $oprog $cprog + + + + + +# inform the user that the build process is starting +# ------------------------------------------------- +if [ $printnotice = yes ]; then + tsec=10 + cat < /dev/null 2>/dev/null; then + host_cc=1 + on_mac_os=yes +else + on_mac_os=no +fi + + + + + +# Build `flock' +# ------------- +# +# Flock (or file-lock) is a unique program that is necessary to serialize +# the (generally parallel) processing of make when necessary. GNU/Linux +# machines have it as part of their `util-linux' programs. But to be +# consistent in non-GNU/Linux systems, we will be using our own build. +# +# The reason that `flock' is sepecial is that we need it to serialize the +# download process of the software tarballs. +flockversion=$(awk '/flock-version/{print $3}' $depverfile) +flocktar=flock-$flockversion.tar.gz +flockurl=http://github.com/discoteq/flock/releases/download/v$flockversion/ + +# Prepare/download the tarball. +if ! [ -f $tardir/$flocktar ]; then + if [ -f $ddir/$flocktar ]; then + cp $ddir/$flocktar $tardir/$flocktar + else + if ! $downloader $tardir/$flocktar $flockurl/$flocktar; then + rm -f $tardir/$flocktar; + echo + echo "DOWNLOAD ERROR: Couldn't download the 'flock' tarball:" + echo " $flockurl" + echo + echo "You can manually place it in '$ddir' to avoid downloading." + exit 1 + fi + fi +fi + +# If the tarball is newer than the (possibly existing) program (the version +# has changed), then delete the program. +if [ -f .local/bin/flock ]; then + if [ $tardir/$flocktar -nt $ibidir/flock ]; then + rm $ibidir/flock + fi +fi + +# Build `flock' if necessary. +if ! [ -f $ibidir/flock ]; then + cd $tmpblddir + tar xf $tardir/$flocktar + cd flock-$flockversion + ./configure --prefix=$instdir + make + make install + cd $topdir + rm -rf $tmpblddir/flock-$flockversion + echo "Discoteq flock $flockversion" > $ibidir/flock +fi + + + + + +# See if GCC can be built +# ----------------------- +# +# On some GNU/Linux distros, the C compiler is broken into `multilib' (for +# 32-bit and 64-bit support, with their own headers). On these systems, +# `/usr/include/sys/cdefs.h' and `/usr/lib/libc.a' are not available by +# default. So GCC will crash with different ugly errors! The only solution +# is that user manually installs the `multilib' part as root, before +# running the configure script. +# +# Note that `sys/cdefs.h' may be available in other directories (for +# example `/usr/include/x86_64-linux-gnu/') that are automatically included +# in an installed GCC. HOWEVER during the build of GCC, all those other +# directories are ignored. So even if they exist, they are useless. +warningsleep=0 +if [ $host_cc = 0 ]; then + if ! [ -f /usr/include/sys/cdefs.h ]; then + host_cc=1 + warningsleep=1 + cat < /dev/null 2>/dev/null; then hasfc=1; fi + + if [ $hasfc = 0 ]; then + cat < /dev/null; then + if [ $jobs = 0 ]; then + numthreads=$(nproc --all); + else + numthreads=$jobs + fi +else + numthreads=1; +fi + + + + + +# Build basic software +# -------------------- +# +# When building these software we don't have our own un-packing software, +# Bash, Make, or AWK. In this step, we'll install such low-level basic +# tools, but we have to be very portable (and use minimal features in all). +make -f reproduce/software/make/basic.mk \ + rpath_command=$rpath_command \ + static_build=$static_build \ + needs_ldl=$needs_ldl \ + on_mac_os=$on_mac_os \ + numthreads=$numthreads \ + host_cc=$host_cc \ + -j$numthreads + + + + + +# All other software +# ------------------ +# +# We will be making all the dependencies before running the top-level +# Makefile. To make the job easier, we'll do it in a Makefile, not a +# script. Bash and Make were the tools we need to run Makefiles, so we had +# to build them in this script. But after this, we can rely on Makefiles. +if [ $jobs = 0 ]; then + numthreads=$($instdir/bin/nproc --all) +else + numthreads=$jobs +fi +.local/bin/env -i HOME=$bdir \ + .local/bin/make -f reproduce/software/make/high-level.mk \ + rpath_command=$rpath_command \ + static_build=$static_build \ + numthreads=$numthreads \ + on_mac_os=$on_mac_os \ + host_cc=$host_cc \ + -j$numthreads + + + + + +# Make sure TeX Live installed successfully +# ----------------------------------------- +# +# TeX Live is managed over the internet, so if there isn't any, or it +# suddenly gets cut, it can't be built. However, when TeX Live isn't +# installed, the project can do all its processing independent of it. It +# will just stop at the stage when all the processing is complete and it is +# only necessary to build the PDF. So we don't want to stop the project's +# configuration and building if its not present. +if [ -f $itidir/texlive-ready-tlmgr ]; then + texlive_result=$(cat $itidir/texlive-ready-tlmgr) +else + texlive_result="NOT!" +fi +if [ x"$texlive_result" = x"NOT!" ]; then + cat <0 { \ + c++; \ + if(c==1) \ + { \ + if('$num'==1) printf("%s", $0); \ + else printf("%s", $0); \ + } \ + else if(c=='$num') printf(" and %s\n", $0); \ + else printf(", %s", $0) \ + }' + fi +} + +# Report the different software in separate contexts (separating Python and +# TeX packages from the C/C++ programs and libraries). +proglibs=$(prepare_name_version $verdir/proglib/*) +pymodules=$(prepare_name_version $verdir/python/*) +texpkg=$(prepare_name_version $verdir/tex/texlive) + +# Write them as one paragraph for LaTeX. +pkgver=$mtexdir/dependencies.tex +.local/bin/echo "This research was done with the following free" > $pkgver +.local/bin/echo "software programs and libraries: $proglibs." >> $pkgver +if [ x"$pymodules" != x ]; then + .local/bin/echo "Within Python, the following modules" >> $pkgver + echo "were used: $pymodules." >> $pkgver +fi +.local/bin/echo "The \LaTeX{} source of the paper was compiled" >> $pkgver +.local/bin/echo "to make the PDF using the following packages:" >> $pkgver +.local/bin/echo "$texpkg. We are very grateful to all their" >> $pkgver +.local/bin/echo "creators for freely providing this necessary" >> $pkgver +.local/bin/echo "infrastructure. This research (and many " >> $pkgver +.local/bin/echo "others) would not be possible without them." >> $pkgver + +# Prepare the BibTeX entries for the used software (if there are any). +hasentry=0 +bibfiles="$ictdir/*" +for f in $bibfiles; do if [ -f $f ]; then hasentry=1; break; fi; done; + +# Make sure we start with an empty output file. +pkgbib=$mtexdir/dependencies-bib.tex +echo "" > $pkgbib + +# Fill it in with all the BibTeX entries in this directory. We'll just +# avoid writing any comments (usually copyright notices) and also put an +# empty line after each file's contents to make the output more readable. +if [ $hasentry = 1 ]; then + for f in $bibfiles; do + awk '!/^%/{print} END{print ""}' $f >> $pkgbib + done +fi + + + + + +# Clean the temporary build directory +# --------------------------------- +# +# By the time the script reaches here the temporary software build +# directory should be empty, so just delete it. Note `tmpblddir' may be a +# symbolic link to shared memory. So, to work in any scenario, first delete +# the contents of the directory (if it has any), then delete `tmpblddir'. +.local/bin/rm -rf $tmpblddir/* $tmpblddir + + + + + +# Register successful completion +# ------------------------------ +echo `.local/bin/date` > $finaltarget + + + + + + +# Final notice +# ------------ +# +# The configuration is now complete, we can inform the user on the next +# step(s) to take. +if [ x$reproducible_paper_group_name = x ]; then + buildcommand="./project make -j8" +else + buildcommand="./project make --group=$reproducible_paper_group_name -j8" +fi +cat < # diff --git a/reproduce/software/make/atlas-multiple.mk b/reproduce/software/make/atlas-multiple.mk index fef25c7..f5efb35 100755 --- a/reproduce/software/make/atlas-multiple.mk +++ b/reproduce/software/make/atlas-multiple.mk @@ -3,8 +3,8 @@ # ------------------------------------------------------------------------ # !!!!! IMPORTANT NOTES !!!!! # -# This Makefile will be run during the initial `./configure' script. It is -# not included into the reproduction pipe after that. +# This Makefile will be run during the initial `./project configure' +# script. It is not included into the reproduction pipe after that. # # ------------------------------------------------------------------------ # diff --git a/reproduce/software/make/atlas-single.mk b/reproduce/software/make/atlas-single.mk index dde2926..c4621ce 100755 --- a/reproduce/software/make/atlas-single.mk +++ b/reproduce/software/make/atlas-single.mk @@ -3,8 +3,8 @@ # ------------------------------------------------------------------------ # !!!!! IMPORTANT NOTES !!!!! # -# This Makefile will be run during the initial `./configure' script. It is -# not included into the reproduction pipe after that. +# This Makefile will be run during the initial `./project configure' +# script. It is not included into the reproduction pipe after that. # # ------------------------------------------------------------------------ # diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index bab37ed..e4ab55a 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -4,8 +4,8 @@ # ------------------------------------------------------------------------ # !!!!! IMPORTANT NOTES !!!!! # -# This Makefile will be run by the initial `./configure' script. It is not -# included into the project after that. +# This Makefile will be run by the initial `./project configure' script. It +# is not included into the project after that. # # This Makefile builds very low-level and basic tools like GNU Tar, GNU # Bash, GNU Make, GCC and etc. Therefore this is the only Makefile in the @@ -58,7 +58,11 @@ export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) -export DYLD_LIBRARY_PATH := $(ildir):$(DYLD_LIBRARY_PATH) + +# RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is +# ultimately redundant. But on some systems, even having a single value +# causes crashs (see bug #56682). So we'll just give it no value at all. +export DYLD_LIBRARY_PATH := # Define the top-level basic programs (that don't depend on any other). top-level-programs = low-level-links gcc diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index dee23ca..b9a70de 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -3,8 +3,8 @@ # ------------------------------------------------------------------------ # !!!!! IMPORTANT NOTES !!!!! # -# This Makefile will be run by the initial `./configure' script. It is not -# included into the reproduction pipe after that. +# This Makefile will be run by the initial `./project configure' script. It +# is not included into the reproduction pipe after that. # # ------------------------------------------------------------------------ # @@ -76,7 +76,11 @@ export LD_RUN_PATH := $(ildir):$(il64dir) export PKG_CONFIG_PATH := $(ildir)/pkgconfig export LD_LIBRARY_PATH := $(ildir):$(il64dir) export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export DYLD_LIBRARY_PATH := $(ildir):$(il64dir) + +# RPATH is automatically written in macOS, so `DYLD_LIBRARY_PATH' is +# ultimately redundant. But on some systems, even having a single value +# causes crashs (see bug #56682). So we'll just give it no value at all. +export DYLD_LIBRARY_PATH := # Building flags: # @@ -90,6 +94,9 @@ export CXXFLAGS := -liconv endif + + + # We want the download to happen on a single thread. So we need to define a # lock, and call a special script we have written for this job. These are # placed here because we want them both in the `high-level.mk' and diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 22b668f..ed1c87d 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -3,8 +3,8 @@ # ------------------------------------------------------------------------ # !!!!! IMPORTANT NOTES !!!!! # -# This Makefile will be run by the initial `./configure' script. It is not -# included into the reproduction pipe after that. +# This Makefile will be run by the initial `./project configure' script. It +# is not included into the reproduction pipe after that. # # ------------------------------------------------------------------------ # -- cgit v1.2.1