From c6a4aaad62a90af9a72f9bc58650696419474239 Mon Sep 17 00:00:00 2001 From: Mohammadreza Khellat Date: Wed, 26 Aug 2020 14:53:10 +0400 Subject: Machine architecture and byte-order available as LaTeX macro Until now, no machine-related specifications were being documented in the workflow. This information can become helpful when observing differences in the outcome of both software and analysis segments of the workflow by others (some software may behave differently based on host machine). With this commit, the host machine's 'hardware class' and 'byte-order' are collected and now available as LaTeX macros for the authors to use in the paper. Currently it is placed in the acknowledgments, right after mentioning the Maneage commit. Furthermore, the project and configuration scripts are now capable of dealing with input directory names that have SPACE (and other special characters) by putting them inside double-quotes. However, having spaces and metacharacters in the address of the build directory could cause build/install failure for some software source files which are beyond the control of Maneage. So we now check the user's given build directory string, and if the string has any '@', '#', '$', '%', '^', '&', '*', '(', ')', '+', ';', and ' ' (SPACE), it will ask the user to provide a different directory. --- reproduce/analysis/make/paper.mk | 9 +- reproduce/software/make/high-level.mk | 8 +- reproduce/software/shell/configure.sh | 258 ++++++++++++++++++++--------- reproduce/software/shell/pre-make-build.sh | 86 +++++----- 4 files changed, 232 insertions(+), 129 deletions(-) (limited to 'reproduce') diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk index e207337..906b4d9 100644 --- a/reproduce/analysis/make/paper.mk +++ b/reproduce/analysis/make/paper.mk @@ -47,8 +47,9 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex @if [ -f .local/bin/pdflatex ] && [ x"$(pdf-build-final)" != x ]; then # Put a LaTeX input command for all the necessary macro files. + # 'hardware-parameters.tex' is created in 'configure.sh'. rm -f $(mtexdir)/project.tex - for t in $(subst paper,,$(makesrc)); do + for t in $(subst paper,,$(makesrc)) hardware-parameters; do echo "\input{tex/build/macros/$$t.tex}" >> $(mtexdir)/project.tex done else @@ -113,7 +114,7 @@ $(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ # do not use PGFPlots, then you should remove the `-shell-escape' # option for better security. See # https://savannah.nongnu.org/task/?15694 for details. - pdflatex -shell-escape -halt-on-error $$p/paper.tex + pdflatex -shell-escape -halt-on-error "$$p"/paper.tex biber paper fi @@ -142,11 +143,11 @@ paper.pdf: $(mtexdir)/project.tex paper.tex $(texbdir)/paper.bbl cd $(texbdir) # See above for a warning and brief discussion on the the # pdflatex option `-shell-escape'. - pdflatex -shell-escape -halt-on-error $$p/paper.tex + pdflatex -shell-escape -halt-on-error "$$p"/paper.tex # Come back to the top project directory and copy the built PDF # file here. - cd $$p + cd "$$p" cp $(texbdir)/$@ $(final-paper) fi diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 9f434bc..cf9c5a1 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -54,9 +54,9 @@ ibidir = $(BDIR)/software/installed/version-info/proglib # Basic directories (specific to this Makefile). il64dir = $(BDIR)/software/installed/lib64 iidir = $(BDIR)/software/installed/include -shsrcdir = $(shell pwd)/reproduce/software/shell -dtexdir = $(shell pwd)/reproduce/software/bibtex -patchdir = $(shell pwd)/reproduce/software/patches +shsrcdir = "$(shell pwd)"/reproduce/software/shell +dtexdir = "$(shell pwd)"/reproduce/software/bibtex +patchdir = "$(shell pwd)"/reproduce/software/patches itidir = $(BDIR)/software/installed/version-info/tex ictdir = $(BDIR)/software/installed/version-info/cite ipydir = $(BDIR)/software/installed/version-info/python @@ -1411,7 +1411,7 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf tar xf $(tdir)/install-tl-unx.tar.gz cd install-tl-* sed -e's|@installdir[@]|$(idir)|g' \ - $$topdir/reproduce/software/config/texlive.conf \ + "$$topdir"/reproduce/software/config/texlive.conf \ > texlive.conf # TeX Live's installation may fail due to any reason. But TeX Live diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 593b1d9..2b4300b 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -48,7 +48,7 @@ need_gfortran=0 # -------------------- # # These are defined to help make this script more readable. -topdir=$(pwd) +topdir="$(pwd)" optionaldir="/optional/path" adir=reproduce/analysis/config cdir=reproduce/software/config @@ -96,7 +96,8 @@ create_file_with_notice () # an absolute address. With this function we can make sure of that. absolute_dir () { - if stat "$1" 1> /dev/null; then + address="$1" + if stat "$address" 1> /dev/null; then echo "$(cd "$(dirname "$1")" && pwd )/$(basename "$1")" else exit 1; @@ -136,18 +137,18 @@ check_permission () { # Make a `junk' file, activate its executable flag and record its # permissions generally. - local junkfile=$1/check_permission_tmp_file - rm -f $junkfile - echo "Don't let my short life go to waste" > $junkfile - chmod +x $junkfile - local perm_before=$(ls -l $junkfile | awk '{print $1}') + local junkfile="$1"/check_permission_tmp_file + rm -f "$junkfile" + echo "Don't let my short life go to waste" > "$junkfile" + chmod +x "$junkfile" + local perm_before=$(ls -l "$junkfile" | awk '{print $1}') # Now, remove the executable flag and record the permissions. - chmod -x $junkfile - local perm_after=$(ls -l $junkfile | awk '{print $1}') + chmod -x "$junkfile" + local perm_after=$(ls -l "$junkfile" | awk '{print $1}') # Clean up before leaving the function - rm -f $junkfile + rm -f "$junkfile" # If the permissions are equal, the filesystem doesn't allow # permissions. @@ -187,8 +188,8 @@ check_permission () free_space_warning() { fs_threshold=$1 - fs_destpath=$2 - return $(df $fs_destpath \ + IFS='"' fs_destpath="$2" + return $(df "$fs_destpath" \ | awk 'FNR==2 {if($4>'$fs_threshold') print 1; \ else print 0; }') } @@ -224,6 +225,62 @@ fi +# Collect CPU information +# ----------------------- +# +# When the project is built, the type of a machine that built it also has +# to to be documented. This way, if different results or behaviors are +# observed in software-related or analysis-related phases of the project, +# it would be easier to track down the root cause. So far this is just +# later recorded as a LaTeX macro to be put in the final paper, but it +# could be used in a more systematic way to optimize/revise project +# workflow and build. +hw_class=$(uname -m) +if [ x$kernelname = xLinux ]; then + byte_order=$(lscpu \ + | grep 'Byte Order' \ + | awk '{ \ + for(i=3;i /dev/null; then + if [ -d "$build_dir" ]; then + if echo "test" > "$build_dir"/$junkname ; then + rm -f "$build_dir"/$junkname instring="the already existing" - bdir=$(absolute_dir $build_dir) - rm -rf $build_dir/$junkname + bdir="$(absolute_dir "$build_dir")" else echo " ** Can't write in '$build_dir'"; fi else - if mkdir $build_dir 2> /dev/null; then + if mkdir "$build_dir" 2> /dev/null; then instring="the newly created" - bdir=$(absolute_dir $build_dir) + bdir="$(absolute_dir "$build_dir")" else echo " ** Can't create '$build_dir'"; fi fi - # If its given, make sure it isn't a subdirectory of the source + # If it is given, make sure it isn't a subdirectory of the source # directory. if ! [ x"$bdir" = x ]; then if echo "$bdir/" \ - | grep '^'$currentdir 2> /dev/null > /dev/null; then + | grep '^'"$currentdir" 2> /dev/null > /dev/null; then # If it was newly created, it will be empty, so delete it. - if ! [ "$(ls -A $bdir)" ]; then rm --dir $bdir; fi + if ! [ "$(ls -A $bdir)" ]; then rm --dir "$bdir"; fi # Inform the user that this is not acceptable and reset `bdir'. bdir= @@ -793,10 +850,30 @@ EOF fi fi - # If everything is fine until now, see if we're able to manipulate - # file permissions. + # If things are fine so far, make sure it does not contain a space + # or other meta-characters which can cause problems during software + # building. + if ! [ x"$bdir" = x ]; then + hasmeta=0; + case $bdir in *['!'\@\#\$\%\^\&\*\(\)\+\;\ ]* ) hasmeta=1 ;; esac + if [ $hasmeta = 1 ]; then + + # If it was newly created, it will be empty, so delete it. + if ! [ "$(ls -A "$bdir")" ]; then rm --dir "$bdir"; fi + + # Inform the user and set 'bdir' to empty again. + bdir= + echo " ** Build directory should not contain meta-characters" + echo " ** (like SPACE, %, \$, !, ;, or parenthesis, among " + echo " ** others): they can interrup the build for some software." + fi + fi + + # If everything is still fine so far, see if we're able to + # manipulate file permissions in the directory's filesystem and if + # so, see if there is atleast 5GB free space. if ! [ x"$bdir" = x ]; then - if ! $(check_permission $bdir); then + if ! $(IFS='"' check_permission "$bdir"); then # Unable to handle permissions well bdir= echo " ** File permissions can't be modified in this directory" @@ -804,7 +881,7 @@ EOF # Able to handle permissions, now check for 5GB free space # in the given partition (note that the number is in units # of 1024 bytes). If this is not the case, print a warning. - if $(free_space_warning 5000000 $bdir); then + if $(free_space_warning 5000000 "$bdir"); then echo " !! LESS THAN 5GB FREE SPACE IN: $bdir" echo " !! We recommend choosing another partition." echo " !! Build will continue in 5 seconds..." @@ -817,7 +894,7 @@ EOF # reset `build_dir' to blank, so it continues asking for another # directory and let the user know that they must select a new # directory. - if [ x$bdir = x ]; then + if [ x"$bdir" = x ]; then build_dir= echo " ** Please select another directory." echo "" @@ -834,9 +911,9 @@ fi # Input directory # --------------- if [ x"$input_dir" = x ]; then - indir=$optionaldir + indir="$optionaldir" else - indir=$input_dir + indir="$input_dir" fi noninteractive_sleep=2 if [ $rewritepconfig = yes ] && [ x"$input_dir" = x ]; then @@ -878,7 +955,7 @@ EOF # In case an input-directory is given, write it in 'indir'. if [ x$inindir != x ]; then - indir=$(absolute_dir $inindir) + indir="$(absolute_dir "$inindir")" echo " -- Using '$indir'" fi fi @@ -922,7 +999,7 @@ EOF # If given, write the software directory. if [ x"$tmpddir" != x ]; then - ddir=$(absolute_dir $tmpddir) + ddir="$(absolute_dir "$tmpddir")" echo " -- Using '$ddir'" fi fi @@ -946,11 +1023,18 @@ if [ $rewritepconfig = yes ]; then -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \ $pconf.in >> $pconf else - # Read the values from existing configuration file. - inbdir=$(awk '$1=="BDIR" {print $3}' $pconf) - - # Read the software directory. - ddir=$(awk '$1=="DEPENDENCIES-DIR" {print $3}' $pconf) + # Read the values from existing configuration file. Note that the build + # directory may have space characters. Even though we currently check + # against it, we hope to be able to remove this condition in the + # future. + inbdir=$(awk '$1=="BDIR" { for(i=3; i $hwparam +.local/bin/echo "\\newcommand{\\machinebyteorder}{$byte_order}" >> $hwparam +.local/bin/echo "\\newcommand{\\machineaddresssizes}{$address_sizes}" >> $hwparam + + + + + # Clean the temporary build directory # --------------------------------- # diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh index 9188fc9..a18966f 100755 --- a/reproduce/software/shell/pre-make-build.sh +++ b/reproduce/software/shell/pre-make-build.sh @@ -30,32 +30,32 @@ set -e -# Input arguments. -bdir=$1 -ddir=$2 -downloader="$3" -user_backup_urls="$4" +# Input arguments (the 'IFS's are to allow space in the name). +IFS='"' bdir="$1" +IFS='"' ddir="$2" +IFS='"' downloader="$3" +IFS='"' user_backup_urls="$4" # Basic directories/files -topdir=$(pwd) -sdir=$bdir/software -tardir=$sdir/tarballs -instdir=$sdir/installed -tmpblddir=$sdir/build-tmp +topdir="$(pwd)" +sdir="$bdir"/software +tardir="$sdir"/tarballs +instdir="$sdir"/installed +tmpblddir="$sdir"/build-tmp confdir=reproduce/software/config -ibidir=$instdir/version-info/proglib +ibidir="$instdir"/version-info/proglib downloadwrapper=reproduce/analysis/bash/download-multi-try # Derived directories -bindir=$instdir/bin -urlfile=$confdir/urls.conf -versionsfile=$confdir/versions.conf -checksumsfile=$confdir/checksums.conf -backupfile=$confdir/servers-backup.conf +bindir="$instdir"/bin +urlfile="$confdir"/urls.conf +versionsfile="$confdir"/versions.conf +checksumsfile="$confdir"/checksums.conf +backupfile="$confdir"/servers-backup.conf @@ -87,21 +87,21 @@ done # Download the necessary tarball. download_tarball() { # Basic definitions - maneagetar=$tardir/$tarball + maneagetar="$tardir"/"$tarball" # See if the tarball already exists in Maneage. if [ -f "$maneagetar" ]; then just_a_place_holder=1 else - ucname=$tardir/$tarball.unchecked + ucname="$tardir"/"$tarball.unchecked" # If the URL is empty, use the top backup server if [ x$w = x ]; then bservers="$backupservers" - tarballurl=$topbackupserver/$tarball + tarballurl="$topbackupserver"/"$tarball" else bservers="$backupservers_all" - tarballurl=$url/$tarball + tarballurl="$url"/"$tarball" fi # See if it is in the input-software directory, if so, make a link, if @@ -111,19 +111,19 @@ download_tarball() { # link and link to an actual file. if [ -f "$ddir/$tarball" ]; then if type realpath > /dev/null 2> /dev/null; then - ln -sf "$(realpath $ddir/$tarball)" "$ucname" + ln -sf "$(realpath "$ddir/$tarball")" "$ucname" else - cp $ddir/$tarball $ucname + cp "$ddir/$tarball" "$ucname" fi else - $downloadwrapper "$downloader" nolock $tarballurl $ucname \ + $downloadwrapper "$downloader" nolock $tarballurl "$ucname" \ "$bservers" fi # Make sure this is the correct tarball. if type sha512sum > /dev/null 2> /dev/null; then checksum=$(sha512sum "$ucname" | awk '{print $1}') - expectedchecksum=$(awk '/^'$progname'-checksum/{print $3}' $checksumsfile) + expectedchecksum=$(awk '/^'$progname'-checksum/{print $3}' "$checksumsfile") if [ x$checksum = x$expectedchecksum ]; then mv "$ucname" "$maneagetar" else echo "ERROR: Non-matching checksum: $tarball" @@ -137,9 +137,9 @@ download_tarball() { # If the tarball is newer than the (possibly existing) program (the version # has changed), then delete the program. - if [ -f $ibidir/$progname ]; then - if [ $maneagetar -nt $ibidir/$progname ]; then - rm $ibidir/$progname + if [ -f "$ibidir/$progname" ]; then + if [ "$maneagetar" -nt "$ibidir/$progname" ]; then + rm "$ibidir/$progname" fi fi } @@ -151,39 +151,39 @@ download_tarball() { # Build the program from the tarball. This function takes one argument # which is the configure-time options. build_program() { - if ! [ -f $ibidir/$progname ]; then + if ! [ -f "$ibidir/$progname" ]; then # Options configoptions=$1 # Go into the temporary building directory. - cd $tmpblddir + cd "$tmpblddir" unpackdir="$progname"-"$version" # Some implementations of 'tar' don't recognize Lzip, so we need to # manually call Lzip first, then call tar afterwards. - csuffix=$(echo $tarball | sed -e's/\./ /g' | awk '{print $NF}') - rm -rf $unpackdir + csuffix=$(echo "$tarball" | sed -e's/\./ /g' | awk '{print $NF}') + rm -rf "$unpackdir" if [ x$csuffix = xlz ]; then intarrm=1 - intar=$(echo $tarball | sed -e's/.lz//') - lzip -c -d $tardir/$tarball > $intar + intar=$(echo "$tarball" | sed -e's/.lz//') + lzip -c -d "$tardir/$tarball" > $intar else intarrm=0 - intar=$tardir/$tarball + intar="$tardir"/"$tarball" fi # Unpack the tarball and go into it. - tar xf $intar - if [ x$intarrm = x1 ]; then rm $intar; fi - cd $unpackdir + tar xf "$intar" + if [ x$intarrm = x1 ]; then rm "$intar"; fi + cd "$unpackdir" # build the project, either with Make and either without it. if [ x$progname = xlzip ]; then - ./configure --build --check --installdir=$instdir/bin $configoptions + ./configure --build --check --installdir="$instdir/bin" $configoptions else # All others accept the configure script. - ./configure --prefix=$instdir $configoptions + ./configure --prefix="$instdir" $configoptions # To build GNU Make, we don't want to assume the existance of a # Make program, so we use its 'build.sh' script and its own built @@ -198,9 +198,9 @@ build_program() { fi # Clean up the source directory - cd $topdir - rm -rf $tmpblddir/$unpackdir - echo "$progname_tex $version" > $ibidir/$progname + cd "$topdir" + rm -rf "$tmpblddir/$unpackdir" + echo "$progname_tex $version" > "$ibidir/$progname" fi } @@ -219,7 +219,7 @@ build_program() { progname="lzip" progname_tex="Lzip" url=$(awk '/^'$progname'-url/{print $3}' $urlfile) -version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) +version=$(awk '/^'$progname'-version/{print $3}' "$versionsfile") tarball=$progname-$version.tar download_tarball build_program -- cgit v1.2.1