From c6a4aaad62a90af9a72f9bc58650696419474239 Mon Sep 17 00:00:00 2001 From: Mohammadreza Khellat Date: Wed, 26 Aug 2020 14:53:10 +0400 Subject: Machine architecture and byte-order available as LaTeX macro Until now, no machine-related specifications were being documented in the workflow. This information can become helpful when observing differences in the outcome of both software and analysis segments of the workflow by others (some software may behave differently based on host machine). With this commit, the host machine's 'hardware class' and 'byte-order' are collected and now available as LaTeX macros for the authors to use in the paper. Currently it is placed in the acknowledgments, right after mentioning the Maneage commit. Furthermore, the project and configuration scripts are now capable of dealing with input directory names that have SPACE (and other special characters) by putting them inside double-quotes. However, having spaces and metacharacters in the address of the build directory could cause build/install failure for some software source files which are beyond the control of Maneage. So we now check the user's given build directory string, and if the string has any '@', '#', '$', '%', '^', '&', '*', '(', ')', '+', ';', and ' ' (SPACE), it will ask the user to provide a different directory. --- reproduce/software/shell/configure.sh | 258 ++++++++++++++++++++--------- reproduce/software/shell/pre-make-build.sh | 86 +++++----- 2 files changed, 223 insertions(+), 121 deletions(-) (limited to 'reproduce/software/shell') diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 593b1d9..2b4300b 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -48,7 +48,7 @@ need_gfortran=0 # -------------------- # # These are defined to help make this script more readable. -topdir=$(pwd) +topdir="$(pwd)" optionaldir="/optional/path" adir=reproduce/analysis/config cdir=reproduce/software/config @@ -96,7 +96,8 @@ create_file_with_notice () # an absolute address. With this function we can make sure of that. absolute_dir () { - if stat "$1" 1> /dev/null; then + address="$1" + if stat "$address" 1> /dev/null; then echo "$(cd "$(dirname "$1")" && pwd )/$(basename "$1")" else exit 1; @@ -136,18 +137,18 @@ check_permission () { # Make a `junk' file, activate its executable flag and record its # permissions generally. - local junkfile=$1/check_permission_tmp_file - rm -f $junkfile - echo "Don't let my short life go to waste" > $junkfile - chmod +x $junkfile - local perm_before=$(ls -l $junkfile | awk '{print $1}') + local junkfile="$1"/check_permission_tmp_file + rm -f "$junkfile" + echo "Don't let my short life go to waste" > "$junkfile" + chmod +x "$junkfile" + local perm_before=$(ls -l "$junkfile" | awk '{print $1}') # Now, remove the executable flag and record the permissions. - chmod -x $junkfile - local perm_after=$(ls -l $junkfile | awk '{print $1}') + chmod -x "$junkfile" + local perm_after=$(ls -l "$junkfile" | awk '{print $1}') # Clean up before leaving the function - rm -f $junkfile + rm -f "$junkfile" # If the permissions are equal, the filesystem doesn't allow # permissions. @@ -187,8 +188,8 @@ check_permission () free_space_warning() { fs_threshold=$1 - fs_destpath=$2 - return $(df $fs_destpath \ + IFS='"' fs_destpath="$2" + return $(df "$fs_destpath" \ | awk 'FNR==2 {if($4>'$fs_threshold') print 1; \ else print 0; }') } @@ -224,6 +225,62 @@ fi +# Collect CPU information +# ----------------------- +# +# When the project is built, the type of a machine that built it also has +# to to be documented. This way, if different results or behaviors are +# observed in software-related or analysis-related phases of the project, +# it would be easier to track down the root cause. So far this is just +# later recorded as a LaTeX macro to be put in the final paper, but it +# could be used in a more systematic way to optimize/revise project +# workflow and build. +hw_class=$(uname -m) +if [ x$kernelname = xLinux ]; then + byte_order=$(lscpu \ + | grep 'Byte Order' \ + | awk '{ \ + for(i=3;i /dev/null; then + if [ -d "$build_dir" ]; then + if echo "test" > "$build_dir"/$junkname ; then + rm -f "$build_dir"/$junkname instring="the already existing" - bdir=$(absolute_dir $build_dir) - rm -rf $build_dir/$junkname + bdir="$(absolute_dir "$build_dir")" else echo " ** Can't write in '$build_dir'"; fi else - if mkdir $build_dir 2> /dev/null; then + if mkdir "$build_dir" 2> /dev/null; then instring="the newly created" - bdir=$(absolute_dir $build_dir) + bdir="$(absolute_dir "$build_dir")" else echo " ** Can't create '$build_dir'"; fi fi - # If its given, make sure it isn't a subdirectory of the source + # If it is given, make sure it isn't a subdirectory of the source # directory. if ! [ x"$bdir" = x ]; then if echo "$bdir/" \ - | grep '^'$currentdir 2> /dev/null > /dev/null; then + | grep '^'"$currentdir" 2> /dev/null > /dev/null; then # If it was newly created, it will be empty, so delete it. - if ! [ "$(ls -A $bdir)" ]; then rm --dir $bdir; fi + if ! [ "$(ls -A $bdir)" ]; then rm --dir "$bdir"; fi # Inform the user that this is not acceptable and reset `bdir'. bdir= @@ -793,10 +850,30 @@ EOF fi fi - # If everything is fine until now, see if we're able to manipulate - # file permissions. + # If things are fine so far, make sure it does not contain a space + # or other meta-characters which can cause problems during software + # building. + if ! [ x"$bdir" = x ]; then + hasmeta=0; + case $bdir in *['!'\@\#\$\%\^\&\*\(\)\+\;\ ]* ) hasmeta=1 ;; esac + if [ $hasmeta = 1 ]; then + + # If it was newly created, it will be empty, so delete it. + if ! [ "$(ls -A "$bdir")" ]; then rm --dir "$bdir"; fi + + # Inform the user and set 'bdir' to empty again. + bdir= + echo " ** Build directory should not contain meta-characters" + echo " ** (like SPACE, %, \$, !, ;, or parenthesis, among " + echo " ** others): they can interrup the build for some software." + fi + fi + + # If everything is still fine so far, see if we're able to + # manipulate file permissions in the directory's filesystem and if + # so, see if there is atleast 5GB free space. if ! [ x"$bdir" = x ]; then - if ! $(check_permission $bdir); then + if ! $(IFS='"' check_permission "$bdir"); then # Unable to handle permissions well bdir= echo " ** File permissions can't be modified in this directory" @@ -804,7 +881,7 @@ EOF # Able to handle permissions, now check for 5GB free space # in the given partition (note that the number is in units # of 1024 bytes). If this is not the case, print a warning. - if $(free_space_warning 5000000 $bdir); then + if $(free_space_warning 5000000 "$bdir"); then echo " !! LESS THAN 5GB FREE SPACE IN: $bdir" echo " !! We recommend choosing another partition." echo " !! Build will continue in 5 seconds..." @@ -817,7 +894,7 @@ EOF # reset `build_dir' to blank, so it continues asking for another # directory and let the user know that they must select a new # directory. - if [ x$bdir = x ]; then + if [ x"$bdir" = x ]; then build_dir= echo " ** Please select another directory." echo "" @@ -834,9 +911,9 @@ fi # Input directory # --------------- if [ x"$input_dir" = x ]; then - indir=$optionaldir + indir="$optionaldir" else - indir=$input_dir + indir="$input_dir" fi noninteractive_sleep=2 if [ $rewritepconfig = yes ] && [ x"$input_dir" = x ]; then @@ -878,7 +955,7 @@ EOF # In case an input-directory is given, write it in 'indir'. if [ x$inindir != x ]; then - indir=$(absolute_dir $inindir) + indir="$(absolute_dir "$inindir")" echo " -- Using '$indir'" fi fi @@ -922,7 +999,7 @@ EOF # If given, write the software directory. if [ x"$tmpddir" != x ]; then - ddir=$(absolute_dir $tmpddir) + ddir="$(absolute_dir "$tmpddir")" echo " -- Using '$ddir'" fi fi @@ -946,11 +1023,18 @@ if [ $rewritepconfig = yes ]; then -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \ $pconf.in >> $pconf else - # Read the values from existing configuration file. - inbdir=$(awk '$1=="BDIR" {print $3}' $pconf) - - # Read the software directory. - ddir=$(awk '$1=="DEPENDENCIES-DIR" {print $3}' $pconf) + # Read the values from existing configuration file. Note that the build + # directory may have space characters. Even though we currently check + # against it, we hope to be able to remove this condition in the + # future. + inbdir=$(awk '$1=="BDIR" { for(i=3; i $hwparam +.local/bin/echo "\\newcommand{\\machinebyteorder}{$byte_order}" >> $hwparam +.local/bin/echo "\\newcommand{\\machineaddresssizes}{$address_sizes}" >> $hwparam + + + + + # Clean the temporary build directory # --------------------------------- # diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh index 9188fc9..a18966f 100755 --- a/reproduce/software/shell/pre-make-build.sh +++ b/reproduce/software/shell/pre-make-build.sh @@ -30,32 +30,32 @@ set -e -# Input arguments. -bdir=$1 -ddir=$2 -downloader="$3" -user_backup_urls="$4" +# Input arguments (the 'IFS's are to allow space in the name). +IFS='"' bdir="$1" +IFS='"' ddir="$2" +IFS='"' downloader="$3" +IFS='"' user_backup_urls="$4" # Basic directories/files -topdir=$(pwd) -sdir=$bdir/software -tardir=$sdir/tarballs -instdir=$sdir/installed -tmpblddir=$sdir/build-tmp +topdir="$(pwd)" +sdir="$bdir"/software +tardir="$sdir"/tarballs +instdir="$sdir"/installed +tmpblddir="$sdir"/build-tmp confdir=reproduce/software/config -ibidir=$instdir/version-info/proglib +ibidir="$instdir"/version-info/proglib downloadwrapper=reproduce/analysis/bash/download-multi-try # Derived directories -bindir=$instdir/bin -urlfile=$confdir/urls.conf -versionsfile=$confdir/versions.conf -checksumsfile=$confdir/checksums.conf -backupfile=$confdir/servers-backup.conf +bindir="$instdir"/bin +urlfile="$confdir"/urls.conf +versionsfile="$confdir"/versions.conf +checksumsfile="$confdir"/checksums.conf +backupfile="$confdir"/servers-backup.conf @@ -87,21 +87,21 @@ done # Download the necessary tarball. download_tarball() { # Basic definitions - maneagetar=$tardir/$tarball + maneagetar="$tardir"/"$tarball" # See if the tarball already exists in Maneage. if [ -f "$maneagetar" ]; then just_a_place_holder=1 else - ucname=$tardir/$tarball.unchecked + ucname="$tardir"/"$tarball.unchecked" # If the URL is empty, use the top backup server if [ x$w = x ]; then bservers="$backupservers" - tarballurl=$topbackupserver/$tarball + tarballurl="$topbackupserver"/"$tarball" else bservers="$backupservers_all" - tarballurl=$url/$tarball + tarballurl="$url"/"$tarball" fi # See if it is in the input-software directory, if so, make a link, if @@ -111,19 +111,19 @@ download_tarball() { # link and link to an actual file. if [ -f "$ddir/$tarball" ]; then if type realpath > /dev/null 2> /dev/null; then - ln -sf "$(realpath $ddir/$tarball)" "$ucname" + ln -sf "$(realpath "$ddir/$tarball")" "$ucname" else - cp $ddir/$tarball $ucname + cp "$ddir/$tarball" "$ucname" fi else - $downloadwrapper "$downloader" nolock $tarballurl $ucname \ + $downloadwrapper "$downloader" nolock $tarballurl "$ucname" \ "$bservers" fi # Make sure this is the correct tarball. if type sha512sum > /dev/null 2> /dev/null; then checksum=$(sha512sum "$ucname" | awk '{print $1}') - expectedchecksum=$(awk '/^'$progname'-checksum/{print $3}' $checksumsfile) + expectedchecksum=$(awk '/^'$progname'-checksum/{print $3}' "$checksumsfile") if [ x$checksum = x$expectedchecksum ]; then mv "$ucname" "$maneagetar" else echo "ERROR: Non-matching checksum: $tarball" @@ -137,9 +137,9 @@ download_tarball() { # If the tarball is newer than the (possibly existing) program (the version # has changed), then delete the program. - if [ -f $ibidir/$progname ]; then - if [ $maneagetar -nt $ibidir/$progname ]; then - rm $ibidir/$progname + if [ -f "$ibidir/$progname" ]; then + if [ "$maneagetar" -nt "$ibidir/$progname" ]; then + rm "$ibidir/$progname" fi fi } @@ -151,39 +151,39 @@ download_tarball() { # Build the program from the tarball. This function takes one argument # which is the configure-time options. build_program() { - if ! [ -f $ibidir/$progname ]; then + if ! [ -f "$ibidir/$progname" ]; then # Options configoptions=$1 # Go into the temporary building directory. - cd $tmpblddir + cd "$tmpblddir" unpackdir="$progname"-"$version" # Some implementations of 'tar' don't recognize Lzip, so we need to # manually call Lzip first, then call tar afterwards. - csuffix=$(echo $tarball | sed -e's/\./ /g' | awk '{print $NF}') - rm -rf $unpackdir + csuffix=$(echo "$tarball" | sed -e's/\./ /g' | awk '{print $NF}') + rm -rf "$unpackdir" if [ x$csuffix = xlz ]; then intarrm=1 - intar=$(echo $tarball | sed -e's/.lz//') - lzip -c -d $tardir/$tarball > $intar + intar=$(echo "$tarball" | sed -e's/.lz//') + lzip -c -d "$tardir/$tarball" > $intar else intarrm=0 - intar=$tardir/$tarball + intar="$tardir"/"$tarball" fi # Unpack the tarball and go into it. - tar xf $intar - if [ x$intarrm = x1 ]; then rm $intar; fi - cd $unpackdir + tar xf "$intar" + if [ x$intarrm = x1 ]; then rm "$intar"; fi + cd "$unpackdir" # build the project, either with Make and either without it. if [ x$progname = xlzip ]; then - ./configure --build --check --installdir=$instdir/bin $configoptions + ./configure --build --check --installdir="$instdir/bin" $configoptions else # All others accept the configure script. - ./configure --prefix=$instdir $configoptions + ./configure --prefix="$instdir" $configoptions # To build GNU Make, we don't want to assume the existance of a # Make program, so we use its 'build.sh' script and its own built @@ -198,9 +198,9 @@ build_program() { fi # Clean up the source directory - cd $topdir - rm -rf $tmpblddir/$unpackdir - echo "$progname_tex $version" > $ibidir/$progname + cd "$topdir" + rm -rf "$tmpblddir/$unpackdir" + echo "$progname_tex $version" > "$ibidir/$progname" fi } @@ -219,7 +219,7 @@ build_program() { progname="lzip" progname_tex="Lzip" url=$(awk '/^'$progname'-url/{print $3}' $urlfile) -version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) +version=$(awk '/^'$progname'-version/{print $3}' "$versionsfile") tarball=$progname-$version.tar download_tarball build_program -- cgit v1.2.1 From 87948105390c035a0f0ad409db622b3850988ddb Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Thu, 27 Aug 2020 16:47:26 +0100 Subject: Plain text editors: nano in basic, emacs and vim in high-level While a project is under development, the raw analysis software are not the only necessary software in a project. We also need tools to all the edit plain-text files within the Maneaged project. Usually people use their operating system's plain-text editor. However, when working on the project on a new computer, or in a container, the plain-text editors will have different versions, or may not be present at all! This can be very annoying and frustrating! With this commit, Maneage now installs GNU Nano as part of the basic tools. GNU Nano is a very simple and small plain text editor (the installed size is only ~3.5MB, and it is friendly to new users). Therefore, any Maneaged project can assume atleast Nano will be present (in particular when no editor is available on the running system!). GNU Emacs and VIM (both without extra dependencies, in particular without GUI support) are also optionally available in 'high-level.mk' (by adding them to 'TARGETS.conf'). The basic idea for the more advanced editors (Emacs and VIM) is that project authors can add their favorite editor while they are working on the project, but upon publication they can remove them from 'TARGETS.conf'. A few other minor things came up during this work and are now also fixed: - The 'file' program and its libraries like 'libmagic' were linking to system's 'libseccomp'! This dependency then leaked into Nano (which depends on 'libmagic'). But this is just an extra feature of 'file', only for the Linux kernel. Also, we have no dependency on it so far. So 'file' is not configured to not build with 'libseccomp'. - A typo was fixed in the line where the physical core information is being read on macOS. - The top-level directories when running './project shell' are now quoted (in case they have special characters). --- reproduce/software/shell/configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/software/shell') diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 2b4300b..21fe1d6 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -254,7 +254,7 @@ elif [ x$on_mac_os = xyes ]; then if [ x$hw_byteorder = x1234 ]; then byte_order="Little Endian"; elif [ x$hw_byteorder = x4321 ]; then byte_order="Big Endian"; fi - address_size_physical=$(sysctl -n machdep.cpu.address_bits.phyiscal) + address_size_physical=$(sysctl -n machdep.cpu.address_bits.physical) address_size_virtual=$(sysctl -n machdep.cpu.address_bits.virtual) address_sizes="$address_size_physical bits physical, " address_sizes+="$address_size_virtual bits virtual" -- cgit v1.2.1