From 016d938cec703a6d0062377ae8001c8bc7532073 Mon Sep 17 00:00:00 2001 From: Boud Roukema Date: Thu, 24 Jun 2021 21:19:38 +0200 Subject: Configuration: New check to see if /dev/shm allows execution On systems that allow it (like GNU/Linux systems), Maneage will build the necessary software in shared memory (a directory that is actually in the RAM, not on an SSD/HDD, on GNU/Linux systems, it is '/dev/shm'). This allows Maneage to operate faster and not harm the HDD/SSD with all the temporary writing of many small files. Until now, we would only check that this directory exists and that it has enough space. However, some systems also set the 'noexec' flag on shared memory for security reasons [1]. This causes Maneage to crash upon building of the software in later phases. With this commit, at the very start of the configuration step, and after all other shared-memory checks are done, a dummy executable script file is created there and its execution is tested. If it doesn't work, shared memory will not be used at all. In the process, the steps dealing with the software building directory in the configure script have been brought in one place and comments were added to further clarify every step. This commit was initially done by Boud Roukema and later edited by Mohammad Akhlaghi. [1] https://web.archive.org/web/20210624192819/https://serverfault.com/questions/72356/how-useful-is-mounting-tmp-noexec --- reproduce/software/shell/configure.sh | 88 ++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 27 deletions(-) (limited to 'reproduce') diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 12690f7..0f7278f 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -1210,22 +1210,6 @@ if ! [ -d "$ictdir" ]; then mkdir "$ictdir"; fi itidir="$verdir"/tex if ! [ -d "$itidir" ]; then mkdir "$itidir"; fi -# Temporary software un-packing/build directory: if the host has the -# standard `/dev/shm' mounting-point, we'll do it in shared memory (on the -# RAM), to avoid harming/over-using the HDDs/SSDs. The RAM of most systems -# today (>8GB) is large enough for the parallel building of the software. -# -# For the name of the directory under `/dev/shm' (for this project), we'll -# use the names of the two parent directories to the current/running -# directory, separated by a `-' instead of `/'. We'll then appended that -# with the user's name (in case multiple users may be working on similar -# project names). Maybe later, we can use something like `mktemp' to add -# random characters to this name and make it unique to every run (even for -# a single user). -tmpblddir="$sdir"/build-tmp -rm -rf "$tmpblddir"/* "$tmpblddir" # If its a link, we need to empty its - # contents first, then itself. - @@ -1297,29 +1281,75 @@ rm -f .gnuastro -# Set the top-level shared memory location. + + +# Software building directory (possibly in RAM) +# --------------------------------------------- +# +# Building the software for the project will need the creation of many +# small temporary files that will ultimately be deleted. To avoid harming +# HDDs/SSDs and improve speed, it is therefore better to build them in the +# RAM when possible. The RAM of most systems today (>8GB) is large enough +# for the parallel building of the software. + +# Set the top-level shared memory location. Currently there is only one +# standard location (for GNU/Linux OSs), so doing this check here and the +# main job below may seem redundant. However, it is written separately from +# the main code below because later, we expect to add more possible +# mounting locations (for other OSs). if [ -d /dev/shm ]; then shmdir=/dev/shm else shmdir="" fi -# If a shared memory mounted directory exists and there is enough space -# there (in RAM), build a temporary directory for this project. -needed_space=2000000 +# If a shared memory mounted directory exists and has the necessary +# conditions, set that directory to build software. if [ x"$shmdir" != x ]; then + + # Make sure it has enough space. + needed_space=2000000 available_space=$(df "$shmdir" | awk 'NR==2{print $4}') if [ $available_space -gt $needed_space ]; then + + # Set the Maneage-specific directory within the shared + # memory. We'll use the names of the two parent directories to the + # current/running directory, separated by a `-' instead of + # `/'. We'll then appended that with the user's name (in case + # multiple users may be working on similar project names). + # + # Maybe later, we can use something like `mktemp' to add random + # characters to this name and make it unique to every run (even for + # a single user). dirname=$(pwd | sed -e's/\// /g' \ - | awk '{l=NF-1; printf("%s-%s",$l, $NF)}') + | awk '{l=NF-1; printf("%s-%s", $l, $NF)}') tbshmdir="$shmdir"/"$dirname"-$(whoami) if ! [ -d "$tbshmdir" ]; then mkdir "$tbshmdir"; fi + + # Some systems may protect '/dev/shm' against the right to execute + # programs by ordinary users. We thus need to check that the device + # allows execution within this directory by this user. + shmexecfile="$tbshmdir"/shm-execution-check.sh + rm -f $shmexecfile # We also don't want any existing flags. + cat > "$shmexecfile" < /dev/null; then tbshmdir=""; fi + rm "$shmexecfile" fi else tbshmdir="" fi -# If a shared memory directory was created set `build-tmp' to be a -# symbolic link to it. Otherwise, just build the temporary build -# directory under the project build directory. +# If a shared memory directory was created, set the software building +# directory to be a symbolic link to it. Otherwise, just build the +# temporary build directory under the project's build directory. +tmpblddir="$sdir"/build-tmp +rm -rf "$tmpblddir"/* "$tmpblddir" # If it is a link, we need to empty + # its contents first, then itself. if [ x"$tbshmdir" = x ]; then mkdir "$tmpblddir"; else ln -s "$tbshmdir" "$tmpblddir"; fi @@ -1330,6 +1360,9 @@ fi # Inform the user that the build process is starting # ------------------------------------------------- +# +# Everything is ready, let the user know that the building is going to +# start. if [ $printnotice = yes ]; then tsec=10 cat <