From db97ea9a8f32050f74d8c981a9efa21f9e0dc512 Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Wed, 14 Nov 2018 01:33:16 +0000 Subject: ./configure and building of Bash and Make with more basic tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a test by Raúl Infante Sainz, we found out that the configure script and the Make script for Bash and Make are making too many assumptions on more recent versions of both. As a result, it couldn't be built. Therefore, the `configure' script was modified to not use more recent tools like `readlink' (to find the absolute address of a relative one). It was also re-organized to not have to read the configuration parameters from a text file. The parameters are directly read from the command-line and are written into the proper file afterwards. This removes the need to opening a text editor by the user (which also caused problems on Raúl's system). To fix the Make version issue, the building of Bash and Make are now done in a new Makefile (`reproduce/src/make/dependencies-basic.mk'). This file doesn't make many of the assumptions that were made in `dependencies.mk'. So it should hopefully work on any version of Make. To help in debugging, for now, the Makefile of configure, are asked to work on one thread (the `-j' option is commented in the `configure'). But after checks, we'll fix this. --- configure | 417 +++++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 274 insertions(+), 143 deletions(-) (limited to 'configure') diff --git a/configure b/configure index 72d7e1c..67804e8 100755 --- a/configure +++ b/configure @@ -40,6 +40,7 @@ set -e # These are defined to help make this script more readable. topdir=$(pwd) installedlink=.local +lbdir=reproduce/build cdir=reproduce/config pdir=$cdir/pipeline @@ -52,6 +53,58 @@ glconf=$cdir/gnuastro/gnuastro-local.conf +# Delete final target of configuration +# ------------------------------------ +# +# Without the top-level `Makefile' the user can't run `make' in this +# directory. But we only want to make it available when we know everything +# else is set up. So we'll delete it at the start of this configuration and +# reset the link in the absolute end. +rm -f Makefile + + + + + +# Notice for top of generated files +# --------------------------------- +# +# In case someone opens the files output from the configuration scripts in +# a text editor and wants to edit them, it is important to let them know +# that their changes are not going to be permenant. +function create_file_with_notice() { + if echo "# IMPORTANT: file can be RE-WRITTEN after './configure'" > $1 + then + echo "#" >> $1 + echo "# This file was created during the reproduction" >> $1 + echo "# pipeline's configuration ('./configure'). Therefore," >> $1 + echo "# it is not under version control and any manual " >> $1 + echo "# changes to it will be over-written if the pipeline " >> $1 + echo "# is re-configured." >> $1 + echo "#" >> $1 + else + echo; echo "Can't write to $1"; echo; + exit 1 + fi +} + + + + + +# Get absolute address +# -------------------- +# +# Since the build directory will go into a symbolic link, we want it to be +# an absolute address. With this function we can make sure of that. +function absolute_dir() { + echo "$(cd "$(dirname "$inbdir")" && pwd )/$(basename "$inbdir")" +} + + + + + # Inform the user # --------------- # @@ -73,6 +126,36 @@ echo +# What to do with possibly existing configuration file +# ---------------------------------------------------- +# +# `LOCAL.mk' is the top-most local configuration for the pipeline. If it +# already exists when this script is run, we'll make a copy of it as backup +# (for example the user might have ran `./configure' by mistake). +rewritepconfig=yes +rewritegconfig=yes +if [ -f $pconf ] || [ -f $glconf ]; then + + # If it already exits, see what the user wants to do. + echo "Atleast one local configuration file already exists." + echo + while [ "$userread" != "y" -a "$userread" != "n" ] + do + read -p"Re-write existing configuration file(s) (y/n)? " userread + done + + # Set `rewriteconfig'. + if [ $userread = "n" ]; then + if [ -f $pconf ]; then rewritepconfig=no; fi + if [ -f $glconf ]; then rewritegconfig=no; fi + fi + echo +fi + + + + + # Identify the downloader tool # ---------------------------- # @@ -88,157 +171,211 @@ echo # variable. So it is important to first give the output filename after # calling `DOWNLOADER' within the Makefiles, and finish the command with # the web address. -print_downloader_notice=1 -if type wget > /dev/null; then - downloader="wget -O"; -else - echo "=======" - echo "Warning" - echo "=======" - echo "Couldn't find GNU Wget. It is used for downloading necessary " - echo "programs and data if they aren't already present in the specified " - echo "directories. Therefore the pipeline will crash if the necessary " - echo "files are not already present on the system." - echo "=======" - echo - downloader="no-downloader-found" -fi; - - +if [ $rewritepconfig = yes ]; then + if type wget > /dev/null; then + downloader="wget -O"; + else + echo "=======" + echo "Warning" + echo "=======" + echo "Couldn't find GNU Wget. It is used for downloading necessary " + echo "programs and data if they aren't already present in the specified " + echo "directories. Therefore the pipeline will crash if the necessary " + echo "files are not already present on the system." + echo "=======" + echo + downloader="no-downloader-found" + fi; +fi -# Write values obtained so far -# ---------------------------- -# -# We'll start writing of the local configuration file with the values that -# have been found so far. -sed -e 's|@downloader[@]|'"$downloader"'|g' $pconf.in > $ptconf +# Build directory +# --------------- +if [ $rewritepconfig = yes ]; then + cat < /dev/null; then + bdir=$(absolute_dir $inbdir) + echo " -- Build directory: '$bdir'" + rm -rf $inbdir/$junkname + else + echo " -- Can't write in '$inbdir'" + fi + else + if mkdir $inbdir 2> /dev/null; then + bdir=$(absolute_dir $inbdir) + echo " -- Build directory set to (the newly created): '$bdir'" + else + echo " -- Can't create '$inbdir'" + fi + fi + done +fi -# Remove possibly existing configuration file? -# -------------------------------------------- -# -# `LOCAL.mk' is the top-most local configuration for the pipeline. If it -# already exists when this script is run, we'll make a copy of it as backup -# (for example the user might have ran `./configure' by mistake). -if [ -f $pconf ]; then - # If it already exits, see what the user wants to do. - while [ "$userread" != "y" -a "$userread" != "n" ] - do - read -p"A configuration already exists, re-configure (y/n)? " userread - done - # Move the configuration if requested. - if [ $userread = "y" ]; then - if mv $pconf $poconf; then - echo "-- Existing configuration moved to '$poconf'." - fi - else - echo "-- Using existing configuration file." +# Dependency tarball directory +# ---------------------------- +if [ $rewritepconfig = yes ]; then + junkddir="/optional/path" + ddir=$junkddir + cat < /dev/null 2>&1 - do - read -p"Your favorite text editor: " userread - done - echo -fi +# Memory mapping minimum size +# --------------------------- +if [ $rewritegconfig = yes ]; then + defaultminmapsize=1000000000 + minmapsize=$defaultminmapsize + cat < $1 - then - echo "#" >> $1 - echo "# This file was created during the reproduction" >> $1 - echo "# pipeline's configuration ('./configure'). Therefore," >> $1 - echo "# it is not under version control and any manual " >> $1 - echo "# changes to it will be over-written if the pipeline " >> $1 - echo "# is re-configured." >> $1 - echo "#" >> $1 - else - exit 1 + read -p"Minimum memory mapping size in bytes (default: $minmapsize): " \ + tmpminmapsize + if [ x"$tmpminmapsize" != x ]; then + minmapsize=$tmpminmapsize + echo " -- Using '$minmapsize'" fi -} +fi -# --------- Delete for no Gnuastro --------- -# Gnuastro's local configuration settings -# -# The `minmapsize' parameter has been set by the user, so we can read it -# and add it to Gnuastro's local configuration file. -create_file_with_notice $glconf -mm=$(awk '$1=="MINMAPSIZE"{print $3}' $ptconf) -echo "# Minimum number of bytes to use HDD/SSD instead of RAM." >> $glconf -echo " minmapsize $mm" >> $glconf -# ------------------------------------------ +# Write the parameters into the local configuration file. +if [ $rewritepconfig = yes ]; then + create_file_with_notice $pconf + sed -e's|@bdir[@]|'"$bdir"'|' \ + -e's|@ddir[@]|'"$ddir"'|' \ + -e's|@downloader[@]|'"$downloader"'|' \ + $pconf.in >> $pconf +else + # Read the values from existing configuration file. + inbdir=$(awk '$1=="BDIR" {print $NF}' $pconf) + ddir=$(awk '$1=="DEPENDENCIES-DIR" {print $NF}' $pconf) + downloader=$(awk '$1=="DOWNLOADER" {print $NF}' $pconf) + + # Make sure `bdir' is an absolute path and it exists. + bdir=$(absolute_dir $inbdir) + if ! [ -d $bdir ]; then mkdir $bdir; fi +fi -# Final pipeline local settings -# ----------------------------- -# -# Make the final file that will be used and delete the temporary file along -# with a possible file ending with `~' that is put there by some editors. -if ! [ -f $pconf ]; then - create_file_with_notice $pconf - cat $ptconf >> $pconf +# --------- Delete for no Gnuastro --------- +# Get the version of Gnuastro that must be used. +depverfile=reproduce/config/pipeline/dependency-versions.mk +gversion=$(awk '$1=="gnuastro-version" {print $NF}' $depverfile) + +# Gnuastro's local configuration settings +if [ $rewritegconfig = yes ]; then + create_file_with_notice $glconf + echo "# Minimum number of bytes to use HDD/SSD instead of RAM." >> $glconf + echo " minmapsize $minmapsize" >> $glconf + echo >> $glconf + echo "# Version of Gnuastro that must be used." >> $glconf + echo " onlyversion $gversion" >> $glconf +else + ingversion=$(awk '$1=="onlyversion" {print $NF}' $glconf) + if [ x$ingversion != x$gversion ]; then + echo "______________________________________________________" + echo "!!!!!!!!!!!!!!!!!!CONFIGURATION ERROR!!!!!!!!!!!!!!!!!" + echo + echo "Gnuastro's version in '$glconf' ($ingversion) doesn't match the tarball version that this pipeline was designed to use in '$depverfile' ($gversion). Please re-run after removing the former file:" + echo + echo " $ rm $glconf" + echo " $ ./configure" + echo + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo + exit 1 + fi fi -rm -f $ptconf $ptconf"~" +# ------------------------------------------ -# Read the necessary directories and build the top build directory as well -# as the symbolic link to it. -lbdir=reproduce/build -rm -f $lbdir -bdir=$(awk '/BDIR =/ {print $3}' $pconf) -ddir=$(awk '/DEPENDENCIES-DIR =/ {print $3}' $pconf) -if ! [ -d $bdir ]; then mkdir $bdir; fi -absolutebdir=$(readlink -f $bdir) -ln -s $absolutebdir $lbdir +# Setup the top-level directories +# ------------------------------- +rm -f $lbdir +ln -s $bdir $lbdir -depdir=$absolutebdir/dependencies +depdir=$bdir/dependencies if ! [ -d $depdir ]; then mkdir $depdir; fi tardir=$depdir/tarballs @@ -250,56 +387,39 @@ if ! [ -d $instdir ]; then mkdir $instdir; fi rm -f $installedlink ln -s $instdir $installedlink -echo "Build directory ready: $absolutebdir"; -echo "... 'reproduce/build' is a symbolic link to it (for easy access)."; -echo +# --------- Delete for no Gnuastro --------- +rm -f .gnuastro +ln -s $(pwd)/reproduce/config/gnuastro .gnuastro +# ------------------------------------------ -# Notice on build Make and Bash, build top directories -# ---------------------------------------------------- -tsec=0 -echo; -echo; -echo "----------------"; +# Build basic dependencies +# ------------------------ +tsec=10 +echo "----------------" echo "Necessary reproduction pipeline dependencies will be built in $tsec sec." echo echo "NOTE: the built software will NOT BE INSTALLED on your system, they" -echo "are only for local usage by this reproduction pipeline." +echo "are only for local usage by this reproduction pipeline (in '$bdir/dependencies/installed')" echo sleep $tsec -export USE_LOCAL_BASH=no -bindir=$bdir/dependencies/installed/bin -make -f reproduce/src/make/dependencies.mk $bindir/bash $bindir/make -j2 +make -f reproduce/src/make/dependencies-basic.mk #-j2 -# Build the final symbolic links -cd $topdir -rm -f Makefile -ln -s $(pwd)/reproduce/src/make/Top-Makefile Makefile -# --------- Delete for no Gnuastro --------- -rm -f .gnuastro -ln -s $(pwd)/reproduce/config/gnuastro .gnuastro -# ------------------------------------------ - - - - - -# Build all the necsesary dependencies -# ------------------------------------ +# Rest of dependencies +# -------------------- # # We will be making all the dependencies before running the top-level # Makefile. To make the job easier, we'll do it in a Makefile, not a # script. Bash and Make were the tools we need to run Makefiles, so we had # to build them in this script. But after this, we can rely on Makefiles. -export USE_LOCAL_BASH=yes -.local/bin/make -f reproduce/src/make/dependencies.mk -j8 +.local/bin/make -f reproduce/src/make/dependencies.mk #-j8 @@ -311,10 +431,10 @@ export USE_LOCAL_BASH=yes # The configuration is now complete, we can inform the user on the next # step(s) to take. echo -echo "------" +echo "----------------" echo "The reproduction pipeline and its environment are SUCCESSFULLY configured." -echo "Please run the following command to start." echo +echo "Please run the following command to start." echo "(Replace '8' with the number of CPU threads)" echo echo " .local/bin/make -j8" @@ -322,3 +442,14 @@ echo echo "To change the configuration later, please re-run './configure'," echo "DO NOT manually edit the relevant files." echo + + + + + +# Final target: the Makefile +# -------------------------- +# +# To see why this is the last step of the configuration, see above (when we +# delete the top-level Makefile at the start of this script). +ln -s $(pwd)/reproduce/src/make/Top-Makefile Makefile -- cgit v1.2.1