diff options
Diffstat (limited to 'reproduce')
| -rw-r--r-- | reproduce/analysis/make/paper.mk | 11 | ||||
| -rw-r--r-- | reproduce/software/config/checksums.conf | 3 | ||||
| -rw-r--r-- | reproduce/software/config/urls.conf | 3 | ||||
| -rw-r--r-- | reproduce/software/config/versions.conf | 3 | ||||
| -rw-r--r-- | reproduce/software/make/basic.mk | 25 | ||||
| -rw-r--r-- | reproduce/software/make/high-level.mk | 53 | ||||
| -rwxr-xr-x | reproduce/software/shell/configure.sh | 258 | ||||
| -rwxr-xr-x | reproduce/software/shell/pre-make-build.sh | 86 |
8 files changed, 306 insertions, 136 deletions
diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk index 2c08847..f363b6f 100644 --- a/reproduce/analysis/make/paper.mk +++ b/reproduce/analysis/make/paper.mk @@ -47,8 +47,9 @@ $(mtexdir)/project.tex: $(mtexdir)/verify.tex @if [ -f .local/bin/latex ] && [ x"$(pdf-build-final)" != x ]; then # Put a LaTeX input command for all the necessary macro files. + # 'hardware-parameters.tex' is created in 'configure.sh'. rm -f $(mtexdir)/project.tex - for t in $(subst paper,,$(makesrc)); do + for t in $(subst paper,,$(makesrc)) hardware-parameters; do echo "\input{tex/build/macros/$$t.tex}" >> $(mtexdir)/project.tex done else @@ -127,7 +128,7 @@ $(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ # do not use PGFPlots, then you should remove the `-shell-escape' # option for better security. See # https://savannah.nongnu.org/task/?15694 for details. - latex -shell-escape -halt-on-error $$p/paper.tex + latex -shell-escape -halt-on-error "$$p"/paper.tex bibtex paper # Hack: tidy up eprint+doi style that didn't work in .bst file. @@ -143,7 +144,7 @@ $(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ | sed -e 's/EOLINE/\n/g' > paper.bbl # The pre-final run of LaTeX after 'paper.bbl' was created. - latex -shell-escape -halt-on-error $$p/paper.tex + latex -shell-escape -halt-on-error "$$p"/paper.tex fi @@ -171,7 +172,7 @@ paper.pdf: $(mtexdir)/project.tex paper.tex $(texbdir)/paper.bbl # See above for a warning and brief discussion on the the # pdflatex option `-shell-escape'. - latex -shell-escape -halt-on-error $$p/paper.tex + latex -shell-escape -halt-on-error "$$p"/paper.tex # Convert the DVI to PostScript, and the PostScript to PDF. The # `-dNOSAFER' option to GhostScript allows transparencies in the @@ -182,6 +183,6 @@ paper.pdf: $(mtexdir)/project.tex paper.tex $(texbdir)/paper.bbl # Come back to the top project directory and copy the built PDF # file here. - cd $$p + cd "$$p" cp $(texbdir)/$@ $(final-paper) fi diff --git a/reproduce/software/config/checksums.conf b/reproduce/software/config/checksums.conf index b516a61..e7a0e63 100644 --- a/reproduce/software/config/checksums.conf +++ b/reproduce/software/config/checksums.conf @@ -44,6 +44,7 @@ make-checksum = ddf0fdcb9ee1b182ef294c5da70c1275288c99bef60e63a25c0abed2ddd44aba metastore-checksum = b2a5fdde9de5ddc1e6c368d5da1b2e97e4fdbaa138a7be281ccb40a81dd4a9bb1849d36b2d5d3f01205079bace60441f82a7002097ff3a7037340a35b0f1574a mpc-checksum = 72d657958b07c7812dc9c7cbae093118ce0e454c68a585bfb0e2fa559f1bf7c5f49b93906f580ab3f1073e5b595d23c6494d4d76b765d16dde857a18dd239628 mpfr-checksum = d583555d08863bf36c89b289ae26bae353d9a31f08ee3894520992d2c26e5683c4c9c193d7ad139632f71c0a476d85ea76182702a98bf08dde7b6f65a54f8b88 +nano-checksum = d101e7f4802c079254e79340b433749dcd699fa9adec3f96e4218ec12f066a1f6b0954c27254bb6f019bc370ee2116817717870f4e2bc782c552442f2cc75195 ncurses-checksum = 4c1333dcc30e858e8a9525d4b9aefb60000cfc727bc4a1062bace06ffc4639ad9f6e54f6bdda0e3a0e5ea14de995f96b52b3327d9ec633608792c99a1e8d840d openssl-checksum = 1523985ba90f38aa91aa6c2d57652f4e243cb2a095ce6336bf34b39b5a9b5b876804299a6825c758b65990e57948da532cca761aa12b10958c97478d04dd6d34 patchelf-checksum = 39745662651cf0a9915685b2767a611ceab4286f8fa57eace342b3f44248431616e8563d4ac6709c97d8534229c73c05470239e462b7e74b36bf629a876dfbad @@ -86,6 +87,7 @@ cdsclient-checksum = 2d7abf0079189b9dd19cb8919061445fd19ea9f7dfd54e8ceee26b74321 cfitsio-checksum = 08a13931726b0ee15bd4e2ad6dd4debb8268f3b0bc33adadec5c6a29295dd536bcccb3cc949721c6cebac6f43b6118e5e38332ac0ba8a07a43553416d8debae5 cmake-checksum = e0591d5fb234f3e7b74d6d2aad44fbf3e19e69547bd428681ba6ad0461d4f3d2a154605808b4733531d2c66f0e91eb39a179ae0d89a37e92a3f20e9cae691468 eigen-checksum = 34cf600914cce719d61511577ef9cd26fbdcb7a6fad1d0ab8396f98b887fac6a5577d3967e84a8f56225cc50de38f3b91f34f447d14312028383e32b34ea1972 +emacs-checksum = dfb26531d2c19cf9fb56505f03d799654b45e5f9528e777900e8280ed2c1d21e04c52f510528e31e015977c471ae63164cedee6174b7439ebcf479a21fc18064 expat-checksum = 514ff2ef3c93af0b1715b7a08732db33c13a113c4c72422716a22ee26c09235deed71ec55510cee24c33bcd6b2347602bd71ce70a432d5583fb63765ff9e0e09 fftw-checksum = ab918b742a7c7dcb56390a0a0014f517a6dff9a2e4b4591060deeb2c652bf3c6868aa74559a422a276b853289b4b701bdcbd3d4d8c08943acf29167a7be81a38 flex-checksum = b4ef58d4a1d66b213e2f59df06959decf46d26b253cdc3f51cd26e2e2b505461ef23dafa974dd2005b1f0cafa5a83fe9258baf78004b2fdae6dfc299bc17bfd1 @@ -130,6 +132,7 @@ swarp-checksum = 80f4ade59738df3d4c9b47bda04148b53c6ba995d523fa8d1e02fb5d952b607 swig-checksum = 5eaa2e06d8e4197fd02194051db1e518325dbb074a4c55a91099ad9c55193874f577764afc9029409a41bd520a95154095f26e33ef5add5c102bb2c1d98d33eb tides-checksum = c3360ff0d023b43749ba09a33302ca059f017a157b3ce7cdcf4f1a1578e90d3e7fa420077043adbee6b1ebf94bd698c8d6b279012f36d2a05b4de5351e30e108 util-linux-checksum = c95d26b9037d6b877a247e6aeb58d17aa80f7e1bd6b523a4e0fde559fe07b3d924ece6d373300fefb65d1f206b3f990aeddb3a03605040e72ce6d6ee88591021 +vim-checksum = 06ba43386fcf308520d88d7a68e9bc1fabd824b05078b8f9112500a2ad4e50a91f1a1c2925889b7c06dbce34307f12abf508e2172b05fd283f965cc06552eb6d wcslib-checksum = 8c98c4b575056e2d966b77a4bc951256d02ecee3a11847e140fd38d93afd0f76b3e906d590c952dc9fc58ceeb1ba062b19d8e1e676ee0032f5b7ed13a9dfa892 xlsxio-checksum = 22870fda7bd4eefd5fea2a9ad7530c9049135129d9b69805091777e6b54b2fc6c3f0e69c6954f36bce54eebbfeccaf637cce9e271a593221a4296d6632470a6c yaml-checksum = dadd7d8e0d88b5ebab005e5d521d56d541580198aa497370966b98c904586e642a1cd4f3881094eb57624f218d50db77417bbfd0ffdce50340f011e35e8c4c02 diff --git a/reproduce/software/config/urls.conf b/reproduce/software/config/urls.conf index 34c4d89..417f5ed 100644 --- a/reproduce/software/config/urls.conf +++ b/reproduce/software/config/urls.conf @@ -47,6 +47,7 @@ #metastore-url = http://akhlaghi.org/maneage-software #mpc-url = http://ftp.gnu.org/gnu/mpc #mpfr-url = http://www.mpfr.org/mpfr-current +#nano-url = https://www.nano-editor.org/dist/v$(word 1, $(subst ., ,$(nano-version))) #ncurses-url = http://ftp.gnu.org/gnu/ncurses #openssl-url = http://www.openssl.org/source #patchelf-url = http://nixos.org/releases/patchelf/patchelf-$(patchelf-version) @@ -88,6 +89,7 @@ #cfitsio-url = https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c #cmake-url = $(shell majv=$$(echo $(cmake-version) | sed -e's/\./ /' | awk '{printf("%d.%d", $$1, $$2)}'); echo https://cmake.org/files/v$$majv) #eigen-url = http://bitbucket.org/eigen/eigen/get/$(eigen-version).tar.gz +#emacs-url = http://ftp.gnu.org/gnu/emacs #expat-url = $(shell vstr=$$(echo $(expat-version) | sed -e's/\./_/g'); echo https://github.com/libexpat/libexpat/releases/download/R_$$vstr/expat-$(expat-version).tar.lz) #fftw-url = ftp://ftp.fftw.org/pub/fftw #flex-url = https://github.com/westes/flex/files/981163 @@ -133,6 +135,7 @@ #tides-url = http://akhlaghi.org/maneage-software #util-linux-url = https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v2.35/ #valgrind-url = https://sourceware.org/pub/valgrind +#vim-url = ftp://ftp.vim.org/pub/vim/unix #wcslib-url = ftp://ftp.atnf.csiro.au/pub/software/wcslib #xlsxio-url = https://github.com/brechtsanders/xlsxio/archive #yaml-url = pyyaml.org/download/libyaml diff --git a/reproduce/software/config/versions.conf b/reproduce/software/config/versions.conf index 0a9e21e..0b3a4d2 100644 --- a/reproduce/software/config/versions.conf +++ b/reproduce/software/config/versions.conf @@ -44,6 +44,7 @@ make-version = 4.3 metastore-version = 1.1.2-23-fa9170b mpc-version = 1.1.0 mpfr-version = 4.0.2 +nano-version = 5.2 ncurses-version = 6.2 patchelf-version = 0.10 perl-version = 5.32.0 @@ -94,6 +95,7 @@ cdsclient-version = 3.84 cfitsio-version = 3.48 cmake-version = 3.18.1 eigen-version = 3.3.7 +emacs-version = 27.1 expat-version = 2.2.9 fftw-version = 3.3.8 flex-version = 2.6.4-91-ga631f5d @@ -134,6 +136,7 @@ swarp-version = 2.38.0 swig-version = 3.0.12 tides-version = 2.0 util-linux-version = 2.35 +vim-version = 8.2 xlsxio-version = 0.2.21 yaml-version = 0.2.5 diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 0b672d8..934580a 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -56,8 +56,13 @@ ibdir = $(BDIR)/software/installed/bin ildir = $(BDIR)/software/installed/lib ibidir = $(BDIR)/software/installed/version-info/proglib -# Ultimate Makefile target. -targets-proglib = low-level-links gcc-$(gcc-version) +# Ultimate Makefile target. GNU Nano (a simple and very light-weight text +# editor) is installed by default, it is recommended to have it in the +# 'basic.mk', so Maneaged projects can be edited on any system (even when +# there is no command-line text editor is available). +targets-proglib = low-level-links \ + gcc-$(gcc-version) \ + nano-$(nano-version) all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) # Define the shell environment @@ -852,7 +857,8 @@ $(ibidir)/diffutils-$(diffutils-version): \ $(ibidir)/file-$(file-version): $(ibidir)/coreutils-$(coreutils-version) tarball=file-$(file-version).tar.gz $(call import-source, $(file-url), $(file-checksum)) - $(call gbuild, file-$(file-version), static,,V=1) + $(call gbuild, file-$(file-version), static, \ + --disable-libseccomp, V=1) echo "File $(file-version)" > $@ $(ibidir)/findutils-$(findutils-version): \ @@ -1145,11 +1151,22 @@ $(ibidir)/texinfo-$(texinfo-version): \ echo "GNU Texinfo $(texinfo-version)" > $@ $(ibidir)/which-$(which-version): $(ibidir)/coreutils-$(coreutils-version) - tarball=/which-$(which-version).tar.gz + tarball=which-$(which-version).tar.gz $(call import-source, $(which-url), $(which-checksum)) $(call gbuild, which-$(which-version), static) echo "GNU Which $(which-version)" > $@ +# GNU Nano is a very light-weight and small, command-line text editor (in +# total around 3.5 Mb after installation!). It is top-level target in the +# basic tools (nothing depends on it, it just depends on GCC). This is +# because some projects may choose to not have it by manually removing it +# from 'targets-proglib' above (it has no effect on processing after all!). +$(ibidir)/nano-$(nano-version): $(ibidir)/gcc-$(gcc-version) + tarball=nano-$(nano-version).tar.xz + $(call import-source, $(nano-url), $(nano-checksum)) + $(call gbuild, nano-$(nano-version), static) + echo "GNU Nano $(nano-version)" > $@ + diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 9f434bc..3c5e518 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -54,9 +54,9 @@ ibidir = $(BDIR)/software/installed/version-info/proglib # Basic directories (specific to this Makefile). il64dir = $(BDIR)/software/installed/lib64 iidir = $(BDIR)/software/installed/include -shsrcdir = $(shell pwd)/reproduce/software/shell -dtexdir = $(shell pwd)/reproduce/software/bibtex -patchdir = $(shell pwd)/reproduce/software/patches +shsrcdir = "$(shell pwd)"/reproduce/software/shell +dtexdir = "$(shell pwd)"/reproduce/software/bibtex +patchdir = "$(shell pwd)"/reproduce/software/patches itidir = $(BDIR)/software/installed/version-info/tex ictdir = $(BDIR)/software/installed/version-info/cite ipydir = $(BDIR)/software/installed/version-info/python @@ -449,6 +449,25 @@ $(ibidir)/eigen-$(eigen-version): rm -rf $(ddir)/eigen-eigen-* echo "Eigen $(eigen-version)" > $@ +# GNU Emacs is an advanced text editor (among many other things!), so it +# isn't directly related to the analysis phase of a project. However, it +# can be useful during the development of a project on systems that don't +# have it natively. So probably after the project is finished and is ready +# for publication, you can remove it from 'TARGETS.conf'. +# +# However, the full Emacs build has a very large number of dependencies +# which aren't necessary in many scenarios so we are disabling everything +# except the core Emacs functionality (using '--without-all') and we are +# also disabling all graphic user interface features (using '--without-x'). +$(ibidir)/emacs-$(emacs-version): + tarball=emacs-$(emacs-version).tar.xz + $(call import-source, $(emacs-url), $(emacs-checksum)) + $(call gbuild, emacs-$(emacs-version), static, \ + --without-all --without-x \ + --without-gnutls --with-ns=no, \ + -j$(numthreads) V=1) + echo "GNU Emacs $(emacs-version)" > $@ + $(ibidir)/expat-$(expat-version): tarball=expat-$(expat-version).tar.lz $(call import-source, $(expat-url), $(expat-checksum)) @@ -1374,8 +1393,30 @@ $(ibidir)/xlsxio-$(xlsxio-version): \ rm $(ibdir)/example_xlsxio_* echo "XLSX I/O $(xlsxio-version)" > $@ - - +# VIM is a text editor which doesn't directly affect processing but can be +# useful in projects during its development, for more see the comment above +# GNU Emacs. +$(ibidir)/vim-$(vim-version): + tarball=vim-$(vim-version).tar.bz2 + $(call import-source, $(vim-url), $(vim-checksum)) + cd $(ddir) + tar xf $(tdir)/$$tarball + n=$$(echo $(vim-version) | sed -e's|\.||') + cd $(ddir)/vim$$n + ./configure --prefix=$(idir) \ + --disable-canberra \ + --enable-multibyte \ + --disable-netbeans \ + --disable-fontset \ + --disable-gpm \ + --disable-acl \ + --disable-gui \ + --with-x=no + make -j$(numthreads) + make install + cd .. + rm -rf vim$$n + echo "VIM $(vim-version)" > $@ @@ -1411,7 +1452,7 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf tar xf $(tdir)/install-tl-unx.tar.gz cd install-tl-* sed -e's|@installdir[@]|$(idir)|g' \ - $$topdir/reproduce/software/config/texlive.conf \ + "$$topdir"/reproduce/software/config/texlive.conf \ > texlive.conf # TeX Live's installation may fail due to any reason. But TeX Live diff --git a/reproduce/software/shell/configure.sh b/reproduce/software/shell/configure.sh index 6ffb4ff..0d22dcc 100755 --- a/reproduce/software/shell/configure.sh +++ b/reproduce/software/shell/configure.sh @@ -48,7 +48,7 @@ need_gfortran=0 # -------------------- # # These are defined to help make this script more readable. -topdir=$(pwd) +topdir="$(pwd)" optionaldir="/optional/path" adir=reproduce/analysis/config cdir=reproduce/software/config @@ -96,7 +96,8 @@ create_file_with_notice () # an absolute address. With this function we can make sure of that. absolute_dir () { - if stat "$1" 1> /dev/null; then + address="$1" + if stat "$address" 1> /dev/null; then echo "$(cd "$(dirname "$1")" && pwd )/$(basename "$1")" else exit 1; @@ -136,18 +137,18 @@ check_permission () { # Make a `junk' file, activate its executable flag and record its # permissions generally. - local junkfile=$1/check_permission_tmp_file - rm -f $junkfile - echo "Don't let my short life go to waste" > $junkfile - chmod +x $junkfile - local perm_before=$(ls -l $junkfile | awk '{print $1}') + local junkfile="$1"/check_permission_tmp_file + rm -f "$junkfile" + echo "Don't let my short life go to waste" > "$junkfile" + chmod +x "$junkfile" + local perm_before=$(ls -l "$junkfile" | awk '{print $1}') # Now, remove the executable flag and record the permissions. - chmod -x $junkfile - local perm_after=$(ls -l $junkfile | awk '{print $1}') + chmod -x "$junkfile" + local perm_after=$(ls -l "$junkfile" | awk '{print $1}') # Clean up before leaving the function - rm -f $junkfile + rm -f "$junkfile" # If the permissions are equal, the filesystem doesn't allow # permissions. @@ -187,8 +188,8 @@ check_permission () free_space_warning() { fs_threshold=$1 - fs_destpath=$2 - return $(df $fs_destpath \ + IFS='"' fs_destpath="$2" + return $(df "$fs_destpath" \ | awk 'FNR==2 {if($4>'$fs_threshold') print 1; \ else print 0; }') } @@ -224,6 +225,62 @@ fi +# Collect CPU information +# ----------------------- +# +# When the project is built, the type of a machine that built it also has +# to to be documented. This way, if different results or behaviors are +# observed in software-related or analysis-related phases of the project, +# it would be easier to track down the root cause. So far this is just +# later recorded as a LaTeX macro to be put in the final paper, but it +# could be used in a more systematic way to optimize/revise project +# workflow and build. +hw_class=$(uname -m) +if [ x$kernelname = xLinux ]; then + byte_order=$(lscpu \ + | grep 'Byte Order' \ + | awk '{ \ + for(i=3;i<NF;++i) \ + printf "%s ", $i; \ + printf "%s", $NF}') + address_sizes=$(lscpu \ + | grep 'Address sizes' \ + | awk '{ \ + for(i=3;i<NF;++i) \ + printf "%s ", $i; \ + printf "%s", $NF}') +elif [ x$on_mac_os = xyes ]; then + hw_byteorder=$(sysctl -n hw.byteorder) + if [ x$hw_byteorder = x1234 ]; then byte_order="Little Endian"; + elif [ x$hw_byteorder = x4321 ]; then byte_order="Big Endian"; + fi + address_size_physical=$(sysctl -n machdep.cpu.address_bits.physical) + address_size_virtual=$(sysctl -n machdep.cpu.address_bits.virtual) + address_sizes="$address_size_physical bits physical, " + address_sizes+="$address_size_virtual bits virtual" +else + byte_order="unrecognized" + address_sizes="unrecognized" + cat <<EOF +______________________________________________________ +!!!!!!! WARNING !!!!!!! + +Machine byte order and address sizes could not be recognized. You can add +the necessary steps in the 'reproduce/software/shell/configure.sh' script +(just above this error message), or contact us with this web-form: + + https://savannah.nongnu.org/support/?func=additem&group=reproduce + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +EOF + sleep 5 +fi + + + + + # Check for C/C++ compilers # ------------------------- # @@ -717,7 +774,7 @@ fi # Build directory # --------------- -currentdir=$(pwd) +currentdir="$(pwd)" if [ $rewritepconfig = yes ]; then cat <<EOF @@ -746,7 +803,7 @@ directory). The build directory cannot be a subdirectory of the source. EOF bdir= junkname=pure-junk-974adfkj38 - while [ x$bdir = x ] + while [ x"$bdir" = x ] do # Ask the user (if not already set on the command-line). if [ x"$build_dir" = x ]; then @@ -761,31 +818,31 @@ EOF fi # If it exists, see if we can write in it. If not, try making it. - if [ -d $build_dir ]; then - if mkdir $build_dir/$junkname 2> /dev/null; then + if [ -d "$build_dir" ]; then + if echo "test" > "$build_dir"/$junkname ; then + rm -f "$build_dir"/$junkname instring="the already existing" - bdir=$(absolute_dir $build_dir) - rm -rf $build_dir/$junkname + bdir="$(absolute_dir "$build_dir")" else echo " ** Can't write in '$build_dir'"; fi else - if mkdir $build_dir 2> /dev/null; then + if mkdir "$build_dir" 2> /dev/null; then instring="the newly created" - bdir=$(absolute_dir $build_dir) + bdir="$(absolute_dir "$build_dir")" else echo " ** Can't create '$build_dir'"; fi fi - # If its given, make sure it isn't a subdirectory of the source + # If it is given, make sure it isn't a subdirectory of the source # directory. if ! [ x"$bdir" = x ]; then if echo "$bdir/" \ - | grep '^'$currentdir 2> /dev/null > /dev/null; then + | grep '^'"$currentdir" 2> /dev/null > /dev/null; then # If it was newly created, it will be empty, so delete it. - if ! [ "$(ls -A $bdir)" ]; then rm --dir $bdir; fi + if ! [ "$(ls -A $bdir)" ]; then rm --dir "$bdir"; fi # Inform the user that this is not acceptable and reset `bdir'. bdir= @@ -793,10 +850,30 @@ EOF fi fi - # If everything is fine until now, see if we're able to manipulate - # file permissions. + # If things are fine so far, make sure it does not contain a space + # or other meta-characters which can cause problems during software + # building. + if ! [ x"$bdir" = x ]; then + hasmeta=0; + case $bdir in *['!'\@\#\$\%\^\&\*\(\)\+\;\ ]* ) hasmeta=1 ;; esac + if [ $hasmeta = 1 ]; then + + # If it was newly created, it will be empty, so delete it. + if ! [ "$(ls -A "$bdir")" ]; then rm --dir "$bdir"; fi + + # Inform the user and set 'bdir' to empty again. + bdir= + echo " ** Build directory should not contain meta-characters" + echo " ** (like SPACE, %, \$, !, ;, or parenthesis, among " + echo " ** others): they can interrup the build for some software." + fi + fi + + # If everything is still fine so far, see if we're able to + # manipulate file permissions in the directory's filesystem and if + # so, see if there is atleast 5GB free space. if ! [ x"$bdir" = x ]; then - if ! $(check_permission $bdir); then + if ! $(IFS='"' check_permission "$bdir"); then # Unable to handle permissions well bdir= echo " ** File permissions can't be modified in this directory" @@ -804,7 +881,7 @@ EOF # Able to handle permissions, now check for 5GB free space # in the given partition (note that the number is in units # of 1024 bytes). If this is not the case, print a warning. - if $(free_space_warning 5000000 $bdir); then + if $(free_space_warning 5000000 "$bdir"); then echo " !! LESS THAN 5GB FREE SPACE IN: $bdir" echo " !! We recommend choosing another partition." echo " !! Build will continue in 5 seconds..." @@ -817,7 +894,7 @@ EOF # reset `build_dir' to blank, so it continues asking for another # directory and let the user know that they must select a new # directory. - if [ x$bdir = x ]; then + if [ x"$bdir" = x ]; then build_dir= echo " ** Please select another directory." echo "" @@ -834,9 +911,9 @@ fi # Input directory # --------------- if [ x"$input_dir" = x ]; then - indir=$optionaldir + indir="$optionaldir" else - indir=$input_dir + indir="$input_dir" fi noninteractive_sleep=2 if [ $rewritepconfig = yes ] && [ x"$input_dir" = x ]; then @@ -878,7 +955,7 @@ EOF # In case an input-directory is given, write it in 'indir'. if [ x$inindir != x ]; then - indir=$(absolute_dir $inindir) + indir="$(absolute_dir "$inindir")" echo " -- Using '$indir'" fi fi @@ -922,7 +999,7 @@ EOF # If given, write the software directory. if [ x"$tmpddir" != x ]; then - ddir=$(absolute_dir $tmpddir) + ddir="$(absolute_dir "$tmpddir")" echo " -- Using '$ddir'" fi fi @@ -946,11 +1023,18 @@ if [ $rewritepconfig = yes ]; then -e's|@groupname[@]|'"$reproducible_paper_group_name"'|' \ $pconf.in >> $pconf else - # Read the values from existing configuration file. - inbdir=$(awk '$1=="BDIR" {print $3}' $pconf) - - # Read the software directory. - ddir=$(awk '$1=="DEPENDENCIES-DIR" {print $3}' $pconf) + # Read the values from existing configuration file. Note that the build + # directory may have space characters. Even though we currently check + # against it, we hope to be able to remove this condition in the + # future. + inbdir=$(awk '$1=="BDIR" { for(i=3; i<NF; i++) \ + printf "%s ", $i; \ + printf "%s", $NF }' $pconf) + + # Read the software directory (same as 'inbdir' above about space). + ddir=$(awk '$1=="DEPENDENCIES-DIR" { for(i=3; i<NF; i++) \ + printf "%s ", $i; \ + printf "%s", $NF}' $pconf) # The downloader command may contain multiple elements, so we'll just # change the (in memory) first and second tokens to empty space and @@ -968,9 +1052,9 @@ else # Make sure `bdir' is an absolute path and it exists. berr=0 ierr=0 - bdir=$(absolute_dir $inbdir) + bdir="$(absolute_dir "$inbdir")" - if ! [ -d $bdir ]; then if ! mkdir $bdir; then berr=1; err=1; fi; fi + if ! [ -d "$bdir" ]; then if ! mkdir "$bdir"; then berr=1; err=1; fi; fi if [ $err = 1 ]; then cat <<EOF @@ -1013,10 +1097,10 @@ fi # (without the user noticing), in the end of this script we make a file and # we'll delete it here (at the start). Therefore if the script crashed in # the middle that file won't exist. -sdir=$bdir/software -finaltarget=$sdir/configuration-done.txt -if ! [ -d $sdir ]; then mkdir $sdir; fi -rm -f $finaltarget +sdir="$bdir"/software +finaltarget="$sdir"/configuration-done.txt +if ! [ -d "$sdir" ]; then mkdir "$sdir"; fi +rm -f "$finaltarget" @@ -1030,56 +1114,56 @@ rm -f $finaltarget # analysis Makefiles (thus making them hard to read), we are just building # them here # Software tarballs -tardir=$sdir/tarballs -if ! [ -d $tardir ]; then mkdir $tardir; fi +tardir="$sdir"/tarballs +if ! [ -d "$tardir" ]; then mkdir "$tardir"; fi # Installed software -instdir=$sdir/installed -if ! [ -d $instdir ]; then mkdir $instdir; fi +instdir="$sdir"/installed +if ! [ -d "$instdir" ]; then mkdir "$instdir"; fi # To record software versions and citation. -verdir=$instdir/version-info -if ! [ -d $verdir ]; then mkdir $verdir; fi +verdir="$instdir"/version-info +if ! [ -d "$verdir" ]; then mkdir "$verdir"; fi # Program and library versions and citation. -ibidir=$verdir/proglib -if ! [ -d $ibidir ]; then mkdir $ibidir; fi +ibidir="$verdir"/proglib +if ! [ -d "$ibidir" ]; then mkdir "$ibidir"; fi # Python module versions and citation. -ipydir=$verdir/python -if ! [ -d $ipydir ]; then mkdir $ipydir; fi +ipydir="$verdir"/python +if ! [ -d "$ipydir" ]; then mkdir "$ipydir"; fi # Used software BibTeX entries. -ictdir=$verdir/cite -if ! [ -d $ictdir ]; then mkdir $ictdir; fi +ictdir="$verdir"/cite +if ! [ -d "$ictdir" ]; then mkdir "$ictdir"; fi # TeXLive versions. -itidir=$verdir/tex -if ! [ -d $itidir ]; then mkdir $itidir; fi +itidir="$verdir"/tex +if ! [ -d "$itidir" ]; then mkdir "$itidir"; fi # Top-level LaTeX. -texdir=$bdir/tex -if ! [ -d $texdir ]; then mkdir $texdir; fi +texdir="$bdir"/tex +if ! [ -d "$texdir" ]; then mkdir "$texdir"; fi # LaTeX macros. -mtexdir=$texdir/macros -if ! [ -d $mtexdir ]; then mkdir $mtexdir; fi +mtexdir="$texdir"/macros +if ! [ -d "$mtexdir" ]; then mkdir "$mtexdir"; fi # TeX build directory. If built in a group scenario, the TeX build # directory must be separate for each member (so they can work on their # relevant parts of the paper without conflicting with each other). if [ "x$reproducible_paper_group_name" = x ]; then - texbdir=$texdir/build + texbdir="$texdir"/build else user=$(whoami) - texbdir=$texdir/build-$user + texbdir="$texdir"/build-$user fi -if ! [ -d $texbdir ]; then mkdir $texbdir; fi +if ! [ -d "$texbdir" ]; then mkdir "$texbdir"; fi # TiKZ (for building figures within LaTeX). -tikzdir=$texbdir/tikz -if ! [ -d $tikzdir ]; then mkdir $tikzdir; fi +tikzdir="$texbdir"/tikz +if ! [ -d "$tikzdir" ]; then mkdir "$tikzdir"; fi # If 'tex/build' and 'tex/tikz' aren't symbolic links, then we are in the # tarball (not the Git repository), so we'll give them another name and let @@ -1120,14 +1204,14 @@ fi # safe, we are deleting all the links on each re-configure of the project. rm -f .build .local -ln -s $bdir .build -ln -s $instdir .local -ln -s $texdir tex/build -ln -s $tikzdir tex/tikz +ln -s "$bdir" .build +ln -s "$instdir" .local +ln -s "$texdir" tex/build +ln -s "$tikzdir" tex/tikz # --------- Delete for no Gnuastro --------- rm -f .gnuastro -ln -s $topdir/reproduce/analysis/config/gnuastro .gnuastro +ln -s "$topdir"/reproduce/analysis/config/gnuastro .gnuastro # ------------------------------------------ @@ -1143,8 +1227,8 @@ ln -s $topdir/reproduce/analysis/config/gnuastro .gnuastro # project names). Maybe later, we can use something like `mktemp' to add # random characters to this name and make it unique to every run (even for # a single user). -tmpblddir=$sdir/build-tmp -rm -rf $tmpblddir/* $tmpblddir # If its a link, we need to empty its +tmpblddir="$sdir"/build-tmp +rm -rf "$tmpblddir"/* "$tmpblddir" # If its a link, we need to empty its # contents first, then itself. # Set the top-level shared memory location. @@ -1156,12 +1240,12 @@ fi # there (in RAM), build a temporary directory for this project. needed_space=2000000 if [ x"$shmdir" != x ]; then - available_space=$(df $shmdir | awk 'NR==2{print $4}') + available_space=$(df "$shmdir" | awk 'NR==2{print $4}') if [ $available_space -gt $needed_space ]; then dirname=$(pwd | sed -e's/\// /g' \ | awk '{l=NF-1; printf("%s-%s",$l, $NF)}') - tbshmdir=$shmdir/"$dirname"-$(whoami) - if ! [ -d $tbshmdir ]; then mkdir $tbshmdir; fi + tbshmdir="$shmdir"/"$dirname"-$(whoami) + if ! [ -d "$tbshmdir" ]; then mkdir "$tbshmdir"; fi fi else tbshmdir="" @@ -1170,8 +1254,8 @@ fi # If a shared memory directory was created set `build-tmp' to be a # symbolic link to it. Otherwise, just build the temporary build # directory under the project build directory. -if [ x$tbshmdir = x ]; then mkdir $tmpblddir; -else ln -s $tbshmdir $tmpblddir; +if [ x"$tbshmdir" = x ]; then mkdir "$tmpblddir"; +else ln -s "$tbshmdir" "$tmpblddir"; fi @@ -1540,6 +1624,24 @@ fi +# Report machine architecture +# --------------------------- +# +# Report hardware +hwparam="$mtexdir/hardware-parameters.tex" + +# Add the text to the ${hwparam} file. Since harware class might include +# underscore, it must be replaced with '\_', otherwise pdftex would +# complain and break the build process when doing ./project make. +hw_class_fixed="$(echo $hw_class | sed -e 's/_/\\_/')" +.local/bin/echo "\\newcommand{\\machinearchitecture}{$hw_class_fixed}" > $hwparam +.local/bin/echo "\\newcommand{\\machinebyteorder}{$byte_order}" >> $hwparam +.local/bin/echo "\\newcommand{\\machineaddresssizes}{$address_sizes}" >> $hwparam + + + + + # Clean the temporary build directory # --------------------------------- # diff --git a/reproduce/software/shell/pre-make-build.sh b/reproduce/software/shell/pre-make-build.sh index 9188fc9..a18966f 100755 --- a/reproduce/software/shell/pre-make-build.sh +++ b/reproduce/software/shell/pre-make-build.sh @@ -30,32 +30,32 @@ set -e -# Input arguments. -bdir=$1 -ddir=$2 -downloader="$3" -user_backup_urls="$4" +# Input arguments (the 'IFS's are to allow space in the name). +IFS='"' bdir="$1" +IFS='"' ddir="$2" +IFS='"' downloader="$3" +IFS='"' user_backup_urls="$4" # Basic directories/files -topdir=$(pwd) -sdir=$bdir/software -tardir=$sdir/tarballs -instdir=$sdir/installed -tmpblddir=$sdir/build-tmp +topdir="$(pwd)" +sdir="$bdir"/software +tardir="$sdir"/tarballs +instdir="$sdir"/installed +tmpblddir="$sdir"/build-tmp confdir=reproduce/software/config -ibidir=$instdir/version-info/proglib +ibidir="$instdir"/version-info/proglib downloadwrapper=reproduce/analysis/bash/download-multi-try # Derived directories -bindir=$instdir/bin -urlfile=$confdir/urls.conf -versionsfile=$confdir/versions.conf -checksumsfile=$confdir/checksums.conf -backupfile=$confdir/servers-backup.conf +bindir="$instdir"/bin +urlfile="$confdir"/urls.conf +versionsfile="$confdir"/versions.conf +checksumsfile="$confdir"/checksums.conf +backupfile="$confdir"/servers-backup.conf @@ -87,21 +87,21 @@ done # Download the necessary tarball. download_tarball() { # Basic definitions - maneagetar=$tardir/$tarball + maneagetar="$tardir"/"$tarball" # See if the tarball already exists in Maneage. if [ -f "$maneagetar" ]; then just_a_place_holder=1 else - ucname=$tardir/$tarball.unchecked + ucname="$tardir"/"$tarball.unchecked" # If the URL is empty, use the top backup server if [ x$w = x ]; then bservers="$backupservers" - tarballurl=$topbackupserver/$tarball + tarballurl="$topbackupserver"/"$tarball" else bservers="$backupservers_all" - tarballurl=$url/$tarball + tarballurl="$url"/"$tarball" fi # See if it is in the input-software directory, if so, make a link, if @@ -111,19 +111,19 @@ download_tarball() { # link and link to an actual file. if [ -f "$ddir/$tarball" ]; then if type realpath > /dev/null 2> /dev/null; then - ln -sf "$(realpath $ddir/$tarball)" "$ucname" + ln -sf "$(realpath "$ddir/$tarball")" "$ucname" else - cp $ddir/$tarball $ucname + cp "$ddir/$tarball" "$ucname" fi else - $downloadwrapper "$downloader" nolock $tarballurl $ucname \ + $downloadwrapper "$downloader" nolock $tarballurl "$ucname" \ "$bservers" fi # Make sure this is the correct tarball. if type sha512sum > /dev/null 2> /dev/null; then checksum=$(sha512sum "$ucname" | awk '{print $1}') - expectedchecksum=$(awk '/^'$progname'-checksum/{print $3}' $checksumsfile) + expectedchecksum=$(awk '/^'$progname'-checksum/{print $3}' "$checksumsfile") if [ x$checksum = x$expectedchecksum ]; then mv "$ucname" "$maneagetar" else echo "ERROR: Non-matching checksum: $tarball" @@ -137,9 +137,9 @@ download_tarball() { # If the tarball is newer than the (possibly existing) program (the version # has changed), then delete the program. - if [ -f $ibidir/$progname ]; then - if [ $maneagetar -nt $ibidir/$progname ]; then - rm $ibidir/$progname + if [ -f "$ibidir/$progname" ]; then + if [ "$maneagetar" -nt "$ibidir/$progname" ]; then + rm "$ibidir/$progname" fi fi } @@ -151,39 +151,39 @@ download_tarball() { # Build the program from the tarball. This function takes one argument # which is the configure-time options. build_program() { - if ! [ -f $ibidir/$progname ]; then + if ! [ -f "$ibidir/$progname" ]; then # Options configoptions=$1 # Go into the temporary building directory. - cd $tmpblddir + cd "$tmpblddir" unpackdir="$progname"-"$version" # Some implementations of 'tar' don't recognize Lzip, so we need to # manually call Lzip first, then call tar afterwards. - csuffix=$(echo $tarball | sed -e's/\./ /g' | awk '{print $NF}') - rm -rf $unpackdir + csuffix=$(echo "$tarball" | sed -e's/\./ /g' | awk '{print $NF}') + rm -rf "$unpackdir" if [ x$csuffix = xlz ]; then intarrm=1 - intar=$(echo $tarball | sed -e's/.lz//') - lzip -c -d $tardir/$tarball > $intar + intar=$(echo "$tarball" | sed -e's/.lz//') + lzip -c -d "$tardir/$tarball" > $intar else intarrm=0 - intar=$tardir/$tarball + intar="$tardir"/"$tarball" fi # Unpack the tarball and go into it. - tar xf $intar - if [ x$intarrm = x1 ]; then rm $intar; fi - cd $unpackdir + tar xf "$intar" + if [ x$intarrm = x1 ]; then rm "$intar"; fi + cd "$unpackdir" # build the project, either with Make and either without it. if [ x$progname = xlzip ]; then - ./configure --build --check --installdir=$instdir/bin $configoptions + ./configure --build --check --installdir="$instdir/bin" $configoptions else # All others accept the configure script. - ./configure --prefix=$instdir $configoptions + ./configure --prefix="$instdir" $configoptions # To build GNU Make, we don't want to assume the existance of a # Make program, so we use its 'build.sh' script and its own built @@ -198,9 +198,9 @@ build_program() { fi # Clean up the source directory - cd $topdir - rm -rf $tmpblddir/$unpackdir - echo "$progname_tex $version" > $ibidir/$progname + cd "$topdir" + rm -rf "$tmpblddir/$unpackdir" + echo "$progname_tex $version" > "$ibidir/$progname" fi } @@ -219,7 +219,7 @@ build_program() { progname="lzip" progname_tex="Lzip" url=$(awk '/^'$progname'-url/{print $3}' $urlfile) -version=$(awk '/^'$progname'-version/{print $3}' $versionsfile) +version=$(awk '/^'$progname'-version/{print $3}' "$versionsfile") tarball=$progname-$version.tar download_tarball build_program |