From ab811d0952ac93ce608c81ab2cc44d67c7b02dbe Mon Sep 17 00:00:00 2001 From: Mohammad Akhlaghi Date: Fri, 10 Jun 2022 01:43:30 +0200 Subject: Housekeeping: some portability issues fixed; four software updates Until now, there were several portability issues in Maneage: 1. Maneage would crash on older operating systems (checked on Debian 6), where Wget didn't have the '--no-use-server-timestamps'. 2. On a Linux kernel 2.6.32 (of the same Debian 6 above) some features in 'util-linux' (like 'swapon' or 'libmount') wouldn't build and wouldn't let 'util-linux' complete. These features need root permissions to be useful, so the wouldn't be used in Maneage any way! But they wouldn't let Maneage get built 3. The './project shell' command would still read the host's '~/.bashrc', letting the host environment leak-in to Maneage's interactive shell. 4. The building of Flex 2.64 wouldn't complete due to a segmentation fault an Ubuntu, but NetPBM (which depends on Flex) would crash with a wrong usage of 'yyunput'. This had actually caused a non-update to Flex in a previous Maneage software update. 5. The update Astrometry.net would assume SExtractor's executable name is 'source-extractor'; causing a crash in usage. This forced the users to manually create a 'source-extractor' symbolic link in the '.local/bin' directory. 6. The 'reproduce/software/shell/tarball-prepare.sh' script (that is used for making Maneage-standard tarballs) wouldn't accept option values with an '=' between the option name and value! It also didnt' print sufficiently informative messages and errors (for example it would say "skipping ..." (making the user think there is a problem!), but it was actually that the file already existed! 7. The 'reproduce/analysis/make/prepare.mk' and 'reproduce/analysis/make/verify.mk' Makefiles that needed to reject some of the 'makesrc' sub-Makefiles would simply substitute their names with nothing. But this would cause problems when the name is part of the name of another sub-Makefile. 8. On the Debian 6 system mentioned above the raw 'df' command's output wasn't in the expected format; so Maneage would fail to properly detect the free space in the disk. With these commit, all the issues above have been solved: for 1, A check has been added to avoid using that option. For 2, those 'util-linux' features have been disabled. For 3, the '--norc' and '--noprofile' options have beed added to the call to Bash. For 4, see below. For 5, the symbolic link is now automatically made with SExtractor. For 6, the option reading components of that script have been fully re-written and more robust sanity checks are also added, with more informative warnings. For 7, the 'subst' function of Make was replaced with 'filter-out' and this fixed the problem. For 8, 'df' is called with the '-P' option so it has a unified format in all versions. For 4, the versions of 'flex' and 'netpbm' have been updated. Since they were the dependency of 'astrometrynet', that has also been updated. In the process, we discovered that 'lzip' has a new version which claims to be faster, so that is also updated. lzip 1.22 --> 1.23 astrometrynet 0.85 --> 0.89 flex 2.6.4 --> 2.6.4-410-74a89fd netpbm 10.73.39 --> 10.73.39 NetPBM needed some manual manipulation in its source (to remove the extra line), so the necessary steps have been added to its build recipe in 'reproduce/software/make/high-level.mk'. --- reproduce/analysis/config/INPUTS.conf | 2 +- reproduce/analysis/make/initialize.mk | 2 +- reproduce/analysis/make/prepare.mk | 6 ++++-- reproduce/analysis/make/verify.mk | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) (limited to 'reproduce/analysis') diff --git a/reproduce/analysis/config/INPUTS.conf b/reproduce/analysis/config/INPUTS.conf index 5a58758..3958153 100644 --- a/reproduce/analysis/config/INPUTS.conf +++ b/reproduce/analysis/config/INPUTS.conf @@ -30,7 +30,7 @@ # # The naming convension is critical for the input files to be properly # imported into the project. In the patterns below, the '%' is the full -# file name (including its prefix): for example in the demo input of this +# file name (including its suffix): for example in the demo input of this # file in the 'maneage' branch, we have 'INPUT-wfpc2.fits-sha256': # therefore, the input file (within the project's '$(indir)') is called # 'wfpc2.fits'. This allows you to simply set '$(indir)/wfpc2.fits' as the diff --git a/reproduce/analysis/make/initialize.mk b/reproduce/analysis/make/initialize.mk index 4e8ee68..6893962 100644 --- a/reproduce/analysis/make/initialize.mk +++ b/reproduce/analysis/make/initialize.mk @@ -138,10 +138,10 @@ curdir := $(shell echo $$(pwd)) # avoid conflicts with existing TeX Live solutions. Later (in 'paper.mk'), # we are also going to overwrite 'TEXINPUTS' just before 'pdflatex'. .ONESHELL: -.SHELLFLAGS = -ec export TEXINPUTS := export CCACHE_DISABLE := 1 export PATH := $(installdir)/bin +.SHELLFLAGS = --noprofile --norc -ec export LDFLAGS := -L$(installdir)/lib export SHELL := $(installdir)/bin/bash export CPPFLAGS := -I$(installdir)/include diff --git a/reproduce/analysis/make/prepare.mk b/reproduce/analysis/make/prepare.mk index ecb6842..aed2b5f 100644 --- a/reproduce/analysis/make/prepare.mk +++ b/reproduce/analysis/make/prepare.mk @@ -1,4 +1,4 @@ -# Basic preparations, called by './project prepare'. +# Basic preparations, called by './project make'. # # Copyright (C) 2019-2022 Mohammad Akhlaghi # @@ -22,7 +22,9 @@ # Final-target # # Without this file, './project make' won't work. -prepare-dep = $(subst prepare, ,$(makesrc)) +# +# We need to remove the 'prepare' word from the list of 'makesrc'. +prepare-dep = $(filter-out prepare, ,$(makesrc)) $(bsdir)/preparation-done.mk: \ $(foreach s, $(prepare-dep), $(mtexdir)/$(s).tex) diff --git a/reproduce/analysis/make/verify.mk b/reproduce/analysis/make/verify.mk index d3f3282..a645713 100644 --- a/reproduce/analysis/make/verify.mk +++ b/reproduce/analysis/make/verify.mk @@ -110,7 +110,7 @@ verify-txt-no-comments-no-space = \ # essentially the same as 'verify-dep', but it has removed # the 'initialize' step (which is information about the # pipeline, not the results). -verify-dep = $(subst verify,,$(subst paper,,$(makesrc))) +verify-dep = $(filter-out verify paper, $(makesrc)) verify-check = $(subst initialize,,$(verify-dep)) $(mtexdir)/verify.tex: $(foreach s, $(verify-dep), $(mtexdir)/$(s).tex) -- cgit v1.2.1