From 7d1960b1675f7764ef0b48776f995e191c99b695 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Mon, 27 May 2019 10:32:29 +0100 Subject: Adding SCons into the project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With this commit, SCons has been included into the project. As in the official webpage of this program says: SCons is an Open Source software construction tool—that is, a next-generation build tool. Think of SCons as an improved, cross-platform substitute for the classic Make utility with integrated functionality similar to autoconf/automake and compiler caches such as ccache. In short, SCons is an easier, more reliable and faster way to build software. The motivation for including this software into the project is because there are some software that use this system for the installation. In particular, the necessity of installing it comes from Imfit. This is an astronomical software that uses this system so that is why it has been included into the project. In principle it is simple, and I have tested it on Mac OS sytem without any problem. --- reproduce/software/make/high-level.mk | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index fe6efd0..14c4a2a 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -130,6 +130,7 @@ tarballs = $(foreach t, astrometry.net-$(astrometrynet-version).tar.gz \ openblas-$(openblas-version).tar.gz \ pixman-$(pixman-version).tar.gz \ scamp-$(scamp-version).tar.lz \ + scons-$(scons-version).tar.gz \ sextractor-$(sextractor-version).tar.lz \ swarp-$(swarp-version).tar.gz \ swig-$(swig-version).tar.gz \ @@ -199,6 +200,9 @@ $(tarballs): $(tdir)/%: | $(lockdir) w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = pixman ]; then w=https://www.cairographics.org/releases elif [ $$n = scamp ]; then w=http://akhlaghi.org/src + elif [ $$n = scons ]; then + mergenames=0 + w=https://sourceforge.net/projects/scons/files/scons/$(scons-version)/scons-$(scons-version).tar.gz/download elif [ $$n = sextractor ]; then w=http://akhlaghi.org/src elif [ $$n = swarp ]; then w=https://www.astromatic.net/download/swarp elif [ $$n = swig ]; then w=https://sourceforge.net/projects/swig/files/swig/swig-$(swig-version) @@ -658,6 +662,18 @@ $(ibidir)/scamp: $(tdir)/scamp-$(scamp-version).tar.lz \ && cp $(dtexdir)/scamp.tex $(ictdir)/ \ && echo "SCAMP $(scamp-version) \citep{scamp}" > $@ +# Since `scons' doesn't use the traditional GNU installation with +# `configure' and `make' it is installed manually using `python'. +$(ibidir)/scons: $(tdir)/scons-$(scons-version).tar.gz \ + $(ibidir)/python + cd $(ddir) \ + && unpackdir=scons-$(scons-version) \ + && rm -rf $$unpackdir \ + && if ! tar xf $<; then echo; echo "Tar error"; exit 1; fi \ + && cd $$unpackdir \ + && python setup.py install \ + && echo "SCons $(scons-version)" > $@ + # Sextractor crashes complaining about not linking with some ATLAS # libraries. But we can override this issue since we have Openblas # installed, it is just necessary to explicity tell sextractor to use it in -- cgit v1.2.1 From 69313eb17ed6e57c1cebe448cd2947665b89b783 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Mon, 27 May 2019 13:29:29 +0100 Subject: Added Imfit into the project With this commit, `imfit' has been included into the project. As described in the official webpage (http://www.mpe.mpg.de/~erwin/code/imfit/): Imfit is a program for fitting astronomical images -- especially images of galaxies, though it can in principle be used for fitting other sources. This program uses the `scons' program to build itself (not the traditional GNU `configure' and `make' system). I have tested it by using an example that can be found in the `examples' directory of the decompressed tarball. It works fine without any error on my Mac OS system. --- reproduce/software/make/high-level.mk | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 14c4a2a..ae0a857 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -119,6 +119,7 @@ tarballs = $(foreach t, astrometry.net-$(astrometrynet-version).tar.gz \ gsl-$(gsl-version).tar.gz \ hdf5-$(hdf5-version).tar.gz \ imagemagick-$(imagemagick-version).tar.xz \ + imfit-$(imfit-version).tar.gz \ install-tl-unx.tar.gz \ jpegsrc.$(libjpeg-version).tar.gz \ lapack-$(lapack-version).tar.gz \ @@ -182,6 +183,9 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = imagemagick ]; then mergenames=0 w=https://www.imagemagick.org/download/releases/ImageMagick-$(imagemagick-version).tar.xz + elif [ $$n = imfit ]; then + mergenames=0 + w=http://www.mpe.mpg.de/~erwin/resources/imfit/imfit-$(imfit-version)-source.tar.gz elif [ $$n = install ]; then w=http://mirror.ctan.org/systems/texlive/tlnet elif [ $$n = jpegsrc ]; then w=http://ijg.org/files elif [ $$n = lapack ]; then w=http://www.netlib.org/lapack @@ -612,6 +616,32 @@ $(ibidir)/imagemagick: $(tdir)/imagemagick-$(imagemagick-version).tar.xz \ --without-x --disable-openmp, V=1) \ && echo "ImageMagick $(imagemagick-version)" > $@ +# `imfit doesn't use the traditional `configure' and `make' to install it. +# Instead of that, it uses `scons'. As a consecuence, the installation is +# manually done by decompressing the tarball, and running `scons' with the +# necessary flags. After that, each compiled program (`imfit', `imfit-mcmc' +# and `makeimage') is copied into the installation directory.' +$(ibidir)/imfit: $(tdir)/imfit-$(imfit-version).tar.gz \ + $(ibidir)/cfitsio \ + $(ibidir)/fftw \ + $(ibidir)/gsl \ + $(ibidir)/scons + cd $(ddir) \ + && unpackdir=imfit-$(imfit-version) \ + && rm -rf $$unpackdir \ + && if ! tar xf $<; then echo; echo "Tar error"; exit 1; fi \ + && cd $$unpackdir \ + && scons --no-openmp --no-nlopt \ + --header-path=$(idir)/inlcude --lib-path=$(idir)/lib imfit \ + && cp imfit $(ibdir) \ + && scons --no-openmp --no-nlopt --header-path=$(idir)/inlcude \ + --lib-path=$(idir)/lib imfit-mcmc \ + && cp imfit-mcmc $(ibdir) \ + && scons --no-openmp --no-nlopt --header-path=$(idir)/inlcude \ + --lib-path=$(idir)/lib makeimage \ + && cp makeimage $(ibdir) \ + && echo "Imfit $(imfit-version)" > $@ + # Netpbm is a prerequisite of Astrometry-net, it contains a lot of programs. # This program has a crazy dialogue installation which is override using the # printf statment. Each `\n' is a new question that the installation process -- cgit v1.2.1 From 0f121e014b10a5862b65841a3cfe582a68a285b2 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Mon, 27 May 2019 14:13:57 +0100 Subject: Added paper citation of Imfit Until this commit, `imfit' paper was not included into the acknowledgements. With this commit, a bib tex file for `imfit' has been added. So, now it is properly cited into the final paper. --- reproduce/software/make/high-level.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index ae0a857..7950c5d 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -640,7 +640,8 @@ $(ibidir)/imfit: $(tdir)/imfit-$(imfit-version).tar.gz \ && scons --no-openmp --no-nlopt --header-path=$(idir)/inlcude \ --lib-path=$(idir)/lib makeimage \ && cp makeimage $(ibdir) \ - && echo "Imfit $(imfit-version)" > $@ + && cp $(dtexdir)/imfit.tex $(ictdir)/ \ + && echo "Imfit $(imfit-version) \citep{imfit2015}" > $@ # Netpbm is a prerequisite of Astrometry-net, it contains a lot of programs. # This program has a crazy dialogue installation which is override using the -- cgit v1.2.1 From 904f1cfce355e38b80b9c60ac91d5a6a7699c87f Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 28 May 2019 12:36:16 +0100 Subject: Corrected typo in Imfit installation, but not working yet With this commit, I have corrected a typo in the installation of `imfit'. In principle it is working on Mac OS system but when I tried to install it in GNU/Linux system it crashed. It complains about a problem with the `cfitsio' library: /pathto/libcfitsio.a: error adding symbols: File format not recognized As in the last commits done by Mohammad in the project, `cfitsio' installation has been upgraded and also the installation has been modified, I am going to try this new version. I will also check if it is a problem about dynamic/static library. It also seems that the installation is taking libraries and variables from the host system, so I will check that everything used is from our own programs/libraries. --- reproduce/software/make/high-level.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 7950c5d..98cdf76 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -632,7 +632,7 @@ $(ibidir)/imfit: $(tdir)/imfit-$(imfit-version).tar.gz \ && if ! tar xf $<; then echo; echo "Tar error"; exit 1; fi \ && cd $$unpackdir \ && scons --no-openmp --no-nlopt \ - --header-path=$(idir)/inlcude --lib-path=$(idir)/lib imfit \ + --header-path=$(idir)/include --lib-path=$(idir)/lib imfit \ && cp imfit $(ibdir) \ && scons --no-openmp --no-nlopt --header-path=$(idir)/inlcude \ --lib-path=$(idir)/lib imfit-mcmc \ -- cgit v1.2.1 From 6fa2c4f35f56420dea960099fed35f5a1f98bfd3 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 31 May 2019 17:44:15 +0100 Subject: Fixing rpath problem in CFITSIO installation `cfitsio' was recently updated and it cames up with a problem when trying to build shared libraries (on Mac OS system laptop). This is because the `configure' script for building `cfitsio' include by default `rpath'. With this commit, this issue is fixed by modifying the installation of `cfitsio'. Using `sed', each ocurrence of `@rpath' in `configure' is replaced by `$(ildir)'. Once it has been done, the installation of `cfitsio' goes as normal. In this commit, the version of the `cfitsio' has been also updated to the most recent one. --- reproduce/software/make/high-level.mk | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 98cdf76..2c35982 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -259,22 +259,22 @@ $(tarballs): $(tdir)/%: | $(lockdir) $(ibidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ $(ibidir)/curl - # CFITSIO hard-codes the absolute address of cURL's `curl-config' - # program (which gives the necessary header and linking - # information) into the configure script. So we'll have to modify - # it manually before doing the standard build. + # CFITSIO hard-codes '@rpath' inside the shared library on + # Mac systems. So we need to change it to our library + # installation path. It doesn't affect GNU/Linux, so we'll + # just do it in any case to keep things clean. topdir=$(pwd); cd $(ddir); tar xf $< customtar=cfitsio-$(cfitsio-version)-custom.tar.gz - sed cfitsio/configure \ - -e's|/usr/bin/curl-config|$(ibdir)/curl-config|g' \ - > cfitsio/configure_tmp - mv cfitsio/configure_tmp cfitsio/configure - chmod +x cfitsio/configure - tar cf $$customtar cfitsio + cd cfitsio-$(cfitsio-version) + sed configure -e's|@rpath|$(ildir)|g' > configure_tmp + mv configure_tmp configure + chmod +x configure + cd .. + tar cf $$customtar cfitsio-$(cfitsio-version) cd $$topdir - # Continue the standard build on the customized tarball. - $(call gbuild, $$customtar, cfitsio, , \ + # Continue the standard build on the customized tarball. + $(call gbuild, $$customtar, cfitsio-$(cfitsio-version), , \ --enable-sse2 --enable-reentrant, , \ make shared) \ && rm $$customtar \ -- cgit v1.2.1 From f2facb9fa47374dff8a1c9505c66c82c6af19b00 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 31 May 2019 18:08:56 +0100 Subject: Building single and double precission float libraries of FFTW Until this commit, `fftw' was building single precission float library because `scamp' had problem on using the default double precission library. However,`imfit' and maybe other programs will use double precission float libraries. With this commit, `fftw' installation is done twice in order to build single and double precission float libraries. It does not complain and goes up to the end without crashing. However, more tests are needed in order to ensure that there is not problem on having both libraries because in principle, they share the same header file and that could cause problems. --- reproduce/software/make/high-level.mk | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 2c35982..61b06d6 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -293,8 +293,16 @@ $(ibidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz && echo "GNU Scientific Library $(gsl-version)" > $@ $(ibidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz + # In order to build single and double precission libraries of + # `fftw', installation of `fftw' is done twice. First time is to + # build single precission float libraries and second time is for + # building the default double precission float libraries $(call gbuild, $<, fftw-$(fftw-version), static, \ - --enable-shared --enable-single) \ + --enable-shared enable-threads \ + --enable-single --enable-type-prefix) \ + && $(call gbuild, $<, fftw-$(fftw-version), static, \ + --enable-shared --enable-threads \ + --enable-type-prefix) \ && cp $(dtexdir)/fftw.tex $(ictdir)/ \ && echo "FFTW $(fftw-version) \citep{fftw}" > $@ -- cgit v1.2.1 From 880379181a36e949a05724229879ea4576a51b29 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Fri, 31 May 2019 18:35:36 +0100 Subject: Modified Imfit installation to not consider default and host paths Until this commit, 'imfit' was installed using proper flags in order to use our own libraries. However, it looks first of all to the default system. As a consecuence, `imfit' program was linked to the host system libraries, even when we told to use our own libraries!! With this commit, the installation of `imfit' has been modified in order to remove the default paths already set in the `SConstruct' script. By doing this, only our paths are take into account for compiling the program. It has been tested on Mac OS laptop and it works fine. Test on GNU/Linux systems remain before merge into the main project branch. --- reproduce/software/make/high-level.mk | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 61b06d6..2ea3534 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -627,8 +627,11 @@ $(ibidir)/imagemagick: $(tdir)/imagemagick-$(imagemagick-version).tar.xz \ # `imfit doesn't use the traditional `configure' and `make' to install it. # Instead of that, it uses `scons'. As a consecuence, the installation is # manually done by decompressing the tarball, and running `scons' with the -# necessary flags. After that, each compiled program (`imfit', `imfit-mcmc' -# and `makeimage') is copied into the installation directory.' +# necessary flags. Despite of that, it is necessary to replace the default +# searching paths in this script by our installation paths. This is done +# with `sed', replacing each ocurrence of `/usr/local' by `$(idir)'. After +# that, each compiled program (`imfit', `imfit-mcmc' and `makeimage') is +# copied into the installation directory. $(ibidir)/imfit: $(tdir)/imfit-$(imfit-version).tar.gz \ $(ibidir)/cfitsio \ $(ibidir)/fftw \ @@ -639,14 +642,18 @@ $(ibidir)/imfit: $(tdir)/imfit-$(imfit-version).tar.gz \ && rm -rf $$unpackdir \ && if ! tar xf $<; then echo; echo "Tar error"; exit 1; fi \ && cd $$unpackdir \ - && scons --no-openmp --no-nlopt \ + && sed -i 's|/usr/local|$(idir)|g' SConstruct \ + && scons --no-openmp --no-nlopt\ + --cc=$(ibdir)/gcc --cpp=$(ibdir)/g++ \ --header-path=$(idir)/include --lib-path=$(idir)/lib imfit \ && cp imfit $(ibdir) \ - && scons --no-openmp --no-nlopt --header-path=$(idir)/inlcude \ - --lib-path=$(idir)/lib imfit-mcmc \ + && scons --no-openmp --no-nlopt\ + --cc=$(ibdir)/gcc --cpp=$(ibdir)/g++ \ + --header-path=$(idir)/include --lib-path=$(idir)/lib imfit-mcmc \ && cp imfit-mcmc $(ibdir) \ - && scons --no-openmp --no-nlopt --header-path=$(idir)/inlcude \ - --lib-path=$(idir)/lib makeimage \ + && scons --no-openmp --no-nlopt\ + --cc=$(ibdir)/gcc --cpp=$(ibdir)/g++ \ + --header-path=$(idir)/include --lib-path=$(idir)/lib makeimage \ && cp makeimage $(ibdir) \ && cp $(dtexdir)/imfit.tex $(ictdir)/ \ && echo "Imfit $(imfit-version) \citep{imfit2015}" > $@ -- cgit v1.2.1 From 7a0d4bf2df6c78015b9dd3ec42cfb23d41252ffb Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 12 Jun 2019 14:21:10 +0100 Subject: Imfit and libcurl: manually adding rpath on GNU/Linux systems While trying to debug the installation of Imfit, we found out that libcurl doesn't have rpath and is thus linking with the host system's libraries. So rpath is now manually added to the build of libcurl. We also found out that Imfit executables don't have rpath, so it was added to them too. --- reproduce/software/make/basic.mk | 6 +++++- reproduce/software/make/high-level.mk | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 5 deletions(-) (limited to 'reproduce/software/make') diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index b21e7b0..f8963bc 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -726,7 +726,8 @@ $(ibidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ # this project is avoid dependency on the host as much as possible. $(ibidir)/curl: $(tdir)/curl-$(curl-version).tar.gz \ $(ibidir)/coreutils \ - $(ibidir)/openssl #Coreutils: only so cURL is built after it. + $(ibidir)/openssl \ + $(needpatchelf) $(call gbuild, $<, curl-$(curl-version), , \ LIBS="-pthread" \ --with-zlib=$(ildir) \ @@ -744,6 +745,9 @@ $(ibidir)/curl: $(tdir)/curl-$(curl-version).tar.gz \ --disable-ldaps \ --disable-ldap \ --without-nss, V=1) \ + && if [ "x$(needpatchelf)" != x ]; then \ + $(ibdir)/patchelf --set-rpath $(ildir) $(ildir)/libcurl.so; \ + fi && echo "cURL $(curl-version)" > $@ diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 2ea3534..2924077 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -158,7 +158,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) : (l==3 ? "%d0\n" \ : (l==2 ? "%d00\n" \ : "%d000\n") ), $$1)}') - w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz + w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio-$(cfitsio-version).tar.gz elif [ $$n = astrometry ]; then w=http://astrometry.net/downloads elif [ $$n = atlas ]; then mergenames=0 @@ -643,19 +643,28 @@ $(ibidir)/imfit: $(tdir)/imfit-$(imfit-version).tar.gz \ && if ! tar xf $<; then echo; echo "Tar error"; exit 1; fi \ && cd $$unpackdir \ && sed -i 's|/usr/local|$(idir)|g' SConstruct \ - && scons --no-openmp --no-nlopt\ + && sed -i 's|/usr/include|$(idir)/include|g' SConstruct \ + && sed -i 's|.append(|.insert(0,|g' SConstruct \ + && scons --no-openmp --no-nlopt \ --cc=$(ibdir)/gcc --cpp=$(ibdir)/g++ \ --header-path=$(idir)/include --lib-path=$(idir)/lib imfit \ && cp imfit $(ibdir) \ && scons --no-openmp --no-nlopt\ --cc=$(ibdir)/gcc --cpp=$(ibdir)/g++ \ - --header-path=$(idir)/include --lib-path=$(idir)/lib imfit-mcmc \ + --header-path=$(idir)/include --lib-path=$(idir)/lib \ + imfit-mcmc \ && cp imfit-mcmc $(ibdir) \ && scons --no-openmp --no-nlopt\ --cc=$(ibdir)/gcc --cpp=$(ibdir)/g++ \ - --header-path=$(idir)/include --lib-path=$(idir)/lib makeimage \ + --header-path=$(idir)/include --lib-path=$(idir)/lib \ + makeimage \ && cp makeimage $(ibdir) \ && cp $(dtexdir)/imfit.tex $(ictdir)/ \ + && if [ "x$(on_mac_os)" != xyes ]; then \ + for p in imfit imfit-mcmc makeimage; do \ + patchelf --set-rpath $(ildir) $(ibdir)/$$p; \ + done; \ + fi \ && echo "Imfit $(imfit-version) \citep{imfit2015}" > $@ # Netpbm is a prerequisite of Astrometry-net, it contains a lot of programs. -- cgit v1.2.1