From e17b5574011890b83d7bf9fcf15be55878bd8ab8 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 16 Apr 2019 06:17:32 +0100 Subject: Symbolic link to Python created, using `python' instead of `pythonX' Until this commit, we were using `python3' when calling Python (because we were using Python version 3.6.8). This will force us to change the name in the future. For example, when `python4' were available and into the pipeline. With this commit, at the end of the Python installation it creates a symbolic link to the Python bin executable with the new name `python'. As a consecuence, whatever version of Python was installed, into the project we will use allways `python' to invoke it. --- reproduce/software/make/python.mk | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 5ec99f0..22c6284 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -214,13 +214,13 @@ $(pytarballs): $(tdir)/%: # While this Makefile is for Python programs, in some cases, we need # certain programs (like Python itself), or libraries for the modules. $(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz - $(call gbuild, $<, libffi-$(libffi-version)) \ + $(call gbuild, $<, libffi-$(libffi-version)) \ echo "Libffi $(libffi-version)" > $@ -$(ibidir)/python3: $(tdir)/python-$(python-version).tar.gz \ - $(ilidir)/libffi - # On Mac systems, the build complains about `clang' specific - # features, so we can't use our own GCC build here. +$(ibidir)/python: $(tdir)/python-$(python-version).tar.gz \ + $(ilidir)/libffi + # On Mac systems, the build complains about `clang' specific + # features, so we can't use our own GCC build here. if [ x$(on_mac_os) = xyes ]; then \ export CC=clang; \ export CXX=clang++; \ @@ -231,7 +231,8 @@ $(ibidir)/python3: $(tdir)/python-$(python-version).tar.gz \ --enable-shared) \ && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') \ - && ln -s $(ildir)/python$$v $(ildir)/python \ + && ln -sf $(ildir)/python$$v $(ildir)/python \ + && ln -sf $(ibdir)/python$$v $(ibdir)/python \ && rm -rf $(ipydir) \ && mkdir $(ipydir) \ && echo "Python $(python-version)" > $@ @@ -259,8 +260,8 @@ pybuild = cd $(ddir); rm -rf $(3); \ -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ $(4) > site.cfg; \ fi; \ - python3 setup.py build \ - && python3 setup.py install \ + python setup.py build \ + && python setup.py install \ && cd .. \ && rm -rf $(3) \ && echo "$(5)" > $@ @@ -475,7 +476,7 @@ $(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \ SecretStorage $(secretstorage-version)) $(ipydir)/setuptools: $(tdir)/setuptools-$(setuptools-version).zip \ - $(ibidir)/python3 \ + $(ibidir)/python \ $(ibidir)/unzip $(call pybuild, unzip, $<, setuptools-$(setuptools-version), ,\ Setuptools $(setuptools-version)) -- cgit v1.2.1 From 967c81fa82179fe151a0d589fa922d896b13b1c8 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 16 Apr 2019 06:33:55 +0100 Subject: Cairo library and its dependency Pixman, are now included In this commit we add Cairo library. The motivation of including this library is because it is a dependency of Astrometry-net (which is a common software used in astronomy projects). In principle it will not be build by default by de pipeline, but we will keep the rule just in case it is needed. Pixman is another library which is a prerequisite of Cairo, it is also built with this commit. --- reproduce/software/config/installation/versions.mk | 2 ++ reproduce/software/make/high-level.mk | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/reproduce/software/config/installation/versions.mk b/reproduce/software/config/installation/versions.mk index 149f9ae..81acacf 100644 --- a/reproduce/software/config/installation/versions.mk +++ b/reproduce/software/config/installation/versions.mk @@ -20,6 +20,7 @@ atlas-version = 3.10.3 bash-version = 5.0 binutils-version = 2.31.1 +cairo-version = 1.16.0 cfitsio-version = 3.45 cmake-version = 3.14.2 coreutils-version = 8.31 @@ -58,6 +59,7 @@ openblas-version = 0.3.5 openmpi-version = 4.0.1 openssl-version = 1.1.1a patchelf-version = 0.9 +pixman-version = 0.38.0 pkgconfig-version = 0.29.2 python-version = 3.7.3 readline-version = 8.0 diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index f7b1f4f..04b6cac 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -50,7 +50,7 @@ ipydir = $(BDIR)/software/installed/version-info/python # high level software depend on it. The current rule for ATLAS is tested # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. -top-level-libraries = # atlas +top-level-libraries = cairo pixman # atlas top-level-python = astroquery matplotlib top-level-programs = gnuastro metastore all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ @@ -116,6 +116,7 @@ include reproduce/software/make/python.mk # downloaded file has our desired format. tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ atlas-$(atlas-version).tar.bz2 \ + cairo-$(cairo-version).tar.bz2 \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ freetype-$(freetype-version).tar.gz \ @@ -134,6 +135,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ metastore-$(metastore-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ openblas-$(openblas-version).tar.gz \ + pixman-$(pixman-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ , $(tdir)/$(t) ) @@ -160,6 +162,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = atlas ]; then mergenames=0 w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download + elif [ $$n = cairo ]; then w=https://www.cairographics.org/releases elif [ $$n = cmake ]; then mergenames=0 majv=$$(echo $(cmake-version) \ @@ -193,6 +196,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) mergenames=0 majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* + elif [ $$n = pixman ]; then w=https://www.cairographics.org/releases elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib else @@ -261,6 +265,13 @@ $(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ && rm $$customtar \ && echo "CFITSIO $(cfitsio-version)" > $@ +$(ilidir)/cairo: $(tdir)/cairo-$(cairo-version).tar.xz \ + $(ilidir)/freetype \ + $(ilidir)/libpng \ + $(ilidir)/pixman + $(call gbuild, $<, cairo-$(cairo-version), static) \ + && echo "Cairo $(cairo-version)" > $@ + $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, $<, gsl-$(gsl-version), static) \ && echo "GNU Scientific Library $(gsl-version)" > $@ @@ -298,6 +309,10 @@ $(ilidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz $(call gbuild, $<, libpng-$(libpng-version), static) \ && echo "Libpng $(libpng-version)" > $@ +$(ilidir)/pixman: $(tdir)/pixman-$(pixman-version).tar.gz + $(call gbuild, $<, pixman-$(pixman-version), static) \ + && echo "Pixman $(pixman-version)" > $@ + $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/libjpeg $(call gbuild, $<, tiff-$(libtiff-version), static, \ -- cgit v1.2.1 From c5ccdd7385499369ca0ca9ed2894658bfe235653 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 16 Apr 2019 06:39:31 +0100 Subject: CC symbolic link to GCC built after GCC installation Until now, we did not have CC. This is an obsolete way of calling C Compiler, nowadays it is GCC. However, there could be some software that is still using this convention (for example, `netpbm'). With this commit, we make the symbolic link `cc' pointing to the installed `gcc'. --- reproduce/software/make/basic.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index cfa05d2..4fc104d 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -854,6 +854,7 @@ $(ibidir)/gcc: $(gcc-prerequisites) \ $(call makelink,gcc); \ $(call makelink,g++,mandatory); \ $(call makelink,gfortran,mandatory); \ + ln -sf $(which gcc) $(ibdir)/cc; \ ccinfo=$$(gcc --version | awk 'NR==1'); \ echo "C compiler (""$$ccinfo"")" > $@; \ else \ @@ -899,5 +900,6 @@ $(ibidir)/gcc: $(gcc-prerequisites) \ fi; \ done; \ fi \ + ln -sf $(ibdir)/gcc $(ibdir)/cc; \ && echo "GNU Compiler Collection (GCC) $(gcc-version)" > $@; \ fi -- cgit v1.2.1 From 335f05386ff8712da2f29a504a4fa42050580b79 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 16 Apr 2019 07:08:00 +0100 Subject: Netpbm and its dependencies are now included With this commit, the installation of `netpbm' has been included. This software has a crazy dialogue installation that we override by giving to it the answers (they differs on GNU/Linux and Mac OS systems). `libxml2' also has been built (a remaining dependency of `netpbm'). In this commit, `libpng' version has been downgraded because it is required by `netpbm'. The motivation of installing `netpbm' is that it is a dependency of Astrometry-net software. --- reproduce/software/config/installation/versions.mk | 7 ++- reproduce/software/make/high-level.mk | 61 ++++++++++++++++++---- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/reproduce/software/config/installation/versions.mk b/reproduce/software/config/installation/versions.mk index 81acacf..557eec2 100644 --- a/reproduce/software/config/installation/versions.mk +++ b/reproduce/software/config/installation/versions.mk @@ -45,9 +45,9 @@ isl-version = 0.18 libbsd-version = 0.9.1 libffi-version = 3.2.1 libjpeg-version = v9b -libpng-version = 1.6.36 libtiff-version = 4.0.10 libtool-version = 2.4.6 +libxml2-version = 2.9.9 lzip-version = 1.20 m4-version = 1.4.18 make-version = 4.2.90 @@ -55,6 +55,7 @@ metastore-version = 1.1.2-23-fa9170b mpfr-version = 4.0.2 mpc-version = 1.1.0 ncurses-version = 6.1 +netpbm-version = 10.47.72 openblas-version = 0.3.5 openmpi-version = 4.0.1 openssl-version = 1.1.1a @@ -77,10 +78,12 @@ zlib-version = 1.2.11 # # When updating the version of these libraries, please look into the build # rule first: In one way or another, the version string becomes necessary -# during their build and must be accounted for. +# during their build and must be accounted for. In particular: +# `libpng' is downgraded because `netpbm' requires `libpng' version < 1.5 bzip2-version = 1.0.6 lapack-version = 3.8.0 libgit2-version = 0.26.0 +libpng-version = 1.4.22 wcslib-version = 6.2 # Python packages diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 04b6cac..8aa9c55 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -50,9 +50,9 @@ ipydir = $(BDIR)/software/installed/version-info/python # high level software depend on it. The current rule for ATLAS is tested # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. -top-level-libraries = cairo pixman # atlas +top-level-libraries = # atlas +top-level-programs = gnuastro metastore netpbm top-level-python = astroquery matplotlib -top-level-programs = gnuastro metastore all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ $(foreach p, $(top-level-python), $(ipydir)/$(p)) \ @@ -132,7 +132,9 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ libbsd-$(libbsd-version).tar.xz \ libpng-$(libpng-version).tar.xz \ libgit2-$(libgit2-version).tar.gz \ + libxml2-$(libxml2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ + netpbm-$(netpbm-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ openblas-$(openblas-version).tar.gz \ pixman-$(pixman-version).tar.gz \ @@ -188,7 +190,11 @@ $(tarballs): $(tdir)/%: | $(lockdir) elif [ $$n = libgit ]; then mergenames=0 w=https://github.com/libgit2/libgit2/archive/v$(libgit2-version).tar.gz + elif [ $$n = libxml ]; then w=ftp://xmlsoft.org/libxml2 elif [ $$n = metastore ]; then w=http://akhlaghi.org/src + elif [ $$n = netpbm ]; then + mergenames=0 + w=https://sourceforge.net/projects/netpbm/files/super_stable/$(netpbm-version)/netpbm-$(netpbm-version).tgz/download elif [ $$n = openblas ]; then mergenames=0 w=https://github.com/xianyi/OpenBLAS/archive/v$(openblas-version).tar.gz @@ -309,6 +315,17 @@ $(ilidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz $(call gbuild, $<, libpng-$(libpng-version), static) \ && echo "Libpng $(libpng-version)" > $@ +$(ilidir)/libxml2: $(tdir)/libxml2-$(libxml2-version).tar.gz + # The libxml2 tarball also contains Python bindings which are built and + # installed to a system directory by default. If you don't need the Python + # bindings, the easiest solution is to compile without Python support: + # ./configure --without-python + # If you really need the Python bindings, try the + # --with-python-install-dir=DIR option + $(call gbuild, $<, libxml2-$(libxml2-version), static, \ + --without-python) \ + && echo "Libxml2 $(libxml2-version)" > $@ + $(ilidir)/pixman: $(tdir)/pixman-$(pixman-version).tar.gz $(call gbuild, $<, pixman-$(pixman-version), static) \ && echo "Pixman $(pixman-version)" > $@ @@ -628,15 +645,37 @@ endif && cp $(dtexdir)/gnuastro.tex $(ictdir)/ \ && echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ - - - - - - - - - +# Netpbm is a prerequisite of Astrometry-net, it contains a lot of programs. +# This program has a crazy dialogue installation which is override using the +# printf statment. Each `\n' is a new question that the installation process +# ask to the user. We give all answers with a pipe to the scripts (configure +# and install). The questions are different depending on the system (tested +# on GNU/Linux and Mac OS). +$(ibidir)/netpbm: $(tdir)/netpbm-$(netpbm-version).tgz \ + $(ilidir)/libjpeg \ + $(ilidir)/libpng \ + $(ilidir)/libtiff \ + $(ilidir)/libxml2 \ + $(ibidir)/unzip + if [ x$(on_mac_os) = xyes ]; then \ + answers='\n\n\n\n\n\n\n\n\n\n\n\nnone\n\n\n'; \ + else \ + answers='\n\n\n\ny\n\n\n\n\n\n\n\n\n\n\n\n\n'; \ + fi; \ + cd $(ddir) \ + && unpackdir=netpbm-$(netpbm-version) \ + && echo "rm -rf $$unpackdir \ + && if ! tar xf $<; then echo; echo "Tar error"; exit 1; fi" \ + && cd $$unpackdir \ + && echo "printf "$$answers" | ./configure \ + && make \ + && rm -rf $(ddir)/$$unpackdir/install \ + && make package pkgdir=$(ddir)/$$unpackdir/install" \ + && printf "$(ddir)/$$unpackdir/install\n$(idir)\n\n\nN\n\n\n\n\nN\n\n" \ + | ./installnetpbm \ + && cd .. \ + && rm -rf $$unpackdir \ + && echo "Netpbm $(netpbm-version)" > $@ -- cgit v1.2.1 From 3ab5e95f2fe2358ccae0a7c1e52b1b3b910140cf Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 16 Apr 2019 07:42:36 +0100 Subject: Astrometry-net and all its dependencies are now in the project With this commit, we include Astrometry-net software and one remaining dependency of it (Swig). In principle it should work as far as we tested all of this software in GNU/Linux, but we need to test also in Mac OS. As a result, this commit is just for setting all the rules and dependencies, but more tests are necessaries to ensure it works properly, specially in Mac OS systems. --- reproduce/software/bibtex/astrometry-net.tex | 25 +++++++++++ reproduce/software/config/installation/versions.mk | 2 + reproduce/software/make/high-level.mk | 51 +++++++++++++++++++--- 3 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 reproduce/software/bibtex/astrometry-net.tex diff --git a/reproduce/software/bibtex/astrometry-net.tex b/reproduce/software/bibtex/astrometry-net.tex new file mode 100644 index 0000000..7e8d0be --- /dev/null +++ b/reproduce/software/bibtex/astrometry-net.tex @@ -0,0 +1,25 @@ +%% Copyright (C) 2019 Raul Infante-Sainz +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{astrometrynet2010, + author = {{Lang}, Dustin and {Hogg}, David W. and {Mierle}, Keir and + {Blanton}, Michael and {Roweis}, Sam}, + title = "{Astrometry.net: Blind Astrometric Calibration of Arbitrary Astronomical Images}", + journal = {\aj}, + keywords = {astrometry, catalogs, instrumentation: miscellaneous, methods: data analysis, methods: statistical, techniques: image processing, Astrophysics - Instrumentation and Methods for Astrophysics}, + year = "2010", + month = "May", + volume = {139}, + number = {5}, + pages = {1782}, + doi = {10.1088/0004-6256/139/5/1782}, +archivePrefix = {arXiv}, + eprint = {0910.2233}, + primaryClass = {astro-ph.IM}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2010AJ....139.1782L}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/config/installation/versions.mk b/reproduce/software/config/installation/versions.mk index 557eec2..5f686a8 100644 --- a/reproduce/software/config/installation/versions.mk +++ b/reproduce/software/config/installation/versions.mk @@ -17,6 +17,7 @@ # . # C/C++ programs and libraries. +astrometry-version = 0.77 atlas-version = 3.10.3 bash-version = 5.0 binutils-version = 2.31.1 @@ -65,6 +66,7 @@ pkgconfig-version = 0.29.2 python-version = 3.7.3 readline-version = 8.0 sed-version = 4.7 +swig-version = 3.0.12 tar-version = 1.32 unzip-version = 6.0 wget-version = 1.20.3 diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 8aa9c55..932aedc 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -51,7 +51,7 @@ ipydir = $(BDIR)/software/installed/version-info/python # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. top-level-libraries = # atlas -top-level-programs = gnuastro metastore netpbm +top-level-programs = astrometry-net gnuastro metastore top-level-python = astroquery matplotlib all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ @@ -114,7 +114,8 @@ include reproduce/software/make/python.mk # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. -tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ +tarballs = $(foreach t, astrometry.net-$(astrometry-version).tar.gz \ + cfitsio-$(cfitsio-version).tar.gz \ atlas-$(atlas-version).tar.bz2 \ cairo-$(cairo-version).tar.bz2 \ cmake-$(cmake-version).tar.gz \ @@ -138,6 +139,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ openblas-$(openblas-version).tar.gz \ pixman-$(pixman-version).tar.gz \ + swig-$(swig-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ , $(tdir)/$(t) ) @@ -161,6 +163,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) : (l==2 ? "%d00\n" \ : "%d000\n") ), $$1)}') w=https://heasarc.gsfc.nasa.gov/FTP/software/fitsio/c/cfitsio$$v.tar.gz + elif [ $$n = astrometry ]; then w=http://astrometry.net/downloads elif [ $$n = atlas ]; then mergenames=0 w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download @@ -203,6 +206,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = pixman ]; then w=https://www.cairographics.org/releases + elif [ $$n = swig ]; then w=https://sourceforge.net/projects/swig/files/swig/swig-$(swig-version) elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib else @@ -495,6 +499,35 @@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ # Programs # -------- # +# Astrometry-net contains a lot of programs. We need to specify the +# installation directory and the Python executable (by default it will look +# for /usr/bin/python) +$(ibidir)/astrometry-net: $(tdir)/astrometry.net-$(astrometry-version).tar.gz \ + $(ilidir)/cairo \ + $(ilidir)/cfitsio \ + $(ilidir)/gsl \ + $(ilidir)/libjpeg \ + $(ilidir)/libpng \ + $(ibidir)/netpbm \ + $(ipydir)/numpy \ + $(ibidir)/python \ + $(ibidir)/swig \ + $(ilidir)/wcslib + cd $(ddir) \ + && if ! tar xf $<; then echo; echo "Tar error"; exit 1; fi \ + && cd astrometry.net-$(astrometry-version) \ + && make \ + && make py \ + && make extra \ + && make install INSTALL_DIR=$(idir) PYTHON_SCRIPT="$(ibdir)/python" \ + && cd .. \ + && rm -rf astrometry.net-$(astrometry-version) \ + && cp $(dtexdir)/astrometry-net.tex $(ictdir)/ \ + && echo "Astrometry-net $(astrometry-version)" > $@ + + + + # CMake can be built with its custom `./bootstrap' script. $(ibidir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ $(ibidir)/curl @@ -664,19 +697,25 @@ $(ibidir)/netpbm: $(tdir)/netpbm-$(netpbm-version).tgz \ fi; \ cd $(ddir) \ && unpackdir=netpbm-$(netpbm-version) \ - && echo "rm -rf $$unpackdir \ - && if ! tar xf $<; then echo; echo "Tar error"; exit 1; fi" \ + && rm -rf $$unpackdir \ + && if ! tar xf $<; then echo; echo "Tar error"; exit 1; fi \ && cd $$unpackdir \ - && echo "printf "$$answers" | ./configure \ + && printf "$$answers" | ./configure \ && make \ && rm -rf $(ddir)/$$unpackdir/install \ - && make package pkgdir=$(ddir)/$$unpackdir/install" \ + && make package pkgdir=$(ddir)/$$unpackdir/install \ && printf "$(ddir)/$$unpackdir/install\n$(idir)\n\n\nN\n\n\n\n\nN\n\n" \ | ./installnetpbm \ && cd .. \ && rm -rf $$unpackdir \ && echo "Netpbm $(netpbm-version)" > $@ +$(ibidir)/swig: $(tdir)/swig-$(swig-version).tar.gz + # Option --without-pcre was a suggestion once the configure step was + # tried and it failed. It was not recommended but it works! + # pcr is a dependency of swig + $(call gbuild, $<, swig-$(swig-version), static, --without-pcre) \ + && echo "Swig $(swig-version)" > $@ -- cgit v1.2.1 From fd88ac3569196ef3ea7f8463af7cf34277d577f5 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 16 Apr 2019 13:09:06 +0100 Subject: Typo corrections in Astrometry-net dependency rules Due to the copy/paste of rules from other files, in some rules there were some spaces insteand of tab. There were also a lack of semicolon in the building of CC, and bad extensions in various tarballs. With this commit, all of these typos have been fixed. --- .file-metadata | Bin 5532 -> 5889 bytes reproduce/software/make/basic.mk | 6 +++--- reproduce/software/make/high-level.mk | 22 +++++++++++----------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.file-metadata b/.file-metadata index e4c87cf..87f4c93 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 4fc104d..8a52038 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -854,7 +854,7 @@ $(ibidir)/gcc: $(gcc-prerequisites) \ $(call makelink,gcc); \ $(call makelink,g++,mandatory); \ $(call makelink,gfortran,mandatory); \ - ln -sf $(which gcc) $(ibdir)/cc; \ + ln -sf $$(which gcc) $(ibdir)/cc; \ ccinfo=$$(gcc --version | awk 'NR==1'); \ echo "C compiler (""$$ccinfo"")" > $@; \ else \ @@ -899,7 +899,7 @@ $(ibidir)/gcc: $(gcc-prerequisites) \ patchelf --set-rpath $(ildir) $$f; \ fi; \ done; \ - fi \ - ln -sf $(ibdir)/gcc $(ibdir)/cc; \ + fi; \ + ln -sf $(ibdir)/gcc $(ibdir)/cc \ && echo "GNU Compiler Collection (GCC) $(gcc-version)" > $@; \ fi diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 932aedc..a904cae 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -117,7 +117,7 @@ include reproduce/software/make/python.mk tarballs = $(foreach t, astrometry.net-$(astrometry-version).tar.gz \ cfitsio-$(cfitsio-version).tar.gz \ atlas-$(atlas-version).tar.bz2 \ - cairo-$(cairo-version).tar.bz2 \ + cairo-$(cairo-version).tar.xz \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ freetype-$(freetype-version).tar.gz \ @@ -135,7 +135,7 @@ tarballs = $(foreach t, astrometry.net-$(astrometry-version).tar.gz \ libgit2-$(libgit2-version).tar.gz \ libxml2-$(libxml2-version).tar.gz \ metastore-$(metastore-version).tar.gz \ - netpbm-$(netpbm-version).tar.gz \ + netpbm-$(netpbm-version).tgz \ openmpi-$(openmpi-version).tar.gz \ openblas-$(openblas-version).tar.gz \ pixman-$(pixman-version).tar.gz \ @@ -279,8 +279,8 @@ $(ilidir)/cairo: $(tdir)/cairo-$(cairo-version).tar.xz \ $(ilidir)/freetype \ $(ilidir)/libpng \ $(ilidir)/pixman - $(call gbuild, $<, cairo-$(cairo-version), static) \ - && echo "Cairo $(cairo-version)" > $@ + $(call gbuild, $<, cairo-$(cairo-version), static) \ + && echo "Cairo $(cairo-version)" > $@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, $<, gsl-$(gsl-version), static) \ @@ -298,7 +298,7 @@ $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ $(call gbuild, $<, freetype-$(freetype-version), static) \ && echo "FreeType $(freetype-version)" > $@ -$(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ +$(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ $(ilidir)/openmpi export CC=mpicc; \ export FC=mpif90; \ @@ -326,13 +326,13 @@ $(ilidir)/libxml2: $(tdir)/libxml2-$(libxml2-version).tar.gz # ./configure --without-python # If you really need the Python bindings, try the # --with-python-install-dir=DIR option - $(call gbuild, $<, libxml2-$(libxml2-version), static, \ - --without-python) \ - && echo "Libxml2 $(libxml2-version)" > $@ + $(call gbuild, $<, libxml2-$(libxml2-version), static, \ + --without-python) \ + && echo "Libxml2 $(libxml2-version)" > $@ $(ilidir)/pixman: $(tdir)/pixman-$(pixman-version).tar.gz - $(call gbuild, $<, pixman-$(pixman-version), static) \ - && echo "Pixman $(pixman-version)" > $@ + $(call gbuild, $<, pixman-$(pixman-version), static) \ + && echo "Pixman $(pixman-version)" > $@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/libjpeg @@ -342,7 +342,7 @@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz $(call gbuild, $<, openmpi-$(openmpi-version), static, , V=1) \ - && echo "OpenMPI $(openmpi-version)" > $@ + && echo "Open MPI $(openmpi-version)" > $@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(tdir)/lapack-$(lapack-version).tar.gz -- cgit v1.2.1 From f95c0a53bb390795acc2c3eed8de5fcbded5cda3 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Tue, 16 Apr 2019 19:09:31 +0100 Subject: Binutils is built on GNU/Linux as GCC dependency, not in MAC OS Until this commit, we didn't install Binutils. However, we need `strip' for installing Netpbm, and `strip' is part of the Binutil software. With this commit, we include Binutils as a dependency of GCC for GNU/Linux systems. For Mac OS systems we create a symbolic link to `strip' just after the generation of the symbolic link to `gcc'. --- .file-metadata | Bin 5889 -> 5854 bytes reproduce/software/make/basic.mk | 2 ++ 2 files changed, 2 insertions(+) diff --git a/.file-metadata b/.file-metadata index 87f4c93..af5792e 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 8a52038..6e6550e 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -830,6 +830,7 @@ ifeq ($(host_cc),1) gcc-prerequisites = else gcc-prerequisites = $(tdir)/gcc-$(gcc-version).tar.xz \ + $(ibidir)/binutils \ $(ilidir)/isl \ $(ilidir)/mpc endif @@ -854,6 +855,7 @@ $(ibidir)/gcc: $(gcc-prerequisites) \ $(call makelink,gcc); \ $(call makelink,g++,mandatory); \ $(call makelink,gfortran,mandatory); \ + $(call makelink,strip,mandatory); \ ln -sf $$(which gcc) $(ibdir)/cc; \ ccinfo=$$(gcc --version | awk 'NR==1'); \ echo "C compiler (""$$ccinfo"")" > $@; \ -- cgit v1.2.1 From bcf9c6f13a0f6cd84b65c6762b8765c3f293bcb3 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 17 Apr 2019 10:42:14 +0100 Subject: SWarp is now into the pipeline SWarp resamples and co-adds together FITS images using any arbitrary astrometric projection defined in the WCS standard. It is a very useful program for astronomy and that is why we added it to the pipeline. With this commit, we also cite the paper for Astrometry-net that until now was missing. --- .file-metadata | Bin 5854 -> 6194 bytes reproduce/software/bibtex/astrometry-net.tex | 25 ------------- reproduce/software/bibtex/astrometrynet.tex | 20 ++++++++++ reproduce/software/bibtex/swarp.tex | 19 ++++++++++ reproduce/software/config/installation/versions.mk | 3 +- reproduce/software/make/high-level.mk | 41 +++++++++++++-------- 6 files changed, 66 insertions(+), 42 deletions(-) delete mode 100644 reproduce/software/bibtex/astrometry-net.tex create mode 100644 reproduce/software/bibtex/astrometrynet.tex create mode 100644 reproduce/software/bibtex/swarp.tex diff --git a/.file-metadata b/.file-metadata index af5792e..95076a8 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/software/bibtex/astrometry-net.tex b/reproduce/software/bibtex/astrometry-net.tex deleted file mode 100644 index 7e8d0be..0000000 --- a/reproduce/software/bibtex/astrometry-net.tex +++ /dev/null @@ -1,25 +0,0 @@ -%% Copyright (C) 2019 Raul Infante-Sainz -%% -%% Copying and distribution of this file, with or without modification, -%% are permitted in any medium without royalty provided the copyright -%% notice and this notice are preserved. This file is offered as-is, -%% without any warranty. - -@ARTICLE{astrometrynet2010, - author = {{Lang}, Dustin and {Hogg}, David W. and {Mierle}, Keir and - {Blanton}, Michael and {Roweis}, Sam}, - title = "{Astrometry.net: Blind Astrometric Calibration of Arbitrary Astronomical Images}", - journal = {\aj}, - keywords = {astrometry, catalogs, instrumentation: miscellaneous, methods: data analysis, methods: statistical, techniques: image processing, Astrophysics - Instrumentation and Methods for Astrophysics}, - year = "2010", - month = "May", - volume = {139}, - number = {5}, - pages = {1782}, - doi = {10.1088/0004-6256/139/5/1782}, -archivePrefix = {arXiv}, - eprint = {0910.2233}, - primaryClass = {astro-ph.IM}, - adsurl = {https://ui.adsabs.harvard.edu/abs/2010AJ....139.1782L}, - adsnote = {Provided by the SAO/NASA Astrophysics Data System} -} diff --git a/reproduce/software/bibtex/astrometrynet.tex b/reproduce/software/bibtex/astrometrynet.tex new file mode 100644 index 0000000..7bfe4c6 --- /dev/null +++ b/reproduce/software/bibtex/astrometrynet.tex @@ -0,0 +1,20 @@ +%% Copyright (C) 2019 Raul Infante-Sainz +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{astrometrynet, + doi = {10.1088/0004-6256/139/5/1782}, + url = {https://doi.org/10.1088%2F0004-6256%2F139%2F5%2F1782}, + year = 2010, + month = {mar}, + publisher = {{IOP} Publishing}, + volume = {139}, + number = {5}, + pages = {1782}, + author = {Dustin Lang and David W. Hogg and Keir Mierle and Michael Blanton and Sam Roweis}, + title = {{ASTROMETRY}.{NET}: {BLIND} {ASTROMETRIC} {CALIBRATION} {OF} {ARBITRARY} {ASTRONOMICAL} {IMAGES}}, + journal = {The Astronomical Journal} +} diff --git a/reproduce/software/bibtex/swarp.tex b/reproduce/software/bibtex/swarp.tex new file mode 100644 index 0000000..03111d6 --- /dev/null +++ b/reproduce/software/bibtex/swarp.tex @@ -0,0 +1,19 @@ +%% Copyright (C) 2019 Raul Infante-Sainz +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@MISC{swarp, + author = {{Bertin}, E.}, + title = "{SWarp: Resampling and Co-adding FITS Images Together}", + keywords = {Software }, +howpublished = {Astrophysics Source Code Library}, + year = 2010, + month = oct, +archivePrefix = "ascl", + eprint = {1010.068}, + adsurl = {http://cdsads.u-strasbg.fr/abs/2010ascl.soft10068B}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/config/installation/versions.mk b/reproduce/software/config/installation/versions.mk index 5f686a8..270eb97 100644 --- a/reproduce/software/config/installation/versions.mk +++ b/reproduce/software/config/installation/versions.mk @@ -17,7 +17,7 @@ # . # C/C++ programs and libraries. -astrometry-version = 0.77 +astrometrynet-version = 0.77 atlas-version = 3.10.3 bash-version = 5.0 binutils-version = 2.31.1 @@ -66,6 +66,7 @@ pkgconfig-version = 0.29.2 python-version = 3.7.3 readline-version = 8.0 sed-version = 4.7 +swarp-version = 2.38.0 swig-version = 3.0.12 tar-version = 1.32 unzip-version = 6.0 diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index a904cae..144057e 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -51,7 +51,7 @@ ipydir = $(BDIR)/software/installed/version-info/python # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. top-level-libraries = # atlas -top-level-programs = astrometry-net gnuastro metastore +top-level-programs = astrometrynet gnuastro metastore swarp top-level-python = astroquery matplotlib all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ @@ -139,6 +139,7 @@ tarballs = $(foreach t, astrometry.net-$(astrometry-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ openblas-$(openblas-version).tar.gz \ pixman-$(pixman-version).tar.gz \ + swarp-$(swarp-version).tar.gz \ swig-$(swig-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ @@ -206,6 +207,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = pixman ]; then w=https://www.cairographics.org/releases + elif [ $$n = swarp ]; then w=https://www.astromatic.net/download/swarp elif [ $$n = swig ]; then w=https://sourceforge.net/projects/swig/files/swig/swig-$(swig-version) elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff elif [ $$n = wcslib ]; then w=ftp://ftp.atnf.csiro.au/pub/software/wcslib @@ -502,28 +504,28 @@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ # Astrometry-net contains a lot of programs. We need to specify the # installation directory and the Python executable (by default it will look # for /usr/bin/python) -$(ibidir)/astrometry-net: $(tdir)/astrometry.net-$(astrometry-version).tar.gz \ - $(ilidir)/cairo \ - $(ilidir)/cfitsio \ - $(ilidir)/gsl \ - $(ilidir)/libjpeg \ - $(ilidir)/libpng \ - $(ibidir)/netpbm \ - $(ipydir)/numpy \ - $(ibidir)/python \ - $(ibidir)/swig \ - $(ilidir)/wcslib +$(ibidir)/astrometrynet: $(tdir)/astrometry.net-$(astrometrynet-version).tar.gz \ + $(ilidir)/cairo \ + $(ilidir)/cfitsio \ + $(ilidir)/gsl \ + $(ilidir)/libjpeg \ + $(ilidir)/libpng \ + $(ibidir)/netpbm \ + $(ipydir)/numpy \ + $(ibidir)/python \ + $(ibidir)/swig \ + $(ilidir)/wcslib cd $(ddir) \ && if ! tar xf $<; then echo; echo "Tar error"; exit 1; fi \ - && cd astrometry.net-$(astrometry-version) \ + && cd astrometry.net-$(astrometrynet-version) \ && make \ && make py \ && make extra \ && make install INSTALL_DIR=$(idir) PYTHON_SCRIPT="$(ibdir)/python" \ && cd .. \ - && rm -rf astrometry.net-$(astrometry-version) \ - && cp $(dtexdir)/astrometry-net.tex $(ictdir)/ \ - && echo "Astrometry-net $(astrometry-version)" > $@ + && rm -rf astrometry.net-$(astrometrynet-version) \ + && cp $(dtexdir)/astrometrynet.tex $(ictdir)/ \ + && echo "Astrometry.net $(astrometrynet-version) \citep{astrometrynet}" > $@ @@ -710,6 +712,13 @@ $(ibidir)/netpbm: $(tdir)/netpbm-$(netpbm-version).tgz \ && rm -rf $$unpackdir \ && echo "Netpbm $(netpbm-version)" > $@ +$(ibidir)/swarp: $(tdir)/swarp-$(swarp-version).tar.gz \ + $(ilidir)/fftw + $(call gbuild, $<, swarp-$(swarp-version), static, \ + --enable-threads) \ + && cp $(dtexdir)/swarp.tex $(ictdir)/ \ + && echo "SWarp $(swarp-version) \citep{swarp}" > $@ + $(ibidir)/swig: $(tdir)/swig-$(swig-version).tar.gz # Option --without-pcre was a suggestion once the configure step was # tried and it failed. It was not recommended but it works! -- cgit v1.2.1 From e9160416416090a67b99f31c2c43f33f8afd6e23 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 17 Apr 2019 13:21:27 +0100 Subject: Corrected bibtex entry for Astrometry-net and Swarp Until now, there were erros in the citation of Astrometry-net and Scamp papers. With this commit, we fix these problems. The Swarp bibtex has also been modify to follow the stetic of the citation style we have right now in the project. We also added the `dependency-bib.tex' as a prerequisite of `paper.bbl'. --- .file-metadata | Bin 6194 -> 6194 bytes configure | 2 +- reproduce/analysis/make/paper.mk | 2 +- reproduce/software/bibtex/astrometrynet.tex | 2 +- reproduce/software/bibtex/swarp.tex | 19 +++++++++---------- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.file-metadata b/.file-metadata index 95076a8..1c09605 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/configure b/configure index 2fa80ea..55ce4dc 100755 --- a/configure +++ b/configure @@ -1106,7 +1106,7 @@ hasentry=0 bibfiles="$ictdir/*" for f in $bibfiles; do if [ -f $f ]; then hasentry=1; break; fi; done; -# Make sure we start with an empty file. +# Make sure we start with an empty output file. pkgbib=$mtexdir/dependencies-bib.tex echo "" > $pkgbib diff --git a/reproduce/analysis/make/paper.mk b/reproduce/analysis/make/paper.mk index 5378ee3..549b7c0 100644 --- a/reproduce/analysis/make/paper.mk +++ b/reproduce/analysis/make/paper.mk @@ -90,7 +90,7 @@ $(mtexdir)/project.tex: $(foreach s, $(subst paper,,$(makesrc)), $(mtexdir)/$(s) # recipe and the `paper.pdf' recipe. But if `tex/src/references.tex' hasn't # been modified, we don't want to re-build the bibliography, only the final # PDF. -$(texbdir)/paper.bbl: tex/src/references.tex \ +$(texbdir)/paper.bbl: tex/src/references.tex $(mtexdir)/dependencies-bib.tex \ | $(tikzdir) $(texbdir) $(mtexdir)/project.tex # If `$(mtexdir)/project.tex' is empty, don't build PDF. @macros=$$(cat $(mtexdir)/project.tex) diff --git a/reproduce/software/bibtex/astrometrynet.tex b/reproduce/software/bibtex/astrometrynet.tex index 7bfe4c6..8bbaceb 100644 --- a/reproduce/software/bibtex/astrometrynet.tex +++ b/reproduce/software/bibtex/astrometrynet.tex @@ -16,5 +16,5 @@ pages = {1782}, author = {Dustin Lang and David W. Hogg and Keir Mierle and Michael Blanton and Sam Roweis}, title = {{ASTROMETRY}.{NET}: {BLIND} {ASTROMETRIC} {CALIBRATION} {OF} {ARBITRARY} {ASTRONOMICAL} {IMAGES}}, - journal = {The Astronomical Journal} + journal = {AJ} } diff --git a/reproduce/software/bibtex/swarp.tex b/reproduce/software/bibtex/swarp.tex index 03111d6..3ff9b91 100644 --- a/reproduce/software/bibtex/swarp.tex +++ b/reproduce/software/bibtex/swarp.tex @@ -5,15 +5,14 @@ %% notice and this notice are preserved. This file is offered as-is, %% without any warranty. -@MISC{swarp, - author = {{Bertin}, E.}, - title = "{SWarp: Resampling and Co-adding FITS Images Together}", - keywords = {Software }, -howpublished = {Astrophysics Source Code Library}, - year = 2010, - month = oct, -archivePrefix = "ascl", - eprint = {1010.068}, - adsurl = {http://cdsads.u-strasbg.fr/abs/2010ascl.soft10068B}, +@ARTICLE{swarp, + author = {{Bertin}, E. and {Mellier}, Y. and {Radovich}, M. and {Missonnier}, G. and + {Didelon}, P. and {Morin}, B.}, + journal = {ASPConf}, + title = "{The TERAPIX Pipeline}", + year = {2002}, + volume = {281}, + pages = {228}, + adsurl = {http://adsabs.harvard.edu/abs/2002ASPC..281..228B}, adsnote = {Provided by the SAO/NASA Astrophysics Data System} } -- cgit v1.2.1 From 3c7d59a46ca2940435258cb2bc1050aea5fd0b87 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Wed, 17 Apr 2019 14:32:40 +0100 Subject: SExtractor is now included into the project With this commit, we add SExtractor to be installed into the project. It is a widely used program to detect object and build catalogues from astronomical images. We had some problems when installing it because it could not link with some ATLAS libraries. But, since we have OpenBLAS installed, we can use it to override the problem with ATLAS. --- .file-metadata | Bin 6194 -> 6282 bytes reproduce/software/bibtex/sextractor.tex | 18 ++++++++++++++++++ reproduce/software/config/installation/versions.mk | 1 + reproduce/software/make/high-level.mk | 19 ++++++++++++++++++- 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 reproduce/software/bibtex/sextractor.tex diff --git a/.file-metadata b/.file-metadata index 1c09605..bbda38e 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/software/bibtex/sextractor.tex b/reproduce/software/bibtex/sextractor.tex new file mode 100644 index 0000000..90d3eff --- /dev/null +++ b/reproduce/software/bibtex/sextractor.tex @@ -0,0 +1,18 @@ +%% Copyright (C) 2019 Raul Infante-Sainz +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@article{sextractor, + author = {{Bertin, E.} and {Arnouts, S.}}, + title = {SExtractor: Software for source extraction}, + DOI= "10.1051/aas:1996164", + url= "https://doi.org/10.1051/aas:1996164", + journal = {AASS}, + year = 1996, + volume = 117, + number = 2, + pages = "393", +} diff --git a/reproduce/software/config/installation/versions.mk b/reproduce/software/config/installation/versions.mk index 270eb97..b7de983 100644 --- a/reproduce/software/config/installation/versions.mk +++ b/reproduce/software/config/installation/versions.mk @@ -66,6 +66,7 @@ pkgconfig-version = 0.29.2 python-version = 3.7.3 readline-version = 8.0 sed-version = 4.7 +sextractor-version = 2.25.0 swarp-version = 2.38.0 swig-version = 3.0.12 tar-version = 1.32 diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 144057e..59287c6 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -51,7 +51,7 @@ ipydir = $(BDIR)/software/installed/version-info/python # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. top-level-libraries = # atlas -top-level-programs = astrometrynet gnuastro metastore swarp +top-level-programs = astrometrynet gnuastro metastore sextractor swarp top-level-python = astroquery matplotlib all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ @@ -139,6 +139,7 @@ tarballs = $(foreach t, astrometry.net-$(astrometry-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ openblas-$(openblas-version).tar.gz \ pixman-$(pixman-version).tar.gz \ + sextractor-$(sextractor-version).tar.lz \ swarp-$(swarp-version).tar.gz \ swig-$(swig-version).tar.gz \ tiff-$(libtiff-version).tar.gz \ @@ -207,6 +208,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = pixman ]; then w=https://www.cairographics.org/releases + elif [ $$n = sextractor ]; then w=http://akhlaghi.org/src elif [ $$n = swarp ]; then w=https://www.astromatic.net/download/swarp elif [ $$n = swig ]; then w=https://sourceforge.net/projects/swig/files/swig/swig-$(swig-version) elif [ $$n = tiff ]; then w=https://download.osgeo.org/libtiff @@ -712,6 +714,21 @@ $(ibidir)/netpbm: $(tdir)/netpbm-$(netpbm-version).tgz \ && rm -rf $$unpackdir \ && echo "Netpbm $(netpbm-version)" > $@ +# Sextractor crashes complaining about not linking with some ATLAS +# libraries. But we can override this issue since we have Openblas +# installed, it is just necessary to explicity tell sextractor to use it in +# the configuration step. +$(ibidir)/sextractor: $(tdir)/sextractor-$(sextractor-version).tar.lz \ + $(ilidir)/openblas \ + $(ilidir)/fftw + $(call gbuild, $<, sextractor-$(sextractor-version), static, \ + --enable-threads --enable-openblas \ + --with-openblas-libdir=$(ildir) \ + --with-openblas-incdir=$(idir)/include) \ + && ln -fs $(ibdir)/sex $(ibdir)/sextractor \ + && cp $(dtexdir)/sextractor.tex $(ictdir)/ \ + && echo "Sextractor $(sextractor-version) \citep{sextractor}" > $@ + $(ibidir)/swarp: $(tdir)/swarp-$(swarp-version).tar.gz \ $(ilidir)/fftw $(call gbuild, $<, swarp-$(swarp-version), static, \ -- cgit v1.2.1 From 1847aa309dc01bf2843430d98376cb015b4cf440 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Mon, 22 Apr 2019 10:57:29 +0100 Subject: SCAMP and cdsclient have been added into the project With this commit, we add SCAMP into the project. This program is part of the Astromatic software and the main purpose is to make the fine astrometry and the distorsion correction astronomical images. In principle, SCAMP needs ATLAS library to be able to work (that is what the documentation says). However, we had some problems with ATLAS libraries on Mac OS system, and due to that, we set the option `--enable-openblas' in the configure step. By doing that, we are be able to install SCAMP with no problems. One dependency of SCAMP is `cdsclient', it has been also added with this commit. The `cdsclient' package is a set of C and shell routines which can be built on Unix stations or PCs running Linux, which once compiled allow to query some databases located at CDS or on mirrors over the network. --- .file-metadata | Bin 6282 -> 6365 bytes reproduce/software/bibtex/scamp.tex | 18 ++++++++ reproduce/software/config/installation/versions.mk | 2 + reproduce/software/make/high-level.mk | 49 ++++++++++++++++++--- 4 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 reproduce/software/bibtex/scamp.tex diff --git a/.file-metadata b/.file-metadata index bbda38e..35d6599 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/software/bibtex/scamp.tex b/reproduce/software/bibtex/scamp.tex new file mode 100644 index 0000000..8d825d7 --- /dev/null +++ b/reproduce/software/bibtex/scamp.tex @@ -0,0 +1,18 @@ +%% Copyright (C) 2019 Raul Infante-Sainz +%% +%% Copying and distribution of this file, with or without modification, +%% are permitted in any medium without royalty provided the copyright +%% notice and this notice are preserved. This file is offered as-is, +%% without any warranty. + +@ARTICLE{scamp, + author = {{Bertin}, E.}, + title = "{Automatic Astrometric and Photometric Calibration with SCAMP}", + year = 2006, + journal= {ASPConf}, + volume = 351, + month = jul, + pages = {112}, + adsurl = {http://cdsads.u-strasbg.fr/abs/2006ASPC..351..112B}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} diff --git a/reproduce/software/config/installation/versions.mk b/reproduce/software/config/installation/versions.mk index b7de983..5761165 100644 --- a/reproduce/software/config/installation/versions.mk +++ b/reproduce/software/config/installation/versions.mk @@ -22,6 +22,7 @@ atlas-version = 3.10.3 bash-version = 5.0 binutils-version = 2.31.1 cairo-version = 1.16.0 +cdsclient-version = 3.84 cfitsio-version = 3.45 cmake-version = 3.14.2 coreutils-version = 8.31 @@ -65,6 +66,7 @@ pixman-version = 0.38.0 pkgconfig-version = 0.29.2 python-version = 3.7.3 readline-version = 8.0 +scamp-version = 2.6.7 sed-version = 4.7 sextractor-version = 2.25.0 swarp-version = 2.38.0 diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 59287c6..55ae090 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -51,7 +51,7 @@ ipydir = $(BDIR)/software/installed/version-info/python # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. top-level-libraries = # atlas -top-level-programs = astrometrynet gnuastro metastore sextractor swarp +top-level-programs = astrometrynet gnuastro metastore scamp sextractor swarp top-level-python = astroquery matplotlib all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ @@ -114,10 +114,11 @@ include reproduce/software/make/python.mk # convention, but include the name/version in their tarball names with # another format, we'll do the modification before the download so the # downloaded file has our desired format. -tarballs = $(foreach t, astrometry.net-$(astrometry-version).tar.gz \ - cfitsio-$(cfitsio-version).tar.gz \ +tarballs = $(foreach t, astrometry.net-$(astrometrynet-version).tar.gz \ atlas-$(atlas-version).tar.bz2 \ cairo-$(cairo-version).tar.xz \ + cdsclient-$(cdsclient-version).tar.gz \ + cfitsio-$(cfitsio-version).tar.gz \ cmake-$(cmake-version).tar.gz \ curl-$(curl-version).tar.gz \ freetype-$(freetype-version).tar.gz \ @@ -139,6 +140,7 @@ tarballs = $(foreach t, astrometry.net-$(astrometry-version).tar.gz \ openmpi-$(openmpi-version).tar.gz \ openblas-$(openblas-version).tar.gz \ pixman-$(pixman-version).tar.gz \ + scamp-$(scamp-version).tar.lz \ sextractor-$(sextractor-version).tar.lz \ swarp-$(swarp-version).tar.gz \ swig-$(swig-version).tar.gz \ @@ -170,6 +172,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) mergenames=0 w=https://sourceforge.net/projects/math-atlas/files/Stable/$(atlas-version)/atlas$(atlas-version).tar.bz2/download elif [ $$n = cairo ]; then w=https://www.cairographics.org/releases + elif [ $$n = cdsclient ]; then w=http://cdsarc.u-strasbg.fr/ftp/pub/sw elif [ $$n = cmake ]; then mergenames=0 majv=$$(echo $(cmake-version) \ @@ -208,6 +211,7 @@ $(tarballs): $(tdir)/%: | $(lockdir) majorver=$$(echo $(openmpi-version) | sed -e 's/\./ /g' | awk '{printf("%d.%d", $$1, $$2)}') w=https://download.open-mpi.org/release/open-mpi/v$$majorver/$* elif [ $$n = pixman ]; then w=https://www.cairographics.org/releases + elif [ $$n = scamp ]; then w=http://akhlaghi.org/src elif [ $$n = sextractor ]; then w=http://akhlaghi.org/src elif [ $$n = swarp ]; then w=https://www.astromatic.net/download/swarp elif [ $$n = swig ]; then w=https://sourceforge.net/projects/swig/files/swig/swig-$(swig-version) @@ -529,8 +533,24 @@ $(ibidir)/astrometrynet: $(tdir)/astrometry.net-$(astrometrynet-version).tar.gz && cp $(dtexdir)/astrometrynet.tex $(ictdir)/ \ && echo "Astrometry.net $(astrometrynet-version) \citep{astrometrynet}" > $@ - - +# cdsclient is a set of software written in c to interact with astronomical +# database servers. It is a dependency of `scamp' to be able to download +# reference catalogues. +# NOTE: we do not use a convencional `gbuild' installation because the +# programs are scripts and we need to touch them before installing. +# Otherwise this software will be re-built each time the configure step is +# invoked. +$(ibidir)/cdsclient: $(tdir)/cdsclient-$(cdsclient-version).tar.gz + cd $(ddir) \ + && tar xf $< \ + && cd cdsclient-$(cdsclient-version) \ + && touch * \ + && ./configure --prefix=$(idir) \ + && make \ + && make install \ + && cd .. \ + && rm -rf cdsclient-$(cdsclient-version) \ + && echo "cdsclient $(cdsclient-version)" > $@ # CMake can be built with its custom `./bootstrap' script. $(ibidir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ @@ -714,6 +734,25 @@ $(ibidir)/netpbm: $(tdir)/netpbm-$(netpbm-version).tgz \ && rm -rf $$unpackdir \ && echo "Netpbm $(netpbm-version)" > $@ +# SCAMP documentation says ATLAS is a mandatory prerequisite for using +# SCAMP. We have ATLAS into the project but there are some problems with the +# libraries that are not yet solved. However, we tried to install it with +# the option --enable-openblas and it worked (same issue happened with +# `sextractor'. +$(ibidir)/scamp: $(tdir)/scamp-$(scamp-version).tar.lz \ + $(ilidir)/atlas \ + $(ilidir)/fftw \ + $(ilidir)/openblas \ + $(ibidir)/cdsclient + $(call gbuild, $<, scamp-$(scamp-version), static, \ + --enable-threads --enable-openblas \ + --with-fftw-libdir=$(idir) \ + --with-fftw-incdir=$(idir)/include \ + --with-openblas-libdir=$(ildir) \ + --with-openblas-incdir=$(idir)/include) \ + && cp $(dtexdir)/scamp.tex $(ictdir)/ \ + && echo "SCAMP $(scamp-version) \citep{scamp}" > $@ + # Sextractor crashes complaining about not linking with some ATLAS # libraries. But we can override this issue since we have Openblas # installed, it is just necessary to explicity tell sextractor to use it in -- cgit v1.2.1 From bc1011005dc58787fc26f9f091a9675fe88baeb7 Mon Sep 17 00:00:00 2001 From: Raul Infante-Sainz Date: Mon, 22 Apr 2019 14:12:00 +0100 Subject: Removed ATLAS as a prerequisite of Scamp Until this commit, ATLAS was a prerequisite of Scamp. The documentation says that. However, we have been able to install Scamp without having ATLAS installed. As a consecuence, ATLAS does not have to be a prerequisite of Scamp anymore. With this commit, we remove ATLAS as prerequisite of Scamp. We also put available software in the template as optional because not all people are going to use these software. --- .file-metadata | Bin 6365 -> 6365 bytes reproduce/software/make/high-level.mk | 16 ++++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.file-metadata b/.file-metadata index 35d6599..6deab7a 100644 Binary files a/.file-metadata and b/.file-metadata differ diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 55ae090..cff2ff8 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -50,9 +50,18 @@ ipydir = $(BDIR)/software/installed/version-info/python # high level software depend on it. The current rule for ATLAS is tested # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. -top-level-libraries = # atlas -top-level-programs = astrometrynet gnuastro metastore scamp sextractor swarp -top-level-python = astroquery matplotlib + +# About available software/libraries: currently the template has rules for +# installing software that are widely used in science, and in particular in +# astrophysics. However, not all of these software will be used for all +# people interested in this template. Due to that, we put some of what we +# consider the main software as optional software of the template (to see a +# complete list of all software/libraries, look at the version number +# Makefile). If that software is needed, just remove the comment `#' to +# install it. +top-level-libraries = # atlas +top-level-programs = gnuastro metastore # astrometrynet scamp sextractor swarp +top-level-python = numpy # astropy astroquery matplotlib scipy all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ $(foreach p, $(top-level-python), $(ipydir)/$(p)) \ @@ -740,7 +749,6 @@ $(ibidir)/netpbm: $(tdir)/netpbm-$(netpbm-version).tgz \ # the option --enable-openblas and it worked (same issue happened with # `sextractor'. $(ibidir)/scamp: $(tdir)/scamp-$(scamp-version).tar.lz \ - $(ilidir)/atlas \ $(ilidir)/fftw \ $(ilidir)/openblas \ $(ibidir)/cdsclient -- cgit v1.2.1