diff options
Diffstat (limited to 'reproduce/software/make')
| -rw-r--r-- | reproduce/software/make/README.md | 2 | ||||
| -rw-r--r-- | reproduce/software/make/atlas-multiple.mk | 2 | ||||
| -rw-r--r-- | reproduce/software/make/atlas-single.mk | 2 | ||||
| -rw-r--r-- | reproduce/software/make/basic.mk | 344 | ||||
| -rw-r--r-- | reproduce/software/make/build-rules.mk | 37 | ||||
| -rw-r--r-- | reproduce/software/make/high-level.mk | 479 | ||||
| -rw-r--r-- | reproduce/software/make/python.mk | 624 | ||||
| -rw-r--r-- | reproduce/software/make/r-cran.mk | 5 | ||||
| -rw-r--r-- | reproduce/software/make/xorg.mk | 6 |
9 files changed, 1141 insertions, 360 deletions
diff --git a/reproduce/software/make/README.md b/reproduce/software/make/README.md index 76ab5c1..00afef4 100644 --- a/reproduce/software/make/README.md +++ b/reproduce/software/make/README.md @@ -1,7 +1,7 @@ Software building instructions ------------------------------ -Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org>\ +Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org>\ See the end of the file for license conditions. This directory contains Makefiles that are called by the high-level diff --git a/reproduce/software/make/atlas-multiple.mk b/reproduce/software/make/atlas-multiple.mk index b92fbfc..6e7d415 100644 --- a/reproduce/software/make/atlas-multiple.mk +++ b/reproduce/software/make/atlas-multiple.mk @@ -8,7 +8,7 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/make/atlas-single.mk b/reproduce/software/make/atlas-single.mk index 720bad6..2b68677 100644 --- a/reproduce/software/make/atlas-single.mk +++ b/reproduce/software/make/atlas-single.mk @@ -8,7 +8,7 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index d90d945..4b18c29 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -21,9 +21,9 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2019-2022 Raul Infante-Sainz <infantesainz@gmail.com> -# Copyright (C) 2022 Pedram Ashofteh Ardakani <pedramardakani@pm.me> +# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2022-2025 Pedram Ashofteh Ardakani <pedramardakani@pm.me> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,7 +39,7 @@ # along with this Makefile. If not, see <http://www.gnu.org/licenses/>. # Top level environment -include reproduce/software/config/LOCAL.conf +include .build/software/config/LOCAL.conf include reproduce/software/make/build-rules.mk include reproduce/software/config/versions.conf include reproduce/software/config/checksums.conf @@ -49,22 +49,27 @@ include reproduce/software/config/checksums.conf include reproduce/software/config/urls.conf # Basic directories -lockdir = $(BDIR)/software/locks -tdir = $(BDIR)/software/tarballs -ddir = $(BDIR)/software/build-tmp -idir = $(BDIR)/software/installed -ibdir = $(BDIR)/software/installed/bin -ildir = $(BDIR)/software/installed/lib -ibidir = $(BDIR)/software/installed/version-info/proglib +lockdir = $(BDIR)/software/locks +tdir = $(BDIR)/software/tarballs +ddir = $(BDIR)/software/build-tmp +idir = $(BDIR)/software/installed +ibdir = $(BDIR)/software/installed/bin +ildir = $(BDIR)/software/installed/lib +iidir = $(BDIR)/software/installed/include +shsrcdir = "$(shell pwd)"/reproduce/software/shell +ibidir = $(BDIR)/software/installed/version-info/proglib # Ultimate Makefile target. GNU Nano (a simple and very light-weight text # editor) is installed by default, it is recommended to have it in the # 'basic.mk', so Maneaged projects can be edited on any system (even when # there is no command-line text editor available). +# +# The recipe is '@echo > /dev/null' so Make does not print "make: Nothing +# to be done for 'all'." targets-proglib = low-level-links \ gcc-$(gcc-version) \ nano-$(nano-version) -all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) +all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)); @echo > /dev/null # Define the shell environment # ---------------------------- @@ -87,13 +92,18 @@ export SHELL := $(ibdir)/dash export PATH := $(ibdir):$(PATH) export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) \ - -Wno-nullability-completeness export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) # Disable built-in rules (which are not needed here!) .SUFFIXES: +# See description of '-Wno-nullability-completeness' in +# 'reproduce/software/shell/configure.sh'. +ifeq ($(on_mac_os),yes) + noccwarnings=-Wno-nullability-completeness +endif +export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) $(noccwarnings) + # This is the "basic" tools where we are relying on the host operating # system, but are slowly populating our basic software envirnoment. To run # (system or template) programs, 'LD_LIBRARY_PATH' is necessary, so here, @@ -101,6 +111,15 @@ export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) # 'LD_LIBRARY_PATH', then we'll add our own newly installed libraries. We # will also make sure that there is no "current directory" in it (by # removing a starting or trailing ':' and any occurance of '::'. +# +# But first: in case LD_LIBRARY_PATH is empty, give it the default value of +# $(sys_library_sh_path) (which was the location of the libraries needed by +# the host's shell). This is because after we add the Maneage's library +# path, on some systems, no other libraries will be checked except those +# that are in 'LD_LIBRARY_PATH'. +ifeq ($(strip $(LD_LIBRARY_PATH)),) +export LD_LIBRARY_PATH=$(sys_library_sh_path) +endif export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH):$(ildir) \ | sed -e's/::/:/g' -e's/^://' -e's/:$$//') @@ -121,17 +140,24 @@ export DYLD_LIBRARY_PATH := # Afer putting everything together, we use the first server as the # reference for all software if their '-url' variable isn't defined (in # 'reproduce/software/config/urls.conf'). -downloadwrapper = ./reproduce/analysis/bash/download-multi-try +downloadwrapper = ./reproduce/analysis/bash/download-multi-try.sh maneage_backup_urls := $(shell awk '!/^#/{printf "%s ", $$1}' \ reproduce/software/config/servers-backup.conf) backupservers_all = $(user_backup_urls) $(maneage_backup_urls) topbackupserver = $(word 1, $(backupservers_all)) backupservers = $(filter-out $(topbackupserver),$(backupservers_all)) - - - - +# When building in Apptainer containers, as of 2025-04-18, we need to +# configure Maneage as root (within the container). In such cases, we need +# to activate the 'FORCE_UNSAFE_CONFIGURE' environment variable to build +# some of the software. The 'if' statement is here to make sure we are in +# Apptainer: in other situations, the "unsafe" configure script shouldn't +# be activated. Note that this doesn't happen in Docker (where the Maneage +# source is in the same directory) because we build a non-root ('maneager' +# user there who executes the configure command. +unsafe-config = if [ $$(pwd) = "/home/maneager/source" ] \ + && [ $$(whoami) = root ]; then \ + export FORCE_UNSAFE_CONFIGURE=1; fi @@ -202,6 +228,7 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ $(call makelink,xcrun) $(call makelink,sysctl) $(call makelink,sw_vers) + $(call makelink,codesign) $(call makelink,dsymutil) $(call makelink,install_name_tool) @@ -222,6 +249,10 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ fi done +# Useful tools: 'ldd' (list libraries linked by binary on GNU +# systems) + $(call makelink,ldd) + # We want this to be empty (so it doesn't interefere with the other # files in 'ibidir'. touch $@ @@ -240,11 +271,8 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ # # The first set of programs to be built are those that we need to unpack # the source code tarballs of each program. We have already installed Lzip -# before calling 'basic.mk', so it is present and working. Hence we first -# build the Lzipped tarball of Gzip, then use our own Gzip to unpack the -# tarballs of the other compression programs. Once all the compression -# programs/libraries are complete, we build our own GNU Tar and continue -# with other software. +# before calling 'basic.mk', so it is present and working. So the only +# prerequisites of these (until reaching Tar) is the necessary directories. $(lockdir): | $(BDIR); mkdir $@ $(ibidir)/gzip-$(gzip-version): | $(ibdir) $(ildir) $(lockdir) tarball=gzip-$(gzip-version).tar.lz @@ -252,13 +280,13 @@ $(ibidir)/gzip-$(gzip-version): | $(ibdir) $(ildir) $(lockdir) $(call gbuild, gzip-$(gzip-version), static, , V=1) echo "GNU Gzip $(gzip-version)" > $@ -$(ibidir)/xz-$(xz-version): $(ibidir)/gzip-$(gzip-version) +$(ibidir)/xz-$(xz-version): | $(ibdir) $(ildir) $(lockdir) tarball=xz-$(xz-version).tar.lz $(call import-source, $(xz-url), $(xz-checksum)) $(call gbuild, xz-$(xz-version), static) echo "XZ Utils $(xz-version)" > $@ -$(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) +$(ibidir)/bzip2-$(bzip2-version): | $(ibdir) $(ildir) $(lockdir) # Download the tarball. tarball=bzip2-$(bzip2-version).tar.lz @@ -287,8 +315,9 @@ $(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) fi cd $(ddir) rm -rf $$tdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$tdir + $(shsrcdir)/prep-source.sh $(ibdir) sed -e 's@\(ln -s -f \)$$(PREFIX)/bin/@\1@' Makefile \ > Makefile.sed mv Makefile.sed Makefile @@ -303,32 +332,12 @@ $(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) ln -fs libbz2.so.$(bzip2-version) libbz2.so echo "Bzip2 $(bzip2-version)" > $@ -$(ibidir)/unzip-$(unzip-version): $(ibidir)/gzip-$(gzip-version) - tarball=unzip-$(unzip-version).tar.lz - $(call import-source, $(unzip-url), $(unzip-checksum)) - $(call gbuild, unzip-$(unzip-version), static,, \ - -f unix/Makefile generic \ - CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ - -f unix/Makefile generic \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) - echo "Unzip $(unzip-version)" > $@ - -$(ibidir)/zip-$(zip-version): $(ibidir)/gzip-$(gzip-version) - tarball=zip-$(zip-version).tar.lz - $(call import-source, $(zip-url), $(zip-checksum)) - $(call gbuild, zip-$(zip-version), static,, \ - -f unix/Makefile generic \ - CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ - -f unix/Makefile generic \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) - echo "Zip $(zip-version)" > $@ - # Some programs (like Wget and CMake) that use zlib need it to be dynamic # so they use our custom build. So we won't force a static-only build. # # Note for a static-only build: Zlib's './configure' doesn't use Autoconf's # configure script, it just accepts a direct '--static' option. -$(ibidir)/zlib-$(zlib-version): $(ibidir)/gzip-$(gzip-version) +$(ibidir)/zlib-$(zlib-version): | $(ibdir) $(ildir) $(lockdir) tarball=zlib-$(zlib-version).tar.lz $(call import-source, $(zlib-url), $(zlib-checksum)) $(call gbuild, zlib-$(zlib-version)) @@ -341,16 +350,15 @@ $(ibidir)/zlib-$(zlib-version): $(ibidir)/gzip-$(gzip-version) # software to be built). $(ibidir)/tar-$(tar-version): \ $(ibidir)/xz-$(xz-version) \ - $(ibidir)/zip-$(zip-version) \ $(ibidir)/gzip-$(gzip-version) \ $(ibidir)/zlib-$(zlib-version) \ - $(ibidir)/bzip2-$(bzip2-version) \ - $(ibidir)/unzip-$(unzip-version) + $(ibidir)/bzip2-$(bzip2-version) # Since all later programs depend on Tar, the configuration will hit # a bottleneck here: only making Tar. So its more efficient to built # it on multiple threads (even when the user's Make doesn't pass down # the number of threads). + $(call unsafe-config) tarball=tar-$(tar-version).tar.lz $(call import-source, $(tar-url), $(tar-checksum)) $(call gbuild, tar-$(tar-version), , , -j$(numthreads) V=1) @@ -418,6 +426,7 @@ $(ibidir)/ncurses-$(ncurses-version): $(ibidir)/patchelf-$(patchelf-version) rm -f $(ibdir)/bash* $(ibdir)/awk* $(ibdir)/gawk* # Standard build process. + export CFLAGS="-std=gnu17 $$CFLAGS" $(call gbuild, ncurses-$(ncurses-version), static, \ --with-shared --enable-rpath --without-normal \ --without-debug --with-cxx-binding \ @@ -461,6 +470,9 @@ $(ibidir)/ncurses-$(ncurses-version): $(ibidir)/patchelf-$(patchelf-version) # # 5. A link is made to also be able to include files from the # 'ncurses' headers. +# +# 6. Top-level symbolic links are made for the 'include' (.h) files. +# if [ x$(on_mac_os) = xyes ]; then so="dylib"; else so="so"; fi if [ -f $(ildir)/libncursesw.$$so ]; then @@ -485,6 +497,11 @@ $(ibidir)/ncurses-$(ncurses-version): $(ibidir)/patchelf-$(patchelf-version) ln -fs $(ildir)/pkgconfig/ncursesw.pc pkgconfig/cursesw.pc ln -fs $(idir)/include/ncursesw $(idir)/include/ncurses + +# Add symbolic links for the ncursesw/*.h 'include' files +# so that they can be found in the top-level include/ directory. + cd "$(iidir)" + ln -fsv $(idir)/include/ncursesw/*.h . echo "GNU NCURSES $(ncurses-version)" > $@ else exit 1 @@ -548,7 +565,7 @@ $(ibidir)/bash-$(bash-version): \ if [ "x$(static_build)" = xyes ]; then stopt="--enable-static-link" else stopt="" fi; - export CFLAGS="$$CFLAGS \ + export CFLAGS="$$CFLAGS -std=gnu17 \ -DDEFAULT_PATH_VALUE='\"$(ibdir)\"' \ -DSTANDARD_UTILS_PATH='\"$(ibdir)\"' \ -DSYS_BASHRC='\"$(BASH_ENV)\"' " @@ -607,8 +624,9 @@ $(ibidir)/perl-$(perl-version): $(ibidir)/patchelf-$(patchelf-version) | awk '{printf("%d.%d", $$1, $$2)}') cd $(ddir) rm -rf perl-$(perl-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd perl-$(perl-version) + $(shsrcdir)/prep-source.sh $(ibdir) ./Configure -des \ -Dusethreads \ -Duseshrplib \ @@ -629,7 +647,7 @@ $(ibidir)/perl-$(perl-version): $(ibidir)/patchelf-$(patchelf-version) -Dcccdlflags='-fPIC' \ $(perl-conflddlflags) \ -Dldflags="$$LDFLAGS" - make -j$(numthreads) + make -j$(numthreads) V=1 make install cd .. rm -rf perl-$(perl-version) @@ -667,20 +685,16 @@ $(ibidir)/coreutils-$(coreutils-version): \ $(ibidir)/openssl-$(openssl-version) # Import, unpack and enter the source directory. + $(call unsafe-config) tarball=coreutils-$(coreutils-version).tar.lz $(call import-source, $(coreutils-url), $(coreutils-checksum)) + +# Unpack and enter the source. cd $(ddir) rm -rf coreutils-$(coreutils-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd coreutils-$(coreutils-version) - -# Set the configure script to use our shell, note that we can't -# assume GNU SED here yet (it installs after Coreutils). - sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ - configure > configure-tmp - mv configure-tmp configure - chmod +x configure + $(shsrcdir)/prep-source.sh $(ibdir) # Configure, build and install Coreutils. ./configure --prefix=$(idir) SHELL=$(ibdir)/bash \ @@ -692,7 +706,7 @@ $(ibidir)/coreutils-$(coreutils-version): \ # Fix RPATH if necessary. if [ -f $(ibdir)/patchelf ]; then make SHELL=$(ibdir)/bash install DESTDIR=junkinst - unalias ls || true # avoid decorated 'ls' commands with extra characters + unalias ls || true # Not decorated 'ls' (with extra characters). instprogs=$$(ls junkinst/$(ibdir)) for f in $$instprogs; do $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/$$f @@ -717,8 +731,9 @@ $(ibidir)/podlators-$(podlators-version): $(ibidir)/perl-$(perl-version) $(call import-source, $(podlators-url), $(podlators-checksum)) cd $(ddir) rm -rf podlators-$(podlators-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd podlators-$(podlators-version) + $(shsrcdir)/prep-source.sh $(ibdir) perl Makefile.PL make make install @@ -741,7 +756,7 @@ $(ibidir)/openssl-$(openssl-version): $(ibidir)/podlators-$(podlators-version) \ # First download the certificates and copy them into the # installation directory. tarball=cert.pem-$(certpem-version) - $(call import-source, $(cert-url), $(cert-checksum)) + $(call import-source, $(certpem-url), $(certpem-checksum)) cp $(tdir)/cert.pem-$(certpem-version) $(idir)/etc/ssl/cert.pem # Now download the OpenSSL tarball. @@ -807,6 +822,8 @@ $(ibidir)/curl-$(curl-version): $(ibidir)/coreutils-$(coreutils-version) --without-librtmp \ --without-libidn2 \ --without-wolfssl \ + --without-nghttp2 \ + --without-nghttp3 \ --without-brotli \ --without-gnutls \ --without-cyassl \ @@ -814,6 +831,7 @@ $(ibidir)/curl-$(curl-version): $(ibidir)/coreutils-$(coreutils-version) --without-axtls \ --disable-ldaps \ --disable-ldap \ + --without-zstd \ --without-nss, V=1) if [ -f $(ibdir)/patchelf ]; then @@ -875,6 +893,12 @@ $(ibidir)/wget-$(wget-version): \ # process of the higher-level programs and libraries. Note that during the # building of those higher-level programs (after this Makefile finishes), # there is no access to the system's PATH. +$(ibidir)/bison-$(bison-version): $(ibidir)/help2man-$(help2man-version) + tarball=bison-$(bison-version).tar.lz + $(call import-source, $(bison-url), $(bison-checksum)) + $(call gbuild, bison-$(bison-version), static, ,V=1 -j$(numthreads)) + echo "GNU Bison $(bison-version)" > $@ + $(ibidir)/diffutils-$(diffutils-version): \ $(ibidir)/coreutils-$(coreutils-version) tarball=diffutils-$(diffutils-version).tar.lz @@ -928,6 +952,13 @@ $(ibidir)/gawk-$(gawk-version): \ # Build final target. echo "GNU AWK $(gawk-version)" > $@ +$(ibidir)/help2man-$(help2man-version): \ + $(ibidir)/coreutils-$(coreutils-version) + tarball=help2man-$(help2man-version).tar.lz + $(call import-source, $(help2man-url), $(help2man-checksum)) + $(call gbuild, help2man-$(help2man-version), static, ,V=1) + echo "Help2man $(Help2man-version)" > $@ + $(ibidir)/libiconv-$(libiconv-version): \ $(ibidir)/pkg-config-$(pkgconfig-version) tarball=libiconv-$(libiconv-version).tar.lz @@ -987,9 +1018,10 @@ $(ibidir)/gmp-$(gmp-version): \ $(ibidir)/coreutils-$(coreutils-version) tarball=gmp-$(gmp-version).tar.lz $(call import-source, $(gmp-url), $(gmp-checksum)) + export CFLAGS="-std=gnu17 $$CFLAGS" $(call gbuild, gmp-$(gmp-version), static, \ --enable-cxx --enable-fat, \ - -j$(numthreads) ,make check) + -j$(numthreads)) echo "GNU Multiple Precision Arithmetic Library $(gmp-version)" > $@ # Less is useful with Git (to view the diffs within a minimal container) @@ -998,7 +1030,19 @@ $(ibidir)/gmp-$(gmp-version): \ $(ibidir)/less-$(less-version): $(ibidir)/ncurses-$(ncurses-version) tarball=less-$(less-version).tar.lz $(call import-source, $(less-url), $(less-checksum)) - $(call gbuild, less-$(less-version), static,,-j$(numthreads)) + +# Without the '--with-regex=posix' option, the build will depend on +# PCRE (perl compatible regular expressions) which are not available +# on some systems/compilers and can cause a crash. Maneage was +# successfully built with the POSIX regular expression (regex), and +# 'less' is generally, an interactive software, not a batch-mode +# software (it is just added in 'basic.mk' because Git uses it to +# display things. Again, this is an interactive meta-operation in +# maneage (operations you only do when you are developing Maneage +# within Maneage interactively, and will not affect into the actual +# reproducible analysis!) + $(call gbuild, less-$(less-version), static, \ + --with-regex=posix,-j$(numthreads)) if [ -f $(ibdir)/patchelf ]; then $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/less; fi @@ -1017,22 +1061,34 @@ $(ibidir)/libtool-$(libtool-version): $(ibidir)/m4-$(m4-version) $(ibidir)/grep-$(grep-version): $(ibidir)/coreutils-$(coreutils-version) tarball=grep-$(grep-version).tar.lz $(call import-source, $(grep-url), $(grep-checksum)) - $(call gbuild, grep-$(grep-version), static,,V=1) + $(call gbuild, grep-$(grep-version), static,, \ + -j$(numthreads) V=1) echo "GNU Grep $(grep-version)" > $@ # M4 doesn't depend on PatchELF, but just to be consistent with the # levels/phases introduced here (where the compressors are level 1, # PatchELF is level 2, and ...), we'll set it as a dependency. +# +# The '--with-syscmd-shell' is used as the default shell and if not given, +# 'm4' will use '/bin/sh' (which is not under Maneage control and can cause +# problems in 'high-level.mk' because it closes off the system's +# LD_LIBRARY_PATH and if the system's '/bin/sh' needs a special system +# library, the high-level programs will not be built). We are setting this +# default shell to Dash because M4 is built before our own Bash. Recall +# that Dash is built before we enter this Makefile. $(ibidir)/m4-$(m4-version): $(ibidir)/patchelf-$(patchelf-version) tarball=m4-$(m4-version).tar.lz $(call import-source, $(m4-url), $(m4-checksum)) - $(call gbuild, m4-$(m4-version), static,,V=1) + export CFLAGS="-std=gnu17 $$CFLAGS" + $(call gbuild, m4-$(m4-version), static, \ + --with-syscmd-shell=$(ibdir)/dash, \ + -j$(numthreads) V=1) echo "GNU M4 $(m4-version)" > $@ $(ibidir)/mpfr-$(mpfr-version): $(ibidir)/gmp-$(gmp-version) tarball=mpfr-$(mpfr-version).tar.lz $(call import-source, $(mpfr-url), $(mpfr-checksum)) - $(call gbuild, mpfr-$(mpfr-version), static, , , make check) + $(call gbuild, mpfr-$(mpfr-version), static) echo "GNU Multiple Precision Floating-Point Reliably $(mpfr-version)" > $@ $(ibidir)/pkg-config-$(pkgconfig-version): $(ibidir)/patchelf-$(patchelf-version) @@ -1056,6 +1112,7 @@ $(ibidir)/pkg-config-$(pkgconfig-version): $(ibidir)/patchelf-$(patchelf-version if [ x$(on_mac_os) = xyes ]; then export compiler="CC=clang" else export compiler="" fi + export CFLAGS="-std=gnu17 $$CFLAGS" $(call gbuild, pkg-config-$(pkgconfig-version), static, \ $$compiler --with-internal-glib \ --with-pc-path=$(ildir)/pkgconfig, V=1) @@ -1070,6 +1127,17 @@ $(ibidir)/sed-$(sed-version): $(ibidir)/coreutils-$(coreutils-version) $(ibidir)/texinfo-$(texinfo-version): \ $(ibidir)/perl-$(perl-version) \ $(ibidir)/gettext-$(gettext-version) + +# Setting for the XS sub-package. "This is because in theory, the XS +# module could be built with a different compiler to the rest of the +# project, needing completely different flags" (part of [1]) +# +# [1] https://lists.gnu.org/archive/html/bug-texinfo/2022-08/msg00068.html + export PERL="$(ibdir)/perl" + export PERL_EXT_LDFLAGS="-L$(ildir)" + export PERL_EXT_CPPFLAGS="-I$(iidir)" + +# Basic build commands. tarball=texinfo-$(texinfo-version).tar.lz $(call import-source, $(texinfo-url), $(texinfo-checksum)) $(call gbuild, texinfo-$(texinfo-version), static) @@ -1105,7 +1173,7 @@ $(ibidir)/mpc-$(mpc-version): $(ibidir)/mpfr-$(mpfr-version) echo "" > $@ else $(call gbuild, mpc-$(mpc-version), static, , \ - -j$(numthreads), make check) + -j$(numthreads)) echo "GNU Multiple Precision Complex library" > $@ fi @@ -1136,6 +1204,7 @@ $(ibidir)/binutils-$(binutils-version): \ $(ibidir)/gawk-$(gawk-version) \ $(ibidir)/grep-$(grep-version) \ $(ibidir)/wget-$(wget-version) \ + $(ibidir)/bison-$(bison-version) \ $(ibidir)/which-$(which-version) \ $(ibidir)/libtool-$(libtool-version) \ $(ibidir)/texinfo-$(texinfo-version) \ @@ -1163,8 +1232,9 @@ $(ibidir)/binutils-$(binutils-version): \ # Build binutils with the standard 'gbuild' function. $(call gbuild, binutils-$(binutils-version), static, \ - --with-lib-path=$(sys_library_path), \ - -j$(numthreads) ) + --with-lib-path=$(sys_library_path) \ + --enable-gprofng=no, \ + -j$(numthreads) V=1) # The 'ld' linker of Binutils needs several '*crt*.o' files from # the host's GNU C Library to run. On some systems these object @@ -1287,20 +1357,49 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) current_dir=$$(pwd) # By default 'ddir' (where GCC is decompressed and built) is in the -# RAM (on systems that support '/dev/shm'). This is done to avoid -# building so many small/temporary files and possibly harming the -# hard-drive or SSD. But if the RAM doesn't have enough space, we -# should use the hard-drive or SSD. During its build GCC's build -# directory will become about 7GB (multiple of 1024 bytes, for GCC -# 11.2.0). So at this step, we are making sure we have more than -# 7.5GiB (multiple of 1000 bytes, which corresponds to 7.32GB) -# before GCC starts to build. Note that the 4th column of 'df' is -# the "available" space at the time of running, not the full -# space. So the background RAM that the OS will be using during -# Maneage is accounted for. Also consider that GCC is built alone -# (no other Maneage software is built at the same time as GCC). - in_ram=$$(df $(ddir) \ - | awk 'NR==2{print ($$4>7500000) ? "yes" : "no"}'); \ +# RAM (on systems that support a '/dev/shm' RAM disk). This is done +# to avoid building so many small/temporary files and possibly +# harming the hard-drive or SSD. But if the RAM doesn't have enough +# space, we should use the hard-drive or SSD. During its build, +# GCC's build directory will become several gigabytes and the build +# also needs RAM. You can track the RAM usage of the system with a +# 1-second resolution (if no other RAM consuming program is running +# while building GCC) with the command below (example outputs can +# be seen in https://savannah.nongnu.org/task/index.php?16623). +# +# c=1; while true; do POSIXLY_CORRECT=1 df -P /dev/shm/maneage-* | awk 'NR==2{print '$c', $3}'; c=$((c+1)); sleep 1; done > mem-usage.txt +# asttable mem-usage.txt -c1,'arith $2 512 x 1024 / 1024 / 1024 /' -o mem.fits +# +# For POSIX portability and longevity (default sizes might change), +# we use the '-P' option, and we use the environment variable +# POSIXLY_CORRECT=1, so the 'block size' is 512 bytes. In this way, +# to get the actual GiB amount, multiply the value returned above +# by 512 (B/block), then divide by 1024^3 (B/GiB). +# +# To get the final value to use here, get the maximum used value +# after GCC is fully built and you have stopped the 'while true' +# command above. You can do this with the command below (assumes +# you have Gnuastro). +# +# aststatistics mem-usage.txt -c2 --maximum | asttable -c'arith $1 7000000 +' -Afixed -B0 +# +# The extra space is because we will assume an extra 3 GiB = 3GiB * +# 1024^3 (B/GiB) / 512 (B/block) = 6291456 blocks are necessary for +# the building (let's round it to 7000000!). +# +# Therefore, we need to make sure that the running system more than +# the necessary amount of space in the RAM. To do this, we use 'df' +# below. +# +# The 4th column of 'df' is the "available" space at the time of +# running, not the full space. So the 'RAM disk' that the OS +# will be using as "pretend" disk space (e.g. using 'tmpfs'; this +# is physically RAM, but appears as if it is disk space) +# during this stage of Maneage is accounted for. GCC is built +# alone - no other Maneage software is built at the same time as +# GCC - so this amount of RAM should be enough. + in_ram=$$(POSIXLY_CORRECT=1 df -P $(ddir) \ + | awk 'NR==2{print ($$4>26613216) ? "yes" : "no"}'); \ if [ $$in_ram = "yes" ]; then odir=$(ddir) else odir=$(BDIR)/software/build-tmp-gcc-due-to-lack-of-space @@ -1308,15 +1407,46 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) mkdir $$odir fi -# Go into the proper directory, unpack GCC and prepare the 'build' -# directory inside it for all the built files. +# Go into the directory to uncompress GCC. cd $$odir + +# Unpack GCC and prepare the 'build' directory inside it for all +# the built files. rm -rf gcc-$(gcc-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions if [ $$odir != $(ddir) ]; then ln -s $$odir/gcc-$(gcc-version) $(ddir)/gcc-$(gcc-version) fi cd gcc-$(gcc-version) + $(shsrcdir)/prep-source.sh $(ibdir) + +# Unfortunately binutils installs headers like 'ansidecl.h' that +# have been seen to conflict with GCC's internal versions of those +# headers. For example in the 'ansidecl.h' of Binutils 2.39, the +# 'PTR' macro isn't defined, while the same file in GCC 12.1.0 has +# defined it. Therefore, without this change, GCC will include the +# file installed from Binutils, not find what it needs and crash! +# Therefore, with the 'CPPFLAGS' modification below, we tell GCC to +# first look into its own 'include' directory before anything else. + export CPPFLAGS="-I$$(pwd)/include $(CPPFLAGS)" + +# In the GNU C Library 2.36 (which is more recent than GCC 12.1.0), +# the 'linux/mount.h' (loaded by 'linux/fs.h', which is loaded by +# 'libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cpp' +# in GCC) conflicts with 'sys/mount.h' which is directly loaded by +# the same file! This is a known conflict in glibc 2.36 (see +# [1]). As described in [1], one solution is the final job done in +# [2]. We therefore do this process here: 1) Not loading +# 'linux/fs.h', and adding the necessary macros directly. +# +# [1] https://sourceware.org/glibc/wiki/Release/2.36#Usage_of_.3Clinux.2Fmount.h.3E_and_.3Csys.2Fmount.h.3E +# [2] https://reviews.llvm.org/D129471 + sed -e's|\#include <linux/fs.h>||' \ + -e"s|FS_IOC_GETFLAGS;|_IOR('f', 1, long);|" \ + -e"s|FS_IOC_GETVERSION;|_IOR('v', 1, long);|" \ + -e"s|FS_IOC_SETFLAGS;|_IOW('f', 2, long);|" \ + -e"s|FS_IOC_SETVERSION;|_IOW('v', 2, long);|" \ + -i libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cpp # Set the build directory for the processing. mkdir build @@ -1396,12 +1526,39 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) +# Level 6: need re-compilation +# ---------------------------- +# +# The initial build of these was done with the host's settings, which will +# cause problems later when we completely close-off the host environment. +$(ibidir)/make-$(make-version): $(ibidir)/gcc-$(gcc-version) + tarball=make-$(make-version).tar.lz + $(call import-source, $(make-url), $(make-checksum)) + $(call gbuild, make-$(make-version), static, \ + --disable-dependency-tracking --without-guile) + echo "GNU Make $(make-version)" > $@ + +$(ibidir)/lzip-$(lzip-version): $(ibidir)/gcc-$(gcc-version) + tarball=lzip-$(lzip-version).tar + unpackdir=lzip-$(lzip-version) + cd $(ddir) + rm -rf $$unpackdir + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) + ./configure --build --check --installdir="$(ibdir)" + if [ -f $(ibdir)/patchelf ]; then + $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/lzip; + fi + cd .. + rm -rf $$unpackdir + echo "Lzip $(lzip-version)" > $@ -# Level 6: Basic text editor +# Level 7: Basic text editor # -------------------------- # # If the project is built in a minimal environment, there is no text @@ -1418,7 +1575,8 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) # nano (and use their own optional high-level text editor). To do this, you # can just have to manually remove 'nano' from 'targets-proglib' above and # add their optional text editor in 'TARGETS.conf'. -$(ibidir)/nano-$(nano-version): $(ibidir)/gcc-$(gcc-version) +$(ibidir)/nano-$(nano-version): $(ibidir)/lzip-$(lzip-version) \ + $(ibidir)/make-$(make-version) tarball=nano-$(nano-version).tar.lz $(call import-source, $(nano-url), $(nano-checksum)) $(call gbuild, nano-$(nano-version), static) diff --git a/reproduce/software/make/build-rules.mk b/reproduce/software/make/build-rules.mk index c25dfb1..463fbbf 100644 --- a/reproduce/software/make/build-rules.mk +++ b/reproduce/software/make/build-rules.mk @@ -3,7 +3,7 @@ # imported into 'basic.mk' and 'high-level.mk'. They should be activated # with Make's 'Call' function. # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -160,7 +160,7 @@ uncompress = csuffix=$$(echo $$utarball \ intarrm=0; \ intar=$$utarball; \ fi; \ - if tar -xf $$intar; then \ + if tar -xf $$intar --no-same-owner --no-same-permissions; then \ if [ x$$intarrm = x1 ]; then rm $$intar; fi; \ else \ echo; echo "Tar error"; exit 1; \ @@ -207,17 +207,10 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ else confscript="$(strip $(6))"; \ fi; \ \ - if [ -f $(ibdir)/bash ]; then \ - if [ -f "$$confscript" ]; then \ - sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ - $$confscript > $$confscript-tmp; \ - mv $$confscript-tmp $$confscript; \ - chmod +x $$confscript; \ - fi; \ + $(shsrcdir)/prep-source.sh $(ibdir); \ + if [ -f $(ibdir)/bash ]; then \ shellop="SHELL=$(ibdir)/bash"; \ - elif [ -f /bin/bash ]; then shellop="SHELL=/bin/bash"; \ - else shellop="SHELL=/bin/sh"; \ + else shellop="SHELL=$(ibdir)/dash"; \ fi; \ \ if [ x$$gbuild_prefix = x ]; then prefixdir="$(idir)"; \ @@ -255,10 +248,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ # CMake # ----- # -# According to the link below, in CMake '/bin/sh' is hardcoded, so there is -# no way to change it unfortunately! -# -# https://stackoverflow.com/questions/21167014/how-to-set-shell-variable-in-makefiles-generated-by-cmake +# Used by packages that are built with CMake. cbuild = if [ x$(static_build) = xyes ] && [ $(2)x = staticx ]; then \ export LDFLAGS="$$LDFLAGS -static"; \ opts="-DBUILD_SHARED_LIBS=OFF"; \ @@ -268,13 +258,18 @@ cbuild = if [ x$(static_build) = xyes ] && [ $(2)x = staticx ]; then \ utarball=$(tdir)/$$tarball; \ $(call uncompress); \ cd $(1); \ - rm -rf project-build; \ - mkdir project-build; \ - cd project-build; \ + $(shsrcdir)/prep-source.sh $(ibdir); \ + if [ -f $(ibdir)/bash ]; then \ + shellop="SHELL=$(ibdir)/bash"; \ + else shellop="SHELL=$(ibdir)/dash"; \ + fi; \ + rm -rf maneage-build; \ + mkdir maneage-build; \ + cd maneage-build; \ cmake .. -DCMAKE_LIBRARY_PATH=$(ildir) \ -DCMAKE_INSTALL_PREFIX=$(idir) \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON $$opts $(3); \ - make; \ - make install; \ + make $$shellop; \ + make $$shellop install; \ cd ../..; \ rm -rf $(1) diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 966ea9b..67ca8b6 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -12,8 +12,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2019-2022 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Raul Infante-Sainz <infantesainz@gmail.com> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,7 +29,7 @@ # along with this Makefile. If not, see <http://www.gnu.org/licenses/>. # Top level environment (same as 'basic.mk') -include reproduce/software/config/LOCAL.conf +include .build/software/config/LOCAL.conf include reproduce/software/make/build-rules.mk include reproduce/software/config/versions.conf include reproduce/software/config/checksums.conf @@ -49,6 +49,7 @@ ddir = $(BDIR)/software/build-tmp idir = $(BDIR)/software/installed ibdir = $(BDIR)/software/installed/bin ildir = $(BDIR)/software/installed/lib +iidir = $(BDIR)/software/installed/include ibidir = $(BDIR)/software/installed/version-info/proglib # Basic directories (specific to this Makefile). @@ -63,6 +64,12 @@ ipydir = $(BDIR)/software/installed/version-info/python ircrandir = $(BDIR)/software/installed/version-info/r-cran ilibrcrandir = $(BDIR)/software/installed/lib/R/library +# Special files. +makewshell = $(ibdir)/make-with-shell + + + + # Targets to build. ifeq ($(strip $(all_highlevel)),1) @@ -111,11 +118,17 @@ else endif -# Ultimate Makefile target. +# Disable the TeXLive target if `--offline` +ifneq ($(strip $(offline)),1) + target-texlive := $(itidir)/texlive +endif + +# Ultimate Makefile target. The recipe is '@echo > /dev/null' so Make does +# not print "make: Nothing to be done for 'all'." all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) \ $(foreach p, $(targets-python), $(ipydir)/$(p)) \ $(foreach p, $(targets-r-cran), $(ircrandir)/$(p)) \ - $(itidir)/texlive + $(target-texlive); @echo > /dev/null # Define the shell environment # ---------------------------- @@ -147,7 +160,6 @@ export SHELL := $(ibdir)/bash .SHELLFLAGS := --noprofile --norc -ec export LDFLAGS := $(rpath_command) -L$(ildir) export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export CPPFLAGS := -I$(idir)/include -Wno-nullability-completeness export PKG_CONFIG_PATH := $(ildir)/pkgconfig:$(idir)/share/pkgconfig # Disable built-in rules (which are not needed here!) @@ -160,6 +172,13 @@ export F77 := $(ibdir)/gfortran export LD_RUN_PATH := $(ildir):$(il64dir) export LD_LIBRARY_PATH := $(ildir):$(il64dir) +# See description of '-Wno-nullability-completeness' in +# 'reproduce/software/shell/configure.sh'. +ifeq ($(on_mac_os),yes) + noccwarnings=-Wno-nullability-completeness +endif +export CPPFLAGS := -I$(idir)/include $(noccwarnings) + # In macOS, if a directory exists in both 'C_INCLUDE_PATH' and 'CPPFLAGS' # it will be ignored in 'CPPFLAGS' (which has higher precedence). So, we # should not define 'C_INCLUDE_PATH' on macOS. This happened with clang @@ -225,7 +244,7 @@ $(idircustom):; mkdir $@ # Afer putting everything together, we use the first server as the # reference for all software if their '-url' variable isn't defined (in # 'reproduce/software/config/urls.conf'). -downloadwrapper = ./reproduce/analysis/bash/download-multi-try +downloadwrapper = ./reproduce/analysis/bash/download-multi-try.sh maneage_backup_urls := $(shell awk '!/^#/{printf "%s ", $$1}' \ reproduce/software/config/servers-backup.conf) backupservers_all = $(user_backup_urls) $(maneage_backup_urls) @@ -330,8 +349,10 @@ $(ibidir)/atlas-$(atlas-version): # 'rpath_command'. export LDFLAGS=-L$(ildir) cd $(ddir) - tar -xf $(tdir)/atlas-$(atlas-version).tar.lz + tar -xf $(tdir)/atlas-$(atlas-version).tar.lz \ + --no-same-owner --no-same-permissions cd ATLAS + $(shsrcdir)/prep-source.sh $(ibdir) rm -rf build mkdir build cd build @@ -386,8 +407,9 @@ $(ibidir)/boost-$(boost-version): \ rm -rf $(ddir)/$$unpackdir topdir=$(pwd) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) ./bootstrap.sh --prefix=$(idir) --with-libraries=all \ --with-python=python3 echo "using mpi ;" > project-config.jam @@ -407,12 +429,11 @@ $(ibidir)/cfitsio-$(cfitsio-version): # systems. So we need to change it to our library installation # path. It doesn't affect GNU/Linux, so we'll just do it in any case # to keep things clean. - topdir=$(pwd); cd $(ddir); tar -xf $(tdir)/$$tarball + topdir=$(pwd); cd $(ddir) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions customtar=cfitsio-$(cfitsio-version)-custom.tar.gz cd cfitsio-$(cfitsio-version) - sed configure -e's|@rpath|$(ildir)|g' > configure_tmp - mv configure_tmp configure - chmod +x configure + sed -i -e's|@rpath|$(ildir)|g' configure cd .. tar cf $$customtar cfitsio-$(cfitsio-version) cd $$topdir @@ -420,15 +441,19 @@ $(ibidir)/cfitsio-$(cfitsio-version): # Continue the standard build on the customized tarball. Note that # with the installation of CFITSIO, 'fpack' and 'funpack' are not # installed by default. Because of that, they are added explicity. +# +# Note that older versions of CFITSIO (before 4.4.0) require a +# specific 'shared' target for the building of the shared libraries. export gbuild_tar=$(ddir)/$$customtar $(call gbuild, cfitsio-$(cfitsio-version), , \ --enable-sse2 --enable-reentrant \ --with-bzip2=$(idir), , \ - make shared fpack funpack) + make fpack funpack) rm $$customtar echo "CFITSIO $(cfitsio-version)" > $@ $(ibidir)/cairo-$(cairo-version): \ + $(ibidir)/libxt-$(libxt-version) \ $(ibidir)/pixman-$(pixman-version) \ $(ibidir)/libpng-$(libpng-version) \ $(ibidir)/freetype-$(freetype-version) @@ -445,10 +470,12 @@ $(ibidir)/eigen-$(eigen-version): tarball=eigen-$(eigen-version).tar.lz $(call import-source, $(eigen-url), $(eigen-checksum)) rm -rf $(ddir)/eigen-eigen-* - topdir=$(pwd); cd $(ddir); tar -xf $(tdir)/$$tarball + topdir=$(pwd); cd $(ddir) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd eigen-$(eigen-version) if ! [ -d $(iidir)/eigen3 ]; then mkdir $(iidir)/eigen3; fi - cp -r Eigen/* $(iidir)/eigen3/ + cp -r Eigen/* $(iidir)/eigen3/ # Some expect 'eigen3'. + ln -s $(iidir)/eigen3 $(iidir)/Eigen # Others expect 'Eigen'. cd $$topdir rm -rf $(ddir)/eigen-$(eigen-version) echo "Eigen $(eigen-version)" > $@ @@ -509,6 +536,12 @@ $(ibidir)/fftw-$(fftw-version): echo "FFTW $(fftw-version) \citep{fftw}" > $@ $(ibidir)/freetype-$(freetype-version): $(ibidir)/libpng-$(libpng-version) +# As of version 2.13.2, FreeType doesn't account for the 'SHELL' +# environment variable. The issue has been reported to the +# developers. But until future versions, the work-around was +# discoverd to be setting the 'GNUMAKE' environment variable so it +# includes 'SHELL'. + export GNUMAKE="$(makewshell)" tarball=freetype-$(freetype-version).tar.lz $(call import-source, $(freetype-url), $(freetype-checksum)) $(call gbuild, freetype-$(freetype-version), static) @@ -529,7 +562,7 @@ $(ibidir)/gsl-$(gsl-version): $(ibidir)/hdf5-$(hdf5-version): $(ibidir)/openmpi-$(openmpi-version) export CC=mpicc export FC=mpif90 - tarball=hdf5-$(hdf5-version).tar.gz + tarball=hdf5-$(hdf5-version).tar.lz $(call import-source, $(hdf5-url), $(hdf5-checksum)) $(call gbuild, hdf5-$(hdf5-version), static, \ --enable-parallel \ @@ -568,8 +601,10 @@ $(ibidir)/healpix-$(healpix-version): $(healpix-python-dep) \ fi rm -rf $(ddir)/Healpix_$(healpix-version) topdir=$(pwd); cd $(ddir); - tar -xf $(tdir)/$$tarball - cd Healpix_$(healpix-version)/src/C/autotools/ + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd Healpix_$(healpix-version) + $(shsrcdir)/prep-source.sh $(ibdir) + cd src/C/autotools autoreconf --install ./configure --prefix=$(idir) make V=1 -j$(numthreads) SHELL=$(ibdir)/bash @@ -658,9 +693,10 @@ $(ibidir)/libpaper-$(libpaper-version): \ # Unpack, build the configure system, build and install. cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions unpackdir=libpaper-$(libpaper-version) cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) autoreconf -fi ./configure --prefix=$(idir) --sysconfdir=$(idir)/etc \ --disable-static @@ -783,17 +819,37 @@ $(ibidir)/metastore-$(metastore-version): \ echo "" > $@ fi +# The Ninja build system (https://ninja-build.org) is also known as simply +# "Ninja". But other package managers (for example Debian) use +# "ninja-build" (the old "ninja" name has become obsolete there). Also, +# their own URL is called "ninja-build". So we use the same convention in +# Maneage. +$(ibidir)/ninjabuild-$(ninjabuild-version): $(ibidir)/cmake-$(cmake-version) + tarball=ninjabuild-$(ninjabuild-version).tar.lz + $(call import-source, $(ninjabuild-url), $(ninjabuild-checksum)) + cd $(ddir) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd ninjabuild-$(ninjabuild-version) + cmake -Bbuild-cmake + cmake --build build-cmake -j$(numthreads) + ./build-cmake/ninja_test + cp -pv build-cmake/ninja $(ibdir)/ + cd .. + rm -rf ninjabuild-$(ninjabuild-version) + echo "Ninja build system $(ninjabuild-version)" > $@ + $(ibidir)/openblas-$(openblas-version): - tarball=OpenBLAS-$(openblas-version).tar.lz + tarball=openblas-$(openblas-version).tar.lz $(call import-source, $(openblas-url), $(openblas-checksum)) if [ x$(on_mac_os) = xyes ]; then export CC=clang; fi cd $(ddir) - tar -xf $(tdir)/$$tarball - cd OpenBLAS-$(openblas-version) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd openblas-$(openblas-version) + $(shsrcdir)/prep-source.sh $(ibdir) make -j$(numthreads) make PREFIX=$(idir) install cd .. - rm -rf OpenBLAS-$(openblas-version) + rm -rf openblas-$(openblas-version) echo "OpenBLAS $(openblas-version)" > $@ $(ibidir)/openmpi-$(openmpi-version): @@ -896,8 +952,8 @@ $(ibidir)/libgit2-$(libgit2-version): $(ibidir)/cmake-$(cmake-version) -DUSE_SSH=OFF -DBUILD_CLAR=OFF \ -DTHREADSAFE=ON -DUSE_ICONV=OFF ) if [ x$(on_mac_os) = xyes ]; then - install_name_tool -id $(ildir)/libgit2.1.3.dylib \ - $(ildir)/libgit2.1.3.dylib + install_name_tool -id $(ildir)/libgit2.1.9.dylib \ + $(ildir)/libgit2.1.9.dylib fi echo "Libgit2 $(libgit2-version)" > $@ @@ -907,20 +963,27 @@ $(ibidir)/wcslib-$(wcslib-version): $(ibidir)/cfitsio-$(cfitsio-version) tarball=wcslib-$(wcslib-version).tar.lz $(call import-source, $(wcslib-url), $(wcslib-checksum)) -# If Fortran isn't present, don't build WCSLIB with it. - if type gfortran &> /dev/null; then fortranopt=""; - else fortranopt="--disable-fortran" - fi - -# Build WCSLIB. +# Build WCSLIB while disabling some features: +# - Fortran is disabled because as of May 2023, on macOS systems +# where we do not install GCC (and thus a standard 'gfortran'), the +# LLVM Fortran compiler is not complete and will cause a crash. If +# you use the Fortran features of WCSLIB, feel free to remove +# '--disable-fortran', but be careful with portability on other +# systems. Hopefully some time in the future, GCC will also be +# install-able on macOS within Maneage or LLVM's 'gfortran' will +# also be complete and will not cause a crash. +# - PGPLOT is disabled because it has many manual steps in its +# installation. Therefore, we currently do not build PGPlots in +# Maneage. Once (if) it is added, we can remove '--without-pgplot'. $(call gbuild, wcslib-$(wcslib-version), , \ LIBS="-pthread -lcurl -lm" \ + --without-pgplot \ + --disable-fortran \ --with-cfitsiolib=$(ildir) \ - --with-cfitsioinc=$(idir)/include \ - --without-pgplot $$fortranopt) + --with-cfitsioinc=$(idir)/include) if [ x$(on_mac_os) = xyes ]; then - install_name_tool -id $(ildir)/libwcs.7.7.dylib \ - $(ildir)/libwcs.7.7.dylib + install_name_tool -id $(ildir)/libwcs.$(wcslib-version).dylib \ + $(ildir)/libwcs.$(wcslib-version).dylib fi echo "WCSLIB $(wcslib-version)" > $@ @@ -938,15 +1001,44 @@ $(ibidir)/wcslib-$(wcslib-version): $(ibidir)/cfitsio-$(cfitsio-version) # # Astrometry-net contains a lot of programs. We need to specify the # installation directory and the Python executable (by default it will look -# for /usr/bin/python) +# for /usr/bin/python). +# +# An optional dependency is 'netpbm' but it has many dependencies and a +# crazy build system. So, it is not in the default preprequisites. If you +# need it you can add it as a prerequisite and pray that it will work. +# +# A consequence of not having 'netpbm' is that for obtaining the +# astrometric solution of one image using 'solve-field', it is necessary to +# build a catalog of sources with image coordinates and flux (x,y,flux) in +# advance. The catalog has to be sorted by flux. Finally, when invoking +# 'solve-field', the width and heigh of the original image have to be +# provided. +# +# More explicitly, raw basic steps using Gnuastro: +# +# - Obtain a catalog with image coordinates and flux (x,y,brightness): +# $ astnoisechisel img.fits -o det.fits +# $ astsegment det.fits -o seg.fits +# $ astmkcatalog seg.fits --clumpscat --x --y --brightness -o cat-raw.fits +# +# - Sort by flux: +# $ asttable cat-raw.fits --hdu 2 --sort brightness --descending \ +# --output cat.fits +# +# - Get the x-size and y-size from the header: +# $ xsize=$(astfits img.fits --keyvalue NAXIS1 --quiet) +# $ ysize=$(astfits img.fits --keyvalue NAXIS2 --quiet) +# +# - Run 'solve-field' to obtain the astrometric solution: +# $ solve-field cat.fits --width $xsize --height $ysize [--other-parameters] $(ibidir)/astrometrynet-$(astrometrynet-version): \ $(ibidir)/gsl-$(gsl-version) \ $(ibidir)/swig-$(swig-version) \ $(ipydir)/numpy-$(numpy-version) \ $(ibidir)/cairo-$(cairo-version) \ $(ibidir)/libpng-$(libpng-version) \ - $(ibidir)/netpbm-$(netpbm-version) \ $(ibidir)/wcslib-$(wcslib-version) \ + $(ipydir)/astropy-$(astropy-version) \ $(ibidir)/cfitsio-$(cfitsio-version) \ $(ibidir)/libjpeg-$(libjpeg-version) @@ -960,8 +1052,9 @@ $(ibidir)/astrometrynet-$(astrometrynet-version): \ # 'astrometrynet' cd $(ddir) rm -rf astrometry.net-$(astrometrynet-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd astrometry.net-$(astrometrynet-version) + $(shsrcdir)/prep-source.sh $(ibdir) sed -e 's|cat /proc/cpuinfo|echo "Ignoring CPU info"|' \ -e 's|-free|echo "Ignoring RAM info"|' Makefile > Makefile.tmp mv Makefile.tmp Makefile @@ -986,12 +1079,6 @@ $(ibidir)/automake-$(automake-version): $(ibidir)/autoconf-$(autoconf-version) $(call gbuild, automake-$(automake-version), static, ,V=1) echo "GNU Automake $(automake-version)" > $@ -$(ibidir)/bison-$(bison-version): $(ibidir)/help2man-$(help2man-version) - tarball=bison-$(bison-version).tar.lz - $(call import-source, $(bison-url), $(bison-checksum)) - $(call gbuild, bison-$(bison-version), static, ,V=1 -j$(numthreads)) - echo "GNU Bison $(bison-version)" > $@ - # cdsclient is a set of software written in c to interact with astronomical # database servers. It is a dependency of 'scamp' to be able to download # reference catalogues. @@ -1004,8 +1091,9 @@ $(ibidir)/cdsclient-$(cdsclient-version): tarball=cdsclient-$(cdsclient-version).tar.lz $(call import-source, $(cdsclient-url), $(cdsclient-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd cdsclient-$(cdsclient-version) + $(shsrcdir)/prep-source.sh $(ibdir) touch * ./configure --prefix=$(idir) make @@ -1022,25 +1110,39 @@ $(ibidir)/cmake-$(cmake-version): tarball=cmake-$(cmake-version).tar.lz $(call import-source, $(cmake-url), $(cmake-checksum)) -# After searching in 'bootstrap', I couldn't find 'LIBS', only -# 'LDFLAGS'. So the extra libraries are being added to 'LDFLAGS', not -# 'LIBS'. -# # On Mac systems, the build complains about 'clang' specific # features, so we can't use our own GCC build here. if [ x$(on_mac_os) = xyes ]; then export CC=clang export CXX=clang++ fi + +# CMake wants a single executable for 'MAKE', so we can't use 'make +# SHELL=$(SHELL) and we have defined this script. + export MAKE="$(makewshell)" + +# Go into the unpacked directory and prepare CMake. cd $(ddir) rm -rf cmake-$(cmake-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd cmake-$(cmake-version) - ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ - --system-bzip2 --system-liblzma --no-qt-gui \ + $(shsrcdir)/prep-source.sh $(ibdir) + +# Bootstrap, build and install CMake: +# - With the '--no-system-libs' option, CMake builds and statically +# links all the libraries it needs. Even though some of those (like +# liblzma, libcurl, zlib or bzip2) are within Maneage, we +# discovered that CMake can get confused and use out-of-Maneage +# libraries (https://savannah.nongnu.org/bugs/?63043). + ./bootstrap --no-qt-gui \ + --prefix=$(idir) \ + --no-system-libs \ --parallel=$(numthreads) - make -j$(numthreads) LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 - make install + $(makewshell) VERBOSE=1 LIBS="$$LIBS -lssl -lcrypto -lz" \ + -j$(numthreads) + $(makewshell) install + +# Clean up. cd .. rm -rf cmake-$(cmake-version) echo "CMake $(cmake-version)" > $@ @@ -1048,7 +1150,11 @@ $(ibidir)/cmake-$(cmake-version): $(ibidir)/flex-$(flex-version): $(ibidir)/bison-$(bison-version) tarball=flex-$(flex-version).tar.lz $(call import-source, $(flex-url), $(flex-checksum)) - $(call gbuild, flex-$(flex-version), static, ,V=1 -j$(numthreads)) + $(call gbuild, flex-$(flex-version), static, \ + --with-libiconv-prefix=$(idir) \ + --with-libintl-prefix=$(idir) \ + --disable-dependency-tracking, \ + V=1 -j$(numthreads)) echo "Flex $(flex-version)" > $@ $(ibidir)/gdb-$(gdb-version): $(ibidir)/python-$(python-version) @@ -1084,13 +1190,16 @@ $(ibidir)/ghostscript-$(ghostscript-version): \ # '-DPNG_ARM_NEON_OPT=0' prevents an arm64 'neon' library from being # required at compile time. cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd ghostscript-$(ghostscript-version) + $(shsrcdir)/prep-source.sh $(ibdir) ./configure --prefix=$(idir) \ --disable-cups \ --enable-dynamic \ --disable-compile-inits \ - CFLAGS="-DPNG_ARM_NEON_OPT=0" + --disable-hidden-visibility \ + CFLAGS="-DPNG_ARM_NEON_OPT=0" \ + LDFLAGS=-Wl,--copy-dt-needed-entries # Build and install the program and the shared libraries. make V=1 -j$(numthreads) @@ -1104,9 +1213,9 @@ $(ibidir)/ghostscript-$(ghostscript-version): \ # Install the fonts. tar -xvf $(tdir)/ghostscript-fonts-std-$(ghostscript-fonts-std-version).tar.lz \ - -C $(idir)/share/ghostscript + -C $(idir)/share/ghostscript --no-same-owner --no-same-permissions tar -xvf $(tdir)/ghostscript-fonts-gnu-$(ghostscript-fonts-gnu-version).tar.lz \ - -C $(idir)/share/ghostscript + -C $(idir)/share/ghostscript --no-same-owner --no-same-permissions fc-cache -v $(idir)/share/ghostscript/fonts/ echo; echo "Ghostscript fonts added to Fontconfig."; echo; @@ -1129,12 +1238,6 @@ $(ibidir)/gnuastro-$(gnuastro-version): \ cp $(dtexdir)/gnuastro.tex $(ictdir)/ echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ -$(ibidir)/help2man-$(help2man-version): - tarball=help2man-$(help2man-version).tar.lz - $(call import-source, $(help2man-url), $(help2man-checksum)) - $(call gbuild, help2man-$(help2man-version), static, ,V=1) - echo "Help2man $(Help2man-version)" > $@ - $(ibidir)/icu-$(icu-version): $(ibidir)/python-$(python-version) # First, we need to remove any possibly existing ICU installation @@ -1149,9 +1252,11 @@ $(ibidir)/icu-$(icu-version): $(ibidir)/python-$(python-version) tarball=icu-$(icu-version).tar.lz $(call import-source, $(icu-url), $(icu-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions unpackdir=icu-$(icu-version) - cd $$unpackdir/icu4c/source + cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) + cd icu4c/source ./configure --enable-static --prefix=$(idir) make -j$(numthreads) V=1 make install @@ -1167,8 +1272,25 @@ $(ibidir)/imagemagick-$(imagemagick-version): \ tarball=ImageMagick-$(imagemagick-version).tar.lz $(call import-source, $(imagemagick-url), $(imagemagick-checksum)) $(call gbuild, ImageMagick-$(imagemagick-version), static, \ - --with-gslib --without-x --disable-openmp, \ + --without-x \ + --with-gslib \ + --disable-openmp \ + --with-libstdc=$(ildir), \ V=1 -j$(numthreads)) + +# On macOS, an executable and several libraries are not properly +# linked with the Ghostscript library (libgs), so we need to fix it +# manually. + if [ x$(on_mac_os) = xyes ]; then + gsversion=$$(echo $(ghostscript-version) \ + | awk -F'.' '{print $$1"."$$2}') + libMagicks=$$(ls -l $(ildir)/libMagick*.dylib \ + | awk '/^-/{print $$NF}') + libMagicks_all="$(ibdir)/magick $$libMagicks" + for f in $$libMagicks_all; do \ + install_name_tool -change libgs.dylib.$$gsversion \ + $(ildir)/libgs.dylib.$$gsversion $$f; done + fi echo "ImageMagick $(imagemagick-version)" > $@ # 'imfit' doesn't use the traditional 'configure' and 'make' to install @@ -1199,8 +1321,9 @@ $(ibidir)/imfit-$(imfit-version): \ cd $(ddir) unpackdir=imfit-$(imfit-version) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) sed -i 's|/usr/local|$(idir)|g' SConstruct sed -i 's|/usr/include|$(idir)/include|g' SConstruct sed -i 's|.append(|.insert(0,|g' SConstruct @@ -1246,8 +1369,10 @@ $(ibidir)/minizip-$(minizip-version): $(ibidir)/automake-$(automake-version) unpackdir=minizip-$(minizip-version) rm -rf $$unpackdir mkdir $$unpackdir - tar -xf $(tdir)/$$tarball -C$$unpackdir --strip-components=1 + tar -xf $(tdir)/$$tarball -C$$unpackdir --strip-components=1 \ + --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) ./configure --prefix=$(idir) make cd contrib/minizip @@ -1280,12 +1405,12 @@ $(ibidir)/missfits-$(missfits-version): cp $(dtexdir)/missfits.tex $(ictdir)/ echo "MissFITS $(missfits-version) \citep{missfits}" > $@ -# Netpbm is a prerequisite of Astrometry-net, it contains a lot of programs. -# This program has a crazy dialogue installation which is override using the -# printf statment. Each '\n' is a new question that the installation process -# ask to the user. We give all answers with a pipe to the scripts (configure -# and install). The questions are different depending on the system (tested -# on GNU/Linux and Mac OS). +# Netpbm is an optional prerequisite of Astrometry-net, it contains a lot +# of programs. This program has a crazy dialogue installation which is +# override using the printf statment. Each '\n' is a new question that the +# installation process ask to the user. We give all answers with a pipe to +# the scripts (configure and install). The questions are different +# depending on the system (tested on GNU/Linux and Mac OS). $(ibidir)/netpbm-$(netpbm-version): \ $(ibidir)/flex-$(flex-version) \ $(ibidir)/libpng-$(libpng-version) \ @@ -1307,8 +1432,9 @@ $(ibidir)/netpbm-$(netpbm-version): \ cd $(ddir) unpackdir=netpbm-$(netpbm-version) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) # As of NetPBM 10.73.39 and Flex 2.6.4-410-74a89fd (commit 74a89fd in # Flex's Git that is 410 commits after version 2.6.4), there is the @@ -1351,15 +1477,43 @@ $(ibidir)/pcre-$(pcre-version): , V=1 -j$(numthreads)) echo "Perl Compatible Regular Expressions $(pcre-version)" > $@ -# 2022-01-01 The rules for building R - identified as r-cran to avoid the -# difficulties in searching text for a one-letter string - were shifted to -# 'r-cran.mk'. +# On macOS 12.3 Monterey with AppleClang 13.1.6.13160021, Plplot 5.15.0 +# needs the 'finite' function of 'math.h' which has been deprecated in +# macOS. By manually adding "#define finite isfinite" in 'math.h' like +# below, we fixed this problem but still it can't find 'exit' during the +# configuration phase so we stopped trying to port it to macOS. It means +# that on macOS Plplot is not available. For other OSs it should be fine. +# On macOS, the file 'tmath.h' can be found with 'xcrun --show-sdk-path'. +# +# cp /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/math.h \ +# $(iidir)/math.h +# awk '{if($$0 ~ /#define isinf\(x\)/) {print "#define finite isfinite"; print $$0} else print $$0 }' \ +# $(iidir)/math.h > $(iidir)/math-tmp.h +# mv $(iidir)/math-tmp.h $(iidir)/math.h +$(ibidir)/plplot-$(plplot-version): \ + $(ibidir)/cairo-$(cairo-version) \ + $(ibidir)/freetype-$(freetype-version) + tarball=plplot-$(plplot-version).tar.lz + $(call import-source, $(plplot-url), $(plplot-checksum)) + $(call cbuild, plplot-$(plplot-version), static, \ + -DDEFAULT_NO_BINDINGS:BOOL=ON \ + -DENABLE_cxx:BOOL=ON \ + -DENABLE_fortran:BOOL=ON \ + -DDEFAULT_NO_QT_DEVICES:BOOL=ON) + echo "PLplot $(pcre-version)" > $@ # SCAMP documentation says ATLAS is a mandatory prerequisite for using # SCAMP. We have ATLAS into the project but there are some problems with the # libraries that are not yet solved. However, we tried to install it with # the option --enable-openblas and it worked (same issue happened with # 'sextractor'. +# +# Plplot in SCAMP cannot be built for macOS. See comments on top of Plplot. +# But if you are not using macOS and want to enable plots, follow the +# steps below: +# +# 1. Add '$(ibidir)/plplot-$(plplot-version)' to the prerequisites +# 2. Remove the option '--enable-plplot=no' from the call of 'gbuild' $(ibidir)/scamp-$(scamp-version): \ $(ibidir)/fftw-$(fftw-version) \ $(ibidir)/openblas-$(openblas-version) \ @@ -1398,8 +1552,9 @@ $(ibidir)/scons-$(scons-version): $(ibidir)/python-$(python-version) cd $(ddir) unpackdir=scons-$(scons-version) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) # Unfortuantely SCons hard-codes its search PATH in its source (to # use POSIX operating system defaults)! So the only way to modify it @@ -1429,32 +1584,56 @@ $(ibidir)/scons-$(scons-version): $(ibidir)/python-$(python-version) echo "SCons $(scons-version)" > $@ # Sextractor crashes complaining about not linking with some ATLAS -# libraries. But we can override this issue since we have Openblas +# libraries. But we can override this issue since we have OpenBLAS # installed, it is just necessary to explicity tell sextractor to use it in # the configuration step. -# -# The '-fcommon' is a necessary C compilation flag for GCC 10 and above. It -# is necessary for astromatic libraries, otherwise their build will crash. $(ibidir)/sextractor-$(sextractor-version): \ $(ibidir)/fftw-$(fftw-version) \ $(ibidir)/openblas-$(openblas-version) +# Import the source. tarball=sextractor-$(sextractor-version).tar.lz $(call import-source, $(sextractor-url), $(sextractor-checksum)) +# Unpack the tarball and enter the directory. + unpackdir=sextractor-$(sextractor-version) + cd $(ddir) + rm -rf $$unpackdir + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd $$unpackdir + # See comment above 'missfits' for '-fcommon'. - $(call gbuild, sextractor-$(sextractor-version), static, \ - CFLAGS="-fcommon" \ - --enable-threads \ - --enable-openblas \ - --with-openblas-libdir=$(ildir) \ - --with-openblas-incdir=$(idir)/include) + ./configure --prefix="$(idir)" \ + CFLAGS="-fcommon" \ + --enable-threads \ + --enable-openblas \ + --libdir=$(ildir) \ + --includedir=$(iidir) \ + --with-openblas-libdir=$(ildir) \ + --with-openblas-incdir=$(iidir) + +# On macOS we need to manually change 'finite' to 'isfinite' in the +# header file 'src/levmar/compiler.h'. Until this problem is +# hopefully fixed in next releases, we are doing it manually using +# 'sed'. Consequently we are not installing it using 'gbuild'. Once +# this is fixed upstream, we can use the standard 'gbuild'. + sed -i -e's|define LM_FINITE finite |define LM_FINITE isfinite |' \ + src/levmar/compiler.h + +# Build, install and delete the temporary files. + make V=1 + make install + cd .. + rm -rf $$unpackdir + +# Make links for other possibly used names, copy citation and build +# the final target. ln -fs $(ibdir)/sex $(ibdir)/sextractor ln -fs $(ibdir)/sex $(ibdir)/source-extractor cp $(dtexdir)/sextractor.tex $(ictdir)/ echo "SExtractor $(sextractor-version) \citep{sextractor}" > $@ $(ibidir)/swarp-$(swarp-version): $(ibidir)/fftw-$(fftw-version) - tarball=swarp-$(swarp-version).tar.gz + tarball=swarp-$(swarp-version).tar.lz $(call import-source, $(swarp-url), $(swarp-checksum)) # See comment above 'missfits' for '-fcommon'. @@ -1507,7 +1686,10 @@ $(ibidir)/swig-$(swig-version): # '$(ibdir)'. If any program does need 'util-linux' libraries, they can # simply add the proper directories to the environment variables, see # 'fontconfig' for example. -$(ibidir)/util-linux-$(util-linux-version): | $(idircustom) +$(ibidir)/util-linux-$(util-linux-version): \ + $(ibidir)/autoconf-$(autoconf-version) \ + $(ibidir)/automake-$(automake-version) \ + | $(idircustom) # Import the source. tarball=util-linux-$(util-linux-version).tar.lz @@ -1517,12 +1699,39 @@ $(ibidir)/util-linux-$(util-linux-version): | $(idircustom) # explained above). As shown below, later, we'll put a symbolic link # of all the necessary binaries in the main '$(idir)/bin'. cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd util-linux-$(util-linux-version) + $(shsrcdir)/prep-source.sh $(ibdir) + +# If a patch is necessary, apply it. + if [ -f $(patchdir)/util-linux-$(util-linux-version)-macos.patch ]; then + cp $(patchdir)/util-linux-$(util-linux-version)-macos.patch \ + util-linux-$(util-linux-version)-macos.patch + git apply util-linux-$(util-linux-version)-macos.patch + fi + +# The 'mkswap' feature needs low-level file system and kernel headers +# that are not always available (in particular on older Linux +# kernels). Also, creating SWAP space will need root permissions, so +# its not something a Maneager may need! Unfortunately there is no +# configuration option to disable this so we'll have to disable it +# manually by commenting the relevant files in the +# 'configure.ac'. + sed -e's|UL_BUILD_INIT(\[mkswap\], \[yes\])|UL_BUILD_INIT(\[mkswap\], \[no\])|' \ + -i configure.ac + +# Having updated 'configure.ac', we need to re-generate the +# './configure' script with 'autoreconf' (which is part of Autoconf +# and needs Automake; hence why they are dependencies. + autoreconf -f + +# Configure Util-linux + export CONFIG_SHELL=$(ibdir)/bash ./configure --prefix=$(idircustom)/util-linux \ --disable-dependency-tracking \ --enable-libmount-support-mtab \ --disable-silent-rules \ + --disable-liblastlog2 \ --disable-mountpoint \ --disable-libmount \ --disable-unshare \ @@ -1592,12 +1801,13 @@ $(ibidir)/xlsxio-$(xlsxio-version): \ # useful in projects during its development, for more see the comment above # GNU Emacs. $(ibidir)/vim-$(vim-version): - tarball=vim-$(vim-version).tar.bz2 + tarball=vim-$(vim-version).tar.lz $(call import-source, $(vim-url), $(vim-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball - n=$$(echo $(vim-version) | sed -e's|\.||') - cd $(ddir)/vim$$n + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + unpackdir=vim-$(vim-version) + cd $(ddir)/$$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) ./configure --prefix=$(idir) \ --disable-canberra \ --enable-multibyte \ @@ -1610,9 +1820,29 @@ $(ibidir)/vim-$(vim-version): make -j$(numthreads) make install cd .. - rm -rf vim$$n + rm -rf $$unpackdir echo "VIM $(vim-version)" > $@ +$(ibidir)/unzip-$(unzip-version): $(ibidir)/gzip-$(gzip-version) + tarball=unzip-$(unzip-version).tar.lz + $(call import-source, $(unzip-url), $(unzip-checksum)) + $(call gbuild, unzip-$(unzip-version), static,, \ + -f unix/Makefile generic \ + CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ + -f unix/Makefile generic \ + BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) + echo "Unzip $(unzip-version)" > $@ + +$(ibidir)/zip-$(zip-version): $(ibidir)/gzip-$(gzip-version) + tarball=zip-$(zip-version).tar.lz + $(call import-source, $(zip-url), $(zip-checksum)) + $(call gbuild, zip-$(zip-version), static,, \ + -f unix/Makefile generic \ + CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ + -f unix/Makefile generic \ + BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) + echo "Zip $(zip-version)" > $@ + @@ -1644,12 +1874,19 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf @topdir=$$(pwd) cd $(ddir) rm -rf install-tl-* - tar -xf $(tdir)/install-tl-unx.tar.gz + tar -xf $(tdir)/install-tl-unx.tar.gz \ + --no-same-owner --no-same-permissions cd install-tl-* + $(shsrcdir)/prep-source.sh $(ibdir) sed -e's|@installdir[@]|$(idir)|g' \ "$$topdir"/reproduce/software/config/texlive.conf \ > texlive.conf +# We do not build TeXLive from source and for its installation it +# downloads components from the web internally; and those components +# can use '/bin/sh' (which will need '$(sys_library_sh_path)'). + export LD_LIBRARY_PATH="$(sys_library_sh_path):$$LD_LIBRARY_PATH" + # TeX Live's installation may fail due to any reason. But TeX Live is # optional (only necessary for building the final PDF). So we don't # want the configure script to fail if it can't run. Possible error @@ -1718,13 +1955,17 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf tarballurl=$$url/install-tl-unx.tar.gz touch $(lockdir)/download downloader="wget --no-use-server-timestamps -O" - if $(downloadwrapper) "$$downloader" $(lockdir)/download \ - $$tarballurl "$(tdir)/install-tl-unx.tar.gz" \ - "$(backupservers)"; then + if $(downloadwrapper) "$$downloader" \ + $(lockdir)/download \ + $$tarballurl \ + "$(tdir)/install-tl-unx.tar.gz" \ + "$(backupservers)"; then cd $(ddir) rm -rf install-tl-* - tar -xf $(tdir)/install-tl-unx.tar.gz + tar -xf $(tdir)/install-tl-unx.tar.gz \ + --no-same-owner --no-same-permissions cd install-tl-* + $(shsrcdir)/prep-source.sh $(ibdir) sed -e's|@installdir[@]|$(idir)|g' \ $$topdir/reproduce/software/config/texlive.conf \ > texlive.conf @@ -1755,10 +1996,19 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf # Live itself (only very basic TeX and LaTeX) and the installation of its # necessary packages into two packages. # -# Note that Biber needs to link with libraries like libnsl. However, we -# don't currently build biber from source. So we can't choose the library -# version. But we have the source and build instructions for the 'nsl' -# library. When we later build biber from source, we can easily use them. +# Note that we do not build the TeXLive executables (like Biber) from +# source. So in case they need special libraries, we can't choose the +# library version here (for example see [1] and [2]). In such cases there +# is no solution but to manually add the location necessary library to +# LD_LIBRARY_PATH when calling the respective LaTeX command in +# 'reproduce/analysis/make/paper.mk'. Fortunately as of Biber 2.20, it does +# not depend on anything except the C library (all dependencies are now +# statically linked), so problems [1] and [2] will not happen. But this can +# generally happen for any other tool/OS, so it is important to build +# TeXLive from source as soon as possible [3]. +# [1] https://github.com/plk/biber/issues/445 +# [2] https://savannah.nongnu.org/bugs/index.php?63175 +# [3] https://savannah.nongnu.org/task/?15267 $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ $(itidir)/texlive-ready-tlmgr @@ -1768,6 +2018,11 @@ $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ echo "" > $@ else +# We do not build TeXLive from source and for its installation it +# downloads components from the web internally; and those +# components can use '/bin/sh' (which needs 'sys_library_sh_path'). + export LD_LIBRARY_PATH="$(sys_library_sh_path)" + # To update itself, tlmgr needs a backup directory. backupdir=$(idir)/texlive/backups mkdir -p $$backupdir @@ -1791,6 +2046,12 @@ $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ # directory so we don't have to modify 'PATH'. ln -fs $(idir)/texlive/maneage/bin/*/* $(ibdir)/ +# Correct any reference to '/bin/sh' within the installed LaTeX +# files (this is because we do no yet install LaTeX from source): + cdir=$$(pwd) + cd $(idir)/texlive + cd $$cdir + # Get all the necessary versions. texlive=$$(pdflatex --version \ | awk 'NR==1' \ diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 6766744..c994e3f 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -12,8 +12,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2019-2022 Raul Infante-Sainz <infantesainz@gmail.com> -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -51,6 +51,8 @@ export MPI_PYTHON3_SITEARCH := # Python-specific installation directories. python-major-version = $(shell echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') +# This is for 'installer' (the python installer) itself: +ilibpymajorversion = $(idir)/lib/python$(python-major-version) @@ -93,13 +95,48 @@ $(ibidir)/python-$(python-version): $(ibidir)/libffi-$(libffi-version) export CC=clang export CXX=clang++ fi - $(call gbuild, python-$(python-version),, \ - --without-ensurepip \ - --with-system-ffi \ - --enable-shared, -j$(numthreads)) + +# Unpack the tarball (see below for the necessary modification). + cd $(ddir) + unpackdir=python-$(python-version) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) + +# Python's 'setup.py' uses 'os.system' to run shell scripts. On the +# other hand 'os.system' only runs '/bin/sh' (which has its own +# libraries to link to and those are blocked at this level). So we +# need to add an extra line on top of the 'os.system' funciton and +# put '/usr/lib' in 'LD_LIBRARY_PATH' within Python's environment for +# system calls (with 'os.putenv'). As of Python 3.13.2 the tarball no +# longer has an 'setup.py'. But when it did, the change below was +# necessary. + if [ -f setup.py ]; then + awk '{if(/os.system\(/) \ + { print " os.putenv(\"LD_LIBRARY_PATH\", \"$$LD_LIBRARY_PATH:$(sys_library_sh_path)\");"; \ + print $$0;} \ + else print $$0}' \ + setup.py > setup-tmp.py + mv setup-tmp.py setup.py + fi + +# Do the basic installation and delete the temporary directory. + ./configure SHELL=$(ibdir)/bash \ + --enable-optimizations \ + --without-ensurepip \ + --prefix="$(idir)" \ + --with-system-ffi \ + --enable-shared + $(makewshell) -j$(numthreads) + $(makewshell) install -j$(numthreads) + cd .. + rm -rf $$unpackdir + +# Set the necessary environment variables and finish the build. ln -sf $(ildir)/python$(python-major-version) $(ildir)/python ln -sf $(ibdir)/python$(python-major-version) $(ibdir)/python - ln -sf $(iidir)/python$(python-major-version)m $(iidir)/python$(python-major-version) + ln -sf $(iidir)/python$(python-major-version)m \ + $(iidir)/python$(python-major-version) rm -rf $(ipydir) mkdir $(ipydir) echo "Python $(python-version)" > $@ @@ -108,39 +145,138 @@ $(ibidir)/python-$(python-version): $(ibidir)/libffi-$(libffi-version) -# Non-PiP Python module installation +# Non-pip Python module installation # ---------------------------------- # -# To build Python packages with direct access to a 'setup.py' (if no direct -# access to 'setup.py' is needed, pip can be used). Note that the -# software's packaged source code is the first prerequisite that is in the -# 'tdir' directory. +# Build strategy for python modules as of February 2025, for python 3.13.2. + +# This strategy is mostly based on recommendations by E Schwartz +# (ztrawhcse) on #python (Libera Chat), in October 2022 and February +# 2025. Some discussions are on documented in Savannah tasks [1][2]. The +# build strategy for 'python-installer' is inspired by the gentoo script +# 'python_domodule' [3]. + +# Bootstrap-step: 'gpep517' [4], motivated by PEP 517 [5], together with +# 'python-installer' (module called 'installer') are built without +# dependences on other python packages apart from python itself. The build +# rules for these two packages do python byte compilation and copy the .py +# and .pyc files into the python install directory. These two packages are +# considered to be 'frontends'. + +# Once these two frontends are available, other packages that do building +# tasks, including both backends and alternative frontends or a mix of +# these (in particular: setuptools, meson [6]/ninja-build [7] , flit-core, +# and meson-python), can be built with the 'python-installer' and +# 'gpep517'. The aims of the various build tools are diverse, include +# ecosystem resilience, reproducibility, build speed and convenience in +# building bigger packages such as numpy, scipy and astropy. + +# The python.mk script now includes only three methods: the boot +# build methods of 'python-installer' and 'gpep517'; and the gpep517 +# frontend. No method is provided for using 'python-installer' directly; +# it is invoked indirectly by source files of many packages, which +# also give metadata describing information for build methods. + +# Why not pip? We do not build any python packages with 'pip' because we +# want to have a fully documented pipeline of (i) the original upstream +# locations of tarballs, (ii) the tarballs' checksums, and (iii) the exact +# sequence of build commands. + +# For an alternative viewpoint on a python build strategy, see [8]. + +# Prerequisite for the pybuild script here: the package's source code +# (tarball) must already be located in the directory 'tdir'. # -# Arguments of this function are the numbers +# Arguments: # 1) Unpack command # 2) Unpacked directory name after unpacking the tarball # 3) site.cfg file (optional). -# 4) Official software name (for paper). +# 4) Official software name (for paper.tex). +# 5) Obligatory parameter: build method (see below): +# BOOT_INSTALLER - only for 'python-installer' +# BOOT_GPEP517 - only for 'gpep517' +# GPEP517 - for any other python package # # Hooks: # pyhook_before: optional steps before running 'python setup.py build' # pyhook_after: optional steps after running 'python setup.py install' -pybuild = cd $(ddir); rm -rf $(2); \ - if ! $(1) $(tdir)/$$tarball; then \ + +# [1] https://savannah.nongnu.org/task/?16268 +# [2] https://savannah.nongnu.org/task/?16625 +# [3] https://gitweb.gentoo.org/repo/gentoo.git/tree/eclass/python-utils-r1.eclass#n646 +# [4] https://pypi.org/project/gpep517 +# [5] https://peps.python.org/pep-0517 +# [6] https://mesonbuild.com +# [7] https://ninja-build.org +# [8] https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html + + +pybuild = cd $(ddir); \ + packagedir=$(strip $(2)); \ + if (printf "$$packagedir" | grep "[a-z][a-z]"); then rm -rf $$packagedir; fi; \ + printf "\nStarting to install python package with maneage pybuild rule: $(4)\n ..."; \ + if ! $(1) $(tdir)/$$tarball --no-same-owner --no-same-permissions; then \ echo; echo "Tar error"; exit 1; \ fi; \ - cd $(2); \ + cd $$packagedir; \ if [ "x$(strip $(3))" != x ]; then \ sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ $(3) > site.cfg; \ fi; \ if type pyhook_before &>/dev/null; then pyhook_before; fi; \ - python setup.py build; \ - python setup.py install; \ + printf "pybuild option5 = __ %s __\n" "$(strip $(5))"; \ + if [ "x$(strip $(5))" = xBOOT_INSTALLER ]; then \ + chmod 0644 src/installer/*.py; \ + mkdir -p $(ilibpymajorversion)/installer; \ + mkdir -p $(ilibpymajorversion)/installer/__pycache__ ; \ + mkdir -p $(ilibpymajorversion)/installer/_scripts; \ + mkdir -p $(ilibpymajorversion)/installer/_scripts/__pycache__ ; \ + cp -pv src/installer/*.py $(ilibpymajorversion)/installer/; \ + cp -pv src/installer/_scripts/__init__.py $(ilibpymajorversion)/installer/_scripts/; \ + cd src/installer/; \ + python -m compileall -o 1 -o 2 -l -f \ + -d $(ilibpymajorversion)/installer/ .; \ + chmod 0644 __pycache__/*.pyc; \ + cp -pv __pycache__/*.pyc \ + $(ilibpymajorversion)/installer/__pycache__; \ + cd -; \ + cd src/installer/_scripts/; \ + python -m compileall -o 1 -o 2 -l -f \ + -d $(ilibpymajorversion)/installer/_scripts/ __init__.py; \ + chmod 0644 __pycache__/*.pyc; \ + cp -pv __pycache__/*.pyc \ + $(ilibpymajorversion)/installer/_scripts/__pycache__; \ + cd -; \ + elif [ "x$(strip $(5))" = xBOOT_GPEP517 ]; then \ + chmod 0644 gpep517/*.py; \ + mkdir -p $(ilibpymajorversion)/gpep517; \ + mkdir -p $(ilibpymajorversion)/gpep517/__pycache__ ; \ + cp -pv gpep517/*.py $(ilibpymajorversion)/gpep517/; \ + cd gpep517/; \ + python -m compileall -o 1 -o 2 -l -f \ + -d $(ilibpymajorversion)/gpep517/ .; \ + chmod 0644 __pycache__/*.pyc; \ + cp -pv __pycache__/*.pyc \ + $(ilibpymajorversion)/gpep517/__pycache__; \ + cd -; \ + elif [ "x$(strip $(5))" = xGPEP517 ]; then \ + printf "\n\n\n=== python build method: gpep517 ====== "; pwd; \ + printf "...............\n\n\n"; \ + python -m gpep517 install-from-source \ + --prefix "" \ + --destdir $(idir) \ + --optimize all; \ + else \ + printf "Error: Unknown pybuild method for $$packagedir: __ $(strip $(5)) __\n"; \ + printf "The pybuild 5th parameter should very likely be set "; \ + printf "to GPEP517 after checking the build rule and "; \ + printf "upgrading if needed.\n"; \ + exit 1; \ + fi; \ if type pyhook_after &>/dev/null; then pyhook_after; fi; \ cd ..; \ - rm -rf $(2); \ + if (printf "$$packagedir" | grep "[a-z][a-z]"); then rm -fr $$packagedir; fi; \ echo "$(4)" > $@ @@ -149,9 +285,11 @@ pybuild = cd $(ddir); rm -rf $(2); \ # Python modules # --------------- -# + + # All the necessary Python modules go here. -$(ipydir)/asn1crypto-$(asn1crypto-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/asn1crypto-$(asn1crypto-version): \ + $(ipydir)/setuptools-$(setuptools-version) tarball=asn1crypto-$(asn1crypto-version).tar.gz $(call import-source, $(asn1crypto-url), $(asn1crypto-checksum)) $(call pybuild, tar -xf, asn1crypto-$(asn1crypto-version), , \ @@ -172,23 +310,58 @@ $(ipydir)/astroquery-$(astroquery-version): \ $(call pybuild, tar -xf, astroquery-$(astroquery-version), , \ Astroquery $(astroquery-version)) +# Astropy: points to consider about dependencies: +# +# The optional dependency 'h5py' that is necessary for writting tables in +# HDF5 format has been removed from Astropy because on macOS it cannot be +# installed. +# +# 2022-or-older dependencies: +# $(ibidir)/expat-$(expat-version) \ +# $(ipydir)/jinja2-$(jinja2-version) \ +# $(ipydir)/html5lib-$(html5lib-version) \ +# $(ipydir)/beautifulsoup4-$(beautifulsoup4-version) \ +# +# While the astropy pyproject.toml file says that the astropy build depends +# on numpy, not scipy, and does not depend on matplotlib; the +# runtime is recommended to depend on both scipy and matplotlib. +# In practice, users of astropy will generally expect scipy and matplotlib +# to be available at runtime, so we set these as prerequisites. $(ipydir)/astropy-$(astropy-version): \ - $(ipydir)/h5py-$(h5py-version) \ - $(ibidir)/expat-$(expat-version) \ $(ipydir)/scipy-$(scipy-version) \ - $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/pyyaml-$(pyyaml-version) \ - $(ipydir)/jinja2-$(jinja2-version) \ $(ipydir)/pyerfa-$(pyerfa-version) \ - $(ipydir)/html5lib-$(html5lib-version) \ - $(ipydir)/beautifulsoup4-$(beautifulsoup4-version) \ + $(ipydir)/pyyaml-$(pyyaml-version) \ + $(ipydir)/matplotlib-$(matplotlib-version) \ + $(ipydir)/astropy-iers-data-$(astropy-iers-data-version) \ $(ipydir)/extension-helpers-$(extension-helpers-version) + +# Tarball and its preparation. tarball=astropy-$(astropy-version).tar.lz $(call import-source, $(astropy-url), $(astropy-checksum)) - $(call pybuild, tar -xf, astropy-$(astropy-version)) - cp $(dtexdir)/astropy.tex $(ictdir)/ + +# Conservatively purge old version (but not astropy_iers, +# astropy-iers): + rm -fvr $(idir)/lib/python*/site-packages/astropy/ + rm -fvr $(idir)/lib/python*/site-packages/astropy-[0-9]*-info/ + rm -fv $(idir)/bin/fits{diff,check,header,info,2bitmap} + rm -fv $(idir)/bin/{samp_hub,showtable,volint,wcslint} + +# Do the basic build. + $(call pybuild, tar -xf, astropy-$(astropy-version),,, \ + GPEP517) + cp -pv $(dtexdir)/astropy.tex $(ictdir)/ echo "Astropy $(astropy-version) \citep{astropy2013,astropy2018}" > $@ +$(ipydir)/astropy-iers-data-$(astropy-iers-data-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=astropy-iers-data-$(astropy-iers-data-version).tar.lz + $(call import-source, $(astropy-iers-data-url), \ + $(astropy-iers-data-checksum)) + $(call pybuild, tar -xf, \ + astropy-iers-data-$(astropy-iers-data-version),,, \ + GPEP517) + echo "Astropy-Iers-Data $(astropy-iers-data-version)" > $@ + $(ipydir)/beautifulsoup4-$(beautifulsoup4-version): \ $(ipydir)/soupsieve-$(soupsieve-version) tarball=beautifulsoup4-$(beautifulsoup4-version).tar.lz @@ -196,13 +369,17 @@ $(ipydir)/beautifulsoup4-$(beautifulsoup4-version): \ $(call pybuild, tar -xf, beautifulsoup4-$(beautifulsoup4-version), , \ BeautifulSoup $(beautifulsoup4-version)) -$(ipydir)/beniget-$(beniget-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/beniget-$(beniget-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=beniget-$(beniget-version).tar.lz $(call import-source, $(beniget-url), $(beniget-checksum)) $(call pybuild, tar -xf, beniget-$(beniget-version), , \ Beniget $(beniget-version)) -$(ipydir)/certifi-$(certifi-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/certifi-$(certifi-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=certifi-$(certifi-version).tar.gz $(call import-source, $(certifi-url), $(certifi-checksum)) $(call pybuild, tar -xf, certifi-$(certifi-version), , \ @@ -215,12 +392,23 @@ $(ipydir)/cffi-$(cffi-version): \ $(call import-source, $(cffi-url), $(cffi-checksum)) $(call pybuild, tar -xf, cffi-$(cffi-version), ,cffi $(cffi-version)) -$(ipydir)/chardet-$(chardet-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/chardet-$(chardet-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=chardet-$(chardet-version).tar.gz $(call import-source, $(chardet-url), $(chardet-checksum)) $(call pybuild, tar -xf, chardet-$(chardet-version), , \ Chardet $(chardet-version)) +$(ipydir)/contourpy-$(contourpy-version): \ + $(ipydir)/pybind11-$(pybind11-version) \ + $(ipydir)/meson-python-$(meson-python-version) + tarball=contourpy-$(contourpy-version).tar.lz + $(call import-source, $(contourpy-url), $(contourpy-checksum)) + $(call pybuild, tar -xf, contourpy-$(contourpy-version), , \ + Contourpy $(contourpy-version), GPEP517) + echo "Contourpy $(contourpy-version)" > $@ + $(ipydir)/corner-$(corner-version): $(ipydir)/matplotlib-$(matplotlib-version) tarball=corner-$(corner-version).tar.gz $(call import-source, $(corner-url), $(corner-checksum)) @@ -229,6 +417,13 @@ $(ipydir)/corner-$(corner-version): $(ipydir)/matplotlib-$(matplotlib-version) cp $(dtexdir)/corner.tex $(ictdir)/ echo "Corner $(corner-version) \citep{corner}" > $@ +$(ipydir)/cppy-$(cppy-version): \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) + tarball=cppy-$(cppy-version).tar.lz + $(call import-source, $(cppy-url), $(cppy-checksum)) + $(call pybuild, tar -xf, cppy-$(cppy-version), , \ + Cppy $(cppy-version), GPEP517) + $(ipydir)/cryptography-$(cryptography-version): \ $(ipydir)/cffi-$(cffi-version) \ $(ipydir)/asn1crypto-$(asn1crypto-version) \ @@ -242,13 +437,17 @@ $(ipydir)/cycler-$(cycler-version): $(ipydir)/six-$(six-version) tarball=cycler-$(cycler-version).tar.lz $(call import-source, $(cycler-url), $(cycler-checksum)) $(call pybuild, tar -xf, cycler-$(cycler-version), , \ - Cycler $(cycler-version)) + Cycler $(cycler-version), GPEP517) + echo "Cycler $(cycler-version)" > $@ -$(ipydir)/cython-$(cython-version): $(ipydir)/setuptools-$(setuptools-version) - tarball=Cython-$(cython-version).tar.lz +$(ipydir)/cython-$(cython-version): \ + $(ipydir)/python-installer-$(python-installer-version) \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=cython-$(cython-version).tar.lz $(call import-source, $(cython-url), $(cython-checksum)) - $(call pybuild, tar -xf, Cython-$(cython-version)) - cp $(dtexdir)/cython.tex $(ictdir)/ + $(call pybuild, tar -xf, cython-$(cython-version),,, GPEP517) + cp -pv $(dtexdir)/cython.tex $(ictdir)/ echo "Cython $(cython-version) \citep{cython2011}" > $@ $(ipydir)/esutil-$(esutil-version): $(ipydir)/numpy-$(numpy-version) @@ -269,25 +468,28 @@ $(ipydir)/eigency-$(eigency-version): \ $(ipydir)/emcee-$(emcee-version): \ $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/setuptools_scm-$(setuptools_scm-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=emcee-$(emcee-version).tar.gz $(call import-source, $(emcee-url), $(emcee-checksum)) $(call pybuild, tar -xf, emcee-$(emcee-version), , \ emcee $(emcee-version)) $(ipydir)/entrypoints-$(entrypoints-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=entrypoints-$(entrypoints-version).tar.gz $(call import-source, $(entrypoints-url), $(entrypoints-checksum)) $(call pybuild, tar -xf, entrypoints-$(entrypoints-version), , \ EntryPoints $(entrypoints-version)) $(ipydir)/extension-helpers-$(extension-helpers-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=extension-helpers-$(extension-helpers-version).tar.lz - $(call import-source, $(extension-helpers-url), $(extension-helpers-checksum)) - $(call pybuild, tar -xf, extension-helpers-$(extension-helpers-version), , \ - Extension-Helpers $(extension-helpers-version)) + $(call import-source, $(extension-helpers-url), \ + $(extension-helpers-checksum)) + $(call pybuild, tar -xf, \ + extension-helpers-$(extension-helpers-version),, \ + Extension-Helpers $(extension-helpers-version), GPEP517) $(ipydir)/flake8-$(flake8-version): \ $(ipydir)/pyflakes-$(pyflakes-version) \ @@ -297,7 +499,29 @@ $(ipydir)/flake8-$(flake8-version): \ $(call pybuild, tar -xf, flake8-$(flake8-version), , \ Flake8 $(flake8-version)) -$(ipydir)/future-$(future-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/flit-core-$(flit-core-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) + tarball=flit-core-$(flit-core-version).tar.lz + $(call import-source, $(flit-core-url), $(flit-core-checksum)) + $(call pybuild, tar -xf, flit-core-$(flit-core-version), , \ + Flit-core $(flit-core-version), GPEP517) + +# Although cython is not an obligatory prerequisite of fonttools, we force +# it as a prerequisite for reproducibility; otherwise build parallelism may +# lead to some builds with and some builds without cython, depending on how +# many cpus the host machine has. +$(ipydir)/fonttools-$(fonttools-version): \ + $(ipydir)/cython-$(cython-version) \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=fonttools-$(fonttools-version).tar.lz + $(call import-source, $(fonttools-url), $(fonttools-checksum)) + $(call pybuild, tar -xf, fonttools-$(fonttools-version), , \ + fonttools $(fonttools-version), GPEP517) + +$(ipydir)/future-$(future-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=future-$(future-version).tar.gz $(call import-source, $(future-url), $(future-checksum)) $(call pybuild, tar -xf, future-$(future-version), , \ @@ -315,12 +539,25 @@ $(ipydir)/galsim-$(galsim-version): \ cp $(dtexdir)/galsim.tex $(ictdir)/ echo "Galsim $(galsim-version) \citep{galsim}" > $@ -$(ipydir)/gast-$(gast-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/gast-$(gast-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=gast-$(gast-version).tar.lz $(call import-source, $(gast-url), $(gast-checksum)) $(call pybuild, tar -xf, gast-$(gast-version), , \ Gast $(gast-version)) +$(ipydir)/gpep517-$(gpep517-version): \ + $(ibidir)/python-$(python-version) + tarball=gpep517-$(gpep517-version).tar.lz + $(call import-source, $(gpep517-url), $(gpep517-checksum)) + $(call pybuild, tar -xf, \ + gpep517-$(gpep517-version), \ + , \ + gpep517 $(gpep517-version), \ + BOOT_GPEP517) + echo "gpep517 $(gpep517-version)" > $@ + $(ipydir)/h5py-$(h5py-version): \ $(ipydir)/six-$(six-version) \ $(ibidir)/hdf5-$(hdf5-version) \ @@ -353,13 +590,17 @@ $(ipydir)/html5lib-$(html5lib-version): \ $(call pybuild, tar -xf, html5lib-$(html5lib-version), , \ HTML5lib $(html5lib-version)) -$(ipydir)/idna-$(idna-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/idna-$(idna-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=idna-$(idna-version).tar.gz $(call import-source, $(idna-url), $(idna-checksum)) $(call pybuild, tar -xf, idna-$(idna-version), , \ idna $(idna-version)) -$(ipydir)/jeepney-$(jeepney-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/jeepney-$(jeepney-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=jeepney-$(jeepney-version).tar.gz $(call import-source, $(jeepney-url), $(jeepney-checksum)) $(call pybuild, tar -xf, jeepney-$(jeepney-version), , \ @@ -374,17 +615,21 @@ $(ipydir)/jinja2-$(jinja2-version): $(ipydir)/markupsafe-$(markupsafe-version) $(ipydir)/keyring-$(keyring-version): \ $(ipydir)/entrypoints-$(entrypoints-version) \ $(ipydir)/secretstorage-$(secretstorage-version) \ - $(ipydir)/setuptools_scm-$(setuptools_scm-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=keyring-$(keyring-version).tar.gz $(call import-source, $(keyring-url), $(keyring-checksum)) $(call pybuild, tar -xf, keyring-$(keyring-version), , \ Keyring $(keyring-version)) -$(ipydir)/kiwisolver-$(kiwisolver-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/kiwisolver-$(kiwisolver-version): \ + $(ipydir)/cppy-$(cppy-version) \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=kiwisolver-$(kiwisolver-version).tar.lz $(call import-source, $(kiwisolver-url), $(kiwisolver-checksum)) $(call pybuild, tar -xf, kiwisolver-$(kiwisolver-version), , \ - Kiwisolver $(kiwisolver-version)) + Kiwisolver $(kiwisolver-version), GPEP517) + cp -pv $(dtexdir)/kiwisolver.tex $(ictdir)/ + echo "Kiwisolver $(kiwisolver-version) \citep{cassowary2001}" > $@ $(ipydir)/lmfit-$(lmfit-version): \ $(ipydir)/six-$(six-version) \ @@ -400,14 +645,17 @@ $(ipydir)/lmfit-$(lmfit-version): \ LMFIT $(lmfit-version)) $(ipydir)/lsstdesccoord-$(lsstdesccoord-version): \ - $(ipydir)/cython-$(cython-version) + $(ipydir)/cffi-$(cffi-version) \ + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/future-$(future-version) tarball=lsstdesccoord-$(lsstdesccoord-version).tar.gz $(call import-source, $(lsstdesccoord-url), $(lsstdesccoord-checksum)) $(call pybuild, tar -xf, LSSTDESC.Coord-$(lsstdesccoord-version), , \ LSSTDESC.Coord $(lsstdesccoord-version)) $(ipydir)/markupsafe-$(markupsafe-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=markupsafe-$(markupsafe-version).tar.lz $(call import-source, $(markupsafe-url), $(markupsafe-checksum)) $(call pybuild, tar -xf, markupsafe-$(markupsafe-version), , \ @@ -418,11 +666,9 @@ $(ipydir)/matplotlib-$(matplotlib-version): \ $(ipydir)/numpy-$(numpy-version) \ $(ipydir)/cycler-$(cycler-version) \ $(ipydir)/pillow-$(pillow-version) \ - $(ibidir)/freetype-$(freetype-version) \ - $(ipydir)/pyparsing-$(pyparsing-version) \ + $(ipydir)/fonttools-$(fonttools-version) \ + $(ipydir)/contourpy-$(contourpy-version) \ $(ipydir)/kiwisolver-$(kiwisolver-version) \ - $(ibidir)/ghostscript-$(ghostscript-version) \ - $(ibidir)/imagemagick-$(imagemagick-version) \ $(ipydir)/python-dateutil-$(python-dateutil-version) # Prepare the source. @@ -435,52 +681,76 @@ $(ipydir)/matplotlib-$(matplotlib-version): \ export CC=clang export CXX=clang++ fi - $(call pybuild, tar -xf, matplotlib-$(matplotlib-version)) + $(call pybuild, tar -xf, matplotlib-$(matplotlib-version),,, GPEP517) cp $(dtexdir)/matplotlib.tex $(ictdir)/ echo "Matplotlib $(matplotlib-version) \citep{matplotlib2007}" > $@ +$(ipydir)/meson-$(meson-version): \ + $(ibidir)/ninjabuild-$(ninjabuild-version) \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=meson-$(meson-version).tar.lz + $(call import-source, $(meson-url), $(meson-checksum)) + $(call pybuild, tar -xf, meson-$(meson-version), , \ + Meson $(meson-version), GPEP517) + echo "Meson $(meson-version)" > $@ + +# The 'meson-python' package may be helpful or requred for some packages. +$(ipydir)/meson-python-$(meson-python-version): \ + $(ipydir)/meson-$(meson-version) \ + $(ipydir)/packaging-$(packaging-version) \ + $(ipydir)/pyproject-metadata-$(pyproject-metadata-version) + tarball=meson-python-$(meson-python-version).tar.lz + $(call import-source, $(meson-python-url), $(meson-python-checksum)) + $(call pybuild, tar -xf, meson-python-$(meson-python-version), , \ + Meson-python $(meson-python-version), GPEP517) + echo "Meson-Python $(meson-python-version)" > $@ + $(ipydir)/mpi4py-$(mpi4py-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ $(ibidir)/openmpi-$(openmpi-version) \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/python-installer-$(python-installer-version) tarball=mpi4py-$(mpi4py-version).tar.lz $(call import-source, $(mpi4py-url), $(mpi4py-checksum)) $(call pybuild, tar -xf, mpi4py-$(mpi4py-version)) cp $(dtexdir)/mpi4py.tex $(ictdir)/ echo "mpi4py $(mpi4py-version) \citep{mpi4py2011}" > $@ -$(ipydir)/mpmath-$(mpmath-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/mpmath-$(mpmath-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=mpmath-$(mpmath-version).tar.gz $(call import-source, $(mpmath-url), $(mpmath-checksum)) $(call pybuild, tar -xf, mpmath-$(mpmath-version), , \ mpmath $(mpmath-version)) +# Until 2025-02-22: we had 'export CFLAGS="--std=c99 $$CFLAGS"' before +# calling pybuild; but that doesn't seem to be necessary. $(ipydir)/numpy-$(numpy-version): \ - $(ibidir)/unzip-$(unzip-version) \ $(ipydir)/cython-$(cython-version) \ $(ibidir)/openblas-$(openblas-version) \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/pybind11-$(pybind11-version) \ + $(ipydir)/meson-python-$(meson-python-version) tarball=numpy-$(numpy-version).tar.lz $(call import-source, $(numpy-url), $(numpy-checksum)) if [ x$(on_mac_os) = xyes ]; then export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle" - else - export LDFLAGS="$(LDFLAGS) -shared" fi - export CFLAGS="--std=c99 $$CFLAGS" conf="$$(pwd)/reproduce/software/config/numpy-scipy.cfg" $(call pybuild, tar -xf, numpy-$(numpy-version),$$conf, \ - Numpy $(numpy-version)) + Numpy $(numpy-version), GPEP517) cp $(dtexdir)/numpy.tex $(ictdir)/ - echo "Numpy $(numpy-version) \citep{numpy2011}" > $@ + echo "Numpy $(numpy-version) \citep{numpy2020}" > $@ $(ipydir)/packaging-$(packaging-version): \ $(ipydir)/pyparsing-$(pyparsing-version) tarball=packaging-$(packaging-version).tar.lz $(call import-source, $(packaging-url), $(packaging-checksum)) $(call pybuild, tar -xf, packaging-$(packaging-version), , \ - Packaging $(packaging-version)) + Packaging $(packaging-version), GPEP517) -$(ipydir)/pexpect-$(pexpect-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/pexpect-$(pexpect-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pexpect-$(pexpect-version).tar.gz $(call import-source, $(pexpect-url), $(pexpect-checksum)) $(call pybuild, tar -xf, pexpect-$(pexpect-version), , \ @@ -488,25 +758,32 @@ $(ipydir)/pexpect-$(pexpect-version): $(ipydir)/setuptools-$(setuptools-version) $(ipydir)/pillow-$(pillow-version): $(ibidir)/libjpeg-$(libjpeg-version) \ $(ipydir)/setuptools-$(setuptools-version) - tarball=Pillow-$(pillow-version).tar.lz + tarball=pillow-$(pillow-version).tar.lz $(call import-source, $(pillow-url), $(pillow-checksum)) - $(call pybuild, tar -xf, Pillow-$(pillow-version), , \ - Pillow $(pillow-version)) - -$(ipydir)/pip-$(pip-version): $(ipydir)/setuptools-$(setuptools-version) - tarball=pip-$(pip-version).tar.gz - $(call import-source, $(pip-url), $(pip-checksum)) - $(call pybuild, tar -xf, pip-$(pip-version), , \ - PiP $(pip-version)) - -$(ipydir)/ply-$(ply-version): $(ipydir)/setuptools-$(setuptools-version) + $(call pybuild, tar -xf, pillow-$(pillow-version), , \ + Pillow $(pillow-version), GPEP517) + +# This should normally not be used, because it's a front-end that obstructs +# reproducibility - source URL; checksum of the tarball; build rule. +# $(ipydir)/pip-$(pip-version): \ +# $(ipydir)/python-installer-$(python-installer-version) \ +# $(ipydir)/wheel-$(wheel-version) +# tarball=pip-$(pip-version).tar.gz +# $(call import-source, $(pip-url), $(pip-checksum)) +# $(call pybuild, tar -xf, pip-$(pip-version), , \ +# PiP $(pip-version)) + +$(ipydir)/ply-$(ply-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=ply-$(ply-version).tar.lz $(call import-source, $(ply-url), $(ply-checksum)) $(call pybuild, tar -xf, ply-$(ply-version), , \ ply $(ply-version)) $(ipydir)/pycodestyle-$(pycodestyle-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pycodestyle-$(pycodestyle-version).tar.gz $(call import-source, $(pycodestyle-url), $(pycodestyle-checksum)) $(call pybuild, tar -xf, pycodestyle-$(pycodestyle-version), , \ @@ -515,16 +792,21 @@ $(ipydir)/pycodestyle-$(pycodestyle-version): \ $(ipydir)/pybind11-$(pybind11-version): \ $(ibidir)/eigen-$(eigen-version) \ $(ibidir)/boost-$(boost-version) \ - $(ipydir)/setuptools-$(setuptools-version) - tarball=pybind11-$(pybind11-version).tar.gz + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) + tarball=pybind11-$(pybind11-version).tar.lz $(call import-source, $(pybind11-url), $(pybind11-checksum)) pyhook_after() { - cp -r include/pybind11 $(iidir)/python$(python-major-version)m/ + cp -pvr pybind11/include/pybind11 \ + $(iidir)/python$(python-major-version)m/ } $(call pybuild, tar -xf, pybind11-$(pybind11-version), , \ - pybind11 $(pybind11-version)) + pybind11 $(pybind11-version), GPEP517) + echo "Pybind11 $(pybind11-version)" > $@ -$(ipydir)/pycparser-$(pycparser-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/pycparser-$(pycparser-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pycparser-$(pycparser-version).tar.gz $(call import-source, $(pycparser-url), $(pycparser-checksum)) $(call pybuild, tar -xf, pycparser-$(pycparser-version), , \ @@ -532,45 +814,83 @@ $(ipydir)/pycparser-$(pycparser-version): $(ipydir)/setuptools-$(setuptools-vers $(ipydir)/pyerfa-$(pyerfa-version): \ $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/packaging-$(packaging-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=pyerfa-$(pyerfa-version).tar.lz $(call import-source, $(pyerfa-url), $(pyerfa-checksum)) $(call pybuild, tar -xf, pyerfa-$(pyerfa-version), , \ - PyERFA $(pyerfa-version)) + PyERFA $(pyerfa-version), GPEP517) -$(ipydir)/pyflakes-$(pyflakes-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/pyflakes-$(pyflakes-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pyflakes-$(pyflakes-version).tar.gz $(call import-source, $(pyflakes-url), $(pyflakes-checksum)) $(call pybuild, tar -xf, pyflakes-$(pyflakes-version), , \ pyflakes $(pyflakes-version)) $(ipydir)/pyparsing-$(pyparsing-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/flit-core-$(flit-core-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pyparsing-$(pyparsing-version).tar.lz $(call import-source, $(pyparsing-url), $(pyparsing-checksum)) $(call pybuild, tar -xf, pyparsing-$(pyparsing-version), , \ - PyParsing $(pyparsing-version)) + PyParsing $(pyparsing-version), GPEP517) -$(ipydir)/pypkgconfig-$(pypkgconfig-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/pypkgconfig-$(pypkgconfig-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pkgconfig-$(pypkgconfig-version).tar.gz $(call import-source, $(pypkgconfig-url), $(pypkgconfig-checksum)) $(call pybuild, tar -xf, pkgconfig-$(pypkgconfig-version), , pkgconfig $(pypkgconfig-version)) +$(ipydir)/pyproject-metadata-$(pyproject-metadata-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/flit-core-$(flit-core-version) \ + $(ipydir)/python-installer-$(python-installer-version) + tarball=pyproject-metadata-$(pyproject-metadata-version).tar.lz + $(call import-source, $(pyproject-metadata-url), $(pyproject-metadata-checksum)) + $(call pybuild, tar -xf, \ + pyproject-metadata-$(pyproject-metadata-version),,, GPEP517) + $(ipydir)/python-dateutil-$(python-dateutil-version): \ - $(ipydir)/six-$(six-version) \ - $(ipydir)/setuptools_scm-$(setuptools_scm-version) - tarball=python-dateutil-$(python-dateutil-version).tar.gz + $(ipydir)/setuptools-scm-$(setuptools-scm-version) + tarball=python-dateutil-$(python-dateutil-version).tar.lz $(call import-source, $(python-dateutil-url), $(python-dateutil-checksum)) $(call pybuild, tar -xf, python-dateutil-$(python-dateutil-version), , \ - python-dateutil $(python-dateutil-version)) + python-dateutil $(python-dateutil-version), GPEP517) + +$(ipydir)/python-installer-$(python-installer-version): \ + $(ibidir)/python-$(python-version) + +# Prepare the tarball. + tarball=python-installer-$(python-installer-version).tar.lz + $(call import-source, $(python-installer-url), $(python-installer-checksum)) + +# Modify the line in the source that will cause a crash when a +# to-be-installed file already exists in the installation path. This +# is very important for Python packages in Maneage (when a dependency +# is updated, the package needs to be re-built, but that would cause +# due to this line). + pyhook_before(){ + mv -v src/installer/destinations.py src/installer/destinations.py.orig; \ + sed -e 's/\(raise FileExistsError.message.\)/## \1/' \ + src/installer/destinations.py.orig > src/installer/destinations.py + } + +# Build the Python installer. + $(call pybuild, tar -xf, \ + python-installer-$(python-installer-version),,, \ + BOOT_INSTALLER) + echo "Python-installer $(python-installer-version)" > $@ $(ipydir)/pythran-$(pythran-version): \ $(ipydir)/ply-$(ply-version) \ $(ipydir)/gast-$(gast-version) \ $(ibidir)/boost-$(boost-version) \ $(ipydir)/beniget-$(beniget-version) \ - $(ipydir)/setuptools_scm-$(setuptools_scm-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=pythran-$(pythran-version).tar.lz $(call import-source, $(pythran-url), $(pythran-checksum)) $(call pybuild, tar -xf, pythran-$(pythran-version), , \ @@ -579,36 +899,59 @@ $(ipydir)/pythran-$(pythran-version): \ $(ipydir)/pyyaml-$(pyyaml-version): \ $(ibidir)/yaml-$(yaml-version) \ $(ipydir)/cython-$(cython-version) - tarball=pyyaml-$(pyyaml-version).tar.gz + tarball=pyyaml-$(pyyaml-version).tar.lz $(call import-source, $(pyyaml-url), $(pyyaml-checksum)) - $(call pybuild, tar -xf, PyYAML-$(pyyaml-version), , \ - PyYAML $(pyyaml-version)) + $(call pybuild, tar -xf, pyyaml-$(pyyaml-version), , \ + PyYAML $(pyyaml-version), GPEP517) $(ipydir)/requests-$(requests-version): $(ipydir)/idna-$(idna-version) \ - $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/certifi-$(certifi-version) \ - $(ipydir)/chardet-$(chardet-version) \ - $(ipydir)/urllib3-$(urllib3-version) + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/certifi-$(certifi-version) \ + $(ipydir)/chardet-$(chardet-version) \ + $(ipydir)/urllib3-$(urllib3-version) tarball=requests-$(requests-version).tar.gz $(call import-source, $(requests-url), $(requests-checksum)) $(call pybuild, tar -xf, requests-$(requests-version), , \ Requests $(requests-version)) +# 'pythran' is disabled in the build of Scipy because of complications it +# caused on some systems. We explicitly disable it using a preprocessor +# directive. 'Pythran' can in principle speed up compilation of scientific +# software [1][2]. +# [1] https://pythran.readthedocs.io/en/latest +# [2] https://docs.scipy.org/doc/scipy-1.15.2/dev/roadmap-detailed.html $(ipydir)/scipy-$(scipy-version): \ $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/pythran-$(pythran-version) \ $(ipydir)/pybind11-$(pybind11-version) tarball=scipy-$(scipy-version).tar.lz $(call import-source, $(scipy-url), $(scipy-checksum)) if [ x$(on_mac_os) = xyes ]; then export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle" else - export LDFLAGS="$(LDFLAGS) -shared" +# Same question as for 'numpy': why '-shared'? This obstructs +# the meson build. +# export LDFLAGS="$(LDFLAGS) -shared" + : fi conf="$$(pwd)/reproduce/software/config/numpy-scipy.cfg" - $(call pybuild, tar -xf, scipy-$(scipy-version),$$conf) + +# Disable pythran: see +# https://docs.scipy.org/doc/scipy-1.15.2/dev/roadmap-detailed.html#use-of-pythran +# export SCIPY_USE_PYTHRAN=0 # deprecated(?) +# Option 1: Hack the source: + pyhook_before() { + mv -iv meson.options meson.options.orig; \ + sed -e 's/\(use-pythran.*value: *\)true/\1false/' \ + meson.options.orig > meson.options + } + +# Option 2: pass the string +# --config-json='{"setup-args": "-Duse-pythran=false"}' +# to gpep517 with correct escaping of single and double quotes. +# Not tried as of 2025-02-25. + $(call pybuild, tar -xf, scipy-$(scipy-version),$$conf,, GPEP517) cp $(dtexdir)/scipy.tex $(ictdir)/ - echo "Scipy $(scipy-version) \citep{scipy2007,scipy2011}" > $@ + echo "Scipy $(scipy-version) \citep{scipy2020}" > $@ $(ipydir)/secretstorage-$(secretstorage-version): \ $(ipydir)/jeepney-$(jeepney-version) \ @@ -618,27 +961,38 @@ $(ipydir)/secretstorage-$(secretstorage-version): \ $(call pybuild, tar -xf, SecretStorage-$(secretstorage-version), , \ SecretStorage $(secretstorage-version)) +$(ipydir)/semantic-version-$(semantic-version-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=semantic-version-$(semantic-version-version).tar.lz + $(call import-source, $(semantic-version-url), \ + $(semantic-version-checksum)) + $(call pybuild, tar -xf, \ + semantic-version-$(semantic-version-version), , \ + Semantic-version $(semantic-version-version), GPEP517) + $(ipydir)/setuptools-$(setuptools-version): \ - $(ibidir)/unzip-$(unzip-version) \ - $(ibidir)/python-$(python-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=setuptools-$(setuptools-version).tar.lz $(call import-source, $(setuptools-url), $(setuptools-checksum)) $(call pybuild, tar -xf, setuptools-$(setuptools-version), , \ - Setuptools $(setuptools-version)) + Setuptools $(setuptools-version), GPEP517) -$(ipydir)/setuptools_scm-$(setuptools_scm-version): \ +$(ipydir)/setuptools-scm-$(setuptools-scm-version): \ $(ipydir)/setuptools-$(setuptools-version) - tarball=setuptools_scm-$(setuptools_scm-version).tar.gz - $(call import-source, $(setuptools_scm-url), $(setuptools_scm-checksum)) - $(call pybuild, tar -xf, setuptools_scm-$(setuptools_scm-version), , \ - Setuptools-scm $(setuptools_scm-version)) + tarball=setuptools-scm-$(setuptools-scm-version).tar.lz + $(call import-source, $(setuptools-scm-url), $(setuptools-scm-checksum)) + $(call pybuild, tar -xf, setuptools-scm-$(setuptools-scm-version), , \ + Setuptools-scm $(setuptools-scm-version), GPEP517) $(ipydir)/setuptools-rust-$(setuptools-rust-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) \ + $(ipydir)/semantic-version-$(semantic-version-version) tarball=setuptools-rust-$(setuptools-rust-version).tar.lz - $(call import-source, $(setuptools-rust-url), $(setuptools-rust-checksum)) + $(call import-source, $(setuptools-rust-url), \ + $(setuptools-rust-checksum)) $(call pybuild, tar -xf, setuptools-rust-$(setuptools-rust-version), , \ - Setuptools-scm $(setuptools-rust-version)) + Setuptools-rust $(setuptools-rust-version), GPEP517) $(ipydir)/sip_tpv-$(sip_tpv-version): \ $(ipydir)/sympy-$(sympy-version) \ @@ -649,14 +1003,17 @@ $(ipydir)/sip_tpv-$(sip_tpv-version): \ cp $(dtexdir)/sip_tpv.tex $(ictdir)/ echo "sip_tpv $(sip_tpv-version) \citep{sip-tpv}" > $@ - -$(ipydir)/six-$(six-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/six-$(six-version): \ + $(ipydir)/setuptools-$(setuptools-version) tarball=six-$(six-version).tar.lz $(call import-source, $(six-url), $(six-checksum)) $(call pybuild, tar -xf, six-$(six-version), , \ - Six $(six-version)) + Six $(six-version), GPEP517) + echo "Six $(six-version)" > $@ -$(ipydir)/soupsieve-$(soupsieve-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/soupsieve-$(soupsieve-version): \ + $(ipydir)/setuptools-$(setuptools-version) \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=soupsieve-$(soupsieve-version).tar.gz $(call import-source, $(soupsieve-url), $(soupsieve-checksum)) $(call pybuild, tar -xf, soupsieve-$(soupsieve-version), , \ @@ -675,21 +1032,30 @@ $(ipydir)/uncertainties-$(uncertainties-version): $(ipydir)/numpy-$(numpy-versio $(call pybuild, tar -xf, uncertainties-$(uncertainties-version), , \ uncertainties $(uncertainties-version)) -$(ipydir)/urllib3-$(urllib3-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/urllib3-$(urllib3-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=urllib3-$(urllib3-version).tar.gz $(call import-source, $(urllib3-url), $(urllib3-checksum)) $(call pybuild, tar -xf, urllib3-$(urllib3-version), , \ Urllib3 $(urllib3-version)) $(ipydir)/webencodings-$(webencodings-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/setuptools-$(setuptools-version) \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=webencodings-$(webencodings-version).tar.gz $(call import-source, $(webencodings-url), $(webencodings-checksum)) $(call pybuild, tar -xf, webencodings-$(webencodings-version), , \ Webencodings $(webencodings-version)) -$(ipydir)/wheel-$(wheel-version): $(ipydir)/setuptools-$(setuptools-version) - tarball=wheel-$(wheel-version).tar.lz +# As of 2025-02, this is only needed if you want 'wheel' on the command +# line; 'setuptools' provides its own version of wheels. +$(ipydir)/wheel-$(wheel-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/flit-core-$(flit-core-version) \ + $(ipydir)/python-installer-$(python-installer-version) +# tarball=wheel-$(wheel-version).tar.lz + tarball=wheel-$(wheel-version).tar.gz $(call import-source, $(wheel-url), $(wheel-checksum)) $(call pybuild, tar -xf, wheel-$(wheel-version), , \ - Wheel $(wheel-version)) + Wheel $(wheel-version), GPEP517) diff --git a/reproduce/software/make/r-cran.mk b/reproduce/software/make/r-cran.mk index 617b8de..7c86c23 100644 --- a/reproduce/software/make/r-cran.mk +++ b/reproduce/software/make/r-cran.mk @@ -12,8 +12,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2022 Boud Roukema <boud@cosmo.torun.pl> -# Copyright (C) 2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2022-2025 Boud Roukema <boud@cosmo.torun.pl> +# Copyright (C) 2022-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -204,6 +204,7 @@ $(ibidir)/r-cran-$(r-cran-version): \ tar -xf $(tdir)/$$tarball unpackdir=R-$(r-cran-version) cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) # We need to manually remove the lines with '~autodetect~', they # cause the configure script to crash in version 4.0.2. They are used diff --git a/reproduce/software/make/xorg.mk b/reproduce/software/make/xorg.mk index dd707e5..864c32a 100644 --- a/reproduce/software/make/xorg.mk +++ b/reproduce/software/make/xorg.mk @@ -14,8 +14,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2021-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2021-2022 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2021-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2021-2025 Raul Infante-Sainz <infantesainz@gmail.com> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -79,7 +79,7 @@ $(ibidir)/libxau-$(libxau-version): $(ibidir)/xorgproto-$(xorgproto-version) # Library implementing the X Display Manager Control Protocol. $(ibidir)/libxdmcp-$(libxdmcp-version): $(ibidir)/libxau-$(libxau-version) - tarball=libXdmcp-$(libxdmcp-version).tar.bz2 + tarball=libXdmcp-$(libxdmcp-version).tar.lz $(call import-source, $(libxdmcp-url), $(libxdmcp-checksum)) $(call gbuild, libXdmcp-$(libxdmcp-version),,$(XORG_CONFIG), V=1) echo "libXdmcp (Xorg) $(libxdmcp-version)" > $@ |