diff options
Diffstat (limited to 'reproduce/software/make')
| -rw-r--r-- | reproduce/software/make/README.md | 2 | ||||
| -rw-r--r-- | reproduce/software/make/atlas-multiple.mk | 2 | ||||
| -rw-r--r-- | reproduce/software/make/atlas-single.mk | 2 | ||||
| -rw-r--r-- | reproduce/software/make/basic.mk | 599 | ||||
| -rw-r--r-- | reproduce/software/make/build-rules.mk | 53 | ||||
| -rw-r--r-- | reproduce/software/make/high-level.mk | 1112 | ||||
| -rw-r--r-- | reproduce/software/make/python.mk | 957 | ||||
| -rw-r--r-- | reproduce/software/make/r-cran.mk | 5 | ||||
| -rw-r--r-- | reproduce/software/make/xorg.mk | 7 |
9 files changed, 2209 insertions, 530 deletions
diff --git a/reproduce/software/make/README.md b/reproduce/software/make/README.md index 76ab5c1..0d1464d 100644 --- a/reproduce/software/make/README.md +++ b/reproduce/software/make/README.md @@ -1,7 +1,7 @@ Software building instructions ------------------------------ -Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org>\ +Copyright (C) 2019-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org>\ See the end of the file for license conditions. This directory contains Makefiles that are called by the high-level diff --git a/reproduce/software/make/atlas-multiple.mk b/reproduce/software/make/atlas-multiple.mk index b92fbfc..a20f558 100644 --- a/reproduce/software/make/atlas-multiple.mk +++ b/reproduce/software/make/atlas-multiple.mk @@ -8,7 +8,7 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/make/atlas-single.mk b/reproduce/software/make/atlas-single.mk index 720bad6..79f5b42 100644 --- a/reproduce/software/make/atlas-single.mk +++ b/reproduce/software/make/atlas-single.mk @@ -8,7 +8,7 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index d90d945..b045bb1 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -21,9 +21,9 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2019-2022 Raul Infante-Sainz <infantesainz@gmail.com> -# Copyright (C) 2022 Pedram Ashofteh Ardakani <pedramardakani@pm.me> +# Copyright (C) 2018-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2026 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2022-2026 Pedram Ashofteh Ardakani <pedramardakani@pm.me> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,7 +39,7 @@ # along with this Makefile. If not, see <http://www.gnu.org/licenses/>. # Top level environment -include reproduce/software/config/LOCAL.conf +include .build/software/config/LOCAL.conf include reproduce/software/make/build-rules.mk include reproduce/software/config/versions.conf include reproduce/software/config/checksums.conf @@ -49,22 +49,27 @@ include reproduce/software/config/checksums.conf include reproduce/software/config/urls.conf # Basic directories -lockdir = $(BDIR)/software/locks -tdir = $(BDIR)/software/tarballs -ddir = $(BDIR)/software/build-tmp -idir = $(BDIR)/software/installed -ibdir = $(BDIR)/software/installed/bin -ildir = $(BDIR)/software/installed/lib -ibidir = $(BDIR)/software/installed/version-info/proglib +lockdir = $(BDIR)/software/locks +tdir = $(BDIR)/software/tarballs +ddir = $(BDIR)/software/build-tmp +idir = $(BDIR)/software/installed +ibdir = $(BDIR)/software/installed/bin +ildir = $(BDIR)/software/installed/lib +iidir = $(BDIR)/software/installed/include +shsrcdir = "$(shell pwd)"/reproduce/software/shell +ibidir = $(BDIR)/software/installed/version-info/proglib # Ultimate Makefile target. GNU Nano (a simple and very light-weight text # editor) is installed by default, it is recommended to have it in the # 'basic.mk', so Maneaged projects can be edited on any system (even when # there is no command-line text editor available). +# +# The recipe is '@echo > /dev/null' so Make does not print "make: Nothing +# to be done for 'all'." targets-proglib = low-level-links \ gcc-$(gcc-version) \ nano-$(nano-version) -all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) +all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)); @echo > /dev/null # Define the shell environment # ---------------------------- @@ -82,27 +87,44 @@ all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) # own build-directory comes first. .ONESHELL: .SHELLFLAGS := -e -c +export LC_ALL := C export CCACHE_DISABLE := 1 export SHELL := $(ibdir)/dash export PATH := $(ibdir):$(PATH) export PKG_CONFIG_PATH := $(ildir)/pkgconfig export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) \ - -Wno-nullability-completeness export LDFLAGS := $(rpath_command) -L$(ildir) $(LDFLAGS) # Disable built-in rules (which are not needed here!) .SUFFIXES: +# See description of '-Wno-nullability-completeness' in +# 'reproduce/software/shell/configure.sh'. +ifeq ($(on_mac_os),yes) + noccwarnings=-Wno-nullability-completeness +endif +export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) $(noccwarnings) + # This is the "basic" tools where we are relying on the host operating -# system, but are slowly populating our basic software envirnoment. To run -# (system or template) programs, 'LD_LIBRARY_PATH' is necessary, so here, -# we'll first tell the programs to look into any possible pre-defined -# 'LD_LIBRARY_PATH', then we'll add our own newly installed libraries. We -# will also make sure that there is no "current directory" in it (by -# removing a starting or trailing ':' and any occurance of '::'. -export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH):$(ildir) \ +# system, but are slowly populating our basic software envirnoment. So +# here, we are also appending the host's 'LD_LIBRARY_PATH' to Maneage's +# installed library path. It is just important to keep Maneage's library +# directories first so when something has been built, the newly built +# software use (link-with) Maneage's libraries, not the host's. We will +# also make sure that there is no "current directory" in it (by removing a +# starting or trailing ':' and any occurance of '::'). +# +# But first: in case LD_LIBRARY_PATH is empty, give it the default value of +# $(sys_library_sh_path) (which was the location of the libraries needed by +# the host's shell). This is because after we add the Maneage's library +# path, on some systems, no other libraries will be checked except those +# that are in 'LD_LIBRARY_PATH'. +ifeq ($(strip $(LD_LIBRARY_PATH)),) +export LD_LIBRARY_PATH=$(sys_library_sh_path) +endif +export LD_LIBRARY_PATH := $(shell echo $(ildir):$(LD_LIBRARY_PATH) \ | sed -e's/::/:/g' -e's/^://' -e's/:$$//') +ldpathorig := $(subst $(ildir):,,$(LD_LIBRARY_PATH)) # RPATH is automatically written in macOS, so 'DYLD_LIBRARY_PATH' is # ultimately redundant. But on some systems, even having a single value @@ -121,17 +143,24 @@ export DYLD_LIBRARY_PATH := # Afer putting everything together, we use the first server as the # reference for all software if their '-url' variable isn't defined (in # 'reproduce/software/config/urls.conf'). -downloadwrapper = ./reproduce/analysis/bash/download-multi-try +downloadwrapper = ./reproduce/analysis/bash/download-multi-try.sh maneage_backup_urls := $(shell awk '!/^#/{printf "%s ", $$1}' \ reproduce/software/config/servers-backup.conf) backupservers_all = $(user_backup_urls) $(maneage_backup_urls) topbackupserver = $(word 1, $(backupservers_all)) backupservers = $(filter-out $(topbackupserver),$(backupservers_all)) - - - - +# When building in Apptainer containers, as of 2025-04-18, we need to +# configure Maneage as root (within the container). In such cases, we need +# to activate the 'FORCE_UNSAFE_CONFIGURE' environment variable to build +# some of the software. The 'if' statement is here to make sure we are in +# Apptainer: in other situations, the "unsafe" configure script shouldn't +# be activated. Note that this doesn't happen in Docker (where the Maneage +# source is in the same directory) because we build a non-root ('maneager' +# user there who executes the configure command. +unsafe-config = if [ $$(pwd) = "/home/maneager/source" ] \ + && [ $$(whoami) = root ]; then \ + export FORCE_UNSAFE_CONFIGURE=1; fi @@ -202,6 +231,7 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ $(call makelink,xcrun) $(call makelink,sysctl) $(call makelink,sw_vers) + $(call makelink,codesign) $(call makelink,dsymutil) $(call makelink,install_name_tool) @@ -222,6 +252,27 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ fi done +# Useful tools: 'ldd' (list libraries linked by binary on GNU +# systems) + $(call makelink,ldd) + +# zic: On some systems like Debian GNU/Linux, 'zic' may be in +# '/usr/sbin' and is available to users. But it is not normally in +# the 'PATH' of a non-root user. This is because the executables +# there are normally system-level commands, and most are executable +# only by the root user. + $(call makelink,zic) + if [ "x$$(command -v zic)" = x ]; then + zicfullpath=$$(PATH=$${PATH}:/usr/sbin command -v zic) + if [ "x$${zicfullpath}" = x ]; then + printf "Warning: 'zic' not found. You may need it for" + printf " higher-level packages such as 'tzdb'. Install" + printf " 'zic' if needed.\n" + else + ln -sf $$(realpath $${zicfullpath}) $(ibdir)/zic + fi + fi + # We want this to be empty (so it doesn't interefere with the other # files in 'ibidir'. touch $@ @@ -240,11 +291,8 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ # # The first set of programs to be built are those that we need to unpack # the source code tarballs of each program. We have already installed Lzip -# before calling 'basic.mk', so it is present and working. Hence we first -# build the Lzipped tarball of Gzip, then use our own Gzip to unpack the -# tarballs of the other compression programs. Once all the compression -# programs/libraries are complete, we build our own GNU Tar and continue -# with other software. +# before calling 'basic.mk', so it is present and working. So the only +# prerequisites of these (until reaching Tar) is the necessary directories. $(lockdir): | $(BDIR); mkdir $@ $(ibidir)/gzip-$(gzip-version): | $(ibdir) $(ildir) $(lockdir) tarball=gzip-$(gzip-version).tar.lz @@ -252,13 +300,13 @@ $(ibidir)/gzip-$(gzip-version): | $(ibdir) $(ildir) $(lockdir) $(call gbuild, gzip-$(gzip-version), static, , V=1) echo "GNU Gzip $(gzip-version)" > $@ -$(ibidir)/xz-$(xz-version): $(ibidir)/gzip-$(gzip-version) +$(ibidir)/xz-$(xz-version): | $(ibdir) $(ildir) $(lockdir) tarball=xz-$(xz-version).tar.lz $(call import-source, $(xz-url), $(xz-checksum)) $(call gbuild, xz-$(xz-version), static) echo "XZ Utils $(xz-version)" > $@ -$(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) +$(ibidir)/bzip2-$(bzip2-version): | $(ibdir) $(ildir) $(lockdir) # Download the tarball. tarball=bzip2-$(bzip2-version).tar.lz @@ -287,8 +335,9 @@ $(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) fi cd $(ddir) rm -rf $$tdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$tdir + $(shsrcdir)/prep-source.sh $(ibdir) sed -e 's@\(ln -s -f \)$$(PREFIX)/bin/@\1@' Makefile \ > Makefile.sed mv Makefile.sed Makefile @@ -303,57 +352,37 @@ $(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) ln -fs libbz2.so.$(bzip2-version) libbz2.so echo "Bzip2 $(bzip2-version)" > $@ -$(ibidir)/unzip-$(unzip-version): $(ibidir)/gzip-$(gzip-version) - tarball=unzip-$(unzip-version).tar.lz - $(call import-source, $(unzip-url), $(unzip-checksum)) - $(call gbuild, unzip-$(unzip-version), static,, \ - -f unix/Makefile generic \ - CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ - -f unix/Makefile generic \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) - echo "Unzip $(unzip-version)" > $@ - -$(ibidir)/zip-$(zip-version): $(ibidir)/gzip-$(gzip-version) - tarball=zip-$(zip-version).tar.lz - $(call import-source, $(zip-url), $(zip-checksum)) - $(call gbuild, zip-$(zip-version), static,, \ - -f unix/Makefile generic \ - CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ - -f unix/Makefile generic \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) - echo "Zip $(zip-version)" > $@ - # Some programs (like Wget and CMake) that use zlib need it to be dynamic # so they use our custom build. So we won't force a static-only build. # # Note for a static-only build: Zlib's './configure' doesn't use Autoconf's # configure script, it just accepts a direct '--static' option. -$(ibidir)/zlib-$(zlib-version): $(ibidir)/gzip-$(gzip-version) +$(ibidir)/zlib-$(zlib-version): | $(ibdir) $(ildir) $(lockdir) tarball=zlib-$(zlib-version).tar.lz $(call import-source, $(zlib-url), $(zlib-checksum)) $(call gbuild, zlib-$(zlib-version)) echo "Zlib $(zlib-version)" > $@ # GNU Tar: When built statically, tar gives a segmentation fault on -# unpacking Bash. So we'll build it dynamically. Note that technically, zip -# and unzip aren't dependencies of Tar, but for a clean build, we'll set -# Tar to be the last compression-related software (the first-set of -# software to be built). +# unpacking Bash. So we'll build it dynamically. $(ibidir)/tar-$(tar-version): \ $(ibidir)/xz-$(xz-version) \ - $(ibidir)/zip-$(zip-version) \ $(ibidir)/gzip-$(gzip-version) \ $(ibidir)/zlib-$(zlib-version) \ - $(ibidir)/bzip2-$(bzip2-version) \ - $(ibidir)/unzip-$(unzip-version) + $(ibidir)/bzip2-$(bzip2-version) -# Since all later programs depend on Tar, the configuration will hit -# a bottleneck here: only making Tar. So its more efficient to built -# it on multiple threads (even when the user's Make doesn't pass down -# the number of threads). +# About the configurations: disabled because they are not relevant in +# the context of Maneage usage. +# - nls: conflicts with iconv on macOS. +# - acl: causing crashes when Tar was compiled with GCC 16.1.1. + $(call unsafe-config) tarball=tar-$(tar-version).tar.lz $(call import-source, $(tar-url), $(tar-checksum)) - $(call gbuild, tar-$(tar-version), , , -j$(numthreads) V=1) + $(call gbuild, tar-$(tar-version), , \ + --disable-nls \ + --disable-acl \ + am_cv_func_iconv=no, \ + -j$(numthreads) V=1) echo "GNU Tar $(tar-version)" > $@ @@ -370,8 +399,8 @@ $(ibidir)/tar-$(tar-version): \ # Patchelf is necessary for some software on GNU/Linux systems, its job is # to manually insert RPATH into the dynamically-linked executable. Since # all the other software depend on Pathelf, to avoid manually repeating as -# a prerequisite (and forgetting in others causing bugs), we'll put it as a -# dependancy of 'tar'. +# a prerequisite (and forgetting in others causing bugs), it is installed +# in this phase (right after 'tar'). $(ibidir)/patchelf-$(patchelf-version): $(ibidir)/tar-$(tar-version) tarball=patchelf-$(patchelf-version).tar.lz $(call import-source, $(patchelf-url), $(patchelf-checksum)) @@ -382,6 +411,84 @@ $(ibidir)/patchelf-$(patchelf-version): $(ibidir)/tar-$(tar-version) echo "PatchELF $(patchelf-version)" > $@ fi +$(ibidir)/pkg-config-$(pkgconfig-version): $(ibidir)/tar-$(tar-version) + +# Download the tarball. + tarball=pkg-config-$(pkgconfig-version).tar.lz + $(call import-source, $(pkgconfig-url), $(pkgconfig-checksum)) + +# An existing 'libiconv' can cause a conflict with 'pkg-config' (this +# is why 'libiconv' depends on 'pkg-config'). On a clean build, +# 'pkg-config' is built first. But when we don't have a clean build +# (and 'libiconv' exists) there will be a problem. So before +# re-building 'pkg-config', we'll remove any installation of +# 'libiconv'. This can create a known bug: +# https://savannah.nongnu.org/bugs/?67001 + rm -f $(ildir)/libiconv* $(idir)/include/iconv.h + +# On GNU/Linux no changes necessary, so we'll start with those. If +# the runner is on a macOS, some changes are necessary. + export compiler="" + confsh=./configure + if [ x$(on_mac_os) = xyes ]; then + +# On macOS we force Clang, because some GCC versions shipped with +# Xcode lack features needed to build Glib. + export compiler="CC=clang" + +# The bundled glib's gatomic.c uses integer-to-pointer conversions +# that newer Clang (Xcode 26+) treats as hard errors on all macOS +# architectures. + CFLAGS="-Wno-int-conversion $$CFLAGS" + +# On macOS ARM64 the bundled glib sub-configure checks for Carbon +# support with the C preprocessor only. Since 'Carbon/Carbon.h' +# exists in the Xcode SDK on Apple Silicon, the check passes and +# '-framework Carbon' is appended to LDFLAGS. But Carbon cannot +# actually be linked on ARM64, so every subsequent linker test in +# glib's configure fails silently and libglib-2.0.la is never +# created. We work around this by passing gbuild a tiny wrapper +# script (instead of ./configure) that patches glib/configure to +# force glib_have_carbon=no before the real configure runs. +# +# NOTE: the initial two SPACES in the SED command are very +# important (see 'glib/configure'): without them, the wrong +# variable will be set. + confsh=$(ddir)/pkg-config-configure + printf '#!/bin/sh\n' > $$confsh + printf 'sed -i.bak "s/^ glib_have_carbon=yes$$/' >> $$confsh + printf ' glib_have_carbon=no/"' >> $$confsh + printf ' glib/configure\n' >> $$confsh + printf 'exec ./configure "$$@"\n' >> $$confsh + chmod +x $$confsh + +# For its 'glib' installation within macOS, pkg-config needs a +# working Python for its build (not affecting its final code), but +# only searches for '/usr/bin/python'. However, some users may have +# Python available in another location, so we'll set it externally +# here to be safe. Note that 'which' is not present on some +# systems, 'command -v' is more portable). + if type python > /dev/null 2> /dev/null; then + pypath=$$(command -v python) + elif type python3 > /dev/null 2> /dev/null; then + pypath=$$(command -v python3) + else + printf "pkg-config: ERROR: a local Python is necessary"; exit 1 + fi + export PYTHON=$$pypath + fi + +# Configuration options: +# --with-internal-glib: the internal glib is necessary because +# without it, we have many problems in macOS. + export CFLAGS="-std=$(std_c_old) $$CFLAGS" + $(call gbuild, pkg-config-$(pkgconfig-version), static, \ + $$compiler \ + --with-internal-glib \ + --with-pc-path=$(ildir)/pkgconfig, V=1, , $$confsh) + if [ -f $$confsh ]; then rm $$confsh; fi # Only on macOS. + echo "pkg-config $(pkgconfig-version)" > $@ + @@ -402,7 +509,10 @@ $(ibidir)/patchelf-$(patchelf-version): $(ibidir)/tar-$(tar-version) # function (for tilde expansion). The first can be disabled with # '--disable-load', but unfortunately I don't know any way to fix the # second. So, we'll have to build it dynamically for now. -$(ibidir)/ncurses-$(ncurses-version): $(ibidir)/patchelf-$(patchelf-version) +$(ibidir)/ncurses-$(ncurses-version): $(ibidir)/patchelf-$(patchelf-version) \ + $(ibidir)/pkg-config-$(pkgconfig-version) + +# Prepare the input. tarball=ncurses-$(ncurses-version).tar.lz $(call import-source, $(ncurses-url), $(ncurses-checksum)) @@ -418,11 +528,18 @@ $(ibidir)/ncurses-$(ncurses-version): $(ibidir)/patchelf-$(patchelf-version) rm -f $(ibdir)/bash* $(ibdir)/awk* $(ibdir)/gawk* # Standard build process. + export CFLAGS="-std=$(std_c_old) $$CFLAGS" $(call gbuild, ncurses-$(ncurses-version), static, \ - --with-shared --enable-rpath --without-normal \ - --without-debug --with-cxx-binding \ - --with-cxx-shared --enable-widec --enable-pc-files \ - --with-pkg-config=$(ildir)/pkgconfig, -j$(numthreads)) + --with-shared \ + --enable-rpath \ + --enable-widec \ + --without-debug \ + --without-normal \ + --enable-pc-files \ + --with-cxx-shared \ + --with-cxx-binding \ + --with-pkg-config=$(ildir)/pkgconfig, \ + -j$(numthreads)) # Unfortunately there are many problems with 'ncurses' using "normal" # (or 8-bit) characters. The standard way that will work is to build @@ -461,6 +578,9 @@ $(ibidir)/ncurses-$(ncurses-version): $(ibidir)/patchelf-$(patchelf-version) # # 5. A link is made to also be able to include files from the # 'ncurses' headers. +# +# 6. Top-level symbolic links are made for the 'include' (.h) files. +# if [ x$(on_mac_os) = xyes ]; then so="dylib"; else so="so"; fi if [ -f $(ildir)/libncursesw.$$so ]; then @@ -485,6 +605,11 @@ $(ibidir)/ncurses-$(ncurses-version): $(ibidir)/patchelf-$(patchelf-version) ln -fs $(ildir)/pkgconfig/ncursesw.pc pkgconfig/cursesw.pc ln -fs $(idir)/include/ncursesw $(idir)/include/ncurses + +# Add symbolic links for the ncursesw/*.h 'include' files +# so that they can be found in the top-level include/ directory. + cd "$(iidir)" + ln -fsv $(idir)/include/ncursesw/*.h . echo "GNU NCURSES $(ncurses-version)" > $@ else exit 1 @@ -523,6 +648,7 @@ $(ibidir)/readline-$(readline-version): \ # pname=bash50-$(printf "%03d" $i); \ # wget http://ftp.gnu.org/gnu/bash/bash-5.0-patches/$pname -O ../$pname;\ # patch -p0 -i ../$pname; \ +# rm ../$pname; \ # done # $ cd .. # $ mv bash-5.0 bash-5.0.$number @@ -551,7 +677,7 @@ $(ibidir)/bash-$(bash-version): \ export CFLAGS="$$CFLAGS \ -DDEFAULT_PATH_VALUE='\"$(ibdir)\"' \ -DSTANDARD_UTILS_PATH='\"$(ibdir)\"' \ - -DSYS_BASHRC='\"$(BASH_ENV)\"' " + -DSYS_BASHRC='\"$(BASH_ENV)\"'" $(call gbuild, bash-$(bash-version),, $$stopt \ --with-installed-readline=$(ildir) \ --with-curses=yes, \ @@ -568,9 +694,6 @@ $(ibidir)/bash-$(bash-version): \ # shell. By convention, 'sh' is just a symbolic link to the preferred # shell executable. So we'll define '$(ibdir)/sh' as a symbolic link # to the Bash that we just built and installed. -# -# Just to be sure that the installation step above went well, before -# making the link, we'll see if the file actually exists there. ln -fs $(ibdir)/bash $(ibdir)/sh echo "GNU Bash $(bash-version)" > $@ @@ -596,44 +719,58 @@ perl-conflddlflags = else perl-conflddlflags = -Dlddlflags="-shared $$LDFLAGS" endif -$(ibidir)/perl-$(perl-version): $(ibidir)/patchelf-$(patchelf-version) +$(ibidir)/perl-$(perl-version): $(ibidir)/tar-$(tar-version) + +# Import tarball. tarball=perl-$(perl-version).tar.lz $(call import-source, $(perl-url), $(perl-checksum)) + +# Extract the version strings. major_version=$$(echo $(perl-version) \ | sed -e's/\./ /g' \ | awk '{printf("%d", $$1)}') base_version=$$(echo $(perl-version) \ | sed -e's/\./ /g' \ | awk '{printf("%d.%d", $$1, $$2)}') + +# Unpack, build and install. cd $(ddir) rm -rf perl-$(perl-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd perl-$(perl-version) + $(shsrcdir)/prep-source.sh $(ibdir) ./Configure -des \ -Dusethreads \ -Duseshrplib \ -Dprefix=$(idir) \ -Dvendorprefix=$(idir) \ - -Dprivlib=$(idir)/share/perl$$major_version/core_perl \ - -Darchlib=$(idir)/lib/perl$$major_version/$$base_version/core_perl \ - -Dsitelib=$(idir)/share/perl$$major_version/site_perl \ - -Dsitearch=$(idir)/lib/perl$$major_version/$$base_version/site_perl \ - -Dvendorlib=$(idir)/share/perl$$major_version/vendor_perl \ - -Dvendorarch=$(idir)/lib/perl$$major_version/$$base_version/vendor_perl \ - -Dscriptdir=$(idir)/bin/core_perl \ - -Dsitescript=$(idir)/bin/site_perl \ - -Dvendorscript=$(idir)/bin/vendor_perl \ + -Dprivlib=$(idir)/share/perl$$major_version/perl-core \ + -Darchlib=$(idir)/lib/perl$$major_version/$$base_version/perl-core \ + -Dsitelib=$(idir)/share/perl$$major_version/perl-site \ + -Dsitearch=$(idir)/lib/perl$$major_version/$$base_version/perl-site \ + -Dvendorlib=$(idir)/share/perl$$major_version/perl-vendor \ + -Dvendorarch=$(idir)/lib/perl$$major_version/$$base_version/perl-vendor \ + -Dscriptdir=$(idir)/bin/perl-core \ + -Dsitescript=$(idir)/bin/perl-site \ + -Dvendorscript=$(idir)/bin/perl-vendor \ -Dinc_version_list=none \ -Dman1ext=1perl \ -Dman3ext=3perl \ -Dcccdlflags='-fPIC' \ $(perl-conflddlflags) \ -Dldflags="$$LDFLAGS" - make -j$(numthreads) + make -j$(numthreads) V=1 make install cd .. rm -rf perl-$(perl-version) - cd $$topdir + +# Symbolic links to necessary Perl programs (that Perl installs in +# the 'perl-core' directory. + ln -sf $(ibdir)/perl-core/xsubpp $(ibdir)/xsubpp + ln -sf $(ibdir)/perl-core/pod2man $(ibdir)/pod2man + ln -sf $(ibdir)/perl-core/pod2text $(ibdir)/pod2text + +# Final target. echo "Perl $(perl-version)" > $@ @@ -667,20 +804,16 @@ $(ibidir)/coreutils-$(coreutils-version): \ $(ibidir)/openssl-$(openssl-version) # Import, unpack and enter the source directory. + $(call unsafe-config) tarball=coreutils-$(coreutils-version).tar.lz $(call import-source, $(coreutils-url), $(coreutils-checksum)) + +# Unpack and enter the source. cd $(ddir) rm -rf coreutils-$(coreutils-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd coreutils-$(coreutils-version) - -# Set the configure script to use our shell, note that we can't -# assume GNU SED here yet (it installs after Coreutils). - sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ - configure > configure-tmp - mv configure-tmp configure - chmod +x configure + $(shsrcdir)/prep-source.sh $(ibdir) # Configure, build and install Coreutils. ./configure --prefix=$(idir) SHELL=$(ibdir)/bash \ @@ -692,7 +825,7 @@ $(ibidir)/coreutils-$(coreutils-version): \ # Fix RPATH if necessary. if [ -f $(ibdir)/patchelf ]; then make SHELL=$(ibdir)/bash install DESTDIR=junkinst - unalias ls || true # avoid decorated 'ls' commands with extra characters + unalias ls || true # Not decorated 'ls' (with extra characters). instprogs=$$(ls junkinst/$(ibdir)) for f in $$instprogs; do $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/$$f @@ -706,28 +839,6 @@ $(ibidir)/coreutils-$(coreutils-version): \ rm -rf coreutils-$(coreutils-version) echo "GNU Coreutils $(coreutils-version)" > $@ -# Podlators -# -# POD is short for "Plain Old Documentation", that is the format used in -# Perl's documentation. Podlators provies two executables pod2man and -# pod2text convert this into the roff format (used in man pages) or pod2 It -# is used by some software like OpenSSL to create their man pages. -$(ibidir)/podlators-$(podlators-version): $(ibidir)/perl-$(perl-version) - tarball=podlators-$(podlators-version).tar.lz - $(call import-source, $(podlators-url), $(podlators-checksum)) - cd $(ddir) - rm -rf podlators-$(podlators-version) - tar -xf $(tdir)/$$tarball - cd podlators-$(podlators-version) - perl Makefile.PL - make - make install - ln -sf $(ibdir)/site_perl/pod2man $(ibdir)/pod2man - ln -sf $(ibdir)/site_perl/pod2text $(ibdir)/pod2text - cd .. - rm -rf podlators-$(podlators-version) - echo "podlators $(podlators-version)" > $@ - # OpenSSL # # Until we find a nice and generic way to create an updated CA file in the @@ -735,13 +846,14 @@ $(ibidir)/podlators-$(podlators-version): $(ibidir)/perl-$(perl-version) # along with the other tarballs. $(idir)/etc:; mkdir $@ $(idir)/etc/ssl: | $(idir)/etc; mkdir $@ -$(ibidir)/openssl-$(openssl-version): $(ibidir)/podlators-$(podlators-version) \ - | $(idir)/etc/ssl +$(ibidir)/openssl-$(openssl-version): $(ibidir)/perl-$(perl-version) \ + $(ibidir)/patchelf-$(patchelf-version) \ + | $(idir)/etc/ssl # First download the certificates and copy them into the # installation directory. tarball=cert.pem-$(certpem-version) - $(call import-source, $(cert-url), $(cert-checksum)) + $(call import-source, $(certpem-url), $(certpem-checksum)) cp $(tdir)/cert.pem-$(certpem-version) $(idir)/etc/ssl/cert.pem # Now download the OpenSSL tarball. @@ -807,6 +919,8 @@ $(ibidir)/curl-$(curl-version): $(ibidir)/coreutils-$(coreutils-version) --without-librtmp \ --without-libidn2 \ --without-wolfssl \ + --without-nghttp2 \ + --without-nghttp3 \ --without-brotli \ --without-gnutls \ --without-cyassl \ @@ -814,6 +928,7 @@ $(ibidir)/curl-$(curl-version): $(ibidir)/coreutils-$(coreutils-version) --without-axtls \ --disable-ldaps \ --disable-ldap \ + --without-zstd \ --without-nss, V=1) if [ -f $(ibdir)/patchelf ]; then @@ -843,6 +958,23 @@ $(ibidir)/wget-$(wget-version): \ tarball=wget-$(wget-version).tar.lz $(call import-source, $(wget-url), $(wget-checksum)) +# Wget needs a working Python for its tests (VERY BAD DESIGN!), this +# does not affect its final operation. But only searches for +# '/usr/bin/python'. However, some users may have Python available in +# another location, so we'll set it externally here to be safe. Note +# that 'which' is not present on some systems, 'command -v' is more +# portable). + if [ x$(on_mac_os) = xyes ]; then + if type python > /dev/null 2> /dev/null; then + pypath=$$(command -v python) + elif type python3 > /dev/null 2> /dev/null; then + pypath=$$(command -v python3) + else + printf "Wget: ERROR: a local Python is necessary"; exit 1 + fi + export PYTHON=$$pypath + fi + # We need to explicitly disable 'libiconv', because of the # 'pkg-config' and 'libiconv' problem. libs="-pthread" @@ -875,6 +1007,12 @@ $(ibidir)/wget-$(wget-version): \ # process of the higher-level programs and libraries. Note that during the # building of those higher-level programs (after this Makefile finishes), # there is no access to the system's PATH. +$(ibidir)/bison-$(bison-version): $(ibidir)/help2man-$(help2man-version) + tarball=bison-$(bison-version).tar.lz + $(call import-source, $(bison-url), $(bison-checksum)) + $(call gbuild, bison-$(bison-version), static, ,V=1 -j$(numthreads)) + echo "GNU Bison $(bison-version)" > $@ + $(ibidir)/diffutils-$(diffutils-version): \ $(ibidir)/coreutils-$(coreutils-version) tarball=diffutils-$(diffutils-version).tar.lz @@ -883,7 +1021,7 @@ $(ibidir)/diffutils-$(diffutils-version): \ echo "GNU Diffutils $(diffutils-version)" > $@ $(ibidir)/file-$(file-version): $(ibidir)/coreutils-$(coreutils-version) - export CFLAGS="-std=c99 $$CFLAGS" + export CFLAGS="-std=$(std_c_old) $$CFLAGS" tarball=file-$(file-version).tar.lz $(call import-source, $(file-url), $(file-checksum)) $(call gbuild, file-$(file-version), static, \ @@ -928,6 +1066,15 @@ $(ibidir)/gawk-$(gawk-version): \ # Build final target. echo "GNU AWK $(gawk-version)" > $@ +$(ibidir)/help2man-$(help2man-version): \ + $(ibidir)/coreutils-$(coreutils-version) + tarball=help2man-$(help2man-version).tar.lz + $(call import-source, $(help2man-url), $(help2man-checksum)) + $(call gbuild, help2man-$(help2man-version), static, ,V=1) + echo "Help2man $(Help2man-version)" > $@ + +# Even though libiconv does not use pkg-config, pkg-config conflicts with +# it, see the comments in pkg-config for more. $(ibidir)/libiconv-$(libiconv-version): \ $(ibidir)/pkg-config-$(pkgconfig-version) tarball=libiconv-$(libiconv-version).tar.lz @@ -943,12 +1090,14 @@ $(ibidir)/libunistring-$(libunistring-version): \ -j$(numthreads)) echo "GNU libunistring $(libunistring-version)" > $@ -$(ibidir)/libxml2-$(libxml2-version): $(ibidir)/patchelf-$(patchelf-version) +$(ibidir)/libxml2-$(libxml2-version): $(ibidir)/tar-$(tar-version) # The libxml2 tarball also contains Python bindings which are built # and installed to a system directory by default. If you don't need # the Python bindings, the easiest solution is to compile without # Python support: './configure --without-python'. If you really need -# the Python bindings, use '--with-python-install-dir=DIR' instead. +# its Python bindings: +# 1. Add the Python dependency. +# 2. Add the '--with-python-install-dir=DIR' configure option. tarball=libxml2-$(libxml2-version).tar.lz $(call import-source, $(libxml2-url), $(libxml2-checksum)) $(call gbuild, libxml2-$(libxml2-version), static, \ @@ -987,9 +1136,10 @@ $(ibidir)/gmp-$(gmp-version): \ $(ibidir)/coreutils-$(coreutils-version) tarball=gmp-$(gmp-version).tar.lz $(call import-source, $(gmp-url), $(gmp-checksum)) + export CFLAGS="-std=$(std_c_old) $$CFLAGS" $(call gbuild, gmp-$(gmp-version), static, \ --enable-cxx --enable-fat, \ - -j$(numthreads) ,make check) + -j$(numthreads)) echo "GNU Multiple Precision Arithmetic Library $(gmp-version)" > $@ # Less is useful with Git (to view the diffs within a minimal container) @@ -998,7 +1148,19 @@ $(ibidir)/gmp-$(gmp-version): \ $(ibidir)/less-$(less-version): $(ibidir)/ncurses-$(ncurses-version) tarball=less-$(less-version).tar.lz $(call import-source, $(less-url), $(less-checksum)) - $(call gbuild, less-$(less-version), static,,-j$(numthreads)) + +# Without the '--with-regex=posix' option, the build will depend on +# PCRE (perl compatible regular expressions) which are not available +# on some systems/compilers and can cause a crash. Maneage was +# successfully built with the POSIX regular expression (regex), and +# 'less' is generally, an interactive software, not a batch-mode +# software (it is just added in 'basic.mk' because Git uses it to +# display things. Again, this is an interactive meta-operation in +# maneage (operations you only do when you are developing Maneage +# within Maneage interactively, and will not affect into the actual +# reproducible analysis!) + $(call gbuild, less-$(less-version), static, \ + --with-regex=posix,-j$(numthreads)) if [ -f $(ibdir)/patchelf ]; then $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/less; fi @@ -1017,50 +1179,31 @@ $(ibidir)/libtool-$(libtool-version): $(ibidir)/m4-$(m4-version) $(ibidir)/grep-$(grep-version): $(ibidir)/coreutils-$(coreutils-version) tarball=grep-$(grep-version).tar.lz $(call import-source, $(grep-url), $(grep-checksum)) - $(call gbuild, grep-$(grep-version), static,,V=1) + $(call gbuild, grep-$(grep-version), static,, \ + -j$(numthreads) V=1) echo "GNU Grep $(grep-version)" > $@ -# M4 doesn't depend on PatchELF, but just to be consistent with the -# levels/phases introduced here (where the compressors are level 1, -# PatchELF is level 2, and ...), we'll set it as a dependency. -$(ibidir)/m4-$(m4-version): $(ibidir)/patchelf-$(patchelf-version) +# The '--with-syscmd-shell' is used as the default shell and if not given, +# 'm4' will use '/bin/sh' (which is not under Maneage control and can cause +# problems in 'high-level.mk' because it closes off the system's +# LD_LIBRARY_PATH and if the system's '/bin/sh' needs a special system +# library, the high-level programs will not be built). We are setting this +# default shell to Dash because M4 is built before our own Bash. Recall +# that Dash is built before we enter this Makefile. +$(ibidir)/m4-$(m4-version): $(ibidir)/libiconv-$(libiconv-version) tarball=m4-$(m4-version).tar.lz $(call import-source, $(m4-url), $(m4-checksum)) - $(call gbuild, m4-$(m4-version), static,,V=1) + $(call gbuild, m4-$(m4-version), static, \ + --with-syscmd-shell=$(ibdir)/dash, \ + -j$(numthreads) V=1) echo "GNU M4 $(m4-version)" > $@ $(ibidir)/mpfr-$(mpfr-version): $(ibidir)/gmp-$(gmp-version) tarball=mpfr-$(mpfr-version).tar.lz $(call import-source, $(mpfr-url), $(mpfr-checksum)) - $(call gbuild, mpfr-$(mpfr-version), static, , , make check) + $(call gbuild, mpfr-$(mpfr-version), static) echo "GNU Multiple Precision Floating-Point Reliably $(mpfr-version)" > $@ -$(ibidir)/pkg-config-$(pkgconfig-version): $(ibidir)/patchelf-$(patchelf-version) - -# Download the tarball. - tarball=pkg-config-$(pkgconfig-version).tar.lz - $(call import-source, $(pkgconfig-url), $(pkgconfig-checksum)) - -# An existing 'libiconv' can cause a conflict with 'pkg-config', this -# is why 'libiconv' depends on 'pkg-config'. On a clean build, -# 'pkg-config' is built first. But when we don't have a clean build -# (and 'libiconv' exists) there will be a problem. So before -# re-building 'pkg-config', we'll remove any installation of -# 'libiconv'. - rm -f $(ildir)/libiconv* $(idir)/include/iconv.h - -# Some Mac OS systems may have a version of the GNU C Compiler (GCC) -# installed that doesn't support some necessary features of building -# Glib (as part of pkg-config). So to be safe, for Mac systems, we'll -# make sure it will use LLVM's Clang. - if [ x$(on_mac_os) = xyes ]; then export compiler="CC=clang" - else export compiler="" - fi - $(call gbuild, pkg-config-$(pkgconfig-version), static, \ - $$compiler --with-internal-glib \ - --with-pc-path=$(ildir)/pkgconfig, V=1) - echo "pkg-config $(pkgconfig-version)" > $@ - $(ibidir)/sed-$(sed-version): $(ibidir)/coreutils-$(coreutils-version) tarball=sed-$(sed-version).tar.lz $(call import-source, $(sed-url), $(sed-checksum)) @@ -1070,6 +1213,17 @@ $(ibidir)/sed-$(sed-version): $(ibidir)/coreutils-$(coreutils-version) $(ibidir)/texinfo-$(texinfo-version): \ $(ibidir)/perl-$(perl-version) \ $(ibidir)/gettext-$(gettext-version) + +# Setting for the XS sub-package. "This is because in theory, the XS +# module could be built with a different compiler to the rest of the +# project, needing completely different flags" (part of [1]) +# +# [1] https://lists.gnu.org/archive/html/bug-texinfo/2022-08/msg00068.html + export PERL="$(ibdir)/perl" + export PERL_EXT_LDFLAGS="-L$(ildir)" + export PERL_EXT_CPPFLAGS="-I$(iidir)" + +# Basic build commands. tarball=texinfo-$(texinfo-version).tar.lz $(call import-source, $(texinfo-url), $(texinfo-checksum)) $(call gbuild, texinfo-$(texinfo-version), static) @@ -1105,7 +1259,7 @@ $(ibidir)/mpc-$(mpc-version): $(ibidir)/mpfr-$(mpfr-version) echo "" > $@ else $(call gbuild, mpc-$(mpc-version), static, , \ - -j$(numthreads), make check) + -j$(numthreads)) echo "GNU Multiple Precision Complex library" > $@ fi @@ -1136,6 +1290,7 @@ $(ibidir)/binutils-$(binutils-version): \ $(ibidir)/gawk-$(gawk-version) \ $(ibidir)/grep-$(grep-version) \ $(ibidir)/wget-$(wget-version) \ + $(ibidir)/bison-$(bison-version) \ $(ibidir)/which-$(which-version) \ $(ibidir)/libtool-$(libtool-version) \ $(ibidir)/texinfo-$(texinfo-version) \ @@ -1156,6 +1311,7 @@ $(ibidir)/binutils-$(binutils-version): \ $(call makelink,ld) $(call makelink,nm) $(call makelink,ps) + $(call makelink,lipo) $(call makelink,strip) $(call makelink,ranlib) echo "" > $@ @@ -1163,8 +1319,9 @@ $(ibidir)/binutils-$(binutils-version): \ # Build binutils with the standard 'gbuild' function. $(call gbuild, binutils-$(binutils-version), static, \ - --with-lib-path=$(sys_library_path), \ - -j$(numthreads) ) + --with-lib-path=$(sys_library_path) \ + --enable-gprofng=no, \ + -j$(numthreads) V=1) # The 'ld' linker of Binutils needs several '*crt*.o' files from # the host's GNU C Library to run. On some systems these object @@ -1287,20 +1444,49 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) current_dir=$$(pwd) # By default 'ddir' (where GCC is decompressed and built) is in the -# RAM (on systems that support '/dev/shm'). This is done to avoid -# building so many small/temporary files and possibly harming the -# hard-drive or SSD. But if the RAM doesn't have enough space, we -# should use the hard-drive or SSD. During its build GCC's build -# directory will become about 7GB (multiple of 1024 bytes, for GCC -# 11.2.0). So at this step, we are making sure we have more than -# 7.5GiB (multiple of 1000 bytes, which corresponds to 7.32GB) -# before GCC starts to build. Note that the 4th column of 'df' is -# the "available" space at the time of running, not the full -# space. So the background RAM that the OS will be using during -# Maneage is accounted for. Also consider that GCC is built alone -# (no other Maneage software is built at the same time as GCC). - in_ram=$$(df $(ddir) \ - | awk 'NR==2{print ($$4>7500000) ? "yes" : "no"}'); \ +# RAM (on systems that support a '/dev/shm' RAM disk). This is done +# to avoid building so many small/temporary files and possibly +# harming the hard-drive or SSD. But if the RAM doesn't have enough +# space, we should use the hard-drive or SSD. During its build, +# GCC's build directory will become several gigabytes and the build +# also needs RAM. You can track the RAM usage of the system with a +# 1-second resolution (if no other RAM consuming program is running +# while building GCC) with the command below (example outputs can +# be seen in https://savannah.nongnu.org/task/index.php?16623). +# +# c=1; while true; do POSIXLY_CORRECT=1 df -P /dev/shm/maneage-* | awk 'NR==2{print '$c', $3}'; c=$((c+1)); sleep 1; done > mem-usage.txt +# asttable mem-usage.txt -c1,'arith $2 512 x 1024 / 1024 / 1024 /' -o mem.fits +# +# For POSIX portability and longevity (default sizes might change), +# we use the '-P' option, and we use the environment variable +# POSIXLY_CORRECT=1, so the 'block size' is 512 bytes. In this way, +# to get the actual GiB amount, multiply the value returned above +# by 512 (B/block), then divide by 1024^3 (B/GiB). +# +# To get the final value to use here, get the maximum used value +# after GCC is fully built and you have stopped the 'while true' +# command above. You can do this with the command below (assumes +# you have Gnuastro). +# +# aststatistics mem-usage.txt -c2 --maximum | asttable -c'arith $1 7000000 +' -Afixed -B0 +# +# The extra space is because we will assume an extra 3 GiB = 3GiB * +# 1024^3 (B/GiB) / 512 (B/block) = 6291456 blocks are necessary for +# the building (let's round it to 7000000!). +# +# Therefore, we need to make sure that the running system more than +# the necessary amount of space in the RAM. To do this, we use 'df' +# below. +# +# The 4th column of 'df' is the "available" space at the time of +# running, not the full space. So the 'RAM disk' that the OS +# will be using as "pretend" disk space (e.g. using 'tmpfs'; this +# is physically RAM, but appears as if it is disk space) +# during this stage of Maneage is accounted for. GCC is built +# alone - no other Maneage software is built at the same time as +# GCC - so this amount of RAM should be enough. + in_ram=$$(POSIXLY_CORRECT=1 df -P $(ddir) \ + | awk 'NR==2{print ($$4>26613216) ? "yes" : "no"}'); \ if [ $$in_ram = "yes" ]; then odir=$(ddir) else odir=$(BDIR)/software/build-tmp-gcc-due-to-lack-of-space @@ -1308,15 +1494,28 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) mkdir $$odir fi -# Go into the proper directory, unpack GCC and prepare the 'build' -# directory inside it for all the built files. +# Go into the directory to uncompress GCC. cd $$odir + +# Unpack GCC and prepare the 'build' directory inside it for all +# the built files. rm -rf gcc-$(gcc-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions if [ $$odir != $(ddir) ]; then ln -s $$odir/gcc-$(gcc-version) $(ddir)/gcc-$(gcc-version) fi cd gcc-$(gcc-version) + $(shsrcdir)/prep-source.sh $(ibdir) + +# Unfortunately binutils installs headers like 'ansidecl.h' that +# have been seen to conflict with GCC's internal versions of those +# headers. For example in the 'ansidecl.h' of Binutils 2.39, the +# 'PTR' macro isn't defined, while the same file in GCC 12.1.0 has +# defined it. Therefore, without this change, GCC will include the +# file installed from Binutils, not find what it needs and crash! +# Therefore, with the 'CPPFLAGS' modification below, we tell GCC to +# first look into its own 'include' directory before anything else. + export CPPFLAGS="-I$$(pwd)/include $(CPPFLAGS)" # Set the build directory for the processing. mkdir build @@ -1396,12 +1595,39 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) +# Level 6: need re-compilation +# ---------------------------- +# +# The initial build of these was done with the host's settings, which will +# cause problems later when we completely close-off the host environment. +$(ibidir)/make-$(make-version): $(ibidir)/gcc-$(gcc-version) + tarball=make-$(make-version).tar.lz + $(call import-source, $(make-url), $(make-checksum)) + $(call gbuild, make-$(make-version), static, \ + --disable-dependency-tracking --without-guile) + echo "GNU Make $(make-version)" > $@ + +$(ibidir)/lzip-$(lzip-version): $(ibidir)/gcc-$(gcc-version) + tarball=lzip-$(lzip-version).tar + unpackdir=lzip-$(lzip-version) + cd $(ddir) + rm -rf $$unpackdir + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) + ./configure --build --check --installdir="$(ibdir)" + if [ -f $(ibdir)/patchelf ]; then + $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/lzip; + fi + cd .. + rm -rf $$unpackdir + echo "Lzip $(lzip-version)" > $@ -# Level 6: Basic text editor +# Level 7: Basic text editor # -------------------------- # # If the project is built in a minimal environment, there is no text @@ -1418,7 +1644,8 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) # nano (and use their own optional high-level text editor). To do this, you # can just have to manually remove 'nano' from 'targets-proglib' above and # add their optional text editor in 'TARGETS.conf'. -$(ibidir)/nano-$(nano-version): $(ibidir)/gcc-$(gcc-version) +$(ibidir)/nano-$(nano-version): $(ibidir)/lzip-$(lzip-version) \ + $(ibidir)/make-$(make-version) tarball=nano-$(nano-version).tar.lz $(call import-source, $(nano-url), $(nano-checksum)) $(call gbuild, nano-$(nano-version), static) diff --git a/reproduce/software/make/build-rules.mk b/reproduce/software/make/build-rules.mk index c25dfb1..454a255 100644 --- a/reproduce/software/make/build-rules.mk +++ b/reproduce/software/make/build-rules.mk @@ -3,7 +3,7 @@ # imported into 'basic.mk' and 'high-level.mk'. They should be activated # with Make's 'Call' function. # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -40,6 +40,14 @@ # Necessary shell variables # 'tarball': This is the name of the actual tarball file without a # directory. +# +# Notes: +# - The LD_LIBRARY_PATH correction is because we build the dependencies of +# the downloaders (Wget or cURL) before they are built them selves. In +# the time between the installation of the downloader dependencies and +# the downloader, the Gnuastro libraries may cause crashes in the host's +# downloader since normally, Gnuastro's library has higher precedence in +# 'LD_LIBRARY_PATH'. import-source = final=$(tdir)/$$tarball; \ url=$(strip $(1)); \ exp_checksum="$(strip $(2))"; \ @@ -82,8 +90,14 @@ import-source = final=$(tdir)/$$tarball; \ if [ x"$$exp_checksum" = x"NO-CHECK-SUM" ]; then \ mv "$$unchecked" "$$final"; \ else \ - if type sha512sum > /dev/null 2>/dev/null; then \ + if type sha512sum > /dev/null 2> /dev/null; then \ + if ! sha512sum --version > /dev/null 2> /dev/null; then \ + export LD_LIBRARY_PATH=$(ldpathorig); \ + fi; \ checksum=$$(sha512sum "$$unchecked" | awk '{print $$1}'); \ + if [ LD_LIBRARY_PATH = $(ldpathorig) ]; then \ + export LD_LIBRARY_PATH=$(LD_LIBRARY_PATH); \ + fi; \ if [ x"$$checksum" = x"$$exp_checksum" ]; then \ mv "$$unchecked" "$$final"; \ else \ @@ -160,7 +174,7 @@ uncompress = csuffix=$$(echo $$utarball \ intarrm=0; \ intar=$$utarball; \ fi; \ - if tar -xf $$intar; then \ + if tar -xf $$intar --no-same-owner --no-same-permissions; then \ if [ x$$intarrm = x1 ]; then rm $$intar; fi; \ else \ echo; echo "Tar error"; exit 1; \ @@ -207,17 +221,10 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ else confscript="$(strip $(6))"; \ fi; \ \ - if [ -f $(ibdir)/bash ]; then \ - if [ -f "$$confscript" ]; then \ - sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ - $$confscript > $$confscript-tmp; \ - mv $$confscript-tmp $$confscript; \ - chmod +x $$confscript; \ - fi; \ + $(shsrcdir)/prep-source.sh $(ibdir); \ + if [ -f $(ibdir)/bash ]; then \ shellop="SHELL=$(ibdir)/bash"; \ - elif [ -f /bin/bash ]; then shellop="SHELL=/bin/bash"; \ - else shellop="SHELL=/bin/sh"; \ + else shellop="SHELL=$(ibdir)/dash"; \ fi; \ \ if [ x$$gbuild_prefix = x ]; then prefixdir="$(idir)"; \ @@ -255,10 +262,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ # CMake # ----- # -# According to the link below, in CMake '/bin/sh' is hardcoded, so there is -# no way to change it unfortunately! -# -# https://stackoverflow.com/questions/21167014/how-to-set-shell-variable-in-makefiles-generated-by-cmake +# Used by packages that are built with CMake. cbuild = if [ x$(static_build) = xyes ] && [ $(2)x = staticx ]; then \ export LDFLAGS="$$LDFLAGS -static"; \ opts="-DBUILD_SHARED_LIBS=OFF"; \ @@ -268,13 +272,18 @@ cbuild = if [ x$(static_build) = xyes ] && [ $(2)x = staticx ]; then \ utarball=$(tdir)/$$tarball; \ $(call uncompress); \ cd $(1); \ - rm -rf project-build; \ - mkdir project-build; \ - cd project-build; \ + $(shsrcdir)/prep-source.sh $(ibdir); \ + if [ -f $(ibdir)/bash ]; then \ + shellop="SHELL=$(ibdir)/bash"; \ + else shellop="SHELL=$(ibdir)/dash"; \ + fi; \ + rm -rf maneage-build; \ + mkdir maneage-build; \ + cd maneage-build; \ cmake .. -DCMAKE_LIBRARY_PATH=$(ildir) \ -DCMAKE_INSTALL_PREFIX=$(idir) \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON $$opts $(3); \ - make; \ - make install; \ + make $$shellop; \ + make $$shellop install; \ cd ../..; \ rm -rf $(1) diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index 966ea9b..c6dcf90 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -12,8 +12,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2019-2022 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2018-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2026 Raul Infante-Sainz <infantesainz@gmail.com> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,7 +29,7 @@ # along with this Makefile. If not, see <http://www.gnu.org/licenses/>. # Top level environment (same as 'basic.mk') -include reproduce/software/config/LOCAL.conf +include .build/software/config/LOCAL.conf include reproduce/software/make/build-rules.mk include reproduce/software/config/versions.conf include reproduce/software/config/checksums.conf @@ -49,6 +49,7 @@ ddir = $(BDIR)/software/build-tmp idir = $(BDIR)/software/installed ibdir = $(BDIR)/software/installed/bin ildir = $(BDIR)/software/installed/lib +iidir = $(BDIR)/software/installed/include ibidir = $(BDIR)/software/installed/version-info/proglib # Basic directories (specific to this Makefile). @@ -63,6 +64,12 @@ ipydir = $(BDIR)/software/installed/version-info/python ircrandir = $(BDIR)/software/installed/version-info/r-cran ilibrcrandir = $(BDIR)/software/installed/lib/R/library +# Special files. +makewshell = $(ibdir)/make-with-shell + + + + # Targets to build. ifeq ($(strip $(all_highlevel)),1) @@ -111,11 +118,17 @@ else endif -# Ultimate Makefile target. +# Disable the TeXLive target if `--offline` +ifneq ($(strip $(offline)),1) + target-texlive := $(itidir)/texlive +endif + +# Ultimate Makefile target. The recipe is '@echo > /dev/null' so Make does +# not print "make: Nothing to be done for 'all'." all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) \ $(foreach p, $(targets-python), $(ipydir)/$(p)) \ $(foreach p, $(targets-r-cran), $(ircrandir)/$(p)) \ - $(itidir)/texlive + $(target-texlive); @echo > /dev/null # Define the shell environment # ---------------------------- @@ -147,19 +160,26 @@ export SHELL := $(ibdir)/bash .SHELLFLAGS := --noprofile --norc -ec export LDFLAGS := $(rpath_command) -L$(ildir) export PKG_CONFIG_LIBDIR := $(ildir)/pkgconfig -export CPPFLAGS := -I$(idir)/include -Wno-nullability-completeness export PKG_CONFIG_PATH := $(ildir)/pkgconfig:$(idir)/share/pkgconfig # Disable built-in rules (which are not needed here!) .SUFFIXES: # Settings specific to this Makefile. +export LC_ALL := C export CC := $(ibdir)/gcc export CXX := $(ibdir)/g++ export F77 := $(ibdir)/gfortran export LD_RUN_PATH := $(ildir):$(il64dir) export LD_LIBRARY_PATH := $(ildir):$(il64dir) +# See description of '-Wno-nullability-completeness' in +# 'reproduce/software/shell/configure.sh'. +ifeq ($(on_mac_os),yes) + noccwarnings=-Wno-nullability-completeness +endif +export CPPFLAGS := -I$(idir)/include $(noccwarnings) + # In macOS, if a directory exists in both 'C_INCLUDE_PATH' and 'CPPFLAGS' # it will be ignored in 'CPPFLAGS' (which has higher precedence). So, we # should not define 'C_INCLUDE_PATH' on macOS. This happened with clang @@ -225,7 +245,7 @@ $(idircustom):; mkdir $@ # Afer putting everything together, we use the first server as the # reference for all software if their '-url' variable isn't defined (in # 'reproduce/software/config/urls.conf'). -downloadwrapper = ./reproduce/analysis/bash/download-multi-try +downloadwrapper = ./reproduce/analysis/bash/download-multi-try.sh maneage_backup_urls := $(shell awk '!/^#/{printf "%s ", $$1}' \ reproduce/software/config/servers-backup.conf) backupservers_all = $(user_backup_urls) $(maneage_backup_urls) @@ -330,8 +350,10 @@ $(ibidir)/atlas-$(atlas-version): # 'rpath_command'. export LDFLAGS=-L$(ildir) cd $(ddir) - tar -xf $(tdir)/atlas-$(atlas-version).tar.lz + tar -xf $(tdir)/atlas-$(atlas-version).tar.lz \ + --no-same-owner --no-same-permissions cd ATLAS + $(shsrcdir)/prep-source.sh $(ibdir) rm -rf build mkdir build cd build @@ -386,8 +408,9 @@ $(ibidir)/boost-$(boost-version): \ rm -rf $(ddir)/$$unpackdir topdir=$(pwd) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) ./bootstrap.sh --prefix=$(idir) --with-libraries=all \ --with-python=python3 echo "using mpi ;" > project-config.jam @@ -407,28 +430,51 @@ $(ibidir)/cfitsio-$(cfitsio-version): # systems. So we need to change it to our library installation # path. It doesn't affect GNU/Linux, so we'll just do it in any case # to keep things clean. - topdir=$(pwd); cd $(ddir); tar -xf $(tdir)/$$tarball + topdir=$(pwd); cd $(ddir) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions customtar=cfitsio-$(cfitsio-version)-custom.tar.gz cd cfitsio-$(cfitsio-version) - sed configure -e's|@rpath|$(ildir)|g' > configure_tmp - mv configure_tmp configure - chmod +x configure + sed -i -e's|@rpath|$(ildir)|g' configure cd .. tar cf $$customtar cfitsio-$(cfitsio-version) cd $$topdir +# Add SSE2 optimization only when we are on a Linux kernel or using +# macOS on an Intel CPU. + arch=$$(uname -m) + unames=$$(uname -s) + if [ "$$unames" = "Linux" ]; then + confopts="$$confopts --enable-sse2" + elif [ "$$unames" = "Darwin" ] && [ "$$arch" = "x86_64" ]; then + confopts="$$confopts --enable-sse2" + fi + +# On macOS, system libraries (e.g., libSystem) are inside the SDK, +# not standard paths. 'xcrun' provides the correct SDK path; use $$ +# so it is evaluated by the shell (not Make). + if [ $(on_mac_os) = yes ]; then + export SDKROOT=$$(xcrun --show-sdk-path) + fi + # Continue the standard build on the customized tarball. Note that # with the installation of CFITSIO, 'fpack' and 'funpack' are not # installed by default. Because of that, they are added explicity. +# +# Note that older versions of CFITSIO (before 4.4.0) require a +# specific 'shared' target for the building of the shared libraries. export gbuild_tar=$(ddir)/$$customtar $(call gbuild, cfitsio-$(cfitsio-version), , \ - --enable-sse2 --enable-reentrant \ + $$confopts \ + --disable-curl \ + --without-fortran \ + --enable-reentrant \ --with-bzip2=$(idir), , \ - make shared fpack funpack) + make fpack funpack) rm $$customtar echo "CFITSIO $(cfitsio-version)" > $@ $(ibidir)/cairo-$(cairo-version): \ + $(ibidir)/libxt-$(libxt-version) \ $(ibidir)/pixman-$(pixman-version) \ $(ibidir)/libpng-$(libpng-version) \ $(ibidir)/freetype-$(freetype-version) @@ -445,10 +491,12 @@ $(ibidir)/eigen-$(eigen-version): tarball=eigen-$(eigen-version).tar.lz $(call import-source, $(eigen-url), $(eigen-checksum)) rm -rf $(ddir)/eigen-eigen-* - topdir=$(pwd); cd $(ddir); tar -xf $(tdir)/$$tarball + topdir=$(pwd); cd $(ddir) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd eigen-$(eigen-version) if ! [ -d $(iidir)/eigen3 ]; then mkdir $(iidir)/eigen3; fi - cp -r Eigen/* $(iidir)/eigen3/ + cp -r Eigen/* $(iidir)/eigen3/ # Some expect 'eigen3'. + ln -s $(iidir)/eigen3 $(iidir)/Eigen # Others expect 'Eigen'. cd $$topdir rm -rf $(ddir)/eigen-$(eigen-version) echo "Eigen $(eigen-version)" > $@ @@ -478,37 +526,69 @@ $(ibidir)/expat-$(expat-version): $(call gbuild, expat-$(expat-version), static) echo "Expat $(expat-version)" > $@ +# FFTW manual customizations: +# +# - To activate the OpenMPI features of FFTW, add OpenMPI as a dependency +# of FFTW: the necessary flags are automatically added if OpenMPI was +# installed before FFTW. +# +# - There are Intel-specific optimizations that can be enabled by adding +# the following two options to 'opts' +# +# --enable-avx --enable-sse2 +# +# However, they cause crashes on non-Intel processors (has been confirmed +# in ARM's aarch64). So in the generic scenario they are +# removed. Checking how these optimizations affect the numerical accuracy +# of the result (and thus optionally adding them for Intel-based +# processors) should be studied before they are optionally added for +# Intel-based CPUs (and ignored for others). $(ibidir)/fftw-$(fftw-version): # Prepare the source tarball. tarball=fftw-$(fftw-version).tar.lz $(call import-source, $(fftw-url), $(fftw-checksum)) -# FFTW's single and double precision libraries must be built -# independently: for the the single-precision library, we need to add -# the '--enable-float' option. We will build this first, then the -# default double-precision library. -# -# There are Intel-specific optimizations that can be enabled by -# adding the following two options to 'confop' -# -# --enable-avx --enable-sse2 -# -# However, they cause crashs on non-Intel processors (has been -# confirmed in ARM's aarch64). So in the generic scenario they are -# removed. Checking how these optimizations affect the numeric -# accuracy of the result (and thus optionally adding them for -# Intel-based processors) should be studied before they are -# optionally added for Intel-based CPUs (and ignored for others). - confop="--enable-shared --enable-threads" - $(call gbuild, fftw-$(fftw-version), static, \ - $$confop --enable-float) - $(call gbuild, fftw-$(fftw-version), static, \ - $$confop) - cp $(dtexdir)/fftw.tex $(ictdir)/ +# Specific customizations: you can add the options mentioned above +# after the 'if' statements, for example: "opts=$$opts EXTRA". + opts="--enable-shared --enable-threads" + if [ x$(on_mac_os) = xyes ]; then opts="$$opts --disable-fortran" + else + if [ -f $(ibidir)/openmpi-$(openmpi-version) ]; then + opts="$$opts --enable-openmp --enable-mpi" + fi + fi + +# FFTW's single, double and long-double precision libraries must be +# built separately (fully independent builds and installs). + $(call gbuild, fftw-$(fftw-version), static, $$opts) + $(call gbuild, fftw-$(fftw-version), static, $$opts \ + --enable-float) + $(call gbuild, fftw-$(fftw-version), static, $$opts \ + --enable-long-double) + cp -pv $(dtexdir)/fftw.tex $(ictdir)/ echo "FFTW $(fftw-version) \citep{fftw}" > $@ +# 2026-02-20 'framel' is needed by the python package lalsuite, which is +# needed by ligo-skymap; currently the compile of 'framel' 8.49.0 fails with +# +# error: initialization of 'void (*)(void)' from incompatible +# pointer type 'void (*)(int, char *)' [-Wincompatible-pointer-types] +# +# Hints: try using gcc -std=gnu99 or -std=gnu11 (default is -std=gnu17). +$(ibidir)/framel-$(framel-version): $(ibidir)/cmake-$(cmake-version) + tarball=framel-$(framel-version).tar.lz + $(call import-source, $(framel-url), $(framel-checksum)) + $(call cbuild, framel-$(framel-version), static) + echo "Framel $(framel-version)" > $@ + $(ibidir)/freetype-$(freetype-version): $(ibidir)/libpng-$(libpng-version) +# As of version 2.13.2, FreeType doesn't account for the 'SHELL' +# environment variable. The issue has been reported to the +# developers. But until future versions, the work-around was +# discoverd to be setting the 'GNUMAKE' environment variable so it +# includes 'SHELL'. + export GNUMAKE="$(makewshell)" tarball=freetype-$(freetype-version).tar.lz $(call import-source, $(freetype-url), $(freetype-checksum)) $(call gbuild, freetype-$(freetype-version), static) @@ -529,7 +609,7 @@ $(ibidir)/gsl-$(gsl-version): $(ibidir)/hdf5-$(hdf5-version): $(ibidir)/openmpi-$(openmpi-version) export CC=mpicc export FC=mpif90 - tarball=hdf5-$(hdf5-version).tar.gz + tarball=hdf5-$(hdf5-version).tar.lz $(call import-source, $(hdf5-url), $(hdf5-checksum)) $(call gbuild, hdf5-$(hdf5-version), static, \ --enable-parallel \ @@ -538,62 +618,114 @@ $(ibidir)/hdf5-$(hdf5-version): $(ibidir)/openmpi-$(openmpi-version) echo "HDF5 library $(hdf5-version)" > $@ # HEALPix includes the source of its C, C++, Python (and several other -# languages) libraries within one tarball. We will include the Python -# installation only when any other Python module is requested (in -# 'TARGETS.conf'). +# languages) libraries within one tarball. However, we do not build the +# python version here; instead we provide 'healpy' as a separate package; +# see 'python.mk' for the build rule. # # Note that the default './configure' script is an interactive script which # is hard to automate. So we need to go into the 'autotools' directory of # the 'C' and 'cxx' directories and configure the GNU Build System (with # 'autoreconf', which uses 'autoconf' and 'automake') to easily build the # HEALPix C/C++ libraries in batch mode. -ifeq ($(strip $(targets-python)),) -healpix-python-dep = -else -healpix-python-dep = $(ipydir)/matplotlib-$(matplotlib-version) \ - $(ipydir)/astropy-$(astropy-version) -endif -$(ibidir)/healpix-$(healpix-version): $(healpix-python-dep) \ +$(ibidir)/healpix-$(healpix-version): \ $(ibidir)/cfitsio-$(cfitsio-version) \ $(ibidir)/autoconf-$(autoconf-version) \ $(ibidir)/automake-$(automake-version) + +# Import the tarball. tarball=healpix-$(healpix-version).tar.lz $(call import-source, $(healpix-url), $(healpix-checksum)) - if [ x"$(healpix-python-dep)" = x ]; then - pycommand1="echo no-healpy-because-no-other-python" - pycommand2="echo no-healpy-because-no-other-python" - else - pycommand1="python setup.py build" - pycommand2="python setup.py install" - fi - rm -rf $(ddir)/Healpix_$(healpix-version) + +# Setup the build directory + rm -rf $(ddir)/healpix-$(healpix-version) topdir=$(pwd); cd $(ddir); - tar -xf $(tdir)/$$tarball - cd Healpix_$(healpix-version)/src/C/autotools/ + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd healpix-$(healpix-version) + $(shsrcdir)/prep-source.sh $(ibdir) + +# Compile and install libsharp: + cd src/common_libraries/libsharp autoreconf --install ./configure --prefix=$(idir) make V=1 -j$(numthreads) SHELL=$(ibdir)/bash make install - cd ../../cxx/autotools/ + +# Compile and install the C version: + cd ../../C/autotools + printf "\n\nDEBUG ONLY\n\n"; /bin/pwd; printf "\nGUBED\n\n\n" + autoreconf --install + ./configure --prefix=$(idir) + make V=1 -j$(numthreads) SHELL=$(ibdir)/bash + make install + +# Compile and install the C++ version: + cd ../../cxx/ + printf "\n\nDEBUG.BBBB ONLY\n\n"; /bin/pwd; printf "\nBBB.GUBED\n\n\n" autoreconf --install ./configure --prefix=$(idir) # With CFITSIO 4.0, the 'CFITSIO_VERSION' macro has three -# components. But this version of Healpix doesn't yet account for -# this. +# components. But this (which?) version of Healpix doesn't yet +# account for this. sed -i -e's/CFITSIO_VERSION/fitsversion/' cxxsupport/fitshandle.cc # Continue with the building. make V=1 -j$(numthreads) SHELL=$(ibdir)/bash make install - cd ../../healpy - $$pycommand1 - $$pycommand2 + +# Finalize the build cd $$topdir - rm -rf $(ddir)/Healpix_$(healpix-version) + rm -rf $(ddir)/healpix-$(healpix-version) cp $(dtexdir)/healpix.tex $(ictdir)/ echo "HEALPix $(healpix-version) \citep{healpix}" > $@ +# 2025-05-24 The build rules here for 'lalsuite' include git initialisation +# with a dummy commit of all the source files since this is required for an +# install unless the .git/ directory (currently 650 Mb) is included in the +# tarball. Our lalsuite tarball also has many big data test files removed, +# since these are unlikely to be needed in typically astronomy usage; this +# brings the .tar.lz file down to 25 Mb from the default of about 950 Mb +# (tar.gz) or 740 Mb (tar.lz). +# +# If you need to do full gravitational wave work with 'lalsuite', then you +# should provide an alternative tarball and edit the three files +# reproduce/software/config/{versions,checksums,urls}.conf in order to use +# your preferred tarball. +$(ipydir)/lalsuite-$(lalsuite-version): \ + $(ibidir)/automake-$(automake-version) \ + $(ibidir)/swig-$(swig-version) \ + $(ibidir)/metaio-$(metaio-version) \ + $(ibidir)/framel-$(framel-version) \ + $(ipydir)/astropy-$(astropy-version) \ + $(ipydir)/igwn-segments-$(igwn-segments-version) \ + $(ipydir)/lscsoft-glue-$(lscsoft-glue-version) \ + $(ipydir)/python-dateutil-$(python-dateutil-version) + tarball=lalsuite-$(lalsuite-version).tar.lz + $(call import-source, $(lalsuite-url), \ + $(lalsuite-checksum)) + +# First: build from source as if it's a C source only. + cd $(ddir) + unpackdir=lalsuite-$(lalsuite-version) + rm -rf $$unpackdir + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) + +# A git commit is needed to generate version-control files; the +# strings in these will be rather arbitrary and should be ignored. + git init && git add . + git commit -a --no-edit -m "Dummy commit" \ + --author "Tarball <tar@ball.only>" + autoreconf + ./configure --prefix="$(idir)" --enable-swig-python + make + make install + +# Clean up and build final tarball + cd $(ddir); rm -fr $$unpackdir + echo "LALSuite $(lalsuite-version)" > $@ + $(ibidir)/libbsd-$(libbsd-version): $(ibidir)/libmd-$(libmd-version) tarball=libbsd-$(libbsd-version).tar.lz $(call import-source, $(libbsd-url), $(libbsd-checksum)) @@ -658,9 +790,10 @@ $(ibidir)/libpaper-$(libpaper-version): \ # Unpack, build the configure system, build and install. cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions unpackdir=libpaper-$(libpaper-version) cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) autoreconf -fi ./configure --prefix=$(idir) --sysconfdir=$(idir)/etc \ --disable-static @@ -695,7 +828,8 @@ $(ibidir)/libtiff-$(libtiff-version): $(ibidir)/libjpeg-$(libjpeg-version) $(call gbuild, libtiff-$(libtiff-version), static, \ --disable-jbig \ --disable-webp \ - --disable-zstd) + --disable-zstd \ + --disable-libdeflate ) echo "Libtiff $(libtiff-version)" > $@ $(ibidir)/libtirpc-$(libtirpc-version): @@ -783,17 +917,80 @@ $(ibidir)/metastore-$(metastore-version): \ echo "" > $@ fi +# The Ninja build system (https://ninja-build.org) is also known as simply +# "Ninja". But other package managers (for example Debian) use +# "ninja-build" (the old "ninja" name has become obsolete there). Also, +# their own URL is called "ninja-build". So we use the same convention in +# Maneage. +$(ibidir)/ninjabuild-$(ninjabuild-version): $(ibidir)/cmake-$(cmake-version) + tarball=ninjabuild-$(ninjabuild-version).tar.lz + $(call import-source, $(ninjabuild-url), $(ninjabuild-checksum)) + cd $(ddir) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd ninjabuild-$(ninjabuild-version) + cmake -Bbuild-cmake + cmake --build build-cmake -j$(numthreads) + ./build-cmake/ninja_test + cp -pv build-cmake/ninja $(ibdir)/ + cd .. + rm -rf ninjabuild-$(ninjabuild-version) + echo "Ninja build system $(ninjabuild-version)" > $@ + $(ibidir)/openblas-$(openblas-version): - tarball=OpenBLAS-$(openblas-version).tar.lz + +# Import the tarball. + tarball=openblas-$(openblas-version).tar.lz $(call import-source, $(openblas-url), $(openblas-checksum)) - if [ x$(on_mac_os) = xyes ]; then export CC=clang; fi + +# If 'gcc' is actually a symbolic link to 'clang', then print the +# following message to guide the user on how to prepare their +# environment. + if $$CC --version | grep clang &> /dev/null; then + +# Print message. + echo; echo + printf "*********************************************\n" + printf "OpenBlas could not be installed with Clang!\n\n" + printf "In order to install OpenBLAS, GCC and Gfortran are " + printf "needed. You can take the following steps to fix " + printf "this:\n\n" + printf "1. Install GCC with Homebrew.\n" + printf "2. Get the full path of 'gcc', 'g++', and 'gfortran' " + printf "(for example with 'which gcc').\n" + printf "3. Comment the 'exit 1' and un-comment (and edit) the " + printf "'export's that are below this message (in the " + printf "'openblas' rule of " + printf "'reproduce/software/make/high-level.mk', within the " + printf "'openblas' rule):\n\n" + printf " exit 1\n" + printf " #export CC=/opt/homebrew/bin/gcc-XX\n" + printf " #export CXX=/opt/homebrew/bin/g++-XX\n" + printf " #export FC=/opt/homebrew/bin/gfortran-XX\n\n" + printf " with the lines:\n\n" + printf " #exit 1\n" + printf " export CC=/opt/homebrew/bin/gcc-15\n" + printf " export CXX=/opt/homebrew/bin/g++-15\n" + printf " export FC=/opt/homebrew/bin/gfortran-15\n\n" + printf "5. Re-configure Maneage.\n" + printf "*********************************************\n" + echo; echo + +# Parts to modify when the message above is printed. + exit 1 + #export CC=/opt/homebrew/bin/gcc-XX + #export CXX=/opt/homebrew/bin/g++-XX + #export FC=/opt/homebrew/bin/gfortran-XX + fi + +# Do the build. cd $(ddir) - tar -xf $(tdir)/$$tarball - cd OpenBLAS-$(openblas-version) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd openblas-$(openblas-version) + $(shsrcdir)/prep-source.sh $(ibdir) make -j$(numthreads) make PREFIX=$(idir) install cd .. - rm -rf OpenBLAS-$(openblas-version) + rm -rf openblas-$(openblas-version) echo "OpenBLAS $(openblas-version)" > $@ $(ibidir)/openmpi-$(openmpi-version): @@ -852,6 +1049,209 @@ $(ibidir)/tides-$(tides-version): cp $(dtexdir)/tides.tex $(ictdir)/ echo "TIDES $(tides-version) \citep{tides}" > $@ +$(ibidir)/sqlite-$(sqlite-version): $(ibidir)/tcl-$(tcl-version) + tarball=sqlite-$(sqlite-version).tar.lz + $(call import-source, $(sqlite-url), $(sqlite-checksum)) + $(call gbuild, sqlite-$(sqlite-version),, \ + --enable-all --enable-static --enable-shared) + echo "Sqlite $(sqlite-version)" > $@ + +$(ibidir)/tcl-$(tcl-version): $(ibidir)/zlib-$(zlib-version) \ + $(ibidir)/tzdb-$(tzdb-version) + +# If both the tcl and tk packages are needed, then they must be built +# together, with identical version numbers. Thus, for simplicity in +# the build system, whether or not we build only tcl, or both tcl and +# tk, we must check that the versions are identical. + if [ "x$(tcl-version)" != "x$(tk-version)" ]; then + printf "Error: This build system requires identical tcl and " + printf "tk versions to be selected **even if** tk is not " + printf "going to be built. However, tcl-version=$(tcl-version) " + printf "while tk-version=$(tk-version)\n" + exit 1 + fi + +# Prepare the input tarball. + tarball=tcl-$(tcl-version).tar.lz + $(call import-source, $(tcl-url), $(tcl-checksum)) + +# Setup the build directory + cd $(ddir) + unpackdir_tcl=tcl-$(tcl-version) + rm -rf $${unpackdir_tcl} + +# Install TCL + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd - # return to the top maneage directory + cd $(ddir)/$${unpackdir_tcl}/unix + $(shsrcdir)/prep-source.sh $(ibdir) + ./configure --prefix="$(idir)" + make -j$(numthreads) + make install + cd - # return to the top maneage directory + +# Setup the 'tclsh' symbolic link. + maj_min_version=$$(printf $(tcl-version)| \ + sed -e "s;^\([0-9]*\.[0-9]*\).*\';\1;") + printf "Will symlink tclsh$${maj_min_version} as tclsh ...\n" + cd $(ibdir) + rm -fv tclsh && ln -sv tclsh$${maj_min_version} tclsh + cd - + cd $(ddir) + +# Finalize. + if (printf "$$unpackdir_tcl" | grep "[a-z][a-z]"); then \ + rm -fr $$unpackdir_tcl + fi + echo "Tcl $(tcl-version)" > $(ibidir)/tcl-$(tcl-version) + + +# The tk library needs the X11 library for building GUIs, and should not +# normally be needed for a non-interactive project. However, GUIs built +# with tk might be convenient during development of a project, since visual +# checks within the maneage shell may be useful. +# +# We need to rebuild tcl temporarily in the temporary build area in order +# to build tk. However, we want the tk target to be dependent on tcl as a +# prerequisite to avoid the possibility of parallel builds that risk +# conflicting within the build area because of duplicate use of the same +# tcl package directory after untarring the archive. +$(ibidir)/tk-$(tk-version): $(ibidir)/zip-$(zip-version) \ + $(ibidir)/tcl-$(tcl-version) \ + $(ibidir)/libx11-$(libx11-version) \ + +# Prepare the tarball. + tarball=tk-$(tk-version).tar.lz + $(call import-source, $(tk-url), $(tk-checksum)) + +# If both the tcl and tk packages are needed, then they must be built +# together, with identical version numbers. Thus, for simplicity in +# the build system, whether or not we build only tcl, or both tcl and +# tk, we must check that the versions are identical. + if [ "x$(tcl-version)" != "x$(tk-version)" ]; then + printf "Error: This build system requires identical tcl and " + printf "tk versions to be selected **even if** tk is not " + printf "going to be built. However, tcl-version=$(tcl-version) " + printf "while tk-version=$(tk-version)\n" + exit 1 + fi + +# Make sure no temporary directory exists. + cd $(ddir) + unpackdir=tk-$(tk-version) + if (printf "$$unpackdir" | grep "[a-z][a-z]"); then + rm -rf $$unpackdir + fi + +# Install TK: zipfs is being disabled because of the problem with +# installing zip (see the comments above zip). + cd $(ddir) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd $$unpackdir/unix + $(shsrcdir)/prep-source.sh $(ibdir) + ./configure --prefix="$(idir)" --with-tcl=$(ildir) + make -j$(numthreads) + make install + +# Finalize. + cd $(ddir) + if (printf "$$unpackdir" | grep "[a-z][a-z]"); then \ + rm -fr $$unpackdir; fi + echo "Tk $(tk-version)" > $(ibidir)/tk-$(tk-version) + + + +# WARNING: Dependence on glibc: tzdb (time zone data base) uses the 'zic' +# program from the GNU C library. Until Maneage builds its own glibc, it is +# imported from the host operating system in the 'low-level-links' step of +# 'basic.mk'. +$(ibidir)/tzdb-$(tzdb-version): + +# Import the tarball + tarball=tzdb-$(tzdb-version).tar.lz + $(call import-source, $(tzdb-url), $(tzdb-checksum)) + +# Unpack the tarball. + cd $(ddir) + unpackdir=tzdb-$(tzdb-version) + rm -rf $$unpackdir + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd $$unpackdir + +# Make necessary corrections. + TZGEN=$(strip $(ddir))/$$unpackdir/tzgen + printf "\n\n\n_____TZGEN=$${TZGEN}_____\n\n\n" + make AWK=gawk BACKWARD="backward" PACKRATDATA=backzone \ + PACKRATLIST=zone.tab VERSION_DEPS= tzdata.zi leapseconds + +# Build the timezone data +# +# The '-b fat' option was created in glibc by around 2020, when +# '-b slim' became the default. Allowing the default of '-b slim' +# could potentially lead to a #Y2038 bug [3]. +# +# Some other hacks are also needed for older glibc. +# +# [1] https://mm.icann.org/pipermail/tz-announce/2020-October/000059.html +# [2] https://github.com/stub42/pytz/issues/48 +# [3] https://en.wikipedia.org/wiki/Year_2038_problem + BFAT_OPTION=$$(if (zic -b fat 2>/dev/null); then \ + printf "%sb fat" "-"; else printf ""; fi) + if [ "x$${BFAT_OPTION}" = "x" ]; then \ + sed -e 's/Sa\([^a-zA-Z]\)/Sat\1/g' \ + -e 's/Su\([^a-zA-Z]\)/Sun\1/g' \ + -e 's/^\(.\|..\) Sat/\1 Sa/' \ + -e "s/lastSu\'/lastSun/" tzdata.zi \ + | grep -v "^L" \ + > tzdata.zi.3letter.days + zic $${BFAT_OPTION} -d $${TZGEN} -L /dev/null \ + tzdata.zi.3letter.days + zic $${BFAT_OPTION} -d $${TZGEN}/right -L leapseconds \ + tzdata.zi.3letter.days + else + zic $${BFAT_OPTION} -d $${TZGEN} -L /dev/null tzdata.zi + zic $${BFAT_OPTION} -d $${TZGEN}/right -L leapseconds \ + tzdata.zi + fi + +# Replace hardlinks by symlinks + if [ "x$${BFAT_OPTION}" = "x" ]; then + +# Older method of creating symbolic links: + grep '^L ' $(ddir)/$$unpackdir/tzdata.zi \ + | while read L target name ; do + absolute_name="$${TZGEN}/$$name" + relative_target=$$(realpath -m -s \ + --relative-to="$${absolute_name%/*}" \ + "$${TZGEN}/$$target") + maybe_newdir=$$(printf $$name | sed -e "s;/[^/]*\';;") + mkdir -p $${TZGEN}/$${maybe_newdir} + mkdir -p $${TZGEN}/right/$${maybe_newdir} + ln -sf "$$relative_target" "$${TZGEN}/$$name" + ln -sf "$$relative_target" "$${TZGEN}/right/$$name" + done + + else + +# More recent (2025-05-25) method of creating symbolic links: + grep '^L ' $(ddir)/$$unpackdir/tzdata.zi \ + | while read L target name ; do + absolute_name="$${TZGEN}/$$name" + relative_target=$$(realpath -m -s \ + --relative-to="$${absolute_name%/*}" \ + "$${TZGEN}/$$target") + ln -sf "$$relative_target" "$${TZGEN}/$$name" + ln -sf "$$relative_target" "$${TZGEN}/right/$$name" + done + fi + +# Generate a posixrules file (with a UTC timezone) and install. + ln -s Etc/UTC $${TZGEN}/posixrules + mkdir -p $(idir)/share/zoneinfo + cp -rv $${TZGEN}/* $(idir)/share/zoneinfo/ + cd $(ddir); rm -rf $$unpackdir + echo "IANA Time Zone Database (tzdb) $(tzdb-version)" > $@ + $(ibidir)/valgrind-$(valgrind-version): tarball=valgrind-$(valgrind-version).tar.lz $(call import-source, $(valgrind-url), $(valgrind-checksum)) @@ -896,8 +1296,8 @@ $(ibidir)/libgit2-$(libgit2-version): $(ibidir)/cmake-$(cmake-version) -DUSE_SSH=OFF -DBUILD_CLAR=OFF \ -DTHREADSAFE=ON -DUSE_ICONV=OFF ) if [ x$(on_mac_os) = xyes ]; then - install_name_tool -id $(ildir)/libgit2.1.3.dylib \ - $(ildir)/libgit2.1.3.dylib + install_name_tool -id $(ildir)/libgit2.1.9.dylib \ + $(ildir)/libgit2.1.9.dylib fi echo "Libgit2 $(libgit2-version)" > $@ @@ -907,20 +1307,46 @@ $(ibidir)/wcslib-$(wcslib-version): $(ibidir)/cfitsio-$(cfitsio-version) tarball=wcslib-$(wcslib-version).tar.lz $(call import-source, $(wcslib-url), $(wcslib-checksum)) -# If Fortran isn't present, don't build WCSLIB with it. - if type gfortran &> /dev/null; then fortranopt=""; - else fortranopt="--disable-fortran" +# macOS arm64 fix: reserve extra space in binary headers so +# install_name_tool (called bellow, after installation) can safely +# update library paths. Without this, we get the following error: +# "larger updated load commands do not fit". SDKROOT is also +# necessary for system libraries. + if [ x$(on_mac_os) = xyes ]; then + sdk=$$(xcrun --show-sdk-path) + LDFLAGS="$$LDFLAGS -isysroot $$sdk" + export LDFLAGS="$$LDFLAGS -Wl,-headerpad_max_install_names" fi -# Build WCSLIB. +# Build WCSLIB while disabling some features: +# +# - Fortran is disabled because as of May 2023, on macOS systems +# where we do not install GCC (and thus a standard 'gfortran'), the +# LLVM Fortran compiler is not complete and will cause a crash. If +# you use the Fortran features of WCSLIB, feel free to remove +# '--disable-fortran', but be careful with portability on other +# systems. Hopefully some time in the future, GCC will also be +# install-able on macOS within Maneage or LLVM's 'gfortran' will +# also be complete and will not cause a crash. +# +# - PGPLOT is disabled because it has many manual steps in its +# installation. Therefore, we currently do not build PGPlots in +# Maneage. Once (if) it is added, we can remove '--without-pgplot'. +# +# - Flex is disabled because we have not made it a prerequisite for +# the 'wcslib' target, and we don't want the behaviour to differ +# between host OSes that may or may not have 'flex', or may have +# different versions of 'flex'. $(call gbuild, wcslib-$(wcslib-version), , \ LIBS="-pthread -lcurl -lm" \ + --disable-flex \ + --without-pgplot \ + --disable-fortran \ --with-cfitsiolib=$(ildir) \ - --with-cfitsioinc=$(idir)/include \ - --without-pgplot $$fortranopt) + --with-cfitsioinc=$(idir)/include) if [ x$(on_mac_os) = xyes ]; then - install_name_tool -id $(ildir)/libwcs.7.7.dylib \ - $(ildir)/libwcs.7.7.dylib + install_name_tool -id $(ildir)/libwcs.$(wcslib-version).dylib \ + $(ildir)/libwcs.$(wcslib-version).dylib fi echo "WCSLIB $(wcslib-version)" > $@ @@ -938,15 +1364,44 @@ $(ibidir)/wcslib-$(wcslib-version): $(ibidir)/cfitsio-$(cfitsio-version) # # Astrometry-net contains a lot of programs. We need to specify the # installation directory and the Python executable (by default it will look -# for /usr/bin/python) +# for /usr/bin/python). +# +# An optional dependency is 'netpbm' but it has many dependencies and a +# crazy build system. So, it is not in the default preprequisites. If you +# need it you can add it as a prerequisite and pray that it will work. +# +# A consequence of not having 'netpbm' is that for obtaining the +# astrometric solution of one image using 'solve-field', it is necessary to +# build a catalog of sources with image coordinates and flux (x,y,flux) in +# advance. The catalog has to be sorted by flux. Finally, when invoking +# 'solve-field', the width and heigh of the original image have to be +# provided. +# +# More explicitly, raw basic steps using Gnuastro: +# +# - Obtain a catalog with image coordinates and flux (x,y,brightness): +# $ astnoisechisel img.fits -o det.fits +# $ astsegment det.fits -o seg.fits +# $ astmkcatalog seg.fits --clumpscat --x --y --brightness -o cat-raw.fits +# +# - Sort by flux: +# $ asttable cat-raw.fits --hdu 2 --sort brightness --descending \ +# --output cat.fits +# +# - Get the x-size and y-size from the header: +# $ xsize=$(astfits img.fits --keyvalue NAXIS1 --quiet) +# $ ysize=$(astfits img.fits --keyvalue NAXIS2 --quiet) +# +# - Run 'solve-field' to obtain the astrometric solution: +# $ solve-field cat.fits --width $xsize --height $ysize [--other-parameters] $(ibidir)/astrometrynet-$(astrometrynet-version): \ $(ibidir)/gsl-$(gsl-version) \ $(ibidir)/swig-$(swig-version) \ $(ipydir)/numpy-$(numpy-version) \ $(ibidir)/cairo-$(cairo-version) \ $(ibidir)/libpng-$(libpng-version) \ - $(ibidir)/netpbm-$(netpbm-version) \ $(ibidir)/wcslib-$(wcslib-version) \ + $(ipydir)/astropy-$(astropy-version) \ $(ibidir)/cfitsio-$(cfitsio-version) \ $(ibidir)/libjpeg-$(libjpeg-version) @@ -960,8 +1415,9 @@ $(ibidir)/astrometrynet-$(astrometrynet-version): \ # 'astrometrynet' cd $(ddir) rm -rf astrometry.net-$(astrometrynet-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd astrometry.net-$(astrometrynet-version) + $(shsrcdir)/prep-source.sh $(ibdir) sed -e 's|cat /proc/cpuinfo|echo "Ignoring CPU info"|' \ -e 's|-free|echo "Ignoring RAM info"|' Makefile > Makefile.tmp mv Makefile.tmp Makefile @@ -986,12 +1442,6 @@ $(ibidir)/automake-$(automake-version): $(ibidir)/autoconf-$(autoconf-version) $(call gbuild, automake-$(automake-version), static, ,V=1) echo "GNU Automake $(automake-version)" > $@ -$(ibidir)/bison-$(bison-version): $(ibidir)/help2man-$(help2man-version) - tarball=bison-$(bison-version).tar.lz - $(call import-source, $(bison-url), $(bison-checksum)) - $(call gbuild, bison-$(bison-version), static, ,V=1 -j$(numthreads)) - echo "GNU Bison $(bison-version)" > $@ - # cdsclient is a set of software written in c to interact with astronomical # database servers. It is a dependency of 'scamp' to be able to download # reference catalogues. @@ -1004,8 +1454,9 @@ $(ibidir)/cdsclient-$(cdsclient-version): tarball=cdsclient-$(cdsclient-version).tar.lz $(call import-source, $(cdsclient-url), $(cdsclient-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd cdsclient-$(cdsclient-version) + $(shsrcdir)/prep-source.sh $(ibdir) touch * ./configure --prefix=$(idir) make @@ -1022,25 +1473,39 @@ $(ibidir)/cmake-$(cmake-version): tarball=cmake-$(cmake-version).tar.lz $(call import-source, $(cmake-url), $(cmake-checksum)) -# After searching in 'bootstrap', I couldn't find 'LIBS', only -# 'LDFLAGS'. So the extra libraries are being added to 'LDFLAGS', not -# 'LIBS'. -# # On Mac systems, the build complains about 'clang' specific # features, so we can't use our own GCC build here. if [ x$(on_mac_os) = xyes ]; then export CC=clang export CXX=clang++ fi + +# CMake wants a single executable for 'MAKE', so we can't use 'make +# SHELL=$(SHELL) and we have defined this script. + export MAKE="$(makewshell)" + +# Go into the unpacked directory and prepare CMake. cd $(ddir) rm -rf cmake-$(cmake-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd cmake-$(cmake-version) - ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ - --system-bzip2 --system-liblzma --no-qt-gui \ + $(shsrcdir)/prep-source.sh $(ibdir) + +# Bootstrap, build and install CMake: +# - With the '--no-system-libs' option, CMake builds and statically +# links all the libraries it needs. Even though some of those (like +# liblzma, libcurl, zlib or bzip2) are within Maneage, we +# discovered that CMake can get confused and use out-of-Maneage +# libraries (https://savannah.nongnu.org/bugs/?63043). + ./bootstrap --no-qt-gui \ + --prefix=$(idir) \ + --no-system-libs \ --parallel=$(numthreads) - make -j$(numthreads) LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 - make install + $(makewshell) VERBOSE=1 LIBS="$$LIBS -lssl -lcrypto -lz" \ + -j$(numthreads) + $(makewshell) install + +# Clean up. cd .. rm -rf cmake-$(cmake-version) echo "CMake $(cmake-version)" > $@ @@ -1048,7 +1513,11 @@ $(ibidir)/cmake-$(cmake-version): $(ibidir)/flex-$(flex-version): $(ibidir)/bison-$(bison-version) tarball=flex-$(flex-version).tar.lz $(call import-source, $(flex-url), $(flex-checksum)) - $(call gbuild, flex-$(flex-version), static, ,V=1 -j$(numthreads)) + $(call gbuild, flex-$(flex-version), static, \ + --with-libiconv-prefix=$(idir) \ + --with-libintl-prefix=$(idir) \ + --disable-dependency-tracking, \ + V=1 -j$(numthreads)) echo "Flex $(flex-version)" > $@ $(ibidir)/gdb-$(gdb-version): $(ibidir)/python-$(python-version) @@ -1080,16 +1549,25 @@ $(ibidir)/ghostscript-$(ghostscript-version): \ tarball=ghostscript-$(ghostscript-version).tar.lz $(call import-source, $(ghostscript-url), $(ghostscript-checksum)) +# On macOS we use the system compiler and linker. The system linker +# there doesn't support '--copy-dt-needed-entries', while the one on +# Linux crashes if we remove it. So we only activate it on macOS. + if [ x$(on_mac_os) = xno ]; then + export LDFLAGS="$$LDFLAGS -Wl,--copy-dt-needed-entries" + fi + # Unpack it and configure Ghostscript. The option # '-DPNG_ARM_NEON_OPT=0' prevents an arm64 'neon' library from being # required at compile time. cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd ghostscript-$(ghostscript-version) - ./configure --prefix=$(idir) \ - --disable-cups \ + $(shsrcdir)/prep-source.sh $(ibdir) + ./configure --disable-cups \ + --prefix=$(idir) \ --enable-dynamic \ --disable-compile-inits \ + --disable-hidden-visibility \ CFLAGS="-DPNG_ARM_NEON_OPT=0" # Build and install the program and the shared libraries. @@ -1104,9 +1582,9 @@ $(ibidir)/ghostscript-$(ghostscript-version): \ # Install the fonts. tar -xvf $(tdir)/ghostscript-fonts-std-$(ghostscript-fonts-std-version).tar.lz \ - -C $(idir)/share/ghostscript + -C $(idir)/share/ghostscript --no-same-owner --no-same-permissions tar -xvf $(tdir)/ghostscript-fonts-gnu-$(ghostscript-fonts-gnu-version).tar.lz \ - -C $(idir)/share/ghostscript + -C $(idir)/share/ghostscript --no-same-owner --no-same-permissions fc-cache -v $(idir)/share/ghostscript/fonts/ echo; echo "Ghostscript fonts added to Fontconfig."; echo; @@ -1115,25 +1593,62 @@ $(ibidir)/ghostscript-$(ghostscript-version): \ rm -rf ghostscript-$(ghostscript-version) echo "GPL Ghostscript $(ghostscript-version)" > $@ +# Gnuastro can optionally depend on libgit2, but it is not included as a +# dependency here for the two reasons below. If you would like to have it, +# add it as a dependency (its build instruction and dependencies are here +# already), then remove the '--without-libgit2' configure option) and also +# comment the following two generic configuration lines: 'outfitsnocommit' +# and 'outfitsnoconfig'. +# - Within Maneage, we have everything under Git already and users are +# expected to include the version in all their products. +# - libgit2 can only be built with CMake (which takes extremely long to +# compile: possibly even longer than GCC!). +# - Maneage is often run on HPCs that can have network connections +# between the storage and processer and the necessary libgit2 +# operations on every created file can have a performance impact. $(ibidir)/gnuastro-$(gnuastro-version): \ $(ibidir)/gsl-$(gsl-version) \ $(ibidir)/wcslib-$(wcslib-version) \ $(ibidir)/libjpeg-$(libjpeg-version) \ $(ibidir)/libtiff-$(libtiff-version) \ - $(ibidir)/libgit2-$(libgit2-version) \ $(ibidir)/ghostscript-$(ghostscript-version) + +# Generic installation. tarball=gnuastro-$(gnuastro-version).tar.lz $(call import-source, $(gnuastro-url), $(gnuastro-checksum)) - $(call gbuild, gnuastro-$(gnuastro-version), static, , \ - -j$(numthreads)) + $(call gbuild, gnuastro-$(gnuastro-version), static, \ + --without-libgit2, -j$(numthreads)) cp $(dtexdir)/gnuastro.tex $(ictdir)/ - echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ -$(ibidir)/help2man-$(help2man-version): - tarball=help2man-$(help2man-version).tar.lz - $(call import-source, $(help2man-url), $(help2man-checksum)) - $(call gbuild, help2man-$(help2man-version), static, ,V=1) - echo "Help2man $(Help2man-version)" > $@ +# Generally, besides the Git commit, we are also disabling the +# default mode of printing any type of metadata and versions of +# dependencies in output headers (through the installation-wide +# configuration file). This is done because within a large pipeline, +# Gnuastro is used to create many intermediate files (that are +# deleted shortly after being created) and it is not worth the +# overhread to keep this information in those intermediate products: +# it is the pipeline's responsibility to put them in the final +# outputs. We recommend to use the Gnuastro Fits program's keyword +# writing options to add as much contextual metadata on your final +# products as possible. Some tips: +# - The creation date is not good (because it is not reproducible +# and will make simple validation hard). The project's Commit +# should be used instead. +# - Define a keyword to keep the public URL of the repository of +# the Maneage'd project. In this way, a person who gets your +# final product can easily check that for all the information +# (including software versions and configuration options). + gconf=$(idir)/etc/gnuastro/gnuastro.conf + echo "" >> $$gconf + echo "# Maneage specific (see Gnuastro build rule for details)." \ + >> $$gconf + echo " outfitsnocommit = 1" >> $$gconf + echo " outfitsnoconfig = 1" >> $$gconf + echo " outfitsnodate = 1" >> $$gconf + echo " outfitsnoversions = 1" >> $$gconf + +# Final target. + echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ $(ibidir)/icu-$(icu-version): $(ibidir)/python-$(python-version) @@ -1149,9 +1664,11 @@ $(ibidir)/icu-$(icu-version): $(ibidir)/python-$(python-version) tarball=icu-$(icu-version).tar.lz $(call import-source, $(icu-url), $(icu-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions unpackdir=icu-$(icu-version) - cd $$unpackdir/icu4c/source + cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) + cd icu4c/source ./configure --enable-static --prefix=$(idir) make -j$(numthreads) V=1 make install @@ -1167,8 +1684,25 @@ $(ibidir)/imagemagick-$(imagemagick-version): \ tarball=ImageMagick-$(imagemagick-version).tar.lz $(call import-source, $(imagemagick-url), $(imagemagick-checksum)) $(call gbuild, ImageMagick-$(imagemagick-version), static, \ - --with-gslib --without-x --disable-openmp, \ + --without-x \ + --with-gslib \ + --disable-openmp \ + --with-libstdc=$(ildir), \ V=1 -j$(numthreads)) + +# On macOS, an executable and several libraries are not properly +# linked with the Ghostscript library (libgs), so we need to fix it +# manually. + if [ x$(on_mac_os) = xyes ]; then + gsversion=$$(echo $(ghostscript-version) \ + | awk -F'.' '{print $$1"."$$2}') + libMagicks=$$(ls -l $(ildir)/libMagick*.dylib \ + | awk '/^-/{print $$NF}') + libMagicks_all="$(ibdir)/magick $$libMagicks" + for f in $$libMagicks_all; do \ + install_name_tool -change libgs.dylib.$$gsversion \ + $(ildir)/libgs.dylib.$$gsversion $$f; done + fi echo "ImageMagick $(imagemagick-version)" > $@ # 'imfit' doesn't use the traditional 'configure' and 'make' to install @@ -1199,8 +1733,9 @@ $(ibidir)/imfit-$(imfit-version): \ cd $(ddir) unpackdir=imfit-$(imfit-version) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) sed -i 's|/usr/local|$(idir)|g' SConstruct sed -i 's|/usr/include|$(idir)/include|g' SConstruct sed -i 's|.append(|.insert(0,|g' SConstruct @@ -1230,6 +1765,30 @@ $(ibidir)/imfit-$(imfit-version): \ rm -rf $$unpackdir echo "Imfit $(imfit-version) \citep{imfit2015}" > $@ +# jq (to parse JSON files on the command-line) uses the external Oniguruma +# regular expression library that has ended development in 2025-04-24, see +# https://github.com/kkos/oniguruma. If this library ('libonig') is present +# on the host, jq will build with that: causing a leaking of os environment +# into the Maneage'd environment. Fortunately, jq maintains a copy of this +# library in its own source (within its tarball) and will build it with the +# '--with-oniguruma=builtin' configuration option. +$(ibidir)/jq-$(jq-version): + tarball=jq-$(jq-version).tar.lz + $(call import-source, $(jq-url), $(jq-checksum)) + $(call gbuild, jq-$(jq-version), static, \ + --with-oniguruma=builtin, V=1) + echo "jq $(jq-version)" > $@ + +# This is the LIGO lscsoft package 'metaio' [1] (to be distinguished from +# some other packages with the same name). +# +# [1] https://git.ligo.org/lscsoft/metaio +$(ibidir)/metaio-$(metaio-version): + tarball=metaio-$(metaio-version).tar.lz + $(call import-source, $(metaio-url), $(metaio-checksum)) + $(call gbuild, metaio-$(metaio-version), static,, V=1) + echo "Lscsoft Metaio $(metaio-version)" > $@ + # Minizip 1.x is actually distributed within zlib. It doesn't have its own # independent tarball. So we need a custom build, which include the GNU # Autotools (Autoconf and Automake). Note that Minizip 2.x isn't like this @@ -1246,8 +1805,10 @@ $(ibidir)/minizip-$(minizip-version): $(ibidir)/automake-$(automake-version) unpackdir=minizip-$(minizip-version) rm -rf $$unpackdir mkdir $$unpackdir - tar -xf $(tdir)/$$tarball -C$$unpackdir --strip-components=1 + tar -xf $(tdir)/$$tarball -C$$unpackdir --strip-components=1 \ + --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) ./configure --prefix=$(idir) make cd contrib/minizip @@ -1280,12 +1841,12 @@ $(ibidir)/missfits-$(missfits-version): cp $(dtexdir)/missfits.tex $(ictdir)/ echo "MissFITS $(missfits-version) \citep{missfits}" > $@ -# Netpbm is a prerequisite of Astrometry-net, it contains a lot of programs. -# This program has a crazy dialogue installation which is override using the -# printf statment. Each '\n' is a new question that the installation process -# ask to the user. We give all answers with a pipe to the scripts (configure -# and install). The questions are different depending on the system (tested -# on GNU/Linux and Mac OS). +# Netpbm is an optional prerequisite of Astrometry-net, it contains a lot +# of programs. This program has a crazy dialogue installation which is +# override using the printf statment. Each '\n' is a new question that the +# installation process ask to the user. We give all answers with a pipe to +# the scripts (configure and install). The questions are different +# depending on the system (tested on GNU/Linux and Mac OS). $(ibidir)/netpbm-$(netpbm-version): \ $(ibidir)/flex-$(flex-version) \ $(ibidir)/libpng-$(libpng-version) \ @@ -1307,8 +1868,9 @@ $(ibidir)/netpbm-$(netpbm-version): \ cd $(ddir) unpackdir=netpbm-$(netpbm-version) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) # As of NetPBM 10.73.39 and Flex 2.6.4-410-74a89fd (commit 74a89fd in # Flex's Git that is 410 commits after version 2.6.4), there is the @@ -1333,6 +1895,25 @@ $(ibidir)/netpbm-$(netpbm-version): \ rm -rf $$unpackdir echo "Netpbm $(netpbm-version)" > $@ +$(ibidir)/parallel-$(parallel-version): + +# Prepare the tarball and build it. + tarball=parallel-$(parallel-version).tar.lz + $(call import-source, $(parallel-url), $(parallel-checksum)) + $(call gbuild, parallel-$(parallel-version), static) + +# GNU Parallel requires the users to acknowledge that they will cite +# it. Otherwise, it prints a warning on every run: reminding the user +# to cite it. In Maneage, we make sure that all software that request +# citation are cited automatically, so there is no need for a +# Maneager to get this warning and we can automatically ensure the +# developer that it will be cited. + echo "will cite" | $(ibdir)/parallel --citation + +# Copy the BibTeX source of the citation and finish. + cp -pv $(dtexdir)/parallel.tex $(ictdir)/ + echo "GNU Parallel $(parallel-version) \citep{parallel}" > $@ + $(ibidir)/patch-$(patch-version): tarball=patch-$(patch-version).tar.gz $(call import-source, $(patch-url), $(patch-checksum)) @@ -1351,15 +1932,50 @@ $(ibidir)/pcre-$(pcre-version): , V=1 -j$(numthreads)) echo "Perl Compatible Regular Expressions $(pcre-version)" > $@ -# 2022-01-01 The rules for building R - identified as r-cran to avoid the -# difficulties in searching text for a one-letter string - were shifted to -# 'r-cran.mk'. +$(ibidir)/pcre2-$(pcre2-version): + tarball=pcre2-$(pcre2-version).tar.lz + $(call import-source, $(pcre2-url), $(pcre2-checksum)) + $(call gbuild, pcre2-$(pcre2-version), static) + echo "Perl Compatible Regular Expressions 2 $(pcre2-version)" > $@ + + +# On macOS 12.3 Monterey with AppleClang 13.1.6.13160021, Plplot 5.15.0 +# needs the 'finite' function of 'math.h' which has been deprecated in +# macOS. By manually adding "#define finite isfinite" in 'math.h' like +# below, we fixed this problem but still it can't find 'exit' during the +# configuration phase so we stopped trying to port it to macOS. It means +# that on macOS Plplot is not available. For other OSs it should be fine. +# On macOS, the file 'tmath.h' can be found with 'xcrun --show-sdk-path'. +# +# cp /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk/usr/include/math.h \ +# $(iidir)/math.h +# awk '{if($$0 ~ /#define isinf\(x\)/) {print "#define finite isfinite"; print $$0} else print $$0 }' \ +# $(iidir)/math.h > $(iidir)/math-tmp.h +# mv $(iidir)/math-tmp.h $(iidir)/math.h +$(ibidir)/plplot-$(plplot-version): \ + $(ibidir)/cairo-$(cairo-version) \ + $(ibidir)/freetype-$(freetype-version) + tarball=plplot-$(plplot-version).tar.lz + $(call import-source, $(plplot-url), $(plplot-checksum)) + $(call cbuild, plplot-$(plplot-version), static, \ + -DDEFAULT_NO_BINDINGS:BOOL=ON \ + -DENABLE_cxx:BOOL=ON \ + -DENABLE_fortran:BOOL=ON \ + -DDEFAULT_NO_QT_DEVICES:BOOL=ON) + echo "PLplot $(pcre-version)" > $@ # SCAMP documentation says ATLAS is a mandatory prerequisite for using # SCAMP. We have ATLAS into the project but there are some problems with the # libraries that are not yet solved. However, we tried to install it with # the option --enable-openblas and it worked (same issue happened with # 'sextractor'. +# +# Plplot in SCAMP cannot be built for macOS. See comments on top of Plplot. +# But if you are not using macOS and want to enable plots, follow the +# steps below: +# +# 1. Add '$(ibidir)/plplot-$(plplot-version)' to the prerequisites +# 2. Remove the option '--enable-plplot=no' from the call of 'gbuild' $(ibidir)/scamp-$(scamp-version): \ $(ibidir)/fftw-$(fftw-version) \ $(ibidir)/openblas-$(openblas-version) \ @@ -1398,8 +2014,9 @@ $(ibidir)/scons-$(scons-version): $(ibidir)/python-$(python-version) cd $(ddir) unpackdir=scons-$(scons-version) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) # Unfortuantely SCons hard-codes its search PATH in its source (to # use POSIX operating system defaults)! So the only way to modify it @@ -1429,32 +2046,56 @@ $(ibidir)/scons-$(scons-version): $(ibidir)/python-$(python-version) echo "SCons $(scons-version)" > $@ # Sextractor crashes complaining about not linking with some ATLAS -# libraries. But we can override this issue since we have Openblas +# libraries. But we can override this issue since we have OpenBLAS # installed, it is just necessary to explicity tell sextractor to use it in # the configuration step. -# -# The '-fcommon' is a necessary C compilation flag for GCC 10 and above. It -# is necessary for astromatic libraries, otherwise their build will crash. $(ibidir)/sextractor-$(sextractor-version): \ $(ibidir)/fftw-$(fftw-version) \ $(ibidir)/openblas-$(openblas-version) +# Import the source. tarball=sextractor-$(sextractor-version).tar.lz $(call import-source, $(sextractor-url), $(sextractor-checksum)) +# Unpack the tarball and enter the directory. + unpackdir=sextractor-$(sextractor-version) + cd $(ddir) + rm -rf $$unpackdir + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd $$unpackdir + # See comment above 'missfits' for '-fcommon'. - $(call gbuild, sextractor-$(sextractor-version), static, \ - CFLAGS="-fcommon" \ - --enable-threads \ - --enable-openblas \ - --with-openblas-libdir=$(ildir) \ - --with-openblas-incdir=$(idir)/include) + ./configure --prefix="$(idir)" \ + CFLAGS="-fcommon" \ + --enable-threads \ + --enable-openblas \ + --libdir=$(ildir) \ + --includedir=$(iidir) \ + --with-openblas-libdir=$(ildir) \ + --with-openblas-incdir=$(iidir) + +# On macOS we need to manually change 'finite' to 'isfinite' in the +# header file 'src/levmar/compiler.h'. Until this problem is +# hopefully fixed in next releases, we are doing it manually using +# 'sed'. Consequently we are not installing it using 'gbuild'. Once +# this is fixed upstream, we can use the standard 'gbuild'. + sed -i -e's|define LM_FINITE finite |define LM_FINITE isfinite |' \ + src/levmar/compiler.h + +# Build, install and delete the temporary files. + make V=1 + make install + cd .. + rm -rf $$unpackdir + +# Make links for other possibly used names, copy citation and build +# the final target. ln -fs $(ibdir)/sex $(ibdir)/sextractor ln -fs $(ibdir)/sex $(ibdir)/source-extractor cp $(dtexdir)/sextractor.tex $(ictdir)/ echo "SExtractor $(sextractor-version) \citep{sextractor}" > $@ $(ibidir)/swarp-$(swarp-version): $(ibidir)/fftw-$(fftw-version) - tarball=swarp-$(swarp-version).tar.gz + tarball=swarp-$(swarp-version).tar.lz $(call import-source, $(swarp-url), $(swarp-checksum)) # See comment above 'missfits' for '-fcommon'. @@ -1464,15 +2105,13 @@ $(ibidir)/swarp-$(swarp-version): $(ibidir)/fftw-$(fftw-version) cp $(dtexdir)/swarp.tex $(ictdir)/ echo "SWarp $(swarp-version) \citep{swarp}" > $@ -$(ibidir)/swig-$(swig-version): - -# Option --without-pcre was a suggestion once the configure step was -# tried and it failed. It was not recommended but it works! pcr is a -# dependency of swig +$(ibidir)/swig-$(swig-version): \ + $(ibidir)/pcre2-$(pcre2-version) +# Not disabling pcre, since PCRE is needed for lalsuite (python). tarball=swig-$(swig-version).tar.lz $(call import-source, $(swig-url), $(swig-checksum)) $(call gbuild, swig-$(swig-version), static, \ - --without-pcre --without-tcl) + --without-tcl) echo "Swig $(swig-version)" > $@ # The disables: @@ -1507,7 +2146,10 @@ $(ibidir)/swig-$(swig-version): # '$(ibdir)'. If any program does need 'util-linux' libraries, they can # simply add the proper directories to the environment variables, see # 'fontconfig' for example. -$(ibidir)/util-linux-$(util-linux-version): | $(idircustom) +$(ibidir)/util-linux-$(util-linux-version): \ + $(ibidir)/autoconf-$(autoconf-version) \ + $(ibidir)/automake-$(automake-version) \ + | $(idircustom) # Import the source. tarball=util-linux-$(util-linux-version).tar.lz @@ -1517,12 +2159,49 @@ $(ibidir)/util-linux-$(util-linux-version): | $(idircustom) # explained above). As shown below, later, we'll put a symbolic link # of all the necessary binaries in the main '$(idir)/bin'. cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd util-linux-$(util-linux-version) + $(shsrcdir)/prep-source.sh $(ibdir) + +# If a patch is necessary, apply it. + if [ -f $(patchdir)/util-linux-$(util-linux-version)-macos.patch ]; then + cp $(patchdir)/util-linux-$(util-linux-version)-macos.patch \ + util-linux-$(util-linux-version)-macos.patch + git apply util-linux-$(util-linux-version)-macos.patch + fi + +# The 'mkswap' feature needs low-level file system and kernel headers +# that are not always available (in particular on older Linux +# kernels). Also, creating SWAP space will need root permissions, so +# its not something a Maneager may need! Unfortunately there is no +# configuration option to disable this so we'll have to disable it +# manually by commenting the relevant files in the +# 'configure.ac'. + sed -e's|UL_BUILD_INIT(\[mkswap\], \[yes\])|UL_BUILD_INIT(\[mkswap\], \[no\])|' \ + -e's|UL_BUILD_INIT(\[bits\], \[yes\])|UL_BUILD_INIT(\[bits\], \[no\])|' \ + -i configure.ac + +# 'autogen.sh' checks for dependencies, but 'flex' (that is not built +# in Maneage) is not needed when compiling from tarball (see: +# https://github.com/util-linux/util-linux/pull/2531#issuecomment-1798020594) + sed -i '/flex/d' autogen.sh + ./autogen.sh + +# The 'autogen.sh' call resets the 'SHELL' (that was corrected in +# 'prep-source.sh') to '/bin/sh'. Therefore, we need to correct it. + sed -i -e's|SHELL = /bin/sh|SHELL = $(SHELL)|' po/Makefile.in.in + +# Having updated 'configure.ac', we need to re-generate the +# './configure' script with 'autoreconf' (which is part of Autoconf +# and needs Automake; hence why they are dependencies. + autoreconf -f + +# Configure Util-linux + export CONFIG_SHELL=$(ibdir)/bash ./configure --prefix=$(idircustom)/util-linux \ --disable-dependency-tracking \ - --enable-libmount-support-mtab \ --disable-silent-rules \ + --disable-liblastlog2 \ --disable-mountpoint \ --disable-libmount \ --disable-unshare \ @@ -1557,6 +2236,7 @@ $(ibidir)/util-linux-$(util-linux-version): | $(idircustom) $(ibidir)/xlsxio-$(xlsxio-version): \ $(ibidir)/cmake-$(cmake-version) \ $(ibidir)/expat-$(expat-version) \ + $(ibidir)/libbsd-$(libbsd-version) \ $(ibidir)/minizip-$(minizip-version) tarball=xlsxio-$(xlsxio-version).tar.gz $(call import-source, $(xlsxio-url), $(xlsxio-checksum)) @@ -1592,12 +2272,14 @@ $(ibidir)/xlsxio-$(xlsxio-version): \ # useful in projects during its development, for more see the comment above # GNU Emacs. $(ibidir)/vim-$(vim-version): - tarball=vim-$(vim-version).tar.bz2 + tarball=vim-$(vim-version).tar.lz $(call import-source, $(vim-url), $(vim-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball - n=$$(echo $(vim-version) | sed -e's|\.||') - cd $(ddir)/vim$$n + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + unpackdir=vim-$(vim-version) + cd $(ddir)/$$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) + export CFLAGS="-std=gnu17 $$CFLAGS" ./configure --prefix=$(idir) \ --disable-canberra \ --enable-multibyte \ @@ -1610,9 +2292,72 @@ $(ibidir)/vim-$(vim-version): make -j$(numthreads) make install cd .. - rm -rf vim$$n + rm -rf $$unpackdir echo "VIM $(vim-version)" > $@ +$(ibidir)/unzip-$(unzip-version): + +# Prepare the tarball and build directory. + tarball=unzip-$(unzip-version).tar.lz + $(call import-source, $(unzip-url), $(unzip-checksum)) + +# Unpack and go into the directory. + export LDFLAGS="$$LDFLAGS -static" + cd $(ddir) + rm -fr unzip-$(unzip-version)/ + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd unzip-$(unzip-version)/ + +# Remove reference to the old 'gmtime' function and include the +# system's 'time.h instead. + sed -i -e's|struct tm *gmtime(), *localtime();|#include <time.h>|' \ + -e'/gmtime/d; /localtime/d' unix/unxcfg.h + +# The 'NO_DIR' variable causes conflicts: the unzip code has its own +# internal fallback implementation called 'NO_DIR'. The modern Linux +# system already provides a standard implementation using +# 'dirent.h'. These two systems conflict with each other. This causes +# compilation errors such as conflicting types for DIR and invalid +# use of DIR. + sed -i -e's/-DNO_DIR//g' unix/configure + sed -i -e's/#ifdef NO_DIR/#if 0/g' unix/unix.c + sed -i -e's/-DNO_DIR//g' -e's/-U_NO_DIR//g' unix/Makefile + +# Since unzip is no longer maintained, it uses an old C standard. + export CFLAGS="-std=gnu89 -fcommon" + +# Build and install + make SHELL=$(ibdir)/bash -f unix/Makefile generic + make SHELL=$(ibdir)/bash install -f unix/Makefile \ + generic BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 + +# Go back to the top build directory, clean up and finalize. + cd $(ddir) + rm -rf unzip-$(unzip-version)/ + echo "Unzip $(unzip-version)" > $@ + +$(ibidir)/zip-$(zip-version): + +# Prepare the tarball and build directory. + tarball=zip-$(zip-version).tar.lz + $(call import-source, $(zip-url), $(zip-checksum)) + +# Unpack and go into the directory. + export LDFLAGS="$$LDFLAGS -static" + cd $(ddir) + rm -fr zip-$(zip-version)/ + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd zip-$(zip-version)/ + +# Build and install + make SHELL=$(ibdir)/bash -f unix/Makefile generic + make SHELL=$(ibdir)/bash install -f unix/Makefile \ + generic BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 + +# Go back to the top build directory, clean up and finalize. + cd $(ddir) + rm -rf zip-$(zip-version)/ + echo "Zip $(zip-version)" > $@ @@ -1644,12 +2389,19 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf @topdir=$$(pwd) cd $(ddir) rm -rf install-tl-* - tar -xf $(tdir)/install-tl-unx.tar.gz + tar -xf $(tdir)/install-tl-unx.tar.gz \ + --no-same-owner --no-same-permissions cd install-tl-* + $(shsrcdir)/prep-source.sh $(ibdir) sed -e's|@installdir[@]|$(idir)|g' \ "$$topdir"/reproduce/software/config/texlive.conf \ > texlive.conf +# We do not build TeXLive from source and for its installation it +# downloads components from the web internally; and those components +# can use '/bin/sh' (which will need '$(sys_library_sh_path)'). + export LD_LIBRARY_PATH="$(sys_library_sh_path):$$LD_LIBRARY_PATH" + # TeX Live's installation may fail due to any reason. But TeX Live is # optional (only necessary for building the final PDF). So we don't # want the configure script to fail if it can't run. Possible error @@ -1718,13 +2470,17 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf tarballurl=$$url/install-tl-unx.tar.gz touch $(lockdir)/download downloader="wget --no-use-server-timestamps -O" - if $(downloadwrapper) "$$downloader" $(lockdir)/download \ - $$tarballurl "$(tdir)/install-tl-unx.tar.gz" \ - "$(backupservers)"; then + if $(downloadwrapper) "$$downloader" \ + $(lockdir)/download \ + $$tarballurl \ + "$(tdir)/install-tl-unx.tar.gz" \ + "$(backupservers)"; then cd $(ddir) rm -rf install-tl-* - tar -xf $(tdir)/install-tl-unx.tar.gz + tar -xf $(tdir)/install-tl-unx.tar.gz \ + --no-same-owner --no-same-permissions cd install-tl-* + $(shsrcdir)/prep-source.sh $(ibdir) sed -e's|@installdir[@]|$(idir)|g' \ $$topdir/reproduce/software/config/texlive.conf \ > texlive.conf @@ -1755,10 +2511,19 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf # Live itself (only very basic TeX and LaTeX) and the installation of its # necessary packages into two packages. # -# Note that Biber needs to link with libraries like libnsl. However, we -# don't currently build biber from source. So we can't choose the library -# version. But we have the source and build instructions for the 'nsl' -# library. When we later build biber from source, we can easily use them. +# Note that we do not build the TeXLive executables (like Biber) from +# source. So in case they need special libraries, we can't choose the +# library version here (for example see [1] and [2]). In such cases there +# is no solution but to manually add the location necessary library to +# LD_LIBRARY_PATH when calling the respective LaTeX command in +# 'reproduce/analysis/make/paper.mk'. Fortunately as of Biber 2.20, it does +# not depend on anything except the C library (all dependencies are now +# statically linked), so problems [1] and [2] will not happen. But this can +# generally happen for any other tool/OS, so it is important to build +# TeXLive from source as soon as possible [3]. +# [1] https://github.com/plk/biber/issues/445 +# [2] https://savannah.nongnu.org/bugs/index.php?63175 +# [3] https://savannah.nongnu.org/task/?15267 $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ $(itidir)/texlive-ready-tlmgr @@ -1768,6 +2533,11 @@ $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ echo "" > $@ else +# We do not build TeXLive from source and for its installation it +# downloads components from the web internally; and those +# components can use '/bin/sh' (which needs 'sys_library_sh_path'). + export LD_LIBRARY_PATH="$(sys_library_sh_path)" + # To update itself, tlmgr needs a backup directory. backupdir=$(idir)/texlive/backups mkdir -p $$backupdir @@ -1791,6 +2561,12 @@ $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ # directory so we don't have to modify 'PATH'. ln -fs $(idir)/texlive/maneage/bin/*/* $(ibdir)/ +# Correct any reference to '/bin/sh' within the installed LaTeX +# files (this is because we do no yet install LaTeX from source): + cdir=$$(pwd) + cd $(idir)/texlive + cd $$cdir + # Get all the necessary versions. texlive=$$(pdflatex --version \ | awk 'NR==1' \ diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 6766744..6153f0a 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -12,8 +12,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2019-2022 Raul Infante-Sainz <infantesainz@gmail.com> -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2026 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2019-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -51,6 +51,8 @@ export MPI_PYTHON3_SITEARCH := # Python-specific installation directories. python-major-version = $(shell echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') +# This is for 'installer' (the python installer) itself: +ilibpymajorversion = $(idir)/lib/python$(python-major-version) @@ -81,7 +83,11 @@ $(ibidir)/libffi-$(libffi-version): fi echo "Libffi $(libffi-version)" > $@ -$(ibidir)/python-$(python-version): $(ibidir)/libffi-$(libffi-version) +# 2025-05-26 '--with-system-ffi' seems to be obsolete +# TODO: For a recommended list of dependencies, see +# https://github.com/pyenv/pyenv/wiki#suggested-build-environment +$(ibidir)/python-$(python-version): $(ibidir)/libffi-$(libffi-version) \ + $(ibidir)/sqlite-$(sqlite-version) # Download the source. tarball=python-$(python-version).tar.lz @@ -93,13 +99,48 @@ $(ibidir)/python-$(python-version): $(ibidir)/libffi-$(libffi-version) export CC=clang export CXX=clang++ fi - $(call gbuild, python-$(python-version),, \ - --without-ensurepip \ - --with-system-ffi \ - --enable-shared, -j$(numthreads)) + +# Unpack the tarball (see below for the necessary modification). + cd $(ddir) + unpackdir=python-$(python-version) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) + +# Python's 'setup.py' uses 'os.system' to run shell scripts. On the +# other hand 'os.system' only runs '/bin/sh' (which has its own +# libraries to link to and those are blocked at this level). So we +# need to add an extra line on top of the 'os.system' funciton and +# put '/usr/lib' in 'LD_LIBRARY_PATH' within Python's environment for +# system calls (with 'os.putenv'). As of Python 3.13.2 the tarball no +# longer has an 'setup.py'. But when it did, the change below was +# necessary. + if [ -f setup.py ]; then + awk '{if(/os.system\(/) \ + { print " os.putenv(\"LD_LIBRARY_PATH\", \"$$LD_LIBRARY_PATH:$(sys_library_sh_path)\");"; \ + print $$0;} \ + else print $$0}' \ + setup.py > setup-tmp.py + mv setup-tmp.py setup.py + fi + +# Do the basic installation and delete the temporary directory. + ./configure SHELL=$(ibdir)/bash \ + --enable-optimizations \ + --without-ensurepip \ + --prefix="$(idir)" \ + --with-system-ffi \ + --enable-shared + $(makewshell) -j$(numthreads) + $(makewshell) install -j$(numthreads) + cd .. + rm -rf $$unpackdir + +# Set the necessary environment variables and finish the build. ln -sf $(ildir)/python$(python-major-version) $(ildir)/python ln -sf $(ibdir)/python$(python-major-version) $(ibdir)/python - ln -sf $(iidir)/python$(python-major-version)m $(iidir)/python$(python-major-version) + ln -sf $(iidir)/python$(python-major-version)m \ + $(iidir)/python$(python-major-version) rm -rf $(ipydir) mkdir $(ipydir) echo "Python $(python-version)" > $@ @@ -108,39 +149,138 @@ $(ibidir)/python-$(python-version): $(ibidir)/libffi-$(libffi-version) -# Non-PiP Python module installation +# Non-pip Python module installation # ---------------------------------- # -# To build Python packages with direct access to a 'setup.py' (if no direct -# access to 'setup.py' is needed, pip can be used). Note that the -# software's packaged source code is the first prerequisite that is in the -# 'tdir' directory. +# Build strategy for python modules as of February 2025, for python 3.13.2. + +# This strategy is mostly based on recommendations by E Schwartz +# (ztrawhcse) on #python (Libera Chat), in October 2022 and February +# 2025. Some discussions are on documented in Savannah tasks [1][2]. The +# build strategy for 'python-installer' is inspired by the gentoo script +# 'python_domodule' [3]. + +# Bootstrap-step: 'gpep517' [4], motivated by PEP 517 [5], together with +# 'python-installer' (module called 'installer') are built without +# dependences on other python packages apart from python itself. The build +# rules for these two packages do python byte compilation and copy the .py +# and .pyc files into the python install directory. These two packages are +# considered to be 'frontends'. + +# Once these two frontends are available, other packages that do building +# tasks, including both backends and alternative frontends or a mix of +# these (in particular: setuptools, meson [6]/ninja-build [7] , flit-core, +# and meson-python), can be built with the 'python-installer' and +# 'gpep517'. The aims of the various build tools are diverse, include +# ecosystem resilience, reproducibility, build speed and convenience in +# building bigger packages such as numpy, scipy and astropy. + +# The python.mk script now includes only three methods: the boot +# build methods of 'python-installer' and 'gpep517'; and the gpep517 +# frontend. No method is provided for using 'python-installer' directly; +# it is invoked indirectly by source files of many packages, which +# also give metadata describing information for build methods. + +# Why not pip? We do not build any python packages with 'pip' because we +# want to have a fully documented pipeline of (i) the original upstream +# locations of tarballs, (ii) the tarballs' checksums, and (iii) the exact +# sequence of build commands. + +# For an alternative viewpoint on a python build strategy, see [8]. + +# Prerequisite for the pybuild script here: the package's source code +# (tarball) must already be located in the directory 'tdir'. # -# Arguments of this function are the numbers +# Arguments: # 1) Unpack command # 2) Unpacked directory name after unpacking the tarball # 3) site.cfg file (optional). -# 4) Official software name (for paper). +# 4) Official software name (for paper.tex). +# 5) Obligatory parameter: build method (see below): +# BOOT_INSTALLER - only for 'python-installer' +# BOOT_GPEP517 - only for 'gpep517' +# GPEP517 - for any other python package # # Hooks: # pyhook_before: optional steps before running 'python setup.py build' # pyhook_after: optional steps after running 'python setup.py install' -pybuild = cd $(ddir); rm -rf $(2); \ - if ! $(1) $(tdir)/$$tarball; then \ + +# [1] https://savannah.nongnu.org/task/?16268 +# [2] https://savannah.nongnu.org/task/?16625 +# [3] https://gitweb.gentoo.org/repo/gentoo.git/tree/eclass/python-utils-r1.eclass#n646 +# [4] https://pypi.org/project/gpep517 +# [5] https://peps.python.org/pep-0517 +# [6] https://mesonbuild.com +# [7] https://ninja-build.org +# [8] https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html + + +pybuild = cd $(ddir); \ + packagedir=$(strip $(2)); \ + if (printf "$$packagedir" | grep "[a-z][a-z]"); then rm -rf $$packagedir; fi; \ + printf "\nStarting to install python package with maneage pybuild rule: $(4)\n ..."; \ + if ! $(1) $(tdir)/$$tarball --no-same-owner --no-same-permissions; then \ echo; echo "Tar error"; exit 1; \ fi; \ - cd $(2); \ + cd $$packagedir; \ if [ "x$(strip $(3))" != x ]; then \ sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ $(3) > site.cfg; \ fi; \ if type pyhook_before &>/dev/null; then pyhook_before; fi; \ - python setup.py build; \ - python setup.py install; \ + printf "pybuild option5 = __ %s __\n" "$(strip $(5))"; \ + if [ "x$(strip $(5))" = xBOOT_INSTALLER ]; then \ + chmod 0644 src/installer/*.py; \ + mkdir -p $(ilibpymajorversion)/installer; \ + mkdir -p $(ilibpymajorversion)/installer/__pycache__ ; \ + mkdir -p $(ilibpymajorversion)/installer/_scripts; \ + mkdir -p $(ilibpymajorversion)/installer/_scripts/__pycache__ ; \ + cp -pv src/installer/*.py $(ilibpymajorversion)/installer/; \ + cp -pv src/installer/_scripts/__init__.py $(ilibpymajorversion)/installer/_scripts/; \ + cd src/installer/; \ + python -m compileall -o 1 -o 2 -l -f \ + -d $(ilibpymajorversion)/installer/ .; \ + chmod 0644 __pycache__/*.pyc; \ + cp -pv __pycache__/*.pyc \ + $(ilibpymajorversion)/installer/__pycache__; \ + cd -; \ + cd src/installer/_scripts/; \ + python -m compileall -o 1 -o 2 -l -f \ + -d $(ilibpymajorversion)/installer/_scripts/ __init__.py; \ + chmod 0644 __pycache__/*.pyc; \ + cp -pv __pycache__/*.pyc \ + $(ilibpymajorversion)/installer/_scripts/__pycache__; \ + cd -; \ + elif [ "x$(strip $(5))" = xBOOT_GPEP517 ]; then \ + chmod 0644 gpep517/*.py; \ + mkdir -p $(ilibpymajorversion)/gpep517; \ + mkdir -p $(ilibpymajorversion)/gpep517/__pycache__ ; \ + cp -pv gpep517/*.py $(ilibpymajorversion)/gpep517/; \ + cd gpep517/; \ + python -m compileall -o 1 -o 2 -l -f \ + -d $(ilibpymajorversion)/gpep517/ .; \ + chmod 0644 __pycache__/*.pyc; \ + cp -pv __pycache__/*.pyc \ + $(ilibpymajorversion)/gpep517/__pycache__; \ + cd -; \ + elif [ "x$(strip $(5))" = xGPEP517 ]; then \ + printf "\n\n\n=== python build method: gpep517 ====== "; pwd; \ + printf "...............\n\n\n"; \ + python -m gpep517 install-from-source \ + --prefix "" \ + --destdir $(idir) \ + --optimize all; \ + else \ + printf "Error: Unknown pybuild method for $$packagedir: __ $(strip $(5)) __\n"; \ + printf "The pybuild 5th parameter should very likely be set "; \ + printf "to GPEP517 after checking the build rule and "; \ + printf "upgrading if needed.\n"; \ + exit 1; \ + fi; \ if type pyhook_after &>/dev/null; then pyhook_after; fi; \ cd ..; \ - rm -rf $(2); \ + if (printf "$$packagedir" | grep "[a-z][a-z]"); then rm -fr $$packagedir; fi; \ echo "$(4)" > $@ @@ -149,9 +289,11 @@ pybuild = cd $(ddir); rm -rf $(2); \ # Python modules # --------------- -# + + # All the necessary Python modules go here. -$(ipydir)/asn1crypto-$(asn1crypto-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/asn1crypto-$(asn1crypto-version): \ + $(ipydir)/setuptools-$(setuptools-version) tarball=asn1crypto-$(asn1crypto-version).tar.gz $(call import-source, $(asn1crypto-url), $(asn1crypto-checksum)) $(call pybuild, tar -xf, asn1crypto-$(asn1crypto-version), , \ @@ -172,23 +314,73 @@ $(ipydir)/astroquery-$(astroquery-version): \ $(call pybuild, tar -xf, astroquery-$(astroquery-version), , \ Astroquery $(astroquery-version)) +# Astropy: points to consider about dependencies: +# +# The optional dependency 'h5py' that is necessary for writting tables in +# HDF5 format has been removed from Astropy because on macOS it cannot be +# installed. +# +# 2022-or-older dependencies: +# $(ibidir)/expat-$(expat-version) \ +# $(ipydir)/jinja2-$(jinja2-version) \ +# $(ipydir)/html5lib-$(html5lib-version) \ +# $(ipydir)/beautifulsoup4-$(beautifulsoup4-version) \ +# +# While the astropy pyproject.toml file says that the astropy build depends +# on numpy, not scipy, and does not depend on matplotlib; the +# runtime is recommended to depend on both scipy and matplotlib. +# In practice, users of astropy will generally expect scipy and matplotlib +# to be available at runtime, so we set these as prerequisites. $(ipydir)/astropy-$(astropy-version): \ - $(ipydir)/h5py-$(h5py-version) \ - $(ibidir)/expat-$(expat-version) \ $(ipydir)/scipy-$(scipy-version) \ - $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/pyyaml-$(pyyaml-version) \ - $(ipydir)/jinja2-$(jinja2-version) \ $(ipydir)/pyerfa-$(pyerfa-version) \ - $(ipydir)/html5lib-$(html5lib-version) \ - $(ipydir)/beautifulsoup4-$(beautifulsoup4-version) \ + $(ipydir)/pyyaml-$(pyyaml-version) \ + $(ipydir)/matplotlib-$(matplotlib-version) \ + $(ipydir)/astropy-iers-data-$(astropy-iers-data-version) \ $(ipydir)/extension-helpers-$(extension-helpers-version) + +# Tarball and its preparation. tarball=astropy-$(astropy-version).tar.lz $(call import-source, $(astropy-url), $(astropy-checksum)) - $(call pybuild, tar -xf, astropy-$(astropy-version)) - cp $(dtexdir)/astropy.tex $(ictdir)/ + +# Conservatively purge old version (but not astropy_iers, +# astropy-iers): + rm -fvr $(idir)/lib/python*/site-packages/astropy/ + rm -fvr $(idir)/lib/python*/site-packages/astropy-[0-9]*-info/ + rm -fv $(idir)/bin/fits{diff,check,header,info,2bitmap} + rm -fv $(idir)/bin/{samp_hub,showtable,volint,wcslint} + +# Uncomment the 'export' command below only when building from a git +# tarball, e.g. for testing if a bug still occurs: +# export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_ASTROPY=7.2.0.87 + +# Do the basic build. + $(call pybuild, tar -xf, astropy-$(astropy-version),,, \ + GPEP517) + cp -pv $(dtexdir)/astropy.tex $(ictdir)/ echo "Astropy $(astropy-version) \citep{astropy2013,astropy2018}" > $@ +$(ipydir)/astropy-healpix-$(astropy-healpix-version): \ + $(ipydir)/extension-helpers-$(extension-helpers-version) \ + $(ipydir)/numpy-$(numpy-version) + tarball=astropy-healpix-$(astropy-healpix-version).tar.lz + $(call import-source, $(astropy-healpix-url), \ + $(astropy-healpix-checksum)) + $(call pybuild, tar -xf, \ + astropy-healpix-$(astropy-healpix-version),,, \ + GPEP517) + echo "Astropy-healpix $(astropy-healpix-version)" > $@ + +$(ipydir)/astropy-iers-data-$(astropy-iers-data-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=astropy-iers-data-$(astropy-iers-data-version).tar.lz + $(call import-source, $(astropy-iers-data-url), \ + $(astropy-iers-data-checksum)) + $(call pybuild, tar -xf, \ + astropy-iers-data-$(astropy-iers-data-version),,, \ + GPEP517) + echo "Astropy-IERS-data $(astropy-iers-data-version)" > $@ + $(ipydir)/beautifulsoup4-$(beautifulsoup4-version): \ $(ipydir)/soupsieve-$(soupsieve-version) tarball=beautifulsoup4-$(beautifulsoup4-version).tar.lz @@ -196,13 +388,17 @@ $(ipydir)/beautifulsoup4-$(beautifulsoup4-version): \ $(call pybuild, tar -xf, beautifulsoup4-$(beautifulsoup4-version), , \ BeautifulSoup $(beautifulsoup4-version)) -$(ipydir)/beniget-$(beniget-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/beniget-$(beniget-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=beniget-$(beniget-version).tar.lz $(call import-source, $(beniget-url), $(beniget-checksum)) $(call pybuild, tar -xf, beniget-$(beniget-version), , \ Beniget $(beniget-version)) -$(ipydir)/certifi-$(certifi-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/certifi-$(certifi-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=certifi-$(certifi-version).tar.gz $(call import-source, $(certifi-url), $(certifi-checksum)) $(call pybuild, tar -xf, certifi-$(certifi-version), , \ @@ -215,12 +411,30 @@ $(ipydir)/cffi-$(cffi-version): \ $(call import-source, $(cffi-url), $(cffi-checksum)) $(call pybuild, tar -xf, cffi-$(cffi-version), ,cffi $(cffi-version)) -$(ipydir)/chardet-$(chardet-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/chardet-$(chardet-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=chardet-$(chardet-version).tar.gz $(call import-source, $(chardet-url), $(chardet-checksum)) $(call pybuild, tar -xf, chardet-$(chardet-version), , \ Chardet $(chardet-version)) +$(ipydir)/colorama-$(colorama-version): \ + $(ipydir)/hatchling-$(hatchling-version) + tarball=colorama-$(colorama-version).tar.lz + $(call import-source, $(colorama-url), $(colorama-checksum)) + $(call pybuild, tar -xf, \ + colorama-$(colorama-version),,, GPEP517) + +$(ipydir)/contourpy-$(contourpy-version): \ + $(ipydir)/pybind11-$(pybind11-version) \ + $(ipydir)/meson-python-$(meson-python-version) + tarball=contourpy-$(contourpy-version).tar.lz + $(call import-source, $(contourpy-url), $(contourpy-checksum)) + $(call pybuild, tar -xf, contourpy-$(contourpy-version), , \ + Contourpy $(contourpy-version), GPEP517) + echo "Contourpy $(contourpy-version)" > $@ + $(ipydir)/corner-$(corner-version): $(ipydir)/matplotlib-$(matplotlib-version) tarball=corner-$(corner-version).tar.gz $(call import-source, $(corner-url), $(corner-checksum)) @@ -229,6 +443,22 @@ $(ipydir)/corner-$(corner-version): $(ipydir)/matplotlib-$(matplotlib-version) cp $(dtexdir)/corner.tex $(ictdir)/ echo "Corner $(corner-version) \citep{corner}" > $@ +$(ipydir)/cosmodesi-mpytools-$(cosmodesi-mpytools-version): \ + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/scipy-$(scipy-version) + tarball=cosmodesi-mpytools-$(cosmodesi-mpytools-version).tar.lz + $(call import-source, $(cosmodesi-mpytools-url), $(cosmodesi-mpytools-checksum)) + $(call pybuild, tar -xf, cosmodesi-mpytools-$(cosmodesi-mpytools-version), , \ + Cosmodesi-mpytools $(cosmodesi-mpytools-version), GPEP517) + echo "Cosmodesi-mpytools $(cosmodesi-mpytools-version)" > $@ + +$(ipydir)/cppy-$(cppy-version): \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) + tarball=cppy-$(cppy-version).tar.lz + $(call import-source, $(cppy-url), $(cppy-checksum)) + $(call pybuild, tar -xf, cppy-$(cppy-version), , \ + Cppy $(cppy-version), GPEP517) + $(ipydir)/cryptography-$(cryptography-version): \ $(ipydir)/cffi-$(cffi-version) \ $(ipydir)/asn1crypto-$(asn1crypto-version) \ @@ -242,13 +472,17 @@ $(ipydir)/cycler-$(cycler-version): $(ipydir)/six-$(six-version) tarball=cycler-$(cycler-version).tar.lz $(call import-source, $(cycler-url), $(cycler-checksum)) $(call pybuild, tar -xf, cycler-$(cycler-version), , \ - Cycler $(cycler-version)) + Cycler $(cycler-version), GPEP517) + echo "Cycler $(cycler-version)" > $@ -$(ipydir)/cython-$(cython-version): $(ipydir)/setuptools-$(setuptools-version) - tarball=Cython-$(cython-version).tar.lz +$(ipydir)/cython-$(cython-version): \ + $(ipydir)/python-installer-$(python-installer-version) \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=cython-$(cython-version).tar.lz $(call import-source, $(cython-url), $(cython-checksum)) - $(call pybuild, tar -xf, Cython-$(cython-version)) - cp $(dtexdir)/cython.tex $(ictdir)/ + $(call pybuild, tar -xf, cython-$(cython-version),,, GPEP517) + cp -pv $(dtexdir)/cython.tex $(ictdir)/ echo "Cython $(cython-version) \citep{cython2011}" > $@ $(ipydir)/esutil-$(esutil-version): $(ipydir)/numpy-$(numpy-version) @@ -269,25 +503,37 @@ $(ipydir)/eigency-$(eigency-version): \ $(ipydir)/emcee-$(emcee-version): \ $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/setuptools_scm-$(setuptools_scm-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=emcee-$(emcee-version).tar.gz $(call import-source, $(emcee-url), $(emcee-checksum)) $(call pybuild, tar -xf, emcee-$(emcee-version), , \ emcee $(emcee-version)) $(ipydir)/entrypoints-$(entrypoints-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=entrypoints-$(entrypoints-version).tar.gz $(call import-source, $(entrypoints-url), $(entrypoints-checksum)) $(call pybuild, tar -xf, entrypoints-$(entrypoints-version), , \ EntryPoints $(entrypoints-version)) +$(ipydir)/exceptiongroup-$(exceptiongroup-version): \ + $(ipydir)/setuptools-$(setuptools-version) \ + $(ipydir)/flit-scm-$(flit-scm-version) + tarball=exceptiongroup-$(exceptiongroup-version).tar.lz + $(call import-source, $(exceptiongroup-url), $(exceptiongroup-checksum)) + $(call pybuild, tar -xf, exceptiongroup-$(exceptiongroup-version), , \ + Exceptiongroup $(exceptiongroup-version), GPEP517) + + $(ipydir)/extension-helpers-$(extension-helpers-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=extension-helpers-$(extension-helpers-version).tar.lz - $(call import-source, $(extension-helpers-url), $(extension-helpers-checksum)) - $(call pybuild, tar -xf, extension-helpers-$(extension-helpers-version), , \ - Extension-Helpers $(extension-helpers-version)) + $(call import-source, $(extension-helpers-url), \ + $(extension-helpers-checksum)) + $(call pybuild, tar -xf, \ + extension-helpers-$(extension-helpers-version),, \ + Extension-Helpers $(extension-helpers-version), GPEP517) $(ipydir)/flake8-$(flake8-version): \ $(ipydir)/pyflakes-$(pyflakes-version) \ @@ -297,7 +543,39 @@ $(ipydir)/flake8-$(flake8-version): \ $(call pybuild, tar -xf, flake8-$(flake8-version), , \ Flake8 $(flake8-version)) -$(ipydir)/future-$(future-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/flit-core-$(flit-core-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) + tarball=flit-core-$(flit-core-version).tar.lz + $(call import-source, $(flit-core-url), $(flit-core-checksum)) + $(call pybuild, tar -xf, flit-core-$(flit-core-version), , \ + Flit-core $(flit-core-version), GPEP517) + + +$(ipydir)/flit-scm-$(flit-scm-version): \ + $(ipydir)/flit-core-$(flit-core-version) \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) + tarball=flit-scm-$(flit-scm-version).tar.lz + $(call import-source, $(flit-scm-url), $(flit-scm-checksum)) + $(call pybuild, tar -xf, flit-scm-$(flit-scm-version), , \ + flit-scm $(flit-scm-version), GPEP517) + + +# Although cython is not an obligatory prerequisite of fonttools, we force +# it as a prerequisite for reproducibility; otherwise build parallelism may +# lead to some builds with and some builds without cython, depending on how +# many cpus the host machine has. +$(ipydir)/fonttools-$(fonttools-version): \ + $(ipydir)/cython-$(cython-version) \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=fonttools-$(fonttools-version).tar.lz + $(call import-source, $(fonttools-url), $(fonttools-checksum)) + $(call pybuild, tar -xf, fonttools-$(fonttools-version), , \ + fonttools $(fonttools-version), GPEP517) + +$(ipydir)/future-$(future-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=future-$(future-version).tar.gz $(call import-source, $(future-url), $(future-checksum)) $(call pybuild, tar -xf, future-$(future-version), , \ @@ -315,12 +593,42 @@ $(ipydir)/galsim-$(galsim-version): \ cp $(dtexdir)/galsim.tex $(ictdir)/ echo "Galsim $(galsim-version) \citep{galsim}" > $@ -$(ipydir)/gast-$(gast-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/gast-$(gast-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=gast-$(gast-version).tar.lz $(call import-source, $(gast-url), $(gast-checksum)) $(call pybuild, tar -xf, gast-$(gast-version), , \ Gast $(gast-version)) +$(ipydir)/gpep517-$(gpep517-version): \ + $(ibidir)/python-$(python-version) + tarball=gpep517-$(gpep517-version).tar.lz + $(call import-source, $(gpep517-url), $(gpep517-checksum)) + $(call pybuild, tar -xf, \ + gpep517-$(gpep517-version), \ + , \ + gpep517 $(gpep517-version), \ + BOOT_GPEP517) + echo "gpep517 $(gpep517-version)" > $@ + +$(ipydir)/hatchling-$(hatchling-version): \ + $(ipydir)/packaging-$(packaging-version) \ + $(ipydir)/pathspec-$(pathspec-version) \ + $(ipydir)/pluggy-$(pluggy-version) \ + $(ipydir)/trove-classifiers-$(trove-classifiers-version) + tarball=hatchling-$(hatchling-version).tar.lz + $(call import-source, $(hatchling-url), $(hatchling-checksum)) + $(call pybuild, tar -xf, \ + hatchling-$(hatchling-version),,, GPEP517) + +$(ipydir)/hatch-vcs-$(hatch-vcs-version): \ + $(ipydir)/hatchling-$(hatchling-version) + tarball=hatch-vcs-$(hatch-vcs-version).tar.lz + $(call import-source, $(hatch-vcs-url), $(hatch-vcs-checksum)) + $(call pybuild, tar -xf, hatch-vcs-$(hatch-vcs-version), , \ + Hatch-vcs $(hatch-vcs-version), GPEP517) + $(ipydir)/h5py-$(h5py-version): \ $(ipydir)/six-$(six-version) \ $(ibidir)/hdf5-$(hdf5-version) \ @@ -335,15 +643,38 @@ $(ipydir)/h5py-$(h5py-version): \ $(call pybuild, tar -xf, h5py-$(h5py-version), , \ h5py $(h5py-version)) -# 'healpy' is actually installed as part of the HEALPix package. It will be -# installed with its C/C++ libraries if any other Python library is -# requested with HEALPix. So actually calling for 'healpix' (when 'healpix' -# is requested) is not necessary. But some users might not know about this -# and just ask for 'healpy'. To avoid confusion in such cases, we'll just -# set 'healpy' to be dependent on 'healpix' and not download any tarball -# for it, or write anything in the final target. -$(ipydir)/healpy-$(healpy-version): $(ibidir)/healpix-$(healpix-version) - touch $@ +# As of 2025-05-20, there are at least three python healpix packages at +# pypi.org, with versions: +# +# https://pypi.org/project/healpy 1.18.1 +# https://pypi.org/project/healpix 2024.2 +# https://pypi.org/project/astropy-healpix 1.1.2 +# +# Maneage provides both 'healpy' and 'astropy-healpix', independently. We +# ignore pypix 'healpix', whose name risks confusion with the C/C++/fortran +# version of healpix. +# +# For simplicity and error tracing, you should best install either healpy +# or astropy-healpix, not both. For example, astropy_healpix/healpy.py has +# an interface to provide 'healpy' type function names. However, it may be +# possible to use both if your python imports avoid namespace clashes. See +# the documentation and source code of the respective packages to study how +# they relate to one another. +# +# The 'healpy' package requires the C/C++ 'healpix' package to be built +# in order to use the shared 'libsharp' library. +$(ipydir)/healpy-$(healpy-version): \ + $(ibidir)/healpix-$(healpix-version) \ + $(ipydir)/matplotlib-$(matplotlib-version) \ + $(ipydir)/scipy-$(scipy-version) + tarball=healpy-$(healpy-version).tar.lz + $(call import-source, $(healpy-url), \ + $(healpy-checksum)) + $(call pybuild, tar -xf, \ + healpy-$(healpy-version),,, \ + GPEP517) + echo "Healpy $(healpy-version)" > $@ + $(ipydir)/html5lib-$(html5lib-version): \ $(ipydir)/six-$(six-version) \ @@ -353,13 +684,17 @@ $(ipydir)/html5lib-$(html5lib-version): \ $(call pybuild, tar -xf, html5lib-$(html5lib-version), , \ HTML5lib $(html5lib-version)) -$(ipydir)/idna-$(idna-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/idna-$(idna-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=idna-$(idna-version).tar.gz $(call import-source, $(idna-url), $(idna-checksum)) $(call pybuild, tar -xf, idna-$(idna-version), , \ idna $(idna-version)) -$(ipydir)/jeepney-$(jeepney-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/jeepney-$(jeepney-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=jeepney-$(jeepney-version).tar.gz $(call import-source, $(jeepney-url), $(jeepney-checksum)) $(call pybuild, tar -xf, jeepney-$(jeepney-version), , \ @@ -371,20 +706,73 @@ $(ipydir)/jinja2-$(jinja2-version): $(ipydir)/markupsafe-$(markupsafe-version) $(call pybuild, tar -xf, jinja2-$(jinja2-version), , \ Jinja2 $(jinja2-version)) +$(ipydir)/joblib-$(joblib-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=joblib-$(joblib-version).tar.lz + $(call import-source, $(joblib-url), $(joblib-checksum)) + $(call pybuild, tar -xf, \ + joblib-$(joblib-version),,, GPEP517) + $(ipydir)/keyring-$(keyring-version): \ $(ipydir)/entrypoints-$(entrypoints-version) \ $(ipydir)/secretstorage-$(secretstorage-version) \ - $(ipydir)/setuptools_scm-$(setuptools_scm-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=keyring-$(keyring-version).tar.gz $(call import-source, $(keyring-url), $(keyring-checksum)) $(call pybuild, tar -xf, keyring-$(keyring-version), , \ Keyring $(keyring-version)) -$(ipydir)/kiwisolver-$(kiwisolver-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/kiwisolver-$(kiwisolver-version): \ + $(ipydir)/cppy-$(cppy-version) \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=kiwisolver-$(kiwisolver-version).tar.lz $(call import-source, $(kiwisolver-url), $(kiwisolver-checksum)) $(call pybuild, tar -xf, kiwisolver-$(kiwisolver-version), , \ - Kiwisolver $(kiwisolver-version)) + Kiwisolver $(kiwisolver-version), GPEP517) + cp -pv $(dtexdir)/kiwisolver.tex $(ictdir)/ + echo "Kiwisolver $(kiwisolver-version) \citep{cassowary2001}" > $@ + +# 2025-05-21: As of Feb 2025, it appears the lscsoft group is not maintaining +# python-ligo-lw and ligo-segments; the LIGO Computing Group forked these +# to igwn-ligolw and igwn-segments, respectively [1]. +# [1] https://github.com/ahnitz/ligo-segments/issues/1 +$(ipydir)/igwn-segments-$(igwn-segments-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=igwn-segments-$(igwn-segments-version).tar.lz + $(call import-source, $(igwn-segments-url), $(igwn-segments-checksum)) + $(call pybuild, tar -xf, \ + igwn-segments-$(igwn-segments-version),, \ + Igwn-segments $(igwn-segments-version), GPEP517) + + +$(ipydir)/iniconfig-$(iniconfig-version): \ + $(ipydir)/hatch-vcs-$(hatch-vcs-version) + tarball=iniconfig-$(iniconfig-version).tar.lz + $(call import-source, $(iniconfig-url), $(iniconfig-checksum)) + $(call pybuild, tar -xf, iniconfig-$(iniconfig-version), , \ + Iniconfig $(iniconfig-version), GPEP517) + +# Upstream this is called ligo.skymap with a full stop '.'; we replace it +# by a hyphen '-' to reduce the chance of regex errors. +# +# 2025-05-19 Warning: this might have some undetected dependencies. +$(ipydir)/ligo-skymap-$(ligo-skymap-version): \ + $(ibidir)/gsl-$(gsl-version) \ + $(ipydir)/tqdm-$(tqdm-version) \ + $(ibidir)/sqlite-$(sqlite3-version) \ + $(ipydir)/astropy-$(astropy-version) \ + $(ipydir)/lalsuite-$(lalsuite-version) \ + $(ipydir)/igwn-ligolw-$(igwn-ligolw-version) \ + $(ipydir)/igwn-segments-$(igwn-segments-version) \ + $(ipydir)/astropy-healpix-$(astropy-healpix-version) + tarball=ligo-skymap-$(ligo-skymap-version).tar.lz + $(call import-source, $(ligo-skymap-url), \ + $(ligo-skymap-checksum)) + $(call pybuild, tar -xf, \ + ligo-skymap-$(ligo-skymap-version),,, \ + GPEP517) + echo "ligo-skymap $(ligo-skymap-version)" > $@ + $(ipydir)/lmfit-$(lmfit-version): \ $(ipydir)/six-$(six-version) \ @@ -399,30 +787,47 @@ $(ipydir)/lmfit-$(lmfit-version): \ $(call pybuild, tar -xf, lmfit-$(lmfit-version), , \ LMFIT $(lmfit-version)) +$(ipydir)/lscsoft-glue-$(lscsoft-glue-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) + tarball=lscsoft-glue-$(lscsoft-glue-version).tar.lz + $(call import-source, $(lscsoft-glue-url), $(lscsoft-glue-checksum)) + $(call pybuild, tar -xf, \ + lscsoft-glue-$(lscsoft-glue-version),,, GPEP517) + $(ipydir)/lsstdesccoord-$(lsstdesccoord-version): \ - $(ipydir)/cython-$(cython-version) + $(ipydir)/cffi-$(cffi-version) \ + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/future-$(future-version) tarball=lsstdesccoord-$(lsstdesccoord-version).tar.gz $(call import-source, $(lsstdesccoord-url), $(lsstdesccoord-checksum)) $(call pybuild, tar -xf, LSSTDESC.Coord-$(lsstdesccoord-version), , \ LSSTDESC.Coord $(lsstdesccoord-version)) $(ipydir)/markupsafe-$(markupsafe-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=markupsafe-$(markupsafe-version).tar.lz $(call import-source, $(markupsafe-url), $(markupsafe-checksum)) $(call pybuild, tar -xf, markupsafe-$(markupsafe-version), , \ MarkupSafe $(markupsafe-version)) +# As of 2026-04-19: Matplotlib needs internet to download and build its +# custom versions of 'freetype' and 'qhull'. The proposed option to 'python +# -m pip instsall' in [1] for using system libraries was tested, but did +# not work in this setup (our 'pybuild' uses 'python -m gpep517 +# install-from-source', not 'pip'). +# +# [1] https://matplotlib.org/stable/install/dependencies.html $(ipydir)/matplotlib-$(matplotlib-version): \ $(itidir)/texlive \ $(ipydir)/numpy-$(numpy-version) \ $(ipydir)/cycler-$(cycler-version) \ $(ipydir)/pillow-$(pillow-version) \ $(ibidir)/freetype-$(freetype-version) \ - $(ipydir)/pyparsing-$(pyparsing-version) \ + $(ipydir)/fonttools-$(fonttools-version) \ + $(ipydir)/contourpy-$(contourpy-version) \ $(ipydir)/kiwisolver-$(kiwisolver-version) \ - $(ibidir)/ghostscript-$(ghostscript-version) \ - $(ibidir)/imagemagick-$(imagemagick-version) \ $(ipydir)/python-dateutil-$(python-dateutil-version) # Prepare the source. @@ -435,52 +840,98 @@ $(ipydir)/matplotlib-$(matplotlib-version): \ export CC=clang export CXX=clang++ fi - $(call pybuild, tar -xf, matplotlib-$(matplotlib-version)) + $(call pybuild, tar -xf, matplotlib-$(matplotlib-version),,, GPEP517) cp $(dtexdir)/matplotlib.tex $(ictdir)/ echo "Matplotlib $(matplotlib-version) \citep{matplotlib2007}" > $@ +$(ipydir)/meson-$(meson-version): \ + $(ibidir)/ninjabuild-$(ninjabuild-version) \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=meson-$(meson-version).tar.lz + $(call import-source, $(meson-url), $(meson-checksum)) + $(call pybuild, tar -xf, meson-$(meson-version), , \ + Meson $(meson-version), GPEP517) + echo "Meson $(meson-version)" > $@ + +# The 'meson-python' package may be helpful or requred for some packages. +$(ipydir)/meson-python-$(meson-python-version): \ + $(ipydir)/meson-$(meson-version) \ + $(ipydir)/packaging-$(packaging-version) \ + $(ipydir)/pyproject-metadata-$(pyproject-metadata-version) + tarball=meson-python-$(meson-python-version).tar.lz + $(call import-source, $(meson-python-url), $(meson-python-checksum)) + $(call pybuild, tar -xf, meson-python-$(meson-python-version), , \ + Meson-python $(meson-python-version), GPEP517) + echo "Meson-Python $(meson-python-version)" > $@ + $(ipydir)/mpi4py-$(mpi4py-version): \ $(ibidir)/openmpi-$(openmpi-version) \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/cython-$(cython-version) tarball=mpi4py-$(mpi4py-version).tar.lz $(call import-source, $(mpi4py-url), $(mpi4py-checksum)) - $(call pybuild, tar -xf, mpi4py-$(mpi4py-version)) + $(call pybuild, tar -xf, mpi4py-$(mpi4py-version),,, GPEP517) cp $(dtexdir)/mpi4py.tex $(ictdir)/ echo "mpi4py $(mpi4py-version) \citep{mpi4py2011}" > $@ -$(ipydir)/mpmath-$(mpmath-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/mpmath-$(mpmath-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=mpmath-$(mpmath-version).tar.gz $(call import-source, $(mpmath-url), $(mpmath-checksum)) $(call pybuild, tar -xf, mpmath-$(mpmath-version), , \ mpmath $(mpmath-version)) +$(ipydir)/networkx-$(networkx-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=networkx-$(networkx-version).tar.lz + $(call import-source, $(networkx-url), $(networkx-checksum)) + $(call pybuild, tar -xf, \ + networkx-$(networkx-version),,, \ + GPEP517) + echo "networkx $(networkx-version)" > $@ + +# Numpy's MacOS builds using Xcode prior to 26.3 needed the following flags +# added to 'LDFLAGS': '-undefined dynamic_lookup -bundle'. $(ipydir)/numpy-$(numpy-version): \ - $(ibidir)/unzip-$(unzip-version) \ $(ipydir)/cython-$(cython-version) \ $(ibidir)/openblas-$(openblas-version) \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/pybind11-$(pybind11-version) \ + $(ipydir)/meson-python-$(meson-python-version) tarball=numpy-$(numpy-version).tar.lz $(call import-source, $(numpy-url), $(numpy-checksum)) - if [ x$(on_mac_os) = xyes ]; then - export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle" - else - export LDFLAGS="$(LDFLAGS) -shared" - fi - export CFLAGS="--std=c99 $$CFLAGS" conf="$$(pwd)/reproduce/software/config/numpy-scipy.cfg" $(call pybuild, tar -xf, numpy-$(numpy-version),$$conf, \ - Numpy $(numpy-version)) + Numpy $(numpy-version), GPEP517) cp $(dtexdir)/numpy.tex $(ictdir)/ - echo "Numpy $(numpy-version) \citep{numpy2011}" > $@ + echo "Numpy $(numpy-version) \citep{numpy2020}" > $@ $(ipydir)/packaging-$(packaging-version): \ $(ipydir)/pyparsing-$(pyparsing-version) tarball=packaging-$(packaging-version).tar.lz $(call import-source, $(packaging-url), $(packaging-checksum)) $(call pybuild, tar -xf, packaging-$(packaging-version), , \ - Packaging $(packaging-version)) + Packaging $(packaging-version), GPEP517) -$(ipydir)/pexpect-$(pexpect-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/pandas-$(pandas-version): \ + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/versioneer-$(versioneer-version) \ + $(ipydir)/pytz-$(pytz-version) + tarball=pandas-$(pandas-version).tar.lz + $(call import-source, $(pandas-url), $(pandas-checksum)) + $(call pybuild, tar -xf, pandas-$(pandas-version), , \ + Pandas $(pandas-version), GPEP517) + echo "pandas $(pandas-version)" > $@ + +$(ipydir)/pathspec-$(pathspec-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=pathspec-$(pathspec-version).tar.lz + $(call import-source, $(pathspec-url), $(pathspec-checksum)) + $(call pybuild, tar -xf, \ + pathspec-$(pathspec-version),,, GPEP517) + +$(ipydir)/pexpect-$(pexpect-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pexpect-$(pexpect-version).tar.gz $(call import-source, $(pexpect-url), $(pexpect-checksum)) $(call pybuild, tar -xf, pexpect-$(pexpect-version), , \ @@ -488,25 +939,39 @@ $(ipydir)/pexpect-$(pexpect-version): $(ipydir)/setuptools-$(setuptools-version) $(ipydir)/pillow-$(pillow-version): $(ibidir)/libjpeg-$(libjpeg-version) \ $(ipydir)/setuptools-$(setuptools-version) - tarball=Pillow-$(pillow-version).tar.lz + tarball=pillow-$(pillow-version).tar.lz $(call import-source, $(pillow-url), $(pillow-checksum)) - $(call pybuild, tar -xf, Pillow-$(pillow-version), , \ - Pillow $(pillow-version)) - -$(ipydir)/pip-$(pip-version): $(ipydir)/setuptools-$(setuptools-version) - tarball=pip-$(pip-version).tar.gz - $(call import-source, $(pip-url), $(pip-checksum)) - $(call pybuild, tar -xf, pip-$(pip-version), , \ - PiP $(pip-version)) - -$(ipydir)/ply-$(ply-version): $(ipydir)/setuptools-$(setuptools-version) + $(call pybuild, tar -xf, pillow-$(pillow-version), , \ + Pillow $(pillow-version), GPEP517) + +# This should normally not be used, because it's a front-end that obstructs +# reproducibility - source URL; checksum of the tarball; build rule. +# $(ipydir)/pip-$(pip-version): \ +# $(ipydir)/python-installer-$(python-installer-version) \ +# $(ipydir)/wheel-$(wheel-version) +# tarball=pip-$(pip-version).tar.gz +# $(call import-source, $(pip-url), $(pip-checksum)) +# $(call pybuild, tar -xf, pip-$(pip-version), , \ +# PiP $(pip-version)) + +$(ipydir)/pluggy-$(pluggy-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=pluggy-$(pluggy-version).tar.lz + $(call import-source, $(pluggy-url), $(pluggy-checksum)) + $(call pybuild, tar -xf, \ + pluggy-$(pluggy-version),,, GPEP517) + +$(ipydir)/ply-$(ply-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=ply-$(ply-version).tar.lz $(call import-source, $(ply-url), $(ply-checksum)) $(call pybuild, tar -xf, ply-$(ply-version), , \ ply $(ply-version)) $(ipydir)/pycodestyle-$(pycodestyle-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pycodestyle-$(pycodestyle-version).tar.gz $(call import-source, $(pycodestyle-url), $(pycodestyle-checksum)) $(call pybuild, tar -xf, pycodestyle-$(pycodestyle-version), , \ @@ -515,16 +980,21 @@ $(ipydir)/pycodestyle-$(pycodestyle-version): \ $(ipydir)/pybind11-$(pybind11-version): \ $(ibidir)/eigen-$(eigen-version) \ $(ibidir)/boost-$(boost-version) \ - $(ipydir)/setuptools-$(setuptools-version) - tarball=pybind11-$(pybind11-version).tar.gz + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) + tarball=pybind11-$(pybind11-version).tar.lz $(call import-source, $(pybind11-url), $(pybind11-checksum)) pyhook_after() { - cp -r include/pybind11 $(iidir)/python$(python-major-version)m/ + cp -pvr pybind11/include/pybind11 \ + $(iidir)/python$(python-major-version)m/ } $(call pybuild, tar -xf, pybind11-$(pybind11-version), , \ - pybind11 $(pybind11-version)) + pybind11 $(pybind11-version), GPEP517) + echo "Pybind11 $(pybind11-version)" > $@ -$(ipydir)/pycparser-$(pycparser-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/pycparser-$(pycparser-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pycparser-$(pycparser-version).tar.gz $(call import-source, $(pycparser-url), $(pycparser-checksum)) $(call pybuild, tar -xf, pycparser-$(pycparser-version), , \ @@ -532,45 +1002,139 @@ $(ipydir)/pycparser-$(pycparser-version): $(ipydir)/setuptools-$(setuptools-vers $(ipydir)/pyerfa-$(pyerfa-version): \ $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/packaging-$(packaging-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=pyerfa-$(pyerfa-version).tar.lz $(call import-source, $(pyerfa-url), $(pyerfa-checksum)) $(call pybuild, tar -xf, pyerfa-$(pyerfa-version), , \ - PyERFA $(pyerfa-version)) + PyERFA $(pyerfa-version), GPEP517) -$(ipydir)/pyflakes-$(pyflakes-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/pyflakes-$(pyflakes-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pyflakes-$(pyflakes-version).tar.gz $(call import-source, $(pyflakes-url), $(pyflakes-checksum)) $(call pybuild, tar -xf, pyflakes-$(pyflakes-version), , \ pyflakes $(pyflakes-version)) +$(ipydir)/pygments-$(pygments-version): \ + $(ipydir)/hatchling-$(hatchling-version) + tarball=pygments-$(pygments-version).tar.lz + $(call import-source, $(pygments-url), $(pygments-checksum)) + $(call pybuild, tar -xf, pygments-$(pygments-version), , \ + Pygments $(pygments-version), GPEP517) + $(ipydir)/pyparsing-$(pyparsing-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/flit-core-$(flit-core-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pyparsing-$(pyparsing-version).tar.lz $(call import-source, $(pyparsing-url), $(pyparsing-checksum)) $(call pybuild, tar -xf, pyparsing-$(pyparsing-version), , \ - PyParsing $(pyparsing-version)) + PyParsing $(pyparsing-version), GPEP517) -$(ipydir)/pypkgconfig-$(pypkgconfig-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/pypkgconfig-$(pypkgconfig-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pkgconfig-$(pypkgconfig-version).tar.gz $(call import-source, $(pypkgconfig-url), $(pypkgconfig-checksum)) $(call pybuild, tar -xf, pkgconfig-$(pypkgconfig-version), , pkgconfig $(pypkgconfig-version)) +$(ipydir)/pyproject-metadata-$(pyproject-metadata-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/flit-core-$(flit-core-version) \ + $(ipydir)/python-installer-$(python-installer-version) + tarball=pyproject-metadata-$(pyproject-metadata-version).tar.lz + $(call import-source, $(pyproject-metadata-url), $(pyproject-metadata-checksum)) + $(call pybuild, tar -xf, \ + pyproject-metadata-$(pyproject-metadata-version),,, GPEP517) + + +$(ipydir)/pytest-$(pytest-version): \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) \ + $(ipydir)/colorama-$(colorama-version) \ + $(ipydir)/exceptiongroup-$(exceptiongroup-version) \ + $(ipydir)/iniconfig-$(iniconfig-version) \ + $(ipydir)/packaging-$(packaging-version) \ + $(ipydir)/pluggy-$(pluggy-version) \ + $(ipydir)/pygments-$(pygments-version) + tarball=pytest-$(pytest-version).tar.lz + $(call import-source, $(pytest-url), $(pytest-checksum)) + $(call pybuild, tar -xf, \ + pytest-$(pytest-version),,, \ + GPEP517) + echo "pytest $(pytest-version)" > $@ + + $(ipydir)/python-dateutil-$(python-dateutil-version): \ - $(ipydir)/six-$(six-version) \ - $(ipydir)/setuptools_scm-$(setuptools_scm-version) - tarball=python-dateutil-$(python-dateutil-version).tar.gz + $(ipydir)/setuptools-scm-$(setuptools-scm-version) \ + $(ipydir)/six-$(six-version) + tarball=python-dateutil-$(python-dateutil-version).tar.lz $(call import-source, $(python-dateutil-url), $(python-dateutil-checksum)) $(call pybuild, tar -xf, python-dateutil-$(python-dateutil-version), , \ - python-dateutil $(python-dateutil-version)) + python-dateutil $(python-dateutil-version), GPEP517) + +$(ipydir)/python-installer-$(python-installer-version): \ + $(ibidir)/python-$(python-version) + +# Prepare the tarball. + tarball=python-installer-$(python-installer-version).tar.lz + $(call import-source, $(python-installer-url), $(python-installer-checksum)) + +# Modify the line in the source that will cause a crash when a +# to-be-installed file already exists in the installation path. This +# is very important for Python packages in Maneage (when a dependency +# is updated, the package needs to be re-built, but that would cause +# due to this line). + pyhook_before(){ + mv -v src/installer/destinations.py src/installer/destinations.py.orig; \ + sed -e 's/\(raise FileExistsError.message.\)/## \1/' \ + src/installer/destinations.py.orig > src/installer/destinations.py + } + +# Build the Python installer. + $(call pybuild, tar -xf, \ + python-installer-$(python-installer-version),,, \ + BOOT_INSTALLER) + echo "Python-installer $(python-installer-version)" > $@ + +# 2025-05-21: As of Feb 2025, it appears the lscsoft group is not maintaining +# python-ligo-lw and ligo-segments; the LIGO Computing Group forked these +# to igwn-ligolw and igwn-segments, respectively [1]. +# [1] https://github.com/ahnitz/ligo-segments/issues/1 +$(ipydir)/igwn-ligolw-$(igwn-ligolw-version): \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) + tarball=igwn-ligolw-$(igwn-ligolw-version).tar.lz + $(call import-source, $(igwn-ligolw-url), \ + $(igwn-ligolw-checksum)) + $(call pybuild, tar -xf, \ + igwn-ligolw-$(igwn-ligolw-version),,, \ + GPEP517) + echo "igwn-ligolw $(igwn-ligolw-version)" > $@ + +$(ipydir)/pyfftw-$(pyfftw-version): $(ipydir)/numpy-$(numpy-version) \ + $(ibidir)/fftw-$(fftw-version) + tarball=pyfftw-$(pyfftw-version).tar.lz + $(call import-source, $(pyfftw-url), $(pyfftw-checksum)) + $(call pybuild, tar -xf, pyfftw-$(pyfftw-version), , \ + Pyfftw $(pyfftw-version), GPEP517) + + +# Called 'fitsio' in pypi.org: https://pypi.org/project/fitsio +$(ipydir)/python3-fitsio-$(python3-fitsio-version): \ + $(ibidir)/patch-$(patch-version) \ + $(ipydir)/numpy-$(numpy-version) + tarball=python3-fitsio-$(python3-fitsio-version).tar.lz + $(call import-source, $(python3-fitsio-url), $(python3-fitsio-checksum)) + $(call pybuild, tar -xf, python3-fitsio-$(python3-fitsio-version), , \ + Python3 Fitsio $(python3-fitsio-version), GPEP517) $(ipydir)/pythran-$(pythran-version): \ $(ipydir)/ply-$(ply-version) \ $(ipydir)/gast-$(gast-version) \ $(ibidir)/boost-$(boost-version) \ $(ipydir)/beniget-$(beniget-version) \ - $(ipydir)/setuptools_scm-$(setuptools_scm-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=pythran-$(pythran-version).tar.lz $(call import-source, $(pythran-url), $(pythran-checksum)) $(call pybuild, tar -xf, pythran-$(pythran-version), , \ @@ -579,36 +1143,64 @@ $(ipydir)/pythran-$(pythran-version): \ $(ipydir)/pyyaml-$(pyyaml-version): \ $(ibidir)/yaml-$(yaml-version) \ $(ipydir)/cython-$(cython-version) - tarball=pyyaml-$(pyyaml-version).tar.gz + tarball=pyyaml-$(pyyaml-version).tar.lz $(call import-source, $(pyyaml-url), $(pyyaml-checksum)) - $(call pybuild, tar -xf, PyYAML-$(pyyaml-version), , \ - PyYAML $(pyyaml-version)) + $(call pybuild, tar -xf, pyyaml-$(pyyaml-version), , \ + PyYAML $(pyyaml-version), GPEP517) + +$(ipydir)/pytz-$(pytz-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=pytz-$(pytz-version).tar.lz + $(call import-source, $(pytz-url), $(pytz-checksum)) + $(call pybuild, tar -xf, pytz-$(pytz-version), , \ + Pytz $(pytz-version), GPEP517) $(ipydir)/requests-$(requests-version): $(ipydir)/idna-$(idna-version) \ - $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/certifi-$(certifi-version) \ - $(ipydir)/chardet-$(chardet-version) \ - $(ipydir)/urllib3-$(urllib3-version) + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/certifi-$(certifi-version) \ + $(ipydir)/chardet-$(chardet-version) \ + $(ipydir)/urllib3-$(urllib3-version) tarball=requests-$(requests-version).tar.gz $(call import-source, $(requests-url), $(requests-checksum)) $(call pybuild, tar -xf, requests-$(requests-version), , \ Requests $(requests-version)) +# Scipy building notes: +# +# 1. 'pythran' is disabled in the build of Scipy because of complications +# it caused on some systems. 'Pythran' can in principle speed up +# compilation of scientific software [1][2]. We explicitly disable it by +# modifying the source. Technically it should also be possible to pass +# the following option to the gpep517 build recipes with correct +# escaping of single and double quotes, but this has not tried as of +# 2025-02-25. +# --config-json='{"setup-args":"-Duse-pythran=false"}' +# +# 2. Scipy's MacOS builds using Xcode prior to 26.3 needed the following +# flags added to 'LDFLAGS': '-undefined dynamic_lookup -bundle'. +# +# [1] https://pythran.readthedocs.io/en/latest +# [2] https://docs.scipy.org/doc/scipy-1.15.2/dev/roadmap-detailed.html#use-of-pythran $(ipydir)/scipy-$(scipy-version): \ $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/pythran-$(pythran-version) \ $(ipydir)/pybind11-$(pybind11-version) + +# Import the tarball. tarball=scipy-$(scipy-version).tar.lz $(call import-source, $(scipy-url), $(scipy-checksum)) - if [ x$(on_mac_os) = xyes ]; then - export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle" - else - export LDFLAGS="$(LDFLAGS) -shared" - fi + +# Disable pythran (see above). + pyhook_before() { + mv -iv meson.options meson.options.orig; \ + sed -e 's/\(use-pythran.*value: *\)true/\1false/' \ + meson.options.orig > meson.options + } + +# Build Scipy. conf="$$(pwd)/reproduce/software/config/numpy-scipy.cfg" - $(call pybuild, tar -xf, scipy-$(scipy-version),$$conf) + $(call pybuild, tar -xf, scipy-$(scipy-version),$$conf,, GPEP517) cp $(dtexdir)/scipy.tex $(ictdir)/ - echo "Scipy $(scipy-version) \citep{scipy2007,scipy2011}" > $@ + echo "Scipy $(scipy-version) \citep{scipy2020}" > $@ $(ipydir)/secretstorage-$(secretstorage-version): \ $(ipydir)/jeepney-$(jeepney-version) \ @@ -618,27 +1210,38 @@ $(ipydir)/secretstorage-$(secretstorage-version): \ $(call pybuild, tar -xf, SecretStorage-$(secretstorage-version), , \ SecretStorage $(secretstorage-version)) +$(ipydir)/semantic-version-$(semantic-version-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=semantic-version-$(semantic-version-version).tar.lz + $(call import-source, $(semantic-version-url), \ + $(semantic-version-checksum)) + $(call pybuild, tar -xf, \ + semantic-version-$(semantic-version-version), , \ + Semantic-version $(semantic-version-version), GPEP517) + $(ipydir)/setuptools-$(setuptools-version): \ - $(ibidir)/unzip-$(unzip-version) \ - $(ibidir)/python-$(python-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=setuptools-$(setuptools-version).tar.lz $(call import-source, $(setuptools-url), $(setuptools-checksum)) $(call pybuild, tar -xf, setuptools-$(setuptools-version), , \ - Setuptools $(setuptools-version)) + Setuptools $(setuptools-version), GPEP517) -$(ipydir)/setuptools_scm-$(setuptools_scm-version): \ +$(ipydir)/setuptools-scm-$(setuptools-scm-version): \ $(ipydir)/setuptools-$(setuptools-version) - tarball=setuptools_scm-$(setuptools_scm-version).tar.gz - $(call import-source, $(setuptools_scm-url), $(setuptools_scm-checksum)) - $(call pybuild, tar -xf, setuptools_scm-$(setuptools_scm-version), , \ - Setuptools-scm $(setuptools_scm-version)) + tarball=setuptools-scm-$(setuptools-scm-version).tar.lz + $(call import-source, $(setuptools-scm-url), $(setuptools-scm-checksum)) + $(call pybuild, tar -xf, setuptools-scm-$(setuptools-scm-version), , \ + Setuptools-scm $(setuptools-scm-version), GPEP517) $(ipydir)/setuptools-rust-$(setuptools-rust-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) \ + $(ipydir)/semantic-version-$(semantic-version-version) tarball=setuptools-rust-$(setuptools-rust-version).tar.lz - $(call import-source, $(setuptools-rust-url), $(setuptools-rust-checksum)) + $(call import-source, $(setuptools-rust-url), \ + $(setuptools-rust-checksum)) $(call pybuild, tar -xf, setuptools-rust-$(setuptools-rust-version), , \ - Setuptools-scm $(setuptools-rust-version)) + Setuptools-rust $(setuptools-rust-version), GPEP517) $(ipydir)/sip_tpv-$(sip_tpv-version): \ $(ipydir)/sympy-$(sympy-version) \ @@ -649,14 +1252,37 @@ $(ipydir)/sip_tpv-$(sip_tpv-version): \ cp $(dtexdir)/sip_tpv.tex $(ictdir)/ echo "sip_tpv $(sip_tpv-version) \citep{sip-tpv}" > $@ - -$(ipydir)/six-$(six-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/scikit-base-$(scikit-base-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) + tarball=scikit-base-$(scikit-base-version).tar.lz + $(call import-source, $(scikit-base-url), $(scikit-base-checksum)) + $(call pybuild, tar -xf, scikit-base-$(scikit-base-version), , \ + Scikit-base $(scikit-base-version), GPEP517) + echo "scikit-base $(scikit-base-version)" > $@ + +$(ipydir)/scikit-learn-$(scikit-learn-version): \ + $(ipydir)/joblib-$(joblib-version) \ + $(ipydir)/threadpoolctl-$(threadpoolctl-version) \ + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/scipy-$(scipy-version) + tarball=scikit-learn-$(scikit-learn-version).tar.lz + $(call import-source, $(scikit-learn-url), $(scikit-learn-checksum)) + $(call pybuild, tar -xf, scikit-learn-$(scikit-learn-version), , \ + Scikit-learn $(scikit-learn-version), GPEP517) + echo "scikit-learn $(scikit-learn-version)" > $@ + +$(ipydir)/six-$(six-version): \ + $(ipydir)/setuptools-$(setuptools-version) tarball=six-$(six-version).tar.lz $(call import-source, $(six-url), $(six-checksum)) $(call pybuild, tar -xf, six-$(six-version), , \ - Six $(six-version)) + Six $(six-version), GPEP517) + echo "Six $(six-version)" > $@ -$(ipydir)/soupsieve-$(soupsieve-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/soupsieve-$(soupsieve-version): \ + $(ipydir)/setuptools-$(setuptools-version) \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=soupsieve-$(soupsieve-version).tar.gz $(call import-source, $(soupsieve-url), $(soupsieve-checksum)) $(call pybuild, tar -xf, soupsieve-$(soupsieve-version), , \ @@ -669,27 +1295,66 @@ $(ipydir)/sympy-$(sympy-version): $(ipydir)/mpmath-$(mpmath-version) cp $(dtexdir)/sympy.tex $(ictdir)/ echo "SymPy $(sympy-version) \citep{sympy}" > $@ +$(ipydir)/threadpoolctl-$(threadpoolctl-version): \ + $(ipydir)/flit-core-$(flit-core-version) + tarball=threadpoolctl-$(threadpoolctl-version).tar.lz + $(call import-source, $(threadpoolctl-url), $(threadpoolctl-checksum)) + $(call pybuild, tar -xf, threadpoolctl-$(threadpoolctl-version), , \ + threadpoolctl $(threadpoolctl-version), GPEP517) + echo "threadpoolctl $(threadpoolctl-version)" > $@ + +$(ipydir)/tqdm-$(tqdm-version): \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) + tarball=tqdm-$(tqdm-version).tar.lz + $(call import-source, $(tqdm-url), $(tqdm-checksum)) + $(call pybuild, tar -xf, \ + tqdm-$(tqdm-version),, \ + Tqdm $(tqdm-version), GPEP517) + +$(ipydir)/trove-classifiers-$(trove-classifiers-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=trove-classifiers-$(trove-classifiers-version).tar.lz + $(call import-source, $(trove-classifiers-url), $(trove-classifiers-checksum)) + $(call pybuild, tar -xf, \ + trove-classifiers-$(trove-classifiers-version),,, GPEP517) + $(ipydir)/uncertainties-$(uncertainties-version): $(ipydir)/numpy-$(numpy-version) tarball=uncertainties-$(uncertainties-version).tar.lz $(call import-source, $(uncertainties-url), $(uncertainties-checksum)) $(call pybuild, tar -xf, uncertainties-$(uncertainties-version), , \ uncertainties $(uncertainties-version)) -$(ipydir)/urllib3-$(urllib3-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/urllib3-$(urllib3-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=urllib3-$(urllib3-version).tar.gz $(call import-source, $(urllib3-url), $(urllib3-checksum)) $(call pybuild, tar -xf, urllib3-$(urllib3-version), , \ Urllib3 $(urllib3-version)) +$(ipydir)/versioneer-$(versioneer-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=versioneer-$(versioneer-version).tar.lz + $(call import-source, $(versioneer-url), $(versioneer-checksum)) + $(call pybuild, tar -xf, versioneer-$(versioneer-version), , \ + Versioneer $(versioneer-version), GPEP517) + $(ipydir)/webencodings-$(webencodings-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/setuptools-$(setuptools-version) \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=webencodings-$(webencodings-version).tar.gz $(call import-source, $(webencodings-url), $(webencodings-checksum)) $(call pybuild, tar -xf, webencodings-$(webencodings-version), , \ Webencodings $(webencodings-version)) -$(ipydir)/wheel-$(wheel-version): $(ipydir)/setuptools-$(setuptools-version) - tarball=wheel-$(wheel-version).tar.lz +# As of 2025-02, this is only needed if you want 'wheel' on the command +# line; 'setuptools' provides its own version of wheels. +$(ipydir)/wheel-$(wheel-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/flit-core-$(flit-core-version) \ + $(ipydir)/python-installer-$(python-installer-version) +# tarball=wheel-$(wheel-version).tar.lz + tarball=wheel-$(wheel-version).tar.gz $(call import-source, $(wheel-url), $(wheel-checksum)) $(call pybuild, tar -xf, wheel-$(wheel-version), , \ - Wheel $(wheel-version)) + Wheel $(wheel-version), GPEP517) diff --git a/reproduce/software/make/r-cran.mk b/reproduce/software/make/r-cran.mk index 617b8de..7a09915 100644 --- a/reproduce/software/make/r-cran.mk +++ b/reproduce/software/make/r-cran.mk @@ -12,8 +12,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2022 Boud Roukema <boud@cosmo.torun.pl> -# Copyright (C) 2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2022-2026 Boud Roukema <boud@cosmo.torun.pl> +# Copyright (C) 2022-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -204,6 +204,7 @@ $(ibidir)/r-cran-$(r-cran-version): \ tar -xf $(tdir)/$$tarball unpackdir=R-$(r-cran-version) cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) # We need to manually remove the lines with '~autodetect~', they # cause the configure script to crash in version 4.0.2. They are used diff --git a/reproduce/software/make/xorg.mk b/reproduce/software/make/xorg.mk index dd707e5..33847f3 100644 --- a/reproduce/software/make/xorg.mk +++ b/reproduce/software/make/xorg.mk @@ -14,8 +14,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2021-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2021-2022 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2021-2026 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2021-2026 Raul Infante-Sainz <infantesainz@gmail.com> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -79,7 +79,7 @@ $(ibidir)/libxau-$(libxau-version): $(ibidir)/xorgproto-$(xorgproto-version) # Library implementing the X Display Manager Control Protocol. $(ibidir)/libxdmcp-$(libxdmcp-version): $(ibidir)/libxau-$(libxau-version) - tarball=libXdmcp-$(libxdmcp-version).tar.bz2 + tarball=libXdmcp-$(libxdmcp-version).tar.lz $(call import-source, $(libxdmcp-url), $(libxdmcp-checksum)) $(call gbuild, libXdmcp-$(libxdmcp-version),,$(XORG_CONFIG), V=1) echo "libXdmcp (Xorg) $(libxdmcp-version)" > $@ @@ -120,6 +120,7 @@ $(ibidir)/fontconfig-$(fontconfig-version): \ $(ibidir)/libxml2-$(libxml2-version) \ $(ibidir)/freetype-$(freetype-version) \ $(ibidir)/util-linux-$(util-linux-version) + # Import the source. tarball=fontconfig-$(fontconfig-version).tar.lz $(call import-source, $(fontconfig-url), $(fontconfig-checksum)) |