diff options
Diffstat (limited to 'reproduce/software/make')
| -rw-r--r-- | reproduce/software/make/README.md | 2 | ||||
| -rw-r--r-- | reproduce/software/make/atlas-multiple.mk | 2 | ||||
| -rw-r--r-- | reproduce/software/make/atlas-single.mk | 2 | ||||
| -rw-r--r-- | reproduce/software/make/basic.mk | 251 | ||||
| -rw-r--r-- | reproduce/software/make/build-rules.mk | 37 | ||||
| -rw-r--r-- | reproduce/software/make/high-level.mk | 275 | ||||
| -rw-r--r-- | reproduce/software/make/python.mk | 612 | ||||
| -rw-r--r-- | reproduce/software/make/r-cran.mk | 5 | ||||
| -rw-r--r-- | reproduce/software/make/xorg.mk | 6 |
9 files changed, 854 insertions, 338 deletions
diff --git a/reproduce/software/make/README.md b/reproduce/software/make/README.md index 76ab5c1..00afef4 100644 --- a/reproduce/software/make/README.md +++ b/reproduce/software/make/README.md @@ -1,7 +1,7 @@ Software building instructions ------------------------------ -Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org>\ +Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org>\ See the end of the file for license conditions. This directory contains Makefiles that are called by the high-level diff --git a/reproduce/software/make/atlas-multiple.mk b/reproduce/software/make/atlas-multiple.mk index b92fbfc..6e7d415 100644 --- a/reproduce/software/make/atlas-multiple.mk +++ b/reproduce/software/make/atlas-multiple.mk @@ -8,7 +8,7 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/make/atlas-single.mk b/reproduce/software/make/atlas-single.mk index 720bad6..2b68677 100644 --- a/reproduce/software/make/atlas-single.mk +++ b/reproduce/software/make/atlas-single.mk @@ -8,7 +8,7 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/reproduce/software/make/basic.mk b/reproduce/software/make/basic.mk index 14d63cf..4b18c29 100644 --- a/reproduce/software/make/basic.mk +++ b/reproduce/software/make/basic.mk @@ -21,9 +21,9 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2019-2022 Raul Infante-Sainz <infantesainz@gmail.com> -# Copyright (C) 2022 Pedram Ashofteh Ardakani <pedramardakani@pm.me> +# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2022-2025 Pedram Ashofteh Ardakani <pedramardakani@pm.me> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,7 +39,7 @@ # along with this Makefile. If not, see <http://www.gnu.org/licenses/>. # Top level environment -include reproduce/software/config/LOCAL.conf +include .build/software/config/LOCAL.conf include reproduce/software/make/build-rules.mk include reproduce/software/config/versions.conf include reproduce/software/config/checksums.conf @@ -49,23 +49,27 @@ include reproduce/software/config/checksums.conf include reproduce/software/config/urls.conf # Basic directories -lockdir = $(BDIR)/software/locks -tdir = $(BDIR)/software/tarballs -ddir = $(BDIR)/software/build-tmp -idir = $(BDIR)/software/installed -ibdir = $(BDIR)/software/installed/bin -ildir = $(BDIR)/software/installed/lib -iidir = $(BDIR)/software/installed/include -ibidir = $(BDIR)/software/installed/version-info/proglib +lockdir = $(BDIR)/software/locks +tdir = $(BDIR)/software/tarballs +ddir = $(BDIR)/software/build-tmp +idir = $(BDIR)/software/installed +ibdir = $(BDIR)/software/installed/bin +ildir = $(BDIR)/software/installed/lib +iidir = $(BDIR)/software/installed/include +shsrcdir = "$(shell pwd)"/reproduce/software/shell +ibidir = $(BDIR)/software/installed/version-info/proglib # Ultimate Makefile target. GNU Nano (a simple and very light-weight text # editor) is installed by default, it is recommended to have it in the # 'basic.mk', so Maneaged projects can be edited on any system (even when # there is no command-line text editor available). +# +# The recipe is '@echo > /dev/null' so Make does not print "make: Nothing +# to be done for 'all'." targets-proglib = low-level-links \ gcc-$(gcc-version) \ nano-$(nano-version) -all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) +all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)); @echo > /dev/null # Define the shell environment # ---------------------------- @@ -107,6 +111,15 @@ export CPPFLAGS := -I$(idir)/include $(CPPFLAGS) $(noccwarnings) # 'LD_LIBRARY_PATH', then we'll add our own newly installed libraries. We # will also make sure that there is no "current directory" in it (by # removing a starting or trailing ':' and any occurance of '::'. +# +# But first: in case LD_LIBRARY_PATH is empty, give it the default value of +# $(sys_library_sh_path) (which was the location of the libraries needed by +# the host's shell). This is because after we add the Maneage's library +# path, on some systems, no other libraries will be checked except those +# that are in 'LD_LIBRARY_PATH'. +ifeq ($(strip $(LD_LIBRARY_PATH)),) +export LD_LIBRARY_PATH=$(sys_library_sh_path) +endif export LD_LIBRARY_PATH := $(shell echo $(LD_LIBRARY_PATH):$(ildir) \ | sed -e's/::/:/g' -e's/^://' -e's/:$$//') @@ -127,17 +140,24 @@ export DYLD_LIBRARY_PATH := # Afer putting everything together, we use the first server as the # reference for all software if their '-url' variable isn't defined (in # 'reproduce/software/config/urls.conf'). -downloadwrapper = ./reproduce/analysis/bash/download-multi-try +downloadwrapper = ./reproduce/analysis/bash/download-multi-try.sh maneage_backup_urls := $(shell awk '!/^#/{printf "%s ", $$1}' \ reproduce/software/config/servers-backup.conf) backupservers_all = $(user_backup_urls) $(maneage_backup_urls) topbackupserver = $(word 1, $(backupservers_all)) backupservers = $(filter-out $(topbackupserver),$(backupservers_all)) - - - - +# When building in Apptainer containers, as of 2025-04-18, we need to +# configure Maneage as root (within the container). In such cases, we need +# to activate the 'FORCE_UNSAFE_CONFIGURE' environment variable to build +# some of the software. The 'if' statement is here to make sure we are in +# Apptainer: in other situations, the "unsafe" configure script shouldn't +# be activated. Note that this doesn't happen in Docker (where the Maneage +# source is in the same directory) because we build a non-root ('maneager' +# user there who executes the configure command. +unsafe-config = if [ $$(pwd) = "/home/maneager/source" ] \ + && [ $$(whoami) = root ]; then \ + export FORCE_UNSAFE_CONFIGURE=1; fi @@ -251,11 +271,8 @@ $(ibidir)/low-level-links: $(ibidir)/grep-$(grep-version) \ # # The first set of programs to be built are those that we need to unpack # the source code tarballs of each program. We have already installed Lzip -# before calling 'basic.mk', so it is present and working. Hence we first -# build the Lzipped tarball of Gzip, then use our own Gzip to unpack the -# tarballs of the other compression programs. Once all the compression -# programs/libraries are complete, we build our own GNU Tar and continue -# with other software. +# before calling 'basic.mk', so it is present and working. So the only +# prerequisites of these (until reaching Tar) is the necessary directories. $(lockdir): | $(BDIR); mkdir $@ $(ibidir)/gzip-$(gzip-version): | $(ibdir) $(ildir) $(lockdir) tarball=gzip-$(gzip-version).tar.lz @@ -263,49 +280,13 @@ $(ibidir)/gzip-$(gzip-version): | $(ibdir) $(ildir) $(lockdir) $(call gbuild, gzip-$(gzip-version), static, , V=1) echo "GNU Gzip $(gzip-version)" > $@ -# 2022-07-14 B Roukema -# -# xz-5.2.5 fails on (at least) CentOS 7 (Redhat) systems while trying -# to compile 'cmake' in Maneage - this is Maneage bug 62700 [1]. -# -# The fix appears to be just a few lines, although it's not clear -# how robust or long-term it is. Since we don't yet have 'patch' in -# 'basic.mk', this file has to be copied into place rather than patched. - -# xz-5.2.5_src_liblzma_liblzma.map is a patched -# version of xz-5.2.5/src/liblzma/liblzma.map based on discussion at -# [1] + [2] + the patch file [3]. -# -# [1] https://savannah.nongnu.org/bugs/index.php?62700 -# [2] https://github.com/easybuilders/easybuild-easyconfigs/issues/14991 -# [3] https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/bcebb3320ffb63f9804ca8d4d64d1822ec7c9792/easybuild/easyconfigs/x/XZ/XZ-5.2.5_compat-libs.patch -$(ibidir)/xz-$(xz-version): $(ibidir)/gzip-$(gzip-version) - -# Prepare the tarball. +$(ibidir)/xz-$(xz-version): | $(ibdir) $(ildir) $(lockdir) tarball=xz-$(xz-version).tar.lz $(call import-source, $(xz-url), $(xz-checksum)) - -# Until the bug mentioned above is fixed, we'll can't use the generic -# rule. -# $(call gbuild, xz-$(xz-version), static) - -# Configure and build with patched file. - srcdir=$$(pwd) - unpackdir=xz-$(xz-version) - patchedfile=xz-5.2.5_src_liblzma_liblzma.map - cd $(ddir) - rm -rf $$unpackdir - tar -x -f $(tdir)/$$tarball - cd $$unpackdir - cp -pv $$srcdir/reproduce/software/patches/$$patchedfile \ - src/liblzma/liblzma.map # copy the fixed file into place - ./configure --prefix=$(idir) - make install - cd .. - rm -rf $$unpackdir + $(call gbuild, xz-$(xz-version), static) echo "XZ Utils $(xz-version)" > $@ -$(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) +$(ibidir)/bzip2-$(bzip2-version): | $(ibdir) $(ildir) $(lockdir) # Download the tarball. tarball=bzip2-$(bzip2-version).tar.lz @@ -334,8 +315,9 @@ $(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) fi cd $(ddir) rm -rf $$tdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$tdir + $(shsrcdir)/prep-source.sh $(ibdir) sed -e 's@\(ln -s -f \)$$(PREFIX)/bin/@\1@' Makefile \ > Makefile.sed mv Makefile.sed Makefile @@ -355,7 +337,7 @@ $(ibidir)/bzip2-$(bzip2-version): $(ibidir)/gzip-$(gzip-version) # # Note for a static-only build: Zlib's './configure' doesn't use Autoconf's # configure script, it just accepts a direct '--static' option. -$(ibidir)/zlib-$(zlib-version): $(ibidir)/gzip-$(gzip-version) +$(ibidir)/zlib-$(zlib-version): | $(ibdir) $(ildir) $(lockdir) tarball=zlib-$(zlib-version).tar.lz $(call import-source, $(zlib-url), $(zlib-checksum)) $(call gbuild, zlib-$(zlib-version)) @@ -376,6 +358,7 @@ $(ibidir)/tar-$(tar-version): \ # a bottleneck here: only making Tar. So its more efficient to built # it on multiple threads (even when the user's Make doesn't pass down # the number of threads). + $(call unsafe-config) tarball=tar-$(tar-version).tar.lz $(call import-source, $(tar-url), $(tar-checksum)) $(call gbuild, tar-$(tar-version), , , -j$(numthreads) V=1) @@ -443,6 +426,7 @@ $(ibidir)/ncurses-$(ncurses-version): $(ibidir)/patchelf-$(patchelf-version) rm -f $(ibdir)/bash* $(ibdir)/awk* $(ibdir)/gawk* # Standard build process. + export CFLAGS="-std=gnu17 $$CFLAGS" $(call gbuild, ncurses-$(ncurses-version), static, \ --with-shared --enable-rpath --without-normal \ --without-debug --with-cxx-binding \ @@ -486,6 +470,9 @@ $(ibidir)/ncurses-$(ncurses-version): $(ibidir)/patchelf-$(patchelf-version) # # 5. A link is made to also be able to include files from the # 'ncurses' headers. +# +# 6. Top-level symbolic links are made for the 'include' (.h) files. +# if [ x$(on_mac_os) = xyes ]; then so="dylib"; else so="so"; fi if [ -f $(ildir)/libncursesw.$$so ]; then @@ -510,6 +497,11 @@ $(ibidir)/ncurses-$(ncurses-version): $(ibidir)/patchelf-$(patchelf-version) ln -fs $(ildir)/pkgconfig/ncursesw.pc pkgconfig/cursesw.pc ln -fs $(idir)/include/ncursesw $(idir)/include/ncurses + +# Add symbolic links for the ncursesw/*.h 'include' files +# so that they can be found in the top-level include/ directory. + cd "$(iidir)" + ln -fsv $(idir)/include/ncursesw/*.h . echo "GNU NCURSES $(ncurses-version)" > $@ else exit 1 @@ -573,7 +565,7 @@ $(ibidir)/bash-$(bash-version): \ if [ "x$(static_build)" = xyes ]; then stopt="--enable-static-link" else stopt="" fi; - export CFLAGS="$$CFLAGS \ + export CFLAGS="$$CFLAGS -std=gnu17 \ -DDEFAULT_PATH_VALUE='\"$(ibdir)\"' \ -DSTANDARD_UTILS_PATH='\"$(ibdir)\"' \ -DSYS_BASHRC='\"$(BASH_ENV)\"' " @@ -632,8 +624,9 @@ $(ibidir)/perl-$(perl-version): $(ibidir)/patchelf-$(patchelf-version) | awk '{printf("%d.%d", $$1, $$2)}') cd $(ddir) rm -rf perl-$(perl-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd perl-$(perl-version) + $(shsrcdir)/prep-source.sh $(ibdir) ./Configure -des \ -Dusethreads \ -Duseshrplib \ @@ -692,20 +685,16 @@ $(ibidir)/coreutils-$(coreutils-version): \ $(ibidir)/openssl-$(openssl-version) # Import, unpack and enter the source directory. + $(call unsafe-config) tarball=coreutils-$(coreutils-version).tar.lz $(call import-source, $(coreutils-url), $(coreutils-checksum)) + +# Unpack and enter the source. cd $(ddir) rm -rf coreutils-$(coreutils-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd coreutils-$(coreutils-version) - -# Set the configure script to use our shell, note that we can't -# assume GNU SED here yet (it installs after Coreutils). - sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ - configure > configure-tmp - mv configure-tmp configure - chmod +x configure + $(shsrcdir)/prep-source.sh $(ibdir) # Configure, build and install Coreutils. ./configure --prefix=$(idir) SHELL=$(ibdir)/bash \ @@ -717,7 +706,7 @@ $(ibidir)/coreutils-$(coreutils-version): \ # Fix RPATH if necessary. if [ -f $(ibdir)/patchelf ]; then make SHELL=$(ibdir)/bash install DESTDIR=junkinst - unalias ls || true # avoid decorated 'ls' commands with extra characters + unalias ls || true # Not decorated 'ls' (with extra characters). instprogs=$$(ls junkinst/$(ibdir)) for f in $$instprogs; do $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/$$f @@ -742,8 +731,9 @@ $(ibidir)/podlators-$(podlators-version): $(ibidir)/perl-$(perl-version) $(call import-source, $(podlators-url), $(podlators-checksum)) cd $(ddir) rm -rf podlators-$(podlators-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd podlators-$(podlators-version) + $(shsrcdir)/prep-source.sh $(ibdir) perl Makefile.PL make make install @@ -766,7 +756,7 @@ $(ibidir)/openssl-$(openssl-version): $(ibidir)/podlators-$(podlators-version) \ # First download the certificates and copy them into the # installation directory. tarball=cert.pem-$(certpem-version) - $(call import-source, $(cert-url), $(cert-checksum)) + $(call import-source, $(certpem-url), $(certpem-checksum)) cp $(tdir)/cert.pem-$(certpem-version) $(idir)/etc/ssl/cert.pem # Now download the OpenSSL tarball. @@ -1028,9 +1018,10 @@ $(ibidir)/gmp-$(gmp-version): \ $(ibidir)/coreutils-$(coreutils-version) tarball=gmp-$(gmp-version).tar.lz $(call import-source, $(gmp-url), $(gmp-checksum)) + export CFLAGS="-std=gnu17 $$CFLAGS" $(call gbuild, gmp-$(gmp-version), static, \ --enable-cxx --enable-fat, \ - -j$(numthreads) ,make check) + -j$(numthreads)) echo "GNU Multiple Precision Arithmetic Library $(gmp-version)" > $@ # Less is useful with Git (to view the diffs within a minimal container) @@ -1039,7 +1030,19 @@ $(ibidir)/gmp-$(gmp-version): \ $(ibidir)/less-$(less-version): $(ibidir)/ncurses-$(ncurses-version) tarball=less-$(less-version).tar.lz $(call import-source, $(less-url), $(less-checksum)) - $(call gbuild, less-$(less-version), static,,-j$(numthreads)) + +# Without the '--with-regex=posix' option, the build will depend on +# PCRE (perl compatible regular expressions) which are not available +# on some systems/compilers and can cause a crash. Maneage was +# successfully built with the POSIX regular expression (regex), and +# 'less' is generally, an interactive software, not a batch-mode +# software (it is just added in 'basic.mk' because Git uses it to +# display things. Again, this is an interactive meta-operation in +# maneage (operations you only do when you are developing Maneage +# within Maneage interactively, and will not affect into the actual +# reproducible analysis!) + $(call gbuild, less-$(less-version), static, \ + --with-regex=posix,-j$(numthreads)) if [ -f $(ibdir)/patchelf ]; then $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/less; fi @@ -1058,22 +1061,34 @@ $(ibidir)/libtool-$(libtool-version): $(ibidir)/m4-$(m4-version) $(ibidir)/grep-$(grep-version): $(ibidir)/coreutils-$(coreutils-version) tarball=grep-$(grep-version).tar.lz $(call import-source, $(grep-url), $(grep-checksum)) - $(call gbuild, grep-$(grep-version), static,,V=1) + $(call gbuild, grep-$(grep-version), static,, \ + -j$(numthreads) V=1) echo "GNU Grep $(grep-version)" > $@ # M4 doesn't depend on PatchELF, but just to be consistent with the # levels/phases introduced here (where the compressors are level 1, # PatchELF is level 2, and ...), we'll set it as a dependency. +# +# The '--with-syscmd-shell' is used as the default shell and if not given, +# 'm4' will use '/bin/sh' (which is not under Maneage control and can cause +# problems in 'high-level.mk' because it closes off the system's +# LD_LIBRARY_PATH and if the system's '/bin/sh' needs a special system +# library, the high-level programs will not be built). We are setting this +# default shell to Dash because M4 is built before our own Bash. Recall +# that Dash is built before we enter this Makefile. $(ibidir)/m4-$(m4-version): $(ibidir)/patchelf-$(patchelf-version) tarball=m4-$(m4-version).tar.lz $(call import-source, $(m4-url), $(m4-checksum)) - $(call gbuild, m4-$(m4-version), static,,V=1) + export CFLAGS="-std=gnu17 $$CFLAGS" + $(call gbuild, m4-$(m4-version), static, \ + --with-syscmd-shell=$(ibdir)/dash, \ + -j$(numthreads) V=1) echo "GNU M4 $(m4-version)" > $@ $(ibidir)/mpfr-$(mpfr-version): $(ibidir)/gmp-$(gmp-version) tarball=mpfr-$(mpfr-version).tar.lz $(call import-source, $(mpfr-url), $(mpfr-checksum)) - $(call gbuild, mpfr-$(mpfr-version), static, , , make check) + $(call gbuild, mpfr-$(mpfr-version), static) echo "GNU Multiple Precision Floating-Point Reliably $(mpfr-version)" > $@ $(ibidir)/pkg-config-$(pkgconfig-version): $(ibidir)/patchelf-$(patchelf-version) @@ -1097,6 +1112,7 @@ $(ibidir)/pkg-config-$(pkgconfig-version): $(ibidir)/patchelf-$(patchelf-version if [ x$(on_mac_os) = xyes ]; then export compiler="CC=clang" else export compiler="" fi + export CFLAGS="-std=gnu17 $$CFLAGS" $(call gbuild, pkg-config-$(pkgconfig-version), static, \ $$compiler --with-internal-glib \ --with-pc-path=$(ildir)/pkgconfig, V=1) @@ -1157,7 +1173,7 @@ $(ibidir)/mpc-$(mpc-version): $(ibidir)/mpfr-$(mpfr-version) echo "" > $@ else $(call gbuild, mpc-$(mpc-version), static, , \ - -j$(numthreads), make check) + -j$(numthreads)) echo "GNU Multiple Precision Complex library" > $@ fi @@ -1216,7 +1232,8 @@ $(ibidir)/binutils-$(binutils-version): \ # Build binutils with the standard 'gbuild' function. $(call gbuild, binutils-$(binutils-version), static, \ - --with-lib-path=$(sys_library_path), \ + --with-lib-path=$(sys_library_path) \ + --enable-gprofng=no, \ -j$(numthreads) V=1) # The 'ld' linker of Binutils needs several '*crt*.o' files from @@ -1344,21 +1361,35 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) # to avoid building so many small/temporary files and possibly # harming the hard-drive or SSD. But if the RAM doesn't have enough # space, we should use the hard-drive or SSD. During its build, -# GCC's build directory will become about 7GiB (in units of 1024^3 -# bytes, for GCC 12.1.0, which corresponds to 7.5GB, in units of -# 1000^3 bytes). So at this step, we make sure that we have more -# than 12GiB before GCC starts to build. See the figure in the link -# below for GCC's RAM consumption as a function of time: +# GCC's build directory will become several gigabytes and the build +# also needs RAM. You can track the RAM usage of the system with a +# 1-second resolution (if no other RAM consuming program is running +# while building GCC) with the command below (example outputs can +# be seen in https://savannah.nongnu.org/task/index.php?16623). # -# https://savannah.nongnu.org/task/?16244#comment12 +# c=1; while true; do POSIXLY_CORRECT=1 df -P /dev/shm/maneage-* | awk 'NR==2{print '$c', $3}'; c=$((c+1)); sleep 1; done > mem-usage.txt +# asttable mem-usage.txt -c1,'arith $2 512 x 1024 / 1024 / 1024 /' -o mem.fits # # For POSIX portability and longevity (default sizes might change), # we use the '-P' option, and we use the environment variable -# POSIXLY_CORRECT=1, so the 'block size' is 512 bytes. We'll also -# allow for about ~0.5 GB at the start. +# POSIXLY_CORRECT=1, so the 'block size' is 512 bytes. In this way, +# to get the actual GiB amount, multiply the value returned above +# by 512 (B/block), then divide by 1024^3 (B/GiB). +# +# To get the final value to use here, get the maximum used value +# after GCC is fully built and you have stopped the 'while true' +# command above. You can do this with the command below (assumes +# you have Gnuastro). # -# So we need 8 GiB * 1024^3 (B/GiB) / 512 blocks/B = 16777216 -# blocks, in blocks of 512 bytes. +# aststatistics mem-usage.txt -c2 --maximum | asttable -c'arith $1 7000000 +' -Afixed -B0 +# +# The extra space is because we will assume an extra 3 GiB = 3GiB * +# 1024^3 (B/GiB) / 512 (B/block) = 6291456 blocks are necessary for +# the building (let's round it to 7000000!). +# +# Therefore, we need to make sure that the running system more than +# the necessary amount of space in the RAM. To do this, we use 'df' +# below. # # The 4th column of 'df' is the "available" space at the time of # running, not the full space. So the 'RAM disk' that the OS @@ -1368,7 +1399,7 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) # alone - no other Maneage software is built at the same time as # GCC - so this amount of RAM should be enough. in_ram=$$(POSIXLY_CORRECT=1 df -P $(ddir) \ - | awk 'NR==2{print ($$4>16777216) ? "yes" : "no"}'); \ + | awk 'NR==2{print ($$4>26613216) ? "yes" : "no"}'); \ if [ $$in_ram = "yes" ]; then odir=$(ddir) else odir=$(BDIR)/software/build-tmp-gcc-due-to-lack-of-space @@ -1382,11 +1413,12 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) # Unpack GCC and prepare the 'build' directory inside it for all # the built files. rm -rf gcc-$(gcc-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions if [ $$odir != $(ddir) ]; then ln -s $$odir/gcc-$(gcc-version) $(ddir)/gcc-$(gcc-version) fi cd gcc-$(gcc-version) + $(shsrcdir)/prep-source.sh $(ibdir) # Unfortunately binutils installs headers like 'ansidecl.h' that # have been seen to conflict with GCC's internal versions of those @@ -1494,14 +1526,26 @@ $(ibidir)/gcc-$(gcc-version): $(ibidir)/binutils-$(binutils-version) -# Software that need re-compilation (to use our own libraries) +# Level 6: need re-compilation +# ---------------------------- +# +# The initial build of these was done with the host's settings, which will +# cause problems later when we completely close-off the host environment. +$(ibidir)/make-$(make-version): $(ibidir)/gcc-$(gcc-version) + tarball=make-$(make-version).tar.lz + $(call import-source, $(make-url), $(make-checksum)) + $(call gbuild, make-$(make-version), static, \ + --disable-dependency-tracking --without-guile) + echo "GNU Make $(make-version)" > $@ + $(ibidir)/lzip-$(lzip-version): $(ibidir)/gcc-$(gcc-version) tarball=lzip-$(lzip-version).tar unpackdir=lzip-$(lzip-version) cd $(ddir) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) ./configure --build --check --installdir="$(ibdir)" if [ -f $(ibdir)/patchelf ]; then $(ibdir)/patchelf --set-rpath $(ildir) $(ibdir)/lzip; @@ -1514,11 +1558,7 @@ $(ibidir)/lzip-$(lzip-version): $(ibidir)/gcc-$(gcc-version) - - - - -# Level 6: Basic text editor +# Level 7: Basic text editor # -------------------------- # # If the project is built in a minimal environment, there is no text @@ -1535,7 +1575,8 @@ $(ibidir)/lzip-$(lzip-version): $(ibidir)/gcc-$(gcc-version) # nano (and use their own optional high-level text editor). To do this, you # can just have to manually remove 'nano' from 'targets-proglib' above and # add their optional text editor in 'TARGETS.conf'. -$(ibidir)/nano-$(nano-version): $(ibidir)/lzip-$(lzip-version) +$(ibidir)/nano-$(nano-version): $(ibidir)/lzip-$(lzip-version) \ + $(ibidir)/make-$(make-version) tarball=nano-$(nano-version).tar.lz $(call import-source, $(nano-url), $(nano-checksum)) $(call gbuild, nano-$(nano-version), static) diff --git a/reproduce/software/make/build-rules.mk b/reproduce/software/make/build-rules.mk index c25dfb1..463fbbf 100644 --- a/reproduce/software/make/build-rules.mk +++ b/reproduce/software/make/build-rules.mk @@ -3,7 +3,7 @@ # imported into 'basic.mk' and 'high-level.mk'. They should be activated # with Make's 'Call' function. # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -160,7 +160,7 @@ uncompress = csuffix=$$(echo $$utarball \ intarrm=0; \ intar=$$utarball; \ fi; \ - if tar -xf $$intar; then \ + if tar -xf $$intar --no-same-owner --no-same-permissions; then \ if [ x$$intarrm = x1 ]; then rm $$intar; fi; \ else \ echo; echo "Tar error"; exit 1; \ @@ -207,17 +207,10 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ else confscript="$(strip $(6))"; \ fi; \ \ - if [ -f $(ibdir)/bash ]; then \ - if [ -f "$$confscript" ]; then \ - sed -e's|\#\! /bin/sh|\#\! $(ibdir)/bash|' \ - -e's|\#\!/bin/sh|\#\! $(ibdir)/bash|' \ - $$confscript > $$confscript-tmp; \ - mv $$confscript-tmp $$confscript; \ - chmod +x $$confscript; \ - fi; \ + $(shsrcdir)/prep-source.sh $(ibdir); \ + if [ -f $(ibdir)/bash ]; then \ shellop="SHELL=$(ibdir)/bash"; \ - elif [ -f /bin/bash ]; then shellop="SHELL=/bin/bash"; \ - else shellop="SHELL=/bin/sh"; \ + else shellop="SHELL=$(ibdir)/dash"; \ fi; \ \ if [ x$$gbuild_prefix = x ]; then prefixdir="$(idir)"; \ @@ -255,10 +248,7 @@ gbuild = if [ x$(static_build) = xyes ] && [ "x$(2)" = xstatic ]; then \ # CMake # ----- # -# According to the link below, in CMake '/bin/sh' is hardcoded, so there is -# no way to change it unfortunately! -# -# https://stackoverflow.com/questions/21167014/how-to-set-shell-variable-in-makefiles-generated-by-cmake +# Used by packages that are built with CMake. cbuild = if [ x$(static_build) = xyes ] && [ $(2)x = staticx ]; then \ export LDFLAGS="$$LDFLAGS -static"; \ opts="-DBUILD_SHARED_LIBS=OFF"; \ @@ -268,13 +258,18 @@ cbuild = if [ x$(static_build) = xyes ] && [ $(2)x = staticx ]; then \ utarball=$(tdir)/$$tarball; \ $(call uncompress); \ cd $(1); \ - rm -rf project-build; \ - mkdir project-build; \ - cd project-build; \ + $(shsrcdir)/prep-source.sh $(ibdir); \ + if [ -f $(ibdir)/bash ]; then \ + shellop="SHELL=$(ibdir)/bash"; \ + else shellop="SHELL=$(ibdir)/dash"; \ + fi; \ + rm -rf maneage-build; \ + mkdir maneage-build; \ + cd maneage-build; \ cmake .. -DCMAKE_LIBRARY_PATH=$(ildir) \ -DCMAKE_INSTALL_PREFIX=$(idir) \ -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON $$opts $(3); \ - make; \ - make install; \ + make $$shellop; \ + make $$shellop install; \ cd ../..; \ rm -rf $(1) diff --git a/reproduce/software/make/high-level.mk b/reproduce/software/make/high-level.mk index a2474af..67ca8b6 100644 --- a/reproduce/software/make/high-level.mk +++ b/reproduce/software/make/high-level.mk @@ -12,8 +12,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2018-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2019-2022 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2018-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Raul Infante-Sainz <infantesainz@gmail.com> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,7 +29,7 @@ # along with this Makefile. If not, see <http://www.gnu.org/licenses/>. # Top level environment (same as 'basic.mk') -include reproduce/software/config/LOCAL.conf +include .build/software/config/LOCAL.conf include reproduce/software/make/build-rules.mk include reproduce/software/config/versions.conf include reproduce/software/config/checksums.conf @@ -64,6 +64,12 @@ ipydir = $(BDIR)/software/installed/version-info/python ircrandir = $(BDIR)/software/installed/version-info/r-cran ilibrcrandir = $(BDIR)/software/installed/lib/R/library +# Special files. +makewshell = $(ibdir)/make-with-shell + + + + # Targets to build. ifeq ($(strip $(all_highlevel)),1) @@ -112,11 +118,17 @@ else endif -# Ultimate Makefile target. +# Disable the TeXLive target if `--offline` +ifneq ($(strip $(offline)),1) + target-texlive := $(itidir)/texlive +endif + +# Ultimate Makefile target. The recipe is '@echo > /dev/null' so Make does +# not print "make: Nothing to be done for 'all'." all: $(foreach p, $(targets-proglib), $(ibidir)/$(p)) \ $(foreach p, $(targets-python), $(ipydir)/$(p)) \ $(foreach p, $(targets-r-cran), $(ircrandir)/$(p)) \ - $(itidir)/texlive + $(target-texlive); @echo > /dev/null # Define the shell environment # ---------------------------- @@ -232,7 +244,7 @@ $(idircustom):; mkdir $@ # Afer putting everything together, we use the first server as the # reference for all software if their '-url' variable isn't defined (in # 'reproduce/software/config/urls.conf'). -downloadwrapper = ./reproduce/analysis/bash/download-multi-try +downloadwrapper = ./reproduce/analysis/bash/download-multi-try.sh maneage_backup_urls := $(shell awk '!/^#/{printf "%s ", $$1}' \ reproduce/software/config/servers-backup.conf) backupservers_all = $(user_backup_urls) $(maneage_backup_urls) @@ -337,8 +349,10 @@ $(ibidir)/atlas-$(atlas-version): # 'rpath_command'. export LDFLAGS=-L$(ildir) cd $(ddir) - tar -xf $(tdir)/atlas-$(atlas-version).tar.lz + tar -xf $(tdir)/atlas-$(atlas-version).tar.lz \ + --no-same-owner --no-same-permissions cd ATLAS + $(shsrcdir)/prep-source.sh $(ibdir) rm -rf build mkdir build cd build @@ -393,8 +407,9 @@ $(ibidir)/boost-$(boost-version): \ rm -rf $(ddir)/$$unpackdir topdir=$(pwd) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) ./bootstrap.sh --prefix=$(idir) --with-libraries=all \ --with-python=python3 echo "using mpi ;" > project-config.jam @@ -414,12 +429,11 @@ $(ibidir)/cfitsio-$(cfitsio-version): # systems. So we need to change it to our library installation # path. It doesn't affect GNU/Linux, so we'll just do it in any case # to keep things clean. - topdir=$(pwd); cd $(ddir); tar -xf $(tdir)/$$tarball + topdir=$(pwd); cd $(ddir) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions customtar=cfitsio-$(cfitsio-version)-custom.tar.gz cd cfitsio-$(cfitsio-version) - sed configure -e's|@rpath|$(ildir)|g' > configure_tmp - mv configure_tmp configure - chmod +x configure + sed -i -e's|@rpath|$(ildir)|g' configure cd .. tar cf $$customtar cfitsio-$(cfitsio-version) cd $$topdir @@ -427,11 +441,14 @@ $(ibidir)/cfitsio-$(cfitsio-version): # Continue the standard build on the customized tarball. Note that # with the installation of CFITSIO, 'fpack' and 'funpack' are not # installed by default. Because of that, they are added explicity. +# +# Note that older versions of CFITSIO (before 4.4.0) require a +# specific 'shared' target for the building of the shared libraries. export gbuild_tar=$(ddir)/$$customtar $(call gbuild, cfitsio-$(cfitsio-version), , \ --enable-sse2 --enable-reentrant \ --with-bzip2=$(idir), , \ - make shared fpack funpack) + make fpack funpack) rm $$customtar echo "CFITSIO $(cfitsio-version)" > $@ @@ -453,7 +470,8 @@ $(ibidir)/eigen-$(eigen-version): tarball=eigen-$(eigen-version).tar.lz $(call import-source, $(eigen-url), $(eigen-checksum)) rm -rf $(ddir)/eigen-eigen-* - topdir=$(pwd); cd $(ddir); tar -xf $(tdir)/$$tarball + topdir=$(pwd); cd $(ddir) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd eigen-$(eigen-version) if ! [ -d $(iidir)/eigen3 ]; then mkdir $(iidir)/eigen3; fi cp -r Eigen/* $(iidir)/eigen3/ # Some expect 'eigen3'. @@ -518,6 +536,12 @@ $(ibidir)/fftw-$(fftw-version): echo "FFTW $(fftw-version) \citep{fftw}" > $@ $(ibidir)/freetype-$(freetype-version): $(ibidir)/libpng-$(libpng-version) +# As of version 2.13.2, FreeType doesn't account for the 'SHELL' +# environment variable. The issue has been reported to the +# developers. But until future versions, the work-around was +# discoverd to be setting the 'GNUMAKE' environment variable so it +# includes 'SHELL'. + export GNUMAKE="$(makewshell)" tarball=freetype-$(freetype-version).tar.lz $(call import-source, $(freetype-url), $(freetype-checksum)) $(call gbuild, freetype-$(freetype-version), static) @@ -577,8 +601,10 @@ $(ibidir)/healpix-$(healpix-version): $(healpix-python-dep) \ fi rm -rf $(ddir)/Healpix_$(healpix-version) topdir=$(pwd); cd $(ddir); - tar -xf $(tdir)/$$tarball - cd Healpix_$(healpix-version)/src/C/autotools/ + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd Healpix_$(healpix-version) + $(shsrcdir)/prep-source.sh $(ibdir) + cd src/C/autotools autoreconf --install ./configure --prefix=$(idir) make V=1 -j$(numthreads) SHELL=$(ibdir)/bash @@ -667,9 +693,10 @@ $(ibidir)/libpaper-$(libpaper-version): \ # Unpack, build the configure system, build and install. cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions unpackdir=libpaper-$(libpaper-version) cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) autoreconf -fi ./configure --prefix=$(idir) --sysconfdir=$(idir)/etc \ --disable-static @@ -792,17 +819,37 @@ $(ibidir)/metastore-$(metastore-version): \ echo "" > $@ fi +# The Ninja build system (https://ninja-build.org) is also known as simply +# "Ninja". But other package managers (for example Debian) use +# "ninja-build" (the old "ninja" name has become obsolete there). Also, +# their own URL is called "ninja-build". So we use the same convention in +# Maneage. +$(ibidir)/ninjabuild-$(ninjabuild-version): $(ibidir)/cmake-$(cmake-version) + tarball=ninjabuild-$(ninjabuild-version).tar.lz + $(call import-source, $(ninjabuild-url), $(ninjabuild-checksum)) + cd $(ddir) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd ninjabuild-$(ninjabuild-version) + cmake -Bbuild-cmake + cmake --build build-cmake -j$(numthreads) + ./build-cmake/ninja_test + cp -pv build-cmake/ninja $(ibdir)/ + cd .. + rm -rf ninjabuild-$(ninjabuild-version) + echo "Ninja build system $(ninjabuild-version)" > $@ + $(ibidir)/openblas-$(openblas-version): - tarball=OpenBLAS-$(openblas-version).tar.lz + tarball=openblas-$(openblas-version).tar.lz $(call import-source, $(openblas-url), $(openblas-checksum)) if [ x$(on_mac_os) = xyes ]; then export CC=clang; fi cd $(ddir) - tar -xf $(tdir)/$$tarball - cd OpenBLAS-$(openblas-version) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd openblas-$(openblas-version) + $(shsrcdir)/prep-source.sh $(ibdir) make -j$(numthreads) make PREFIX=$(idir) install cd .. - rm -rf OpenBLAS-$(openblas-version) + rm -rf openblas-$(openblas-version) echo "OpenBLAS $(openblas-version)" > $@ $(ibidir)/openmpi-$(openmpi-version): @@ -905,8 +952,8 @@ $(ibidir)/libgit2-$(libgit2-version): $(ibidir)/cmake-$(cmake-version) -DUSE_SSH=OFF -DBUILD_CLAR=OFF \ -DTHREADSAFE=ON -DUSE_ICONV=OFF ) if [ x$(on_mac_os) = xyes ]; then - install_name_tool -id $(ildir)/libgit2.1.3.dylib \ - $(ildir)/libgit2.1.3.dylib + install_name_tool -id $(ildir)/libgit2.1.9.dylib \ + $(ildir)/libgit2.1.9.dylib fi echo "Libgit2 $(libgit2-version)" > $@ @@ -916,20 +963,27 @@ $(ibidir)/wcslib-$(wcslib-version): $(ibidir)/cfitsio-$(cfitsio-version) tarball=wcslib-$(wcslib-version).tar.lz $(call import-source, $(wcslib-url), $(wcslib-checksum)) -# If Fortran isn't present, don't build WCSLIB with it. - if type gfortran &> /dev/null; then fortranopt=""; - else fortranopt="--disable-fortran" - fi - -# Build WCSLIB. +# Build WCSLIB while disabling some features: +# - Fortran is disabled because as of May 2023, on macOS systems +# where we do not install GCC (and thus a standard 'gfortran'), the +# LLVM Fortran compiler is not complete and will cause a crash. If +# you use the Fortran features of WCSLIB, feel free to remove +# '--disable-fortran', but be careful with portability on other +# systems. Hopefully some time in the future, GCC will also be +# install-able on macOS within Maneage or LLVM's 'gfortran' will +# also be complete and will not cause a crash. +# - PGPLOT is disabled because it has many manual steps in its +# installation. Therefore, we currently do not build PGPlots in +# Maneage. Once (if) it is added, we can remove '--without-pgplot'. $(call gbuild, wcslib-$(wcslib-version), , \ LIBS="-pthread -lcurl -lm" \ + --without-pgplot \ + --disable-fortran \ --with-cfitsiolib=$(ildir) \ - --with-cfitsioinc=$(idir)/include \ - --without-pgplot $$fortranopt) + --with-cfitsioinc=$(idir)/include) if [ x$(on_mac_os) = xyes ]; then - install_name_tool -id $(ildir)/libwcs.7.11.dylib \ - $(ildir)/libwcs.7.11.dylib + install_name_tool -id $(ildir)/libwcs.$(wcslib-version).dylib \ + $(ildir)/libwcs.$(wcslib-version).dylib fi echo "WCSLIB $(wcslib-version)" > $@ @@ -998,8 +1052,9 @@ $(ibidir)/astrometrynet-$(astrometrynet-version): \ # 'astrometrynet' cd $(ddir) rm -rf astrometry.net-$(astrometrynet-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd astrometry.net-$(astrometrynet-version) + $(shsrcdir)/prep-source.sh $(ibdir) sed -e 's|cat /proc/cpuinfo|echo "Ignoring CPU info"|' \ -e 's|-free|echo "Ignoring RAM info"|' Makefile > Makefile.tmp mv Makefile.tmp Makefile @@ -1036,8 +1091,9 @@ $(ibidir)/cdsclient-$(cdsclient-version): tarball=cdsclient-$(cdsclient-version).tar.lz $(call import-source, $(cdsclient-url), $(cdsclient-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd cdsclient-$(cdsclient-version) + $(shsrcdir)/prep-source.sh $(ibdir) touch * ./configure --prefix=$(idir) make @@ -1054,25 +1110,39 @@ $(ibidir)/cmake-$(cmake-version): tarball=cmake-$(cmake-version).tar.lz $(call import-source, $(cmake-url), $(cmake-checksum)) -# After searching in 'bootstrap', I couldn't find 'LIBS', only -# 'LDFLAGS'. So the extra libraries are being added to 'LDFLAGS', not -# 'LIBS'. -# # On Mac systems, the build complains about 'clang' specific # features, so we can't use our own GCC build here. if [ x$(on_mac_os) = xyes ]; then export CC=clang export CXX=clang++ fi + +# CMake wants a single executable for 'MAKE', so we can't use 'make +# SHELL=$(SHELL) and we have defined this script. + export MAKE="$(makewshell)" + +# Go into the unpacked directory and prepare CMake. cd $(ddir) rm -rf cmake-$(cmake-version) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd cmake-$(cmake-version) - ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ - --system-bzip2 --system-liblzma --no-qt-gui \ + $(shsrcdir)/prep-source.sh $(ibdir) + +# Bootstrap, build and install CMake: +# - With the '--no-system-libs' option, CMake builds and statically +# links all the libraries it needs. Even though some of those (like +# liblzma, libcurl, zlib or bzip2) are within Maneage, we +# discovered that CMake can get confused and use out-of-Maneage +# libraries (https://savannah.nongnu.org/bugs/?63043). + ./bootstrap --no-qt-gui \ + --prefix=$(idir) \ + --no-system-libs \ --parallel=$(numthreads) - make -j$(numthreads) LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 - make install + $(makewshell) VERBOSE=1 LIBS="$$LIBS -lssl -lcrypto -lz" \ + -j$(numthreads) + $(makewshell) install + +# Clean up. cd .. rm -rf cmake-$(cmake-version) echo "CMake $(cmake-version)" > $@ @@ -1120,13 +1190,16 @@ $(ibidir)/ghostscript-$(ghostscript-version): \ # '-DPNG_ARM_NEON_OPT=0' prevents an arm64 'neon' library from being # required at compile time. cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd ghostscript-$(ghostscript-version) + $(shsrcdir)/prep-source.sh $(ibdir) ./configure --prefix=$(idir) \ --disable-cups \ --enable-dynamic \ --disable-compile-inits \ - CFLAGS="-DPNG_ARM_NEON_OPT=0" + --disable-hidden-visibility \ + CFLAGS="-DPNG_ARM_NEON_OPT=0" \ + LDFLAGS=-Wl,--copy-dt-needed-entries # Build and install the program and the shared libraries. make V=1 -j$(numthreads) @@ -1140,9 +1213,9 @@ $(ibidir)/ghostscript-$(ghostscript-version): \ # Install the fonts. tar -xvf $(tdir)/ghostscript-fonts-std-$(ghostscript-fonts-std-version).tar.lz \ - -C $(idir)/share/ghostscript + -C $(idir)/share/ghostscript --no-same-owner --no-same-permissions tar -xvf $(tdir)/ghostscript-fonts-gnu-$(ghostscript-fonts-gnu-version).tar.lz \ - -C $(idir)/share/ghostscript + -C $(idir)/share/ghostscript --no-same-owner --no-same-permissions fc-cache -v $(idir)/share/ghostscript/fonts/ echo; echo "Ghostscript fonts added to Fontconfig."; echo; @@ -1163,7 +1236,7 @@ $(ibidir)/gnuastro-$(gnuastro-version): \ $(call gbuild, gnuastro-$(gnuastro-version), static, , \ -j$(numthreads)) cp $(dtexdir)/gnuastro.tex $(ictdir)/ - echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro,akhlaghi19}" > $@ + echo "GNU Astronomy Utilities $(gnuastro-version) \citep{gnuastro}" > $@ $(ibidir)/icu-$(icu-version): $(ibidir)/python-$(python-version) @@ -1179,9 +1252,11 @@ $(ibidir)/icu-$(icu-version): $(ibidir)/python-$(python-version) tarball=icu-$(icu-version).tar.lz $(call import-source, $(icu-url), $(icu-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions unpackdir=icu-$(icu-version) - cd $$unpackdir/icu4c/source + cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) + cd icu4c/source ./configure --enable-static --prefix=$(idir) make -j$(numthreads) V=1 make install @@ -1246,8 +1321,9 @@ $(ibidir)/imfit-$(imfit-version): \ cd $(ddir) unpackdir=imfit-$(imfit-version) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) sed -i 's|/usr/local|$(idir)|g' SConstruct sed -i 's|/usr/include|$(idir)/include|g' SConstruct sed -i 's|.append(|.insert(0,|g' SConstruct @@ -1293,8 +1369,10 @@ $(ibidir)/minizip-$(minizip-version): $(ibidir)/automake-$(automake-version) unpackdir=minizip-$(minizip-version) rm -rf $$unpackdir mkdir $$unpackdir - tar -xf $(tdir)/$$tarball -C$$unpackdir --strip-components=1 + tar -xf $(tdir)/$$tarball -C$$unpackdir --strip-components=1 \ + --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) ./configure --prefix=$(idir) make cd contrib/minizip @@ -1354,8 +1432,9 @@ $(ibidir)/netpbm-$(netpbm-version): \ cd $(ddir) unpackdir=netpbm-$(netpbm-version) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) # As of NetPBM 10.73.39 and Flex 2.6.4-410-74a89fd (commit 74a89fd in # Flex's Git that is 410 commits after version 2.6.4), there is the @@ -1473,8 +1552,9 @@ $(ibidir)/scons-$(scons-version): $(ibidir)/python-$(python-version) cd $(ddir) unpackdir=scons-$(scons-version) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) # Unfortuantely SCons hard-codes its search PATH in its source (to # use POSIX operating system defaults)! So the only way to modify it @@ -1518,7 +1598,7 @@ $(ibidir)/sextractor-$(sextractor-version): \ unpackdir=sextractor-$(sextractor-version) cd $(ddir) rm -rf $$unpackdir - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd $$unpackdir # See comment above 'missfits' for '-fcommon'. @@ -1606,7 +1686,10 @@ $(ibidir)/swig-$(swig-version): # '$(ibdir)'. If any program does need 'util-linux' libraries, they can # simply add the proper directories to the environment variables, see # 'fontconfig' for example. -$(ibidir)/util-linux-$(util-linux-version): | $(idircustom) +$(ibidir)/util-linux-$(util-linux-version): \ + $(ibidir)/autoconf-$(autoconf-version) \ + $(ibidir)/automake-$(automake-version) \ + | $(idircustom) # Import the source. tarball=util-linux-$(util-linux-version).tar.lz @@ -1616,28 +1699,31 @@ $(ibidir)/util-linux-$(util-linux-version): | $(idircustom) # explained above). As shown below, later, we'll put a symbolic link # of all the necessary binaries in the main '$(idir)/bin'. cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions cd util-linux-$(util-linux-version) + $(shsrcdir)/prep-source.sh $(ibdir) -# If a patch exists for the current version, apply it. +# If a patch is necessary, apply it. if [ -f $(patchdir)/util-linux-$(util-linux-version)-macos.patch ]; then cp $(patchdir)/util-linux-$(util-linux-version)-macos.patch \ util-linux-$(util-linux-version)-macos.patch git apply util-linux-$(util-linux-version)-macos.patch fi -# The 'mkswap' feature needs low-level file system and kernel headers -# that are not always available (in particular on older Linux -# kernels). Also, creating SWAP space will need root permissions, so -# its not something a Maneager may need! Unfortunately there is no -# configuration option to disable this so we'll have to disable it -# manually by commenting the relevant files in the -# 'configure.ac'. Having a more recent 'configure.ac' will trigger -# the './configure' script to be re-created after the first run, but -# it is pretty fast and not a problem. +# The 'mkswap' feature needs low-level file system and kernel headers +# that are not always available (in particular on older Linux +# kernels). Also, creating SWAP space will need root permissions, so +# its not something a Maneager may need! Unfortunately there is no +# configuration option to disable this so we'll have to disable it +# manually by commenting the relevant files in the +# 'configure.ac'. sed -e's|UL_BUILD_INIT(\[mkswap\], \[yes\])|UL_BUILD_INIT(\[mkswap\], \[no\])|' \ -i configure.ac +# Having updated 'configure.ac', we need to re-generate the +# './configure' script with 'autoreconf' (which is part of Autoconf +# and needs Automake; hence why they are dependencies. + autoreconf -f # Configure Util-linux export CONFIG_SHELL=$(ibdir)/bash @@ -1645,6 +1731,7 @@ $(ibidir)/util-linux-$(util-linux-version): | $(idircustom) --disable-dependency-tracking \ --enable-libmount-support-mtab \ --disable-silent-rules \ + --disable-liblastlog2 \ --disable-mountpoint \ --disable-libmount \ --disable-unshare \ @@ -1717,9 +1804,10 @@ $(ibidir)/vim-$(vim-version): tarball=vim-$(vim-version).tar.lz $(call import-source, $(vim-url), $(vim-checksum)) cd $(ddir) - tar -xf $(tdir)/$$tarball + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions unpackdir=vim-$(vim-version) cd $(ddir)/$$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) ./configure --prefix=$(idir) \ --disable-canberra \ --enable-multibyte \ @@ -1786,12 +1874,19 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf @topdir=$$(pwd) cd $(ddir) rm -rf install-tl-* - tar -xf $(tdir)/install-tl-unx.tar.gz + tar -xf $(tdir)/install-tl-unx.tar.gz \ + --no-same-owner --no-same-permissions cd install-tl-* + $(shsrcdir)/prep-source.sh $(ibdir) sed -e's|@installdir[@]|$(idir)|g' \ "$$topdir"/reproduce/software/config/texlive.conf \ > texlive.conf +# We do not build TeXLive from source and for its installation it +# downloads components from the web internally; and those components +# can use '/bin/sh' (which will need '$(sys_library_sh_path)'). + export LD_LIBRARY_PATH="$(sys_library_sh_path):$$LD_LIBRARY_PATH" + # TeX Live's installation may fail due to any reason. But TeX Live is # optional (only necessary for building the final PDF). So we don't # want the configure script to fail if it can't run. Possible error @@ -1860,13 +1955,17 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf tarballurl=$$url/install-tl-unx.tar.gz touch $(lockdir)/download downloader="wget --no-use-server-timestamps -O" - if $(downloadwrapper) "$$downloader" $(lockdir)/download \ - $$tarballurl "$(tdir)/install-tl-unx.tar.gz" \ - "$(backupservers)"; then + if $(downloadwrapper) "$$downloader" \ + $(lockdir)/download \ + $$tarballurl \ + "$(tdir)/install-tl-unx.tar.gz" \ + "$(backupservers)"; then cd $(ddir) rm -rf install-tl-* - tar -xf $(tdir)/install-tl-unx.tar.gz + tar -xf $(tdir)/install-tl-unx.tar.gz \ + --no-same-owner --no-same-permissions cd install-tl-* + $(shsrcdir)/prep-source.sh $(ibdir) sed -e's|@installdir[@]|$(idir)|g' \ $$topdir/reproduce/software/config/texlive.conf \ > texlive.conf @@ -1897,10 +1996,19 @@ $(itidir)/texlive-ready-tlmgr: reproduce/software/config/texlive.conf # Live itself (only very basic TeX and LaTeX) and the installation of its # necessary packages into two packages. # -# Note that Biber needs to link with libraries like libnsl. However, we -# don't currently build biber from source. So we can't choose the library -# version. But we have the source and build instructions for the 'nsl' -# library. When we later build biber from source, we can easily use them. +# Note that we do not build the TeXLive executables (like Biber) from +# source. So in case they need special libraries, we can't choose the +# library version here (for example see [1] and [2]). In such cases there +# is no solution but to manually add the location necessary library to +# LD_LIBRARY_PATH when calling the respective LaTeX command in +# 'reproduce/analysis/make/paper.mk'. Fortunately as of Biber 2.20, it does +# not depend on anything except the C library (all dependencies are now +# statically linked), so problems [1] and [2] will not happen. But this can +# generally happen for any other tool/OS, so it is important to build +# TeXLive from source as soon as possible [3]. +# [1] https://github.com/plk/biber/issues/445 +# [2] https://savannah.nongnu.org/bugs/index.php?63175 +# [3] https://savannah.nongnu.org/task/?15267 $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ $(itidir)/texlive-ready-tlmgr @@ -1910,6 +2018,11 @@ $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ echo "" > $@ else +# We do not build TeXLive from source and for its installation it +# downloads components from the web internally; and those +# components can use '/bin/sh' (which needs 'sys_library_sh_path'). + export LD_LIBRARY_PATH="$(sys_library_sh_path)" + # To update itself, tlmgr needs a backup directory. backupdir=$(idir)/texlive/backups mkdir -p $$backupdir @@ -1933,6 +2046,12 @@ $(itidir)/texlive: reproduce/software/config/texlive-packages.conf \ # directory so we don't have to modify 'PATH'. ln -fs $(idir)/texlive/maneage/bin/*/* $(ibdir)/ +# Correct any reference to '/bin/sh' within the installed LaTeX +# files (this is because we do no yet install LaTeX from source): + cdir=$$(pwd) + cd $(idir)/texlive + cd $$cdir + # Get all the necessary versions. texlive=$$(pdflatex --version \ | awk 'NR==1' \ diff --git a/reproduce/software/make/python.mk b/reproduce/software/make/python.mk index 546195b..c994e3f 100644 --- a/reproduce/software/make/python.mk +++ b/reproduce/software/make/python.mk @@ -12,8 +12,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2019-2022 Raul Infante-Sainz <infantesainz@gmail.com> -# Copyright (C) 2019-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2019-2025 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2019-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -51,6 +51,8 @@ export MPI_PYTHON3_SITEARCH := # Python-specific installation directories. python-major-version = $(shell echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') +# This is for 'installer' (the python installer) itself: +ilibpymajorversion = $(idir)/lib/python$(python-major-version) @@ -93,13 +95,48 @@ $(ibidir)/python-$(python-version): $(ibidir)/libffi-$(libffi-version) export CC=clang export CXX=clang++ fi - $(call gbuild, python-$(python-version),, \ - --without-ensurepip \ - --with-system-ffi \ - --enable-shared, -j$(numthreads)) + +# Unpack the tarball (see below for the necessary modification). + cd $(ddir) + unpackdir=python-$(python-version) + tar -xf $(tdir)/$$tarball --no-same-owner --no-same-permissions + cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) + +# Python's 'setup.py' uses 'os.system' to run shell scripts. On the +# other hand 'os.system' only runs '/bin/sh' (which has its own +# libraries to link to and those are blocked at this level). So we +# need to add an extra line on top of the 'os.system' funciton and +# put '/usr/lib' in 'LD_LIBRARY_PATH' within Python's environment for +# system calls (with 'os.putenv'). As of Python 3.13.2 the tarball no +# longer has an 'setup.py'. But when it did, the change below was +# necessary. + if [ -f setup.py ]; then + awk '{if(/os.system\(/) \ + { print " os.putenv(\"LD_LIBRARY_PATH\", \"$$LD_LIBRARY_PATH:$(sys_library_sh_path)\");"; \ + print $$0;} \ + else print $$0}' \ + setup.py > setup-tmp.py + mv setup-tmp.py setup.py + fi + +# Do the basic installation and delete the temporary directory. + ./configure SHELL=$(ibdir)/bash \ + --enable-optimizations \ + --without-ensurepip \ + --prefix="$(idir)" \ + --with-system-ffi \ + --enable-shared + $(makewshell) -j$(numthreads) + $(makewshell) install -j$(numthreads) + cd .. + rm -rf $$unpackdir + +# Set the necessary environment variables and finish the build. ln -sf $(ildir)/python$(python-major-version) $(ildir)/python ln -sf $(ibdir)/python$(python-major-version) $(ibdir)/python - ln -sf $(iidir)/python$(python-major-version)m $(iidir)/python$(python-major-version) + ln -sf $(iidir)/python$(python-major-version)m \ + $(iidir)/python$(python-major-version) rm -rf $(ipydir) mkdir $(ipydir) echo "Python $(python-version)" > $@ @@ -108,39 +145,138 @@ $(ibidir)/python-$(python-version): $(ibidir)/libffi-$(libffi-version) -# Non-PiP Python module installation +# Non-pip Python module installation # ---------------------------------- # -# To build Python packages with direct access to a 'setup.py' (if no direct -# access to 'setup.py' is needed, pip can be used). Note that the -# software's packaged source code is the first prerequisite that is in the -# 'tdir' directory. +# Build strategy for python modules as of February 2025, for python 3.13.2. + +# This strategy is mostly based on recommendations by E Schwartz +# (ztrawhcse) on #python (Libera Chat), in October 2022 and February +# 2025. Some discussions are on documented in Savannah tasks [1][2]. The +# build strategy for 'python-installer' is inspired by the gentoo script +# 'python_domodule' [3]. + +# Bootstrap-step: 'gpep517' [4], motivated by PEP 517 [5], together with +# 'python-installer' (module called 'installer') are built without +# dependences on other python packages apart from python itself. The build +# rules for these two packages do python byte compilation and copy the .py +# and .pyc files into the python install directory. These two packages are +# considered to be 'frontends'. + +# Once these two frontends are available, other packages that do building +# tasks, including both backends and alternative frontends or a mix of +# these (in particular: setuptools, meson [6]/ninja-build [7] , flit-core, +# and meson-python), can be built with the 'python-installer' and +# 'gpep517'. The aims of the various build tools are diverse, include +# ecosystem resilience, reproducibility, build speed and convenience in +# building bigger packages such as numpy, scipy and astropy. + +# The python.mk script now includes only three methods: the boot +# build methods of 'python-installer' and 'gpep517'; and the gpep517 +# frontend. No method is provided for using 'python-installer' directly; +# it is invoked indirectly by source files of many packages, which +# also give metadata describing information for build methods. + +# Why not pip? We do not build any python packages with 'pip' because we +# want to have a fully documented pipeline of (i) the original upstream +# locations of tarballs, (ii) the tarballs' checksums, and (iii) the exact +# sequence of build commands. + +# For an alternative viewpoint on a python build strategy, see [8]. + +# Prerequisite for the pybuild script here: the package's source code +# (tarball) must already be located in the directory 'tdir'. # -# Arguments of this function are the numbers +# Arguments: # 1) Unpack command # 2) Unpacked directory name after unpacking the tarball # 3) site.cfg file (optional). -# 4) Official software name (for paper). +# 4) Official software name (for paper.tex). +# 5) Obligatory parameter: build method (see below): +# BOOT_INSTALLER - only for 'python-installer' +# BOOT_GPEP517 - only for 'gpep517' +# GPEP517 - for any other python package # # Hooks: # pyhook_before: optional steps before running 'python setup.py build' # pyhook_after: optional steps after running 'python setup.py install' -pybuild = cd $(ddir); rm -rf $(2); \ - if ! $(1) $(tdir)/$$tarball; then \ + +# [1] https://savannah.nongnu.org/task/?16268 +# [2] https://savannah.nongnu.org/task/?16625 +# [3] https://gitweb.gentoo.org/repo/gentoo.git/tree/eclass/python-utils-r1.eclass#n646 +# [4] https://pypi.org/project/gpep517 +# [5] https://peps.python.org/pep-0517 +# [6] https://mesonbuild.com +# [7] https://ninja-build.org +# [8] https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html + + +pybuild = cd $(ddir); \ + packagedir=$(strip $(2)); \ + if (printf "$$packagedir" | grep "[a-z][a-z]"); then rm -rf $$packagedir; fi; \ + printf "\nStarting to install python package with maneage pybuild rule: $(4)\n ..."; \ + if ! $(1) $(tdir)/$$tarball --no-same-owner --no-same-permissions; then \ echo; echo "Tar error"; exit 1; \ fi; \ - cd $(2); \ + cd $$packagedir; \ if [ "x$(strip $(3))" != x ]; then \ sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ $(3) > site.cfg; \ fi; \ if type pyhook_before &>/dev/null; then pyhook_before; fi; \ - python setup.py build; \ - python setup.py install; \ + printf "pybuild option5 = __ %s __\n" "$(strip $(5))"; \ + if [ "x$(strip $(5))" = xBOOT_INSTALLER ]; then \ + chmod 0644 src/installer/*.py; \ + mkdir -p $(ilibpymajorversion)/installer; \ + mkdir -p $(ilibpymajorversion)/installer/__pycache__ ; \ + mkdir -p $(ilibpymajorversion)/installer/_scripts; \ + mkdir -p $(ilibpymajorversion)/installer/_scripts/__pycache__ ; \ + cp -pv src/installer/*.py $(ilibpymajorversion)/installer/; \ + cp -pv src/installer/_scripts/__init__.py $(ilibpymajorversion)/installer/_scripts/; \ + cd src/installer/; \ + python -m compileall -o 1 -o 2 -l -f \ + -d $(ilibpymajorversion)/installer/ .; \ + chmod 0644 __pycache__/*.pyc; \ + cp -pv __pycache__/*.pyc \ + $(ilibpymajorversion)/installer/__pycache__; \ + cd -; \ + cd src/installer/_scripts/; \ + python -m compileall -o 1 -o 2 -l -f \ + -d $(ilibpymajorversion)/installer/_scripts/ __init__.py; \ + chmod 0644 __pycache__/*.pyc; \ + cp -pv __pycache__/*.pyc \ + $(ilibpymajorversion)/installer/_scripts/__pycache__; \ + cd -; \ + elif [ "x$(strip $(5))" = xBOOT_GPEP517 ]; then \ + chmod 0644 gpep517/*.py; \ + mkdir -p $(ilibpymajorversion)/gpep517; \ + mkdir -p $(ilibpymajorversion)/gpep517/__pycache__ ; \ + cp -pv gpep517/*.py $(ilibpymajorversion)/gpep517/; \ + cd gpep517/; \ + python -m compileall -o 1 -o 2 -l -f \ + -d $(ilibpymajorversion)/gpep517/ .; \ + chmod 0644 __pycache__/*.pyc; \ + cp -pv __pycache__/*.pyc \ + $(ilibpymajorversion)/gpep517/__pycache__; \ + cd -; \ + elif [ "x$(strip $(5))" = xGPEP517 ]; then \ + printf "\n\n\n=== python build method: gpep517 ====== "; pwd; \ + printf "...............\n\n\n"; \ + python -m gpep517 install-from-source \ + --prefix "" \ + --destdir $(idir) \ + --optimize all; \ + else \ + printf "Error: Unknown pybuild method for $$packagedir: __ $(strip $(5)) __\n"; \ + printf "The pybuild 5th parameter should very likely be set "; \ + printf "to GPEP517 after checking the build rule and "; \ + printf "upgrading if needed.\n"; \ + exit 1; \ + fi; \ if type pyhook_after &>/dev/null; then pyhook_after; fi; \ cd ..; \ - rm -rf $(2); \ + if (printf "$$packagedir" | grep "[a-z][a-z]"); then rm -fr $$packagedir; fi; \ echo "$(4)" > $@ @@ -149,9 +285,11 @@ pybuild = cd $(ddir); rm -rf $(2); \ # Python modules # --------------- -# + + # All the necessary Python modules go here. -$(ipydir)/asn1crypto-$(asn1crypto-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/asn1crypto-$(asn1crypto-version): \ + $(ipydir)/setuptools-$(setuptools-version) tarball=asn1crypto-$(asn1crypto-version).tar.gz $(call import-source, $(asn1crypto-url), $(asn1crypto-checksum)) $(call pybuild, tar -xf, asn1crypto-$(asn1crypto-version), , \ @@ -172,25 +310,58 @@ $(ipydir)/astroquery-$(astroquery-version): \ $(call pybuild, tar -xf, astroquery-$(astroquery-version), , \ Astroquery $(astroquery-version)) +# Astropy: points to consider about dependencies: +# # The optional dependency 'h5py' that is necessary for writting tables in # HDF5 format has been removed from Astropy because on macOS it cannot be # installed. +# +# 2022-or-older dependencies: +# $(ibidir)/expat-$(expat-version) \ +# $(ipydir)/jinja2-$(jinja2-version) \ +# $(ipydir)/html5lib-$(html5lib-version) \ +# $(ipydir)/beautifulsoup4-$(beautifulsoup4-version) \ +# +# While the astropy pyproject.toml file says that the astropy build depends +# on numpy, not scipy, and does not depend on matplotlib; the +# runtime is recommended to depend on both scipy and matplotlib. +# In practice, users of astropy will generally expect scipy and matplotlib +# to be available at runtime, so we set these as prerequisites. $(ipydir)/astropy-$(astropy-version): \ - $(ibidir)/expat-$(expat-version) \ $(ipydir)/scipy-$(scipy-version) \ - $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/pyyaml-$(pyyaml-version) \ - $(ipydir)/jinja2-$(jinja2-version) \ $(ipydir)/pyerfa-$(pyerfa-version) \ - $(ipydir)/html5lib-$(html5lib-version) \ - $(ipydir)/beautifulsoup4-$(beautifulsoup4-version) \ + $(ipydir)/pyyaml-$(pyyaml-version) \ + $(ipydir)/matplotlib-$(matplotlib-version) \ + $(ipydir)/astropy-iers-data-$(astropy-iers-data-version) \ $(ipydir)/extension-helpers-$(extension-helpers-version) + +# Tarball and its preparation. tarball=astropy-$(astropy-version).tar.lz $(call import-source, $(astropy-url), $(astropy-checksum)) - $(call pybuild, tar -xf, astropy-$(astropy-version)) - cp $(dtexdir)/astropy.tex $(ictdir)/ + +# Conservatively purge old version (but not astropy_iers, +# astropy-iers): + rm -fvr $(idir)/lib/python*/site-packages/astropy/ + rm -fvr $(idir)/lib/python*/site-packages/astropy-[0-9]*-info/ + rm -fv $(idir)/bin/fits{diff,check,header,info,2bitmap} + rm -fv $(idir)/bin/{samp_hub,showtable,volint,wcslint} + +# Do the basic build. + $(call pybuild, tar -xf, astropy-$(astropy-version),,, \ + GPEP517) + cp -pv $(dtexdir)/astropy.tex $(ictdir)/ echo "Astropy $(astropy-version) \citep{astropy2013,astropy2018}" > $@ +$(ipydir)/astropy-iers-data-$(astropy-iers-data-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=astropy-iers-data-$(astropy-iers-data-version).tar.lz + $(call import-source, $(astropy-iers-data-url), \ + $(astropy-iers-data-checksum)) + $(call pybuild, tar -xf, \ + astropy-iers-data-$(astropy-iers-data-version),,, \ + GPEP517) + echo "Astropy-Iers-Data $(astropy-iers-data-version)" > $@ + $(ipydir)/beautifulsoup4-$(beautifulsoup4-version): \ $(ipydir)/soupsieve-$(soupsieve-version) tarball=beautifulsoup4-$(beautifulsoup4-version).tar.lz @@ -198,13 +369,17 @@ $(ipydir)/beautifulsoup4-$(beautifulsoup4-version): \ $(call pybuild, tar -xf, beautifulsoup4-$(beautifulsoup4-version), , \ BeautifulSoup $(beautifulsoup4-version)) -$(ipydir)/beniget-$(beniget-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/beniget-$(beniget-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=beniget-$(beniget-version).tar.lz $(call import-source, $(beniget-url), $(beniget-checksum)) $(call pybuild, tar -xf, beniget-$(beniget-version), , \ Beniget $(beniget-version)) -$(ipydir)/certifi-$(certifi-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/certifi-$(certifi-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=certifi-$(certifi-version).tar.gz $(call import-source, $(certifi-url), $(certifi-checksum)) $(call pybuild, tar -xf, certifi-$(certifi-version), , \ @@ -217,12 +392,23 @@ $(ipydir)/cffi-$(cffi-version): \ $(call import-source, $(cffi-url), $(cffi-checksum)) $(call pybuild, tar -xf, cffi-$(cffi-version), ,cffi $(cffi-version)) -$(ipydir)/chardet-$(chardet-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/chardet-$(chardet-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=chardet-$(chardet-version).tar.gz $(call import-source, $(chardet-url), $(chardet-checksum)) $(call pybuild, tar -xf, chardet-$(chardet-version), , \ Chardet $(chardet-version)) +$(ipydir)/contourpy-$(contourpy-version): \ + $(ipydir)/pybind11-$(pybind11-version) \ + $(ipydir)/meson-python-$(meson-python-version) + tarball=contourpy-$(contourpy-version).tar.lz + $(call import-source, $(contourpy-url), $(contourpy-checksum)) + $(call pybuild, tar -xf, contourpy-$(contourpy-version), , \ + Contourpy $(contourpy-version), GPEP517) + echo "Contourpy $(contourpy-version)" > $@ + $(ipydir)/corner-$(corner-version): $(ipydir)/matplotlib-$(matplotlib-version) tarball=corner-$(corner-version).tar.gz $(call import-source, $(corner-url), $(corner-checksum)) @@ -231,6 +417,13 @@ $(ipydir)/corner-$(corner-version): $(ipydir)/matplotlib-$(matplotlib-version) cp $(dtexdir)/corner.tex $(ictdir)/ echo "Corner $(corner-version) \citep{corner}" > $@ +$(ipydir)/cppy-$(cppy-version): \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) + tarball=cppy-$(cppy-version).tar.lz + $(call import-source, $(cppy-url), $(cppy-checksum)) + $(call pybuild, tar -xf, cppy-$(cppy-version), , \ + Cppy $(cppy-version), GPEP517) + $(ipydir)/cryptography-$(cryptography-version): \ $(ipydir)/cffi-$(cffi-version) \ $(ipydir)/asn1crypto-$(asn1crypto-version) \ @@ -244,13 +437,17 @@ $(ipydir)/cycler-$(cycler-version): $(ipydir)/six-$(six-version) tarball=cycler-$(cycler-version).tar.lz $(call import-source, $(cycler-url), $(cycler-checksum)) $(call pybuild, tar -xf, cycler-$(cycler-version), , \ - Cycler $(cycler-version)) + Cycler $(cycler-version), GPEP517) + echo "Cycler $(cycler-version)" > $@ -$(ipydir)/cython-$(cython-version): $(ipydir)/setuptools-$(setuptools-version) - tarball=Cython-$(cython-version).tar.lz +$(ipydir)/cython-$(cython-version): \ + $(ipydir)/python-installer-$(python-installer-version) \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=cython-$(cython-version).tar.lz $(call import-source, $(cython-url), $(cython-checksum)) - $(call pybuild, tar -xf, Cython-$(cython-version)) - cp $(dtexdir)/cython.tex $(ictdir)/ + $(call pybuild, tar -xf, cython-$(cython-version),,, GPEP517) + cp -pv $(dtexdir)/cython.tex $(ictdir)/ echo "Cython $(cython-version) \citep{cython2011}" > $@ $(ipydir)/esutil-$(esutil-version): $(ipydir)/numpy-$(numpy-version) @@ -271,25 +468,28 @@ $(ipydir)/eigency-$(eigency-version): \ $(ipydir)/emcee-$(emcee-version): \ $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/setuptools_scm-$(setuptools_scm-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=emcee-$(emcee-version).tar.gz $(call import-source, $(emcee-url), $(emcee-checksum)) $(call pybuild, tar -xf, emcee-$(emcee-version), , \ emcee $(emcee-version)) $(ipydir)/entrypoints-$(entrypoints-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=entrypoints-$(entrypoints-version).tar.gz $(call import-source, $(entrypoints-url), $(entrypoints-checksum)) $(call pybuild, tar -xf, entrypoints-$(entrypoints-version), , \ EntryPoints $(entrypoints-version)) $(ipydir)/extension-helpers-$(extension-helpers-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=extension-helpers-$(extension-helpers-version).tar.lz - $(call import-source, $(extension-helpers-url), $(extension-helpers-checksum)) - $(call pybuild, tar -xf, extension-helpers-$(extension-helpers-version), , \ - Extension-Helpers $(extension-helpers-version)) + $(call import-source, $(extension-helpers-url), \ + $(extension-helpers-checksum)) + $(call pybuild, tar -xf, \ + extension-helpers-$(extension-helpers-version),, \ + Extension-Helpers $(extension-helpers-version), GPEP517) $(ipydir)/flake8-$(flake8-version): \ $(ipydir)/pyflakes-$(pyflakes-version) \ @@ -299,7 +499,29 @@ $(ipydir)/flake8-$(flake8-version): \ $(call pybuild, tar -xf, flake8-$(flake8-version), , \ Flake8 $(flake8-version)) -$(ipydir)/future-$(future-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/flit-core-$(flit-core-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) + tarball=flit-core-$(flit-core-version).tar.lz + $(call import-source, $(flit-core-url), $(flit-core-checksum)) + $(call pybuild, tar -xf, flit-core-$(flit-core-version), , \ + Flit-core $(flit-core-version), GPEP517) + +# Although cython is not an obligatory prerequisite of fonttools, we force +# it as a prerequisite for reproducibility; otherwise build parallelism may +# lead to some builds with and some builds without cython, depending on how +# many cpus the host machine has. +$(ipydir)/fonttools-$(fonttools-version): \ + $(ipydir)/cython-$(cython-version) \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=fonttools-$(fonttools-version).tar.lz + $(call import-source, $(fonttools-url), $(fonttools-checksum)) + $(call pybuild, tar -xf, fonttools-$(fonttools-version), , \ + fonttools $(fonttools-version), GPEP517) + +$(ipydir)/future-$(future-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=future-$(future-version).tar.gz $(call import-source, $(future-url), $(future-checksum)) $(call pybuild, tar -xf, future-$(future-version), , \ @@ -317,12 +539,25 @@ $(ipydir)/galsim-$(galsim-version): \ cp $(dtexdir)/galsim.tex $(ictdir)/ echo "Galsim $(galsim-version) \citep{galsim}" > $@ -$(ipydir)/gast-$(gast-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/gast-$(gast-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=gast-$(gast-version).tar.lz $(call import-source, $(gast-url), $(gast-checksum)) $(call pybuild, tar -xf, gast-$(gast-version), , \ Gast $(gast-version)) +$(ipydir)/gpep517-$(gpep517-version): \ + $(ibidir)/python-$(python-version) + tarball=gpep517-$(gpep517-version).tar.lz + $(call import-source, $(gpep517-url), $(gpep517-checksum)) + $(call pybuild, tar -xf, \ + gpep517-$(gpep517-version), \ + , \ + gpep517 $(gpep517-version), \ + BOOT_GPEP517) + echo "gpep517 $(gpep517-version)" > $@ + $(ipydir)/h5py-$(h5py-version): \ $(ipydir)/six-$(six-version) \ $(ibidir)/hdf5-$(hdf5-version) \ @@ -355,13 +590,17 @@ $(ipydir)/html5lib-$(html5lib-version): \ $(call pybuild, tar -xf, html5lib-$(html5lib-version), , \ HTML5lib $(html5lib-version)) -$(ipydir)/idna-$(idna-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/idna-$(idna-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=idna-$(idna-version).tar.gz $(call import-source, $(idna-url), $(idna-checksum)) $(call pybuild, tar -xf, idna-$(idna-version), , \ idna $(idna-version)) -$(ipydir)/jeepney-$(jeepney-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/jeepney-$(jeepney-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=jeepney-$(jeepney-version).tar.gz $(call import-source, $(jeepney-url), $(jeepney-checksum)) $(call pybuild, tar -xf, jeepney-$(jeepney-version), , \ @@ -376,17 +615,21 @@ $(ipydir)/jinja2-$(jinja2-version): $(ipydir)/markupsafe-$(markupsafe-version) $(ipydir)/keyring-$(keyring-version): \ $(ipydir)/entrypoints-$(entrypoints-version) \ $(ipydir)/secretstorage-$(secretstorage-version) \ - $(ipydir)/setuptools_scm-$(setuptools_scm-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=keyring-$(keyring-version).tar.gz $(call import-source, $(keyring-url), $(keyring-checksum)) $(call pybuild, tar -xf, keyring-$(keyring-version), , \ Keyring $(keyring-version)) -$(ipydir)/kiwisolver-$(kiwisolver-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/kiwisolver-$(kiwisolver-version): \ + $(ipydir)/cppy-$(cppy-version) \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=kiwisolver-$(kiwisolver-version).tar.lz $(call import-source, $(kiwisolver-url), $(kiwisolver-checksum)) $(call pybuild, tar -xf, kiwisolver-$(kiwisolver-version), , \ - Kiwisolver $(kiwisolver-version)) + Kiwisolver $(kiwisolver-version), GPEP517) + cp -pv $(dtexdir)/kiwisolver.tex $(ictdir)/ + echo "Kiwisolver $(kiwisolver-version) \citep{cassowary2001}" > $@ $(ipydir)/lmfit-$(lmfit-version): \ $(ipydir)/six-$(six-version) \ @@ -411,7 +654,8 @@ $(ipydir)/lsstdesccoord-$(lsstdesccoord-version): \ LSSTDESC.Coord $(lsstdesccoord-version)) $(ipydir)/markupsafe-$(markupsafe-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=markupsafe-$(markupsafe-version).tar.lz $(call import-source, $(markupsafe-url), $(markupsafe-checksum)) $(call pybuild, tar -xf, markupsafe-$(markupsafe-version), , \ @@ -422,11 +666,9 @@ $(ipydir)/matplotlib-$(matplotlib-version): \ $(ipydir)/numpy-$(numpy-version) \ $(ipydir)/cycler-$(cycler-version) \ $(ipydir)/pillow-$(pillow-version) \ - $(ibidir)/freetype-$(freetype-version) \ - $(ipydir)/pyparsing-$(pyparsing-version) \ + $(ipydir)/fonttools-$(fonttools-version) \ + $(ipydir)/contourpy-$(contourpy-version) \ $(ipydir)/kiwisolver-$(kiwisolver-version) \ - $(ibidir)/ghostscript-$(ghostscript-version) \ - $(ibidir)/imagemagick-$(imagemagick-version) \ $(ipydir)/python-dateutil-$(python-dateutil-version) # Prepare the source. @@ -439,40 +681,63 @@ $(ipydir)/matplotlib-$(matplotlib-version): \ export CC=clang export CXX=clang++ fi - $(call pybuild, tar -xf, matplotlib-$(matplotlib-version)) + $(call pybuild, tar -xf, matplotlib-$(matplotlib-version),,, GPEP517) cp $(dtexdir)/matplotlib.tex $(ictdir)/ echo "Matplotlib $(matplotlib-version) \citep{matplotlib2007}" > $@ +$(ipydir)/meson-$(meson-version): \ + $(ibidir)/ninjabuild-$(ninjabuild-version) \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=meson-$(meson-version).tar.lz + $(call import-source, $(meson-url), $(meson-checksum)) + $(call pybuild, tar -xf, meson-$(meson-version), , \ + Meson $(meson-version), GPEP517) + echo "Meson $(meson-version)" > $@ + +# The 'meson-python' package may be helpful or requred for some packages. +$(ipydir)/meson-python-$(meson-python-version): \ + $(ipydir)/meson-$(meson-version) \ + $(ipydir)/packaging-$(packaging-version) \ + $(ipydir)/pyproject-metadata-$(pyproject-metadata-version) + tarball=meson-python-$(meson-python-version).tar.lz + $(call import-source, $(meson-python-url), $(meson-python-checksum)) + $(call pybuild, tar -xf, meson-python-$(meson-python-version), , \ + Meson-python $(meson-python-version), GPEP517) + echo "Meson-Python $(meson-python-version)" > $@ + $(ipydir)/mpi4py-$(mpi4py-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ $(ibidir)/openmpi-$(openmpi-version) \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/python-installer-$(python-installer-version) tarball=mpi4py-$(mpi4py-version).tar.lz $(call import-source, $(mpi4py-url), $(mpi4py-checksum)) $(call pybuild, tar -xf, mpi4py-$(mpi4py-version)) cp $(dtexdir)/mpi4py.tex $(ictdir)/ echo "mpi4py $(mpi4py-version) \citep{mpi4py2011}" > $@ -$(ipydir)/mpmath-$(mpmath-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/mpmath-$(mpmath-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=mpmath-$(mpmath-version).tar.gz $(call import-source, $(mpmath-url), $(mpmath-checksum)) $(call pybuild, tar -xf, mpmath-$(mpmath-version), , \ mpmath $(mpmath-version)) +# Until 2025-02-22: we had 'export CFLAGS="--std=c99 $$CFLAGS"' before +# calling pybuild; but that doesn't seem to be necessary. $(ipydir)/numpy-$(numpy-version): \ $(ipydir)/cython-$(cython-version) \ $(ibidir)/openblas-$(openblas-version) \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/pybind11-$(pybind11-version) \ + $(ipydir)/meson-python-$(meson-python-version) tarball=numpy-$(numpy-version).tar.lz $(call import-source, $(numpy-url), $(numpy-checksum)) if [ x$(on_mac_os) = xyes ]; then export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle" - else - export LDFLAGS="$(LDFLAGS) -shared" fi - export CFLAGS="--std=c99 $$CFLAGS" conf="$$(pwd)/reproduce/software/config/numpy-scipy.cfg" $(call pybuild, tar -xf, numpy-$(numpy-version),$$conf, \ - Numpy $(numpy-version)) + Numpy $(numpy-version), GPEP517) cp $(dtexdir)/numpy.tex $(ictdir)/ echo "Numpy $(numpy-version) \citep{numpy2020}" > $@ @@ -481,9 +746,11 @@ $(ipydir)/packaging-$(packaging-version): \ tarball=packaging-$(packaging-version).tar.lz $(call import-source, $(packaging-url), $(packaging-checksum)) $(call pybuild, tar -xf, packaging-$(packaging-version), , \ - Packaging $(packaging-version)) + Packaging $(packaging-version), GPEP517) -$(ipydir)/pexpect-$(pexpect-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/pexpect-$(pexpect-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pexpect-$(pexpect-version).tar.gz $(call import-source, $(pexpect-url), $(pexpect-checksum)) $(call pybuild, tar -xf, pexpect-$(pexpect-version), , \ @@ -491,25 +758,32 @@ $(ipydir)/pexpect-$(pexpect-version): $(ipydir)/setuptools-$(setuptools-version) $(ipydir)/pillow-$(pillow-version): $(ibidir)/libjpeg-$(libjpeg-version) \ $(ipydir)/setuptools-$(setuptools-version) - tarball=Pillow-$(pillow-version).tar.lz + tarball=pillow-$(pillow-version).tar.lz $(call import-source, $(pillow-url), $(pillow-checksum)) - $(call pybuild, tar -xf, Pillow-$(pillow-version), , \ - Pillow $(pillow-version)) - -$(ipydir)/pip-$(pip-version): $(ipydir)/setuptools-$(setuptools-version) - tarball=pip-$(pip-version).tar.gz - $(call import-source, $(pip-url), $(pip-checksum)) - $(call pybuild, tar -xf, pip-$(pip-version), , \ - PiP $(pip-version)) - -$(ipydir)/ply-$(ply-version): $(ipydir)/setuptools-$(setuptools-version) + $(call pybuild, tar -xf, pillow-$(pillow-version), , \ + Pillow $(pillow-version), GPEP517) + +# This should normally not be used, because it's a front-end that obstructs +# reproducibility - source URL; checksum of the tarball; build rule. +# $(ipydir)/pip-$(pip-version): \ +# $(ipydir)/python-installer-$(python-installer-version) \ +# $(ipydir)/wheel-$(wheel-version) +# tarball=pip-$(pip-version).tar.gz +# $(call import-source, $(pip-url), $(pip-checksum)) +# $(call pybuild, tar -xf, pip-$(pip-version), , \ +# PiP $(pip-version)) + +$(ipydir)/ply-$(ply-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=ply-$(ply-version).tar.lz $(call import-source, $(ply-url), $(ply-checksum)) $(call pybuild, tar -xf, ply-$(ply-version), , \ ply $(ply-version)) $(ipydir)/pycodestyle-$(pycodestyle-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pycodestyle-$(pycodestyle-version).tar.gz $(call import-source, $(pycodestyle-url), $(pycodestyle-checksum)) $(call pybuild, tar -xf, pycodestyle-$(pycodestyle-version), , \ @@ -518,16 +792,21 @@ $(ipydir)/pycodestyle-$(pycodestyle-version): \ $(ipydir)/pybind11-$(pybind11-version): \ $(ibidir)/eigen-$(eigen-version) \ $(ibidir)/boost-$(boost-version) \ - $(ipydir)/setuptools-$(setuptools-version) - tarball=pybind11-$(pybind11-version).tar.gz + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) + tarball=pybind11-$(pybind11-version).tar.lz $(call import-source, $(pybind11-url), $(pybind11-checksum)) pyhook_after() { - cp -r include/pybind11 $(iidir)/python$(python-major-version)m/ + cp -pvr pybind11/include/pybind11 \ + $(iidir)/python$(python-major-version)m/ } $(call pybuild, tar -xf, pybind11-$(pybind11-version), , \ - pybind11 $(pybind11-version)) + pybind11 $(pybind11-version), GPEP517) + echo "Pybind11 $(pybind11-version)" > $@ -$(ipydir)/pycparser-$(pycparser-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/pycparser-$(pycparser-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pycparser-$(pycparser-version).tar.gz $(call import-source, $(pycparser-url), $(pycparser-checksum)) $(call pybuild, tar -xf, pycparser-$(pycparser-version), , \ @@ -535,45 +814,83 @@ $(ipydir)/pycparser-$(pycparser-version): $(ipydir)/setuptools-$(setuptools-vers $(ipydir)/pyerfa-$(pyerfa-version): \ $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/packaging-$(packaging-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=pyerfa-$(pyerfa-version).tar.lz $(call import-source, $(pyerfa-url), $(pyerfa-checksum)) $(call pybuild, tar -xf, pyerfa-$(pyerfa-version), , \ - PyERFA $(pyerfa-version)) + PyERFA $(pyerfa-version), GPEP517) -$(ipydir)/pyflakes-$(pyflakes-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/pyflakes-$(pyflakes-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pyflakes-$(pyflakes-version).tar.gz $(call import-source, $(pyflakes-url), $(pyflakes-checksum)) $(call pybuild, tar -xf, pyflakes-$(pyflakes-version), , \ pyflakes $(pyflakes-version)) $(ipydir)/pyparsing-$(pyparsing-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/flit-core-$(flit-core-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pyparsing-$(pyparsing-version).tar.lz $(call import-source, $(pyparsing-url), $(pyparsing-checksum)) $(call pybuild, tar -xf, pyparsing-$(pyparsing-version), , \ - PyParsing $(pyparsing-version)) + PyParsing $(pyparsing-version), GPEP517) -$(ipydir)/pypkgconfig-$(pypkgconfig-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/pypkgconfig-$(pypkgconfig-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=pkgconfig-$(pypkgconfig-version).tar.gz $(call import-source, $(pypkgconfig-url), $(pypkgconfig-checksum)) $(call pybuild, tar -xf, pkgconfig-$(pypkgconfig-version), , pkgconfig $(pypkgconfig-version)) +$(ipydir)/pyproject-metadata-$(pyproject-metadata-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/flit-core-$(flit-core-version) \ + $(ipydir)/python-installer-$(python-installer-version) + tarball=pyproject-metadata-$(pyproject-metadata-version).tar.lz + $(call import-source, $(pyproject-metadata-url), $(pyproject-metadata-checksum)) + $(call pybuild, tar -xf, \ + pyproject-metadata-$(pyproject-metadata-version),,, GPEP517) + $(ipydir)/python-dateutil-$(python-dateutil-version): \ - $(ipydir)/six-$(six-version) \ - $(ipydir)/setuptools_scm-$(setuptools_scm-version) - tarball=python-dateutil-$(python-dateutil-version).tar.gz + $(ipydir)/setuptools-scm-$(setuptools-scm-version) + tarball=python-dateutil-$(python-dateutil-version).tar.lz $(call import-source, $(python-dateutil-url), $(python-dateutil-checksum)) $(call pybuild, tar -xf, python-dateutil-$(python-dateutil-version), , \ - python-dateutil $(python-dateutil-version)) + python-dateutil $(python-dateutil-version), GPEP517) + +$(ipydir)/python-installer-$(python-installer-version): \ + $(ibidir)/python-$(python-version) + +# Prepare the tarball. + tarball=python-installer-$(python-installer-version).tar.lz + $(call import-source, $(python-installer-url), $(python-installer-checksum)) + +# Modify the line in the source that will cause a crash when a +# to-be-installed file already exists in the installation path. This +# is very important for Python packages in Maneage (when a dependency +# is updated, the package needs to be re-built, but that would cause +# due to this line). + pyhook_before(){ + mv -v src/installer/destinations.py src/installer/destinations.py.orig; \ + sed -e 's/\(raise FileExistsError.message.\)/## \1/' \ + src/installer/destinations.py.orig > src/installer/destinations.py + } + +# Build the Python installer. + $(call pybuild, tar -xf, \ + python-installer-$(python-installer-version),,, \ + BOOT_INSTALLER) + echo "Python-installer $(python-installer-version)" > $@ $(ipydir)/pythran-$(pythran-version): \ $(ipydir)/ply-$(ply-version) \ $(ipydir)/gast-$(gast-version) \ $(ibidir)/boost-$(boost-version) \ $(ipydir)/beniget-$(beniget-version) \ - $(ipydir)/setuptools_scm-$(setuptools_scm-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=pythran-$(pythran-version).tar.lz $(call import-source, $(pythran-url), $(pythran-checksum)) $(call pybuild, tar -xf, pythran-$(pythran-version), , \ @@ -582,25 +899,27 @@ $(ipydir)/pythran-$(pythran-version): \ $(ipydir)/pyyaml-$(pyyaml-version): \ $(ibidir)/yaml-$(yaml-version) \ $(ipydir)/cython-$(cython-version) - tarball=pyyaml-$(pyyaml-version).tar.gz + tarball=pyyaml-$(pyyaml-version).tar.lz $(call import-source, $(pyyaml-url), $(pyyaml-checksum)) - $(call pybuild, tar -xf, PyYAML-$(pyyaml-version), , \ - PyYAML $(pyyaml-version)) + $(call pybuild, tar -xf, pyyaml-$(pyyaml-version), , \ + PyYAML $(pyyaml-version), GPEP517) $(ipydir)/requests-$(requests-version): $(ipydir)/idna-$(idna-version) \ - $(ipydir)/numpy-$(numpy-version) \ - $(ipydir)/certifi-$(certifi-version) \ - $(ipydir)/chardet-$(chardet-version) \ - $(ipydir)/urllib3-$(urllib3-version) + $(ipydir)/numpy-$(numpy-version) \ + $(ipydir)/certifi-$(certifi-version) \ + $(ipydir)/chardet-$(chardet-version) \ + $(ipydir)/urllib3-$(urllib3-version) tarball=requests-$(requests-version).tar.gz $(call import-source, $(requests-url), $(requests-checksum)) $(call pybuild, tar -xf, requests-$(requests-version), , \ Requests $(requests-version)) # 'pythran' is disabled in the build of Scipy because of complications it -# caused on some systems. It is explicitly disabled using the environmental -# variable. If for some reason it is needed, set the environment variable -# to 1, and add it as a prerequisite of 'scipy'. +# caused on some systems. We explicitly disable it using a preprocessor +# directive. 'Pythran' can in principle speed up compilation of scientific +# software [1][2]. +# [1] https://pythran.readthedocs.io/en/latest +# [2] https://docs.scipy.org/doc/scipy-1.15.2/dev/roadmap-detailed.html $(ipydir)/scipy-$(scipy-version): \ $(ipydir)/numpy-$(numpy-version) \ $(ipydir)/pybind11-$(pybind11-version) @@ -609,11 +928,28 @@ $(ipydir)/scipy-$(scipy-version): \ if [ x$(on_mac_os) = xyes ]; then export LDFLAGS="$(LDFLAGS) -undefined dynamic_lookup -bundle" else - export LDFLAGS="$(LDFLAGS) -shared" +# Same question as for 'numpy': why '-shared'? This obstructs +# the meson build. +# export LDFLAGS="$(LDFLAGS) -shared" + : fi - export SCIPY_USE_PYTHRAN=0 conf="$$(pwd)/reproduce/software/config/numpy-scipy.cfg" - $(call pybuild, tar -xf, scipy-$(scipy-version),$$conf) + +# Disable pythran: see +# https://docs.scipy.org/doc/scipy-1.15.2/dev/roadmap-detailed.html#use-of-pythran +# export SCIPY_USE_PYTHRAN=0 # deprecated(?) +# Option 1: Hack the source: + pyhook_before() { + mv -iv meson.options meson.options.orig; \ + sed -e 's/\(use-pythran.*value: *\)true/\1false/' \ + meson.options.orig > meson.options + } + +# Option 2: pass the string +# --config-json='{"setup-args": "-Duse-pythran=false"}' +# to gpep517 with correct escaping of single and double quotes. +# Not tried as of 2025-02-25. + $(call pybuild, tar -xf, scipy-$(scipy-version),$$conf,, GPEP517) cp $(dtexdir)/scipy.tex $(ictdir)/ echo "Scipy $(scipy-version) \citep{scipy2020}" > $@ @@ -625,26 +961,38 @@ $(ipydir)/secretstorage-$(secretstorage-version): \ $(call pybuild, tar -xf, SecretStorage-$(secretstorage-version), , \ SecretStorage $(secretstorage-version)) +$(ipydir)/semantic-version-$(semantic-version-version): \ + $(ipydir)/setuptools-$(setuptools-version) + tarball=semantic-version-$(semantic-version-version).tar.lz + $(call import-source, $(semantic-version-url), \ + $(semantic-version-checksum)) + $(call pybuild, tar -xf, \ + semantic-version-$(semantic-version-version), , \ + Semantic-version $(semantic-version-version), GPEP517) + $(ipydir)/setuptools-$(setuptools-version): \ - $(ibidir)/python-$(python-version) + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=setuptools-$(setuptools-version).tar.lz $(call import-source, $(setuptools-url), $(setuptools-checksum)) $(call pybuild, tar -xf, setuptools-$(setuptools-version), , \ - Setuptools $(setuptools-version)) + Setuptools $(setuptools-version), GPEP517) -$(ipydir)/setuptools_scm-$(setuptools_scm-version): \ +$(ipydir)/setuptools-scm-$(setuptools-scm-version): \ $(ipydir)/setuptools-$(setuptools-version) - tarball=setuptools_scm-$(setuptools_scm-version).tar.gz - $(call import-source, $(setuptools_scm-url), $(setuptools_scm-checksum)) - $(call pybuild, tar -xf, setuptools_scm-$(setuptools_scm-version), , \ - Setuptools-scm $(setuptools_scm-version)) + tarball=setuptools-scm-$(setuptools-scm-version).tar.lz + $(call import-source, $(setuptools-scm-url), $(setuptools-scm-checksum)) + $(call pybuild, tar -xf, setuptools-scm-$(setuptools-scm-version), , \ + Setuptools-scm $(setuptools-scm-version), GPEP517) $(ipydir)/setuptools-rust-$(setuptools-rust-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/setuptools-scm-$(setuptools-scm-version) \ + $(ipydir)/semantic-version-$(semantic-version-version) tarball=setuptools-rust-$(setuptools-rust-version).tar.lz - $(call import-source, $(setuptools-rust-url), $(setuptools-rust-checksum)) + $(call import-source, $(setuptools-rust-url), \ + $(setuptools-rust-checksum)) $(call pybuild, tar -xf, setuptools-rust-$(setuptools-rust-version), , \ - Setuptools-scm $(setuptools-rust-version)) + Setuptools-rust $(setuptools-rust-version), GPEP517) $(ipydir)/sip_tpv-$(sip_tpv-version): \ $(ipydir)/sympy-$(sympy-version) \ @@ -655,14 +1003,17 @@ $(ipydir)/sip_tpv-$(sip_tpv-version): \ cp $(dtexdir)/sip_tpv.tex $(ictdir)/ echo "sip_tpv $(sip_tpv-version) \citep{sip-tpv}" > $@ - -$(ipydir)/six-$(six-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/six-$(six-version): \ + $(ipydir)/setuptools-$(setuptools-version) tarball=six-$(six-version).tar.lz $(call import-source, $(six-url), $(six-checksum)) $(call pybuild, tar -xf, six-$(six-version), , \ - Six $(six-version)) + Six $(six-version), GPEP517) + echo "Six $(six-version)" > $@ -$(ipydir)/soupsieve-$(soupsieve-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/soupsieve-$(soupsieve-version): \ + $(ipydir)/setuptools-$(setuptools-version) \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=soupsieve-$(soupsieve-version).tar.gz $(call import-source, $(soupsieve-url), $(soupsieve-checksum)) $(call pybuild, tar -xf, soupsieve-$(soupsieve-version), , \ @@ -681,21 +1032,30 @@ $(ipydir)/uncertainties-$(uncertainties-version): $(ipydir)/numpy-$(numpy-versio $(call pybuild, tar -xf, uncertainties-$(uncertainties-version), , \ uncertainties $(uncertainties-version)) -$(ipydir)/urllib3-$(urllib3-version): $(ipydir)/setuptools-$(setuptools-version) +$(ipydir)/urllib3-$(urllib3-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/python-installer-$(python-installer-version) tarball=urllib3-$(urllib3-version).tar.gz $(call import-source, $(urllib3-url), $(urllib3-checksum)) $(call pybuild, tar -xf, urllib3-$(urllib3-version), , \ Urllib3 $(urllib3-version)) $(ipydir)/webencodings-$(webencodings-version): \ - $(ipydir)/setuptools-$(setuptools-version) + $(ipydir)/setuptools-$(setuptools-version) \ + $(ipydir)/setuptools-scm-$(setuptools-scm-version) tarball=webencodings-$(webencodings-version).tar.gz $(call import-source, $(webencodings-url), $(webencodings-checksum)) $(call pybuild, tar -xf, webencodings-$(webencodings-version), , \ Webencodings $(webencodings-version)) -$(ipydir)/wheel-$(wheel-version): $(ipydir)/setuptools-$(setuptools-version) - tarball=wheel-$(wheel-version).tar.lz +# As of 2025-02, this is only needed if you want 'wheel' on the command +# line; 'setuptools' provides its own version of wheels. +$(ipydir)/wheel-$(wheel-version): \ + $(ipydir)/gpep517-$(gpep517-version) \ + $(ipydir)/flit-core-$(flit-core-version) \ + $(ipydir)/python-installer-$(python-installer-version) +# tarball=wheel-$(wheel-version).tar.lz + tarball=wheel-$(wheel-version).tar.gz $(call import-source, $(wheel-url), $(wheel-checksum)) $(call pybuild, tar -xf, wheel-$(wheel-version), , \ - Wheel $(wheel-version)) + Wheel $(wheel-version), GPEP517) diff --git a/reproduce/software/make/r-cran.mk b/reproduce/software/make/r-cran.mk index 617b8de..7c86c23 100644 --- a/reproduce/software/make/r-cran.mk +++ b/reproduce/software/make/r-cran.mk @@ -12,8 +12,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2022 Boud Roukema <boud@cosmo.torun.pl> -# Copyright (C) 2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2022-2025 Boud Roukema <boud@cosmo.torun.pl> +# Copyright (C) 2022-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -204,6 +204,7 @@ $(ibidir)/r-cran-$(r-cran-version): \ tar -xf $(tdir)/$$tarball unpackdir=R-$(r-cran-version) cd $$unpackdir + $(shsrcdir)/prep-source.sh $(ibdir) # We need to manually remove the lines with '~autodetect~', they # cause the configure script to crash in version 4.0.2. They are used diff --git a/reproduce/software/make/xorg.mk b/reproduce/software/make/xorg.mk index dd707e5..864c32a 100644 --- a/reproduce/software/make/xorg.mk +++ b/reproduce/software/make/xorg.mk @@ -14,8 +14,8 @@ # # ------------------------------------------------------------------------ # -# Copyright (C) 2021-2022 Mohammad Akhlaghi <mohammad@akhlaghi.org> -# Copyright (C) 2021-2022 Raul Infante-Sainz <infantesainz@gmail.com> +# Copyright (C) 2021-2025 Mohammad Akhlaghi <mohammad@akhlaghi.org> +# Copyright (C) 2021-2025 Raul Infante-Sainz <infantesainz@gmail.com> # # This Makefile is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -79,7 +79,7 @@ $(ibidir)/libxau-$(libxau-version): $(ibidir)/xorgproto-$(xorgproto-version) # Library implementing the X Display Manager Control Protocol. $(ibidir)/libxdmcp-$(libxdmcp-version): $(ibidir)/libxau-$(libxau-version) - tarball=libXdmcp-$(libxdmcp-version).tar.bz2 + tarball=libXdmcp-$(libxdmcp-version).tar.lz $(call import-source, $(libxdmcp-url), $(libxdmcp-checksum)) $(call gbuild, libXdmcp-$(libxdmcp-version),,$(XORG_CONFIG), V=1) echo "libXdmcp (Xorg) $(libxdmcp-version)" > $@ |