diff options
Diffstat (limited to 'reproduce/src')
| -rw-r--r-- | reproduce/src/make/dependencies-basic.mk | 223 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies-python.mk | 189 | ||||
| -rw-r--r-- | reproduce/src/make/dependencies.mk | 204 | ||||
| -rw-r--r-- | reproduce/src/make/initialize.mk | 99 |
4 files changed, 351 insertions, 364 deletions
diff --git a/reproduce/src/make/dependencies-basic.mk b/reproduce/src/make/dependencies-basic.mk index 1573f39..d4830ca 100644 --- a/reproduce/src/make/dependencies-basic.mk +++ b/reproduce/src/make/dependencies-basic.mk @@ -49,7 +49,8 @@ tdir = $(BDIR)/dependencies/tarballs idir = $(BDIR)/dependencies/installed ibdir = $(BDIR)/dependencies/installed/bin ildir = $(BDIR)/dependencies/installed/lib -ilidir = $(BDIR)/dependencies/installed/lib/built +ibidir = $(BDIR)/dependencies/installed/version-info/bin +ilidir = $(BDIR)/dependencies/installed/version-info/lib # We'll need the system's PATH for making links to low-level programs we # won't be building ourselves. @@ -67,7 +68,7 @@ export LD_LIBRARY_PATH := $(ildir):$(LD_LIBRARY_PATH) # Define the programs that don't depend on any other. top-level-programs = low-level-links wget gcc -all: $(foreach p, $(top-level-programs), $(ibdir)/$(p)) +all: $(foreach p, $(top-level-programs), $(ibidir)/$(p)) @@ -126,7 +127,7 @@ tarballs = $(foreach t, bash-$(bash-version).tar.gz \ xz-$(xz-version).tar.gz \ zlib-$(zlib-version).tar.gz \ , $(tdir)/$(t) ) -$(tarballs): $(tdir)/%: $(lockdir) +$(tarballs): $(tdir)/%: | $(lockdir) if [ -f $(DEPENDENCIES-DIR)/$* ]; then \ cp $(DEPENDENCIES-DIR)/$* $@; \ else \ @@ -213,7 +214,7 @@ makelink = origpath="$$PATH"; \ if [ x$$a != x ]; then $$c $$a $(ibdir)/$(1); fi; \ export PATH="$$origpath" $(ibdir) $(ildir):; mkdir $@ -$(ibdir)/low-level-links: | $(ibdir) $(ildir) +$(ibidir)/low-level-links: | $(ibdir) $(ildir) # The Assembler $(call makelink,as) @@ -260,7 +261,9 @@ $(ibdir)/low-level-links: | $(ibdir) $(ildir) fi; \ done - echo "Low-level symbolic links are setup" > $@ + # We want this to be empty (so it doesn't interefere with the other + # files in `ibidir'. + touch $@ @@ -277,22 +280,26 @@ $(ibdir)/low-level-links: | $(ibdir) $(ildir) # The first set of programs to be built are those that we need to unpack # the source code tarballs of each program. First, we'll build the # necessary programs, then we'll build GNU Tar. -$(ibdir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz - $(call gbuild, $<, gzip-$(gzip-version), static, , V=1) +$(ibidir)/gzip: $(tdir)/gzip-$(gzip-version).tar.gz + $(call gbuild, $<, gzip-$(gzip-version), static, , V=1) \ + && echo "GNU Gzip $(gzip-version)" > $@ # GNU Lzip: For a static build, the `-static' flag should be given to # LDFLAGS on the command-line (not from the environment). -$(ibdir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz ifeq ($(static_build),yes) - $(call gbuild, $<, lzip-$(lzip-version), , LDFLAGS="-static") +lzipconf="LDFLAGS=-static" else - $(call gbuild, $<, lzip-$(lzip-version)) +lzipconf= endif +$(ibidir)/lzip: $(tdir)/lzip-$(lzip-version).tar.gz + $(call gbuild, $<, lzip-$(lzip-version), , $(lzipconf)) \ + && echo "Lzip $(lzip-version)" > $@ -$(ibdir)/xz: $(tdir)/xz-$(xz-version).tar.gz - $(call gbuild, $<, xz-$(xz-version), static) +$(ibidir)/xz: $(tdir)/xz-$(xz-version).tar.gz + $(call gbuild, $<, xz-$(xz-version), static) \ + && echo "XZ Utils $(xz-version)" > $@ -$(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz +$(ibidir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz # Bzip2 doesn't have a `./configure' script, and its Makefile # doesn't build a shared library. So we can't use the `gbuild' # function here and we need to take some extra steps (inspired @@ -326,20 +333,22 @@ $(ibdir)/bzip2: $(tdir)/bzip2-$(bzip2-version).tar.gz && cd .. \ && rm -rf $$tdir \ && cd $(ildir) \ - && ln -fs libbz2.so.1.0 libbz2.so + && ln -fs libbz2.so.1.0 libbz2.so \ + && echo "Bzip2 $(bzip2-version)" > $@ # GNU Tar: When built statically, tar gives a segmentation fault on # unpacking Bash. So we'll build it dynamically. -$(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \ - $(ibdir)/bzip2 \ - $(ibdir)/lzip \ - $(ibdir)/gzip \ - $(ibdir)/xz +$(ibidir)/tar: $(tdir)/tar-$(tar-version).tar.gz \ + $(ibidir)/bzip2 \ + $(ibidir)/lzip \ + $(ibidir)/gzip \ + $(ibidir)/xz # Since all later programs depend on Tar, the pipeline will be # stuck here, only making Tar. So its more efficient to built it on # multiple threads (when the user's Make doesn't pass down the # number of threads). - $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads)) + $(call gbuild, $<, tar-$(tar-version), , , -j$(numthreads)) \ + && echo "GNU Tar $(tar-version)" > $@ @@ -361,13 +370,14 @@ $(ibdir)/tar: $(tdir)/tar-$(tar-version).tar.gz \ # function (for tilde expansion). The first can be disabled with # `--disable-load', but unfortunately I don't know any way to fix the # second. So, we'll have to build it dynamically for now. -$(ibdir)/make: $(tdir)/make-$(make-version).tar.lz \ - $(ibdir)/tar +$(ibidir)/make: $(tdir)/make-$(make-version).tar.lz \ + $(ibidir)/tar # See Tar's comments for the `-j' option. - $(call gbuild, $<, make-$(make-version), , , -j$(numthreads)) + $(call gbuild, $<, make-$(make-version), , , -j$(numthreads)) \ + && echo "GNU Make $(make-version)" > $@ $(ilidir)/ncurses: $(tdir)/ncurses-$(ncurses-version).tar.gz \ - $(ibdir)/make | $(ilidir) + $(ibidir)/make # Delete the library that will be installed (so we can make sure # the build process completed afterwards and reset the links). @@ -447,7 +457,7 @@ $(ilidir)/ncurses: $(tdir)/ncurses-$(ncurses-version).tar.gz \ ln -fs $(ildir)/pkgconfig/ncursesw.pc pkgconfig/cursesw.pc; \ \ ln -fs $(idir)/include/ncursesw $(idir)/include/ncurses; \ - echo "GNU ncurses is built and ready" > $@; \ + echo "GNU NCURSES $(ncurses-version)" > $@; \ else \ exit 1; \ fi @@ -456,12 +466,13 @@ $(ilidir)/readline: $(tdir)/readline-$(readline-version).tar.gz \ $(ilidir)/ncurses $(call gbuild, $<, readline-$(readline-version), static, \ --with-curses --disable-install-examples, \ - SHLIB_LIBS="-lncursesw" ) && \ - echo "GNU Readline is built and ready" > $@ + SHLIB_LIBS="-lncursesw" ) \ + && echo "GNU Readline $(readline-version)" > $@ -$(ibdir)/patchelf: $(tdir)/patchelf-$(patchelf-version).tar.gz \ - $(ibdir)/make - $(call gbuild, $<, patchelf-$(patchelf-version), static) +$(ibidir)/patchelf: $(tdir)/patchelf-$(patchelf-version).tar.gz \ + $(ibidir)/make + $(call gbuild, $<, patchelf-$(patchelf-version), static) \ + && echo "PatchELF $(patchelf-version)" > $@ # IMPORTANT: Even though we have enabled `rpath', Bash doesn't write the @@ -475,14 +486,14 @@ $(ibdir)/patchelf: $(tdir)/patchelf-$(patchelf-version).tar.gz \ ifeq ($(on_mac_os),yes) needpatchelf = else -needpatchelf = $(ibdir)/patchelf +needpatchelf = $(ibidir)/patchelf endif -$(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ - $(ilidir)/readline \ - $(needpatchelf) +$(ibidir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ + $(ilidir)/readline \ + $(needpatchelf) - # Delete any possibly existing output - rm -f $@ + # Delete the (possibly) existing Bash executable. + rm -f $(ibdir)/bash # Build Bash. Note that we aren't building Bash with # `--with-installed-readline'. This is because (as described above) @@ -522,7 +533,11 @@ $(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ # Just to be sure that the installation step above went well, # before making the link, we'll see if the file actually exists # there. - if [ -f $@ ]; then ln -fs $@ $(ibdir)/sh; else exit 1; fi + if [ -f $(ibdir)/bash ]; then \ + ln -fs $(ibdir)/bash $(ibdir)/sh; \ + echo "GNU Bash $(bash-version)" > $@; \ + else \ + echo "GNU Bash not built!"; exit 1; fi @@ -537,10 +552,10 @@ $(ibdir)/bash: $(tdir)/bash-$(bash-version).tar.gz \ # Note for a static-only build: Zlib's `./configure' doesn't use Autoconf's # configure script, it just accepts a direct `--static' option. $(idir)/etc:; mkdir $@ -$(ilidir): | $(ildir); mkdir $@ $(ilidir)/zlib: $(tdir)/zlib-$(zlib-version).tar.gz \ - $(ibdir)/bash | $(ilidir) - $(call gbuild, $<, zlib-$(zlib-version)) && echo "Zlib is built" > $@ + $(ibidir)/bash + $(call gbuild, $<, zlib-$(zlib-version)) \ + && echo "Zlib $(zlib-version)" > $@ # OpenSSL: Some programs/libraries later need dynamic linking. So we'll # build libssl (and libcrypto) dynamically also. @@ -584,7 +599,7 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ else \ patchelf --set-rpath $(ildir) $(ildir)/libssl.so; \ fi; \ - echo "OpenSSL is built and ready" > $@; \ + echo "OpenSSL $(openssl-version)" > $@; \ fi # GNU Wget @@ -600,9 +615,9 @@ $(ilidir)/openssl: $(tdir)/openssl-$(openssl-version).tar.gz \ # building as part of this pipeline. So to avoid too much dependency on the # host system (especially a crash when these libraries are updated on the # host), they are disabled here. -$(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \ - $(ibdir)/pkg-config \ - $(ilidir)/openssl +$(ibidir)/wget: $(tdir)/wget-$(wget-version).tar.lz \ + $(ibidir)/pkg-config \ + $(ilidir)/openssl libs="-pthread"; \ if [ x$(needs_ldl) = xyes ]; then libs="$$libs -ldl"; fi; \ $(call gbuild, $<, wget-$(wget-version), , \ @@ -616,7 +631,8 @@ $(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \ --without-libidn \ --disable-pcre2 \ --disable-pcre \ - --disable-iri ) + --disable-iri ) \ + && echo "GNU Wget $(wget-version)" > $@ @@ -625,19 +641,22 @@ $(ibdir)/wget: $(tdir)/wget-$(wget-version).tar.lz \ # Basic command-line programs necessary in build process of the # higher-level dependencies: Note that during the building of those # programs, there is no access to the system's PATH. -$(ibdir)/diff: $(tdir)/diffutils-$(diffutils-version).tar.xz \ - $(ibdir)/bash - $(call gbuild, $<, diffutils-$(diffutils-version), static) - -$(ibdir)/find: $(tdir)/findutils-$(findutils-version).tar.lz \ - $(ibdir)/bash - $(call gbuild, $<, findutils-$(findutils-version), static) - -$(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \ - $(ibdir)/bash +$(ibidir)/diffutils: $(tdir)/diffutils-$(diffutils-version).tar.xz \ + $(ibidir)/bash + $(call gbuild, $<, diffutils-$(diffutils-version), static, , V=1) \ + && echo "GNU Diffutils $(diffutils-version)" > $@ + +$(ibidir)/findutils: $(tdir)/findutils-$(findutils-version).tar.lz \ + $(ibidir)/bash + $(call gbuild, $<, findutils-$(findutils-version), static, , V=1) \ + && echo "GNU Findutils $(findutils-version)" > $@ + +$(ibidir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \ + $(ibidir)/bash # Build the main program. $(call gbuild, $<, gawk-$(gawk-version), static, \ - --with-readline=$(idir)); + --with-readline=$(idir)) \ + && echo "GNU AWK $(gawk-version)" > $@ # Since AWK doesn't include RPATH by default, we'll have to # manually include it using the `patchelf' program. Just note that @@ -650,31 +669,36 @@ $(ibdir)/gawk: $(tdir)/gawk-$(gawk-version).tar.lz \ fi; \ fi -$(ibdir)/grep: $(tdir)/grep-$(grep-version).tar.xz \ - $(ibdir)/bash - $(call gbuild, $<, grep-$(grep-version), static) +$(ibidir)/grep: $(tdir)/grep-$(grep-version).tar.xz \ + $(ibidir)/bash + $(call gbuild, $<, grep-$(grep-version), static) \ + && echo "GNU Grep $(grep-version)" > $@ -$(ibdir)/ls: $(tdir)/coreutils-$(coreutils-version).tar.xz \ - $(ilidir)/openssl +$(ibidir)/coreutils: $(tdir)/coreutils-$(coreutils-version).tar.xz \ + $(ilidir)/openssl # Coreutils will use the hashing features of OpenSSL's `libcrypto'. # See Tar's comments for the `-j' option. $(call gbuild, $<, coreutils-$(coreutils-version), static, \ LDFLAGS="$(LDFLAGS)" CPPFLAGS="$(CPPFLAGS)" \ --enable-rpath --disable-silent-rules --with-openssl, \ - -j$(numthreads)) + -j$(numthreads)) \ + && echo "GNU Coreutils $(coreutils-version)" > $@ -$(ibdir)/pkg-config: $(tdir)/pkg-config-$(pkgconfig-version).tar.gz \ - $(ibdir)/bash +$(ibidir)/pkg-config: $(tdir)/pkg-config-$(pkgconfig-version).tar.gz \ + $(ibidir)/bash $(call gbuild, $<, pkg-config-$(pkgconfig-version), static, \ - --with-internal-glib --with-pc-path=$(ildir)/pkgconfig) + --with-internal-glib --with-pc-path=$(ildir)/pkgconfig) \ + && echo "pkg-config $(pkgconfig-version)" > $@ -$(ibdir)/sed: $(tdir)/sed-$(sed-version).tar.xz \ - $(ibdir)/bash - $(call gbuild, $<, sed-$(sed-version), static) +$(ibidir)/sed: $(tdir)/sed-$(sed-version).tar.xz \ + $(ibidir)/bash + $(call gbuild, $<, sed-$(sed-version), static) \ + && echo "GNU Sed $(sed-version)" > $@ -$(ibdir)/which: $(tdir)/which-$(which-version).tar.gz \ - $(ibdir)/bash - $(call gbuild, $<, which-$(which-version), static) +$(ibidir)/which: $(tdir)/which-$(which-version).tar.gz \ + $(ibidir)/bash + $(call gbuild, $<, which-$(which-version), static) \ + && echo "GNU Which $(which-version)" > $@ @@ -685,33 +709,34 @@ $(ibdir)/which: $(tdir)/which-$(which-version).tar.gz \ -# (CURRENTLY IGNORED) GCC prerequisites -# ------------------------------------- +# GCC prerequisites +# ----------------- $(ilidir)/gmp: $(tdir)/gmp-$(gmp-version).tar.lz \ - $(ibdir)/bash | $(ilidir) + $(ibidir)/bash $(call gbuild, $<, gmp-$(gmp-version), static, , , make check) \ - && echo "GNU multiple precision arithmetic library is built" > $@ + && echo "GNU Multiple Precision Arithmetic Library $(gmp-version)" > $@ $(ilidir)/mpfr: $(tdir)/mpfr-$(mpfr-version).tar.xz \ $(ilidir)/gmp $(call gbuild, $<, mpfr-$(mpfr-version), static, , , make check) \ - && echo "GNU MPFR library is built" > $@ + && echo "GNU Multiple Precision Floating-Point Reliably $(mpfr-version)" > $@ $(ilidir)/mpc: $(tdir)/mpc-$(mpc-version).tar.gz \ $(ilidir)/mpfr $(call gbuild, $<, mpc-$(mpc-version), static, , , make check) \ - && echo "GNU MPC library is built" > $@ + && echo "GNU Multiple Precision Complex library" > $@ $(ilidir)/isl: $(tdir)/isl-$(isl-version).tar.bz2 \ $(ilidir)/gmp $(call gbuild, $<, isl-$(isl-version), static) \ - && echo "GCC's ISL library is built" > $@ + && echo "GNU Integer Set Library $(isl-version)" > $@ # Binutils' linker `ld' is apparently only good for GNU/Linux systems and # other OSs have their own. So for now we aren't actually building # Binutils (`ld' isn't a prerequisite of GCC). -$(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz - $(call gbuild, $<, binutils-$(binutils-version), static) +$(ibidir)/binutils: $(tdir)/binutils-$(binutils-version).tar.lz + $(call gbuild, $<, binutils-$(binutils-version), static) \ + && echo "GNU Binutils $(binutils-version)" > $@ @@ -742,22 +767,22 @@ $(ibdir)/ld: $(tdir)/binutils-$(binutils-version).tar.lz # # We are currently having problems installing GCC on macOS, so for the time # being, if the pipeline is being run on a macOS, we'll just set a link. -#ifeq ($(on_mac_os),yes) -#gcc-prerequisites = -#else +ifeq ($(on_mac_os),yes) +gcc-prerequisites = +else gcc-prerequisites = $(tdir)/gcc-$(gcc-version).tar.xz \ $(ilidir)/isl \ $(ilidir)/mpc -#endif -$(ibdir)/gcc: $(gcc-prerequisites) \ - $(ibdir)/ls \ - $(ibdir)/sed \ - $(ibdir)/gawk \ - $(ibdir)/grep \ - $(ibdir)/diff \ - $(ibdir)/find \ - $(ibdir)/bash \ - $(ibdir)/which +endif +$(ibidir)/gcc: $(gcc-prerequisites) \ + $(ibidir)/sed \ + $(ibidir)/gawk \ + $(ibidir)/grep \ + $(ibidir)/bash \ + $(ibidir)/which \ + $(ibidir)/findutils \ + $(ibidir)/diffutils \ + $(ibidir)/coreutils # On a macOS, we (currently!) won't build GCC because of some # errors we are still trying to find. So, we'll just make a @@ -769,14 +794,15 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ # in '$(idir)/lib' by defining the '$(idir)/lib64' as a symbolic # link to '$(idir)/lib'. -# SO FAR WE HAVEN'T BEEN ABLE TO GET A CONSISTENT BUILD OF GCC ON MAC -# (SOMETIMES IT CRASHES IN libiberty with g++) AND SOMETIMES IT FINISHES, -# SO, MORE TESTS ARE NEEDED ON MAC AND WE'LL USE THE HOST'S COMPILER UNTIL -# THEN. + # SO FAR WE HAVEN'T BEEN ABLE TO GET A CONSISTENT BUILD OF GCC ON + # MAC (SOMETIMES IT CRASHES IN libiberty with g++) AND SOMETIMES IT + # FINISHES, SO, MORE TESTS ARE NEEDED ON MAC AND WE'LL USE THE + # HOST'S COMPILER UNTIL THEN. if [ "x$(on_mac_os)" = xyes ]; then \ $(call makelink,g++); \ $(call makelink,gfortran); \ $(call makelink,gcc,copy); \ + echo "" > $@; \ else \ rm -f $(ibdir)/gcc* $(ibdir)/g++ $(ibdir)/gfortran $(ibdir)/gcov*;\ rm -rf $(ildir)/gcc $(ildir)/libcc* $(ildir)/libgcc*; \ @@ -819,5 +845,6 @@ $(ibdir)/gcc: $(gcc-prerequisites) \ patchelf --set-rpath $(ildir) $$f; \ fi; \ done; \ - fi; \ + fi \ + && echo "GNU Compiler Collection (GCC) $(gcc-version)" > $@; \ fi diff --git a/reproduce/src/make/dependencies-python.mk b/reproduce/src/make/dependencies-python.mk index eb5189d..02c6112 100644 --- a/reproduce/src/make/dependencies-python.mk +++ b/reproduce/src/make/dependencies-python.mk @@ -197,47 +197,16 @@ $(pytarballs): $(tdir)/%: else tarballurl=$$h fi - # If the download fails, Wget will write the error message in the - # target file, so Make will think that its done! To avoid this - # problem, we'll rename the output. - echo "Downloading $$tarballurl" - if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then - rm -f $@ - echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1 - fi + # Download using the script specially defined for this job. + touch $(lockdir)/download + $(downloadwrapper) "$$downloader" $(lockdir)/download \ + $$tarballurl $@ fi -# Install without pip -# -------------------- -# -# To build Python packages with direct access to a `setup.py' (if no direct -# access to `setup.py' is needed, pip can be used). -# Arguments of this function are the numbers -# 1) Unpack command -# 2) Package name -# 3) Unpacked directory name after unpacking the tarball -# 4) site.cfg file (optional) -pybuild = cd $(ddir); rm -rf $(3); \ - if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ - cd $(3); \ - if [ "x$(4)" != x ]; then \ - sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ - -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ - $(4) > site.cfg; \ - fi; \ - python3 setup.py build \ - && python3 setup.py install \ - && cd .. && rm -rf $(3) \ - && echo "done!" > $@ - - - - - # Necessary programs and libraries # -------------------------------- # @@ -245,10 +214,10 @@ pybuild = cd $(ddir); rm -rf $(3); \ # certain programs (like Python itself), or libraries for the modules. $(ilidir)/libffi: $(tdir)/libffi-$(libffi-version).tar.gz $(call gbuild, $<, libffi-$(libffi-version)) \ - echo "libffi is built" > $@ + echo "Libffi $(libffi-version)" > $@ -$(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \ - $(ilidir)/libffi +$(ibidir)/python3: $(tdir)/python-$(python-version).tar.gz \ + $(ilidir)/libffi # On Mac systems, the build complains about `clang' specific # features, so we can't use our own GCC build here. if [ x$(on_mac_os) = xyes ]; then \ @@ -262,7 +231,39 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \ && v=$$(echo $(python-version) | awk 'BEGIN{FS="."} \ {printf "%d.%d\n", $$1, $$2}') \ && ln -s $(ildir)/python$$v $(ildir)/python \ - && rm -rf $(ipydir) && mkdir $(ipydir) + && rm -rf $(ipydir) \ + && mkdir $(ipydir) \ + && echo "Python $(python-version)" > $@ + + + + + +# Non-PiP Python module installation +# ---------------------------------- +# +# To build Python packages with direct access to a `setup.py' (if no direct +# access to `setup.py' is needed, pip can be used). +# Arguments of this function are the numbers +# 1) Unpack command +# 2) Package name +# 3) Unpacked directory name after unpacking the tarball +# 4) site.cfg file (optional) +# 5) Official software name.(for paper). +pybuild = cd $(ddir); rm -rf $(3); \ + if ! $(1) $(2); then echo; echo "Tar error"; exit 1; fi; \ + cd $(3); \ + if [ "x$(4)" != x ]; then \ + sed -e 's|@LIBDIR[@]|'"$(ildir)"'|' \ + -e 's|@INCDIR[@]|'"$(idir)/include"'|' \ + $(4) > site.cfg; \ + fi; \ + python3 setup.py build \ + && python3 setup.py install \ + && cd .. \ + && rm -rf $(3) \ + && echo "$(5)" > $@ + @@ -273,7 +274,8 @@ $(ibdir)/python3: $(tdir)/python-$(python-version).tar.gz \ # All the necessary Python modules go here. $(ipydir)/asn1crypto: $(tdir)/asn1crypto-$(asn1crypto-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, asn1crypto-$(asn1crypto-version)) + $(call pybuild, tar xf, $<, asn1crypto-$(asn1crypto-version), , \ + Asn1crypto $(asn1crypto-version)) $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ $(ipydir)/astropy \ @@ -282,46 +284,56 @@ $(ipydir)/astroquery: $(tdir)/astroquery-$(astroquery-version).tar.gz \ $(ipydir)/keyring \ $(ipydir)/numpy \ $(ipydir)/requests - $(call pybuild, tar xf, $<, astroquery-$(astroquery-version)) + $(call pybuild, tar xf, $<, astroquery-$(astroquery-version), ,\ + Astroquery $(astroquery-version)) $(ipydir)/astropy: $(tdir)/astropy-$(astropy-version).tar.gz \ $(ipydir)/h5py \ $(ipydir)/numpy \ $(ipydir)/scipy - $(call pybuild, tar xf, $<, astropy-$(astropy-version)) + $(call pybuild, tar xf, $<, astropy-$(astropy-version), ,\ + "Astropy $(astropy-version) \citep{astropy2013,astropy2018}") $(ipydir)/beautifulsoup4: $(tdir)/beautifulsoup4-$(beautifulsoup4-version).tar.gz \ $(ipydir)/soupsieve - $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version)) + $(call pybuild, tar xf, $<, beautifulsoup4-$(beautifulsoup4-version), ,\ + BeautifulSoup $(beautifulsoup4-version)) $(ipydir)/certifi: $(tdir)/certifi-$(certifi-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, certifi-$(certifi-version)) + $(call pybuild, tar xf, $<, certifi-$(certifi-version), ,\ + Certifi $(certifi-version)) $(ipydir)/cffi: $(tdir)/cffi-$(cffi-version).tar.gz \ $(ilidir)/libffi \ $(ipydir)/pycparser - $(call pybuild, tar xf, $<, cffi-$(cffi-version)) + $(call pybuild, tar xf, $<, cffi-$(cffi-version), ,\ + cffi $(cffi-version)) $(ipydir)/chardet: $(tdir)/chardet-$(chardet-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, chardet-$(chardet-version)) + $(call pybuild, tar xf, $<, chardet-$(chardet-version), ,\ + Chardet $(chardet-version)) $(ipydir)/cryptography: $(tdir)/cryptography-$(cryptography-version).tar.gz \ $(ipydir)/asn1crypto \ $(ipydir)/cffi - $(call pybuild, tar xf, $<, cryptography-$(cryptography-version)) + $(call pybuild, tar xf, $<, cryptography-$(cryptography-version), ,\ + Cryptography $(cryptography-version)) $(ipydir)/cycler: $(tdir)/cycler-$(cycler-version).tar.gz \ $(ipydir)/six - $(call pybuild, tar xf, $<, cycler-$(cycler-version)) + $(call pybuild, tar xf, $<, cycler-$(cycler-version), ,\ + Cycler $(cycler-version)) $(ipydir)/cython: $(tdir)/Cython-$(cython-version).tar.gz - $(call pybuild, tar xf, $<, Cython-$(cython-version)) + $(call pybuild, tar xf, $<, Cython-$(cython-version), ,\ + Cython $(cython-version)) $(ipydir)/entrypoints: $(tdir)/entrypoints-$(entrypoints-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version)) + $(call pybuild, tar xf, $<, entrypoints-$(entrypoints-version), ,\ + EntryPoints $(entrypoints-version)) $(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \ $(ilidir)/hdf5 \ @@ -330,31 +342,37 @@ $(ipydir)/h5py: $(tdir)/h5py-$(h5py-version).tar.gz \ $(ipydir)/setuptools #$(ipydir)/mpi4py # AFTER its problem is fixed. #export HDF5_MPI=ON; # AFTER its problem is fixed. - export HDF5_DIR=$(ildir); \ - $(call pybuild, tar xf, $<, h5py-$(h5py-version)) + export HDF5_DIR=$(ildir); \ + $(call pybuild, tar xf, $<, h5py-$(h5py-version), ,\ + h5py $(h5py-version)) $(ipydir)/html5lib: $(tdir)/html5lib-$(html5lib-version).tar.gz \ $(ipydir)/six \ $(ipydir)/webencodings - $(call pybuild, tar xf, $<, html5lib-$(html5lib-version)) + $(call pybuild, tar xf, $<, html5lib-$(html5lib-version), ,\ + HTML5lib $(html5lib-version)) $(ipydir)/idna: $(tdir)/idna-$(idna-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, idna-$(idna-version)) + $(call pybuild, tar xf, $<, idna-$(idna-version), ,\ + idna $(idna-version)) $(ipydir)/jeepney: $(tdir)/jeepney-$(jeepney-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, jeepney-$(jeepney-version)) + $(call pybuild, tar xf, $<, jeepney-$(jeepney-version), ,\ + Jeepney $(jeepney-version)) $(ipydir)/keyring: $(tdir)/keyring-$(keyring-version).tar.gz \ $(ipydir)/entrypoints \ $(ipydir)/secretstorage \ $(ipydir)/setuptools_scm - $(call pybuild, tar xf, $<, keyring-$(keyring-version)) + $(call pybuild, tar xf, $<, keyring-$(keyring-version), ,\ + Keyring $(keyring-version)) $(ipydir)/kiwisolver: $(tdir)/kiwisolver-$(kiwisolver-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version)) + $(call pybuild, tar xf, $<, kiwisolver-$(kiwisolver-version), ,\ + Kiwisolver $(kiwisolver-version)) $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/cycler \ @@ -363,7 +381,8 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/numpy \ $(ipydir)/pyparsing \ $(ipydir)/python-dateutil - $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version)) + $(call pybuild, tar xf, $<, matplotlib-$(matplotlib-version), ,\ + Matplotlib $(matplotlib-version)) # Currently mpi4py doesn't build because of some conflict with OpenMPI: # @@ -377,7 +396,8 @@ $(ipydir)/matplotlib: $(tdir)/matplotlib-$(matplotlib-version).tar.gz \ $(ipydir)/mpi4py: $(tdir)/mpi4py-$(mpi4py-version).tar.gz \ $(ipydir)/setuptools \ $(ilidir)/openmpi - $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version)) + $(call pybuild, tar xf, $<, mpi4py-$(mpi4py-version), ,\ + mpi4py $(mpi4py-version)) $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ $(ipydir)/setuptools \ @@ -389,27 +409,33 @@ $(ipydir)/numpy: $(tdir)/numpy-$(numpy-version).zip \ export LDFLAGS="$(LDFLAGS) -shared"; \ fi; \ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ - $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf) + $(call pybuild, unzip, $<, numpy-$(numpy-version),$$conf, \ + Numpy $(numpy-version)) -$(ibdir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ - $(ipydir)/setuptools - $(call pybuild, tar xf, $<, pip-$(pip-version)) +$(ibidir)/pip3: $(tdir)/pip-$(pip-version).tar.gz \ + $(ipydir)/setuptools + $(call pybuild, tar xf, $<, pip-$(pip-version), ,\ + PiP $(pip-version)) $(ipydir)/pypkgconfig: $(tdir)/pkgconfig-$(pypkgconfig-version).tar.gz - $(call pybuild, tar xf, $<, pkgconfig-$(pypkgconfig-version)) + $(call pybuild, tar xf, $<, pkgconfig-$(pypkgconfig-version), , + pkgconfig $(pypkgconfig-version)) $(ipydir)/pycparser: $(tdir)/pycparser-$(pycparser-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, pycparser-$(pycparser-version)) + $(call pybuild, tar xf, $<, pycparser-$(pycparser-version), ,\ + pycparser $(pycparser-version)) $(ipydir)/pyparsing: $(tdir)/pyparsing-$(pyparsing-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version)) + $(call pybuild, tar xf, $<, pyparsing-$(pyparsing-version), ,\ + PyParsing $(pyparsing-version)) $(ipydir)/python-dateutil: $(tdir)/python-dateutil-$(python-dateutil-version).tar.gz \ $(ipydir)/setuptools_scm \ $(ipydir)/six - $(call pybuild, tar xf, $<, python-dateutil-$(python-dateutil-version)) + $(call pybuild, tar xf, $<, python-dateutil-$(python-dateutil-version), ,\ + python-dateutil $(python-dateutil-version)) $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ $(ipydir)/certifi \ @@ -417,7 +443,8 @@ $(ipydir)/requests: $(tdir)/requests-$(requests-version).tar.gz \ $(ipydir)/idna \ $(ipydir)/numpy \ $(ipydir)/urllib3 - $(call pybuild, tar xf, $<, requests-$(requests-version)) + $(call pybuild, tar xf, $<, requests-$(requests-version), ,\ + Requests $(requests-version)) $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ $(ipydir)/numpy @@ -427,33 +454,41 @@ $(ipydir)/scipy: $(tdir)/scipy-$(scipy-version).tar.gz \ export LDFLAGS="$(LDFLAGS) -shared"; \ fi; \ conf="$$(pwd)/reproduce/config/pipeline/dependency-numpy-scipy.cfg"; \ - $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf) + $(call pybuild, tar xf, $<, scipy-$(scipy-version),$$conf,\ + Scipy $(scipy-version)) $(ipydir)/secretstorage: $(tdir)/secretstorage-$(secretstorage-version).tar.gz \ $(ipydir)/cryptography \ $(ipydir)/jeepney - $(call pybuild, tar xf, $<, SecretStorage-$(secretstorage-version)) + $(call pybuild, tar xf, $<, SecretStorage-$(secretstorage-version), ,\ + SecretStorage $(secretstorage-version)) $(ipydir)/setuptools: $(tdir)/setuptools-$(setuptools-version).zip \ - $(ibdir)/python3 - $(call pybuild, unzip, $<, setuptools-$(setuptools-version)) + $(ibidir)/python3 + $(call pybuild, unzip, $<, setuptools-$(setuptools-version), ,\ + Setuptools $(setuptools-version)) $(ipydir)/setuptools_scm: $(tdir)/setuptools_scm-$(setuptools_scm-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version)) + $(call pybuild, tar xf, $<, setuptools_scm-$(setuptools_scm-version), ,\ + Setuptools-scm $(setuptools_scm-version)) $(ipydir)/six: $(tdir)/six-$(six-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, six-$(six-version)) + $(call pybuild, tar xf, $<, six-$(six-version), ,\ + Six $(six-version)) $(ipydir)/soupsieve: $(tdir)/soupsieve-$(soupsieve-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version)) + $(call pybuild, tar xf, $<, soupsieve-$(soupsieve-version), ,\ + SoupSieve $(soupsieve-version)) $(ipydir)/urllib3: $(tdir)/urllib3-$(urllib3-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, urllib3-$(urllib3-version)) + $(call pybuild, tar xf, $<, urllib3-$(urllib3-version), ,\ + Urllib3 $(urllib3-version)) $(ipydir)/webencodings: $(tdir)/webencodings-$(webencodings-version).tar.gz \ $(ipydir)/setuptools - $(call pybuild, tar xf, $<, webencodings-$(webencodings-version)) + $(call pybuild, tar xf, $<, webencodings-$(webencodings-version), ,\ + Webencodings $(webencodings-version)) diff --git a/reproduce/src/make/dependencies.mk b/reproduce/src/make/dependencies.mk index fdbfd7f..780a4f0 100644 --- a/reproduce/src/make/dependencies.mk +++ b/reproduce/src/make/dependencies.mk @@ -35,14 +35,16 @@ include reproduce/src/make/dependencies-build-rules.mk include reproduce/config/pipeline/dependency-texlive.mk include reproduce/config/pipeline/dependency-versions.mk - -ddir = $(BDIR)/dependencies -tdir = $(BDIR)/dependencies/tarballs -idir = $(BDIR)/dependencies/installed -ibdir = $(BDIR)/dependencies/installed/bin -ildir = $(BDIR)/dependencies/installed/lib -ilidir = $(BDIR)/dependencies/installed/lib/built -ipydir = $(BDIR)/dependencies/installed/lib/built/python +lockdir = $(BDIR)/locks +ddir = $(BDIR)/dependencies +tdir = $(BDIR)/dependencies/tarballs +idir = $(BDIR)/dependencies/installed +ibdir = $(BDIR)/dependencies/installed/bin +ildir = $(BDIR)/dependencies/installed/lib +ibidir = $(BDIR)/dependencies/installed/version-info/bin +ilidir = $(BDIR)/dependencies/installed/version-info/lib +itidir = $(BDIR)/dependencies/installed/version-info/tex +ipydir = $(BDIR)/dependencies/installed/version-info/python # Define the top-level programs to build (installed in `.local/bin'). # @@ -51,12 +53,12 @@ ipydir = $(BDIR)/dependencies/installed/lib/built/python # successfully on Mac (only static) and GNU/Linux (shared and static). But, # since it takes a few hours to build, it is not currently a target. top-level-libraries = # atlas -top-level-programs = astnoisechisel flock metastore unzip zip +top-level-programs = astnoisechisel metastore unzip zip top-level-python = astroquery matplotlib -all: $(ddir)/texlive-versions.tex \ - $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ - $(foreach p, $(top-level-programs), $(ibdir)/$(p)) \ - $(foreach p, $(top-level-python), $(ipydir)/$(p)) +all: $(foreach p, $(top-level-libraries), $(ilidir)/$(p)) \ + $(foreach p, $(top-level-programs), $(ibidir)/$(p)) \ + $(foreach p, $(top-level-python), $(ipydir)/$(p)) \ + $(itidir)/texlive # Other basic environment settings: We are only including the host # operating system's PATH environment variable (after our own!) for the @@ -85,6 +87,15 @@ export LD_LIBRARY_PATH := $(ildir):$(il64dir) export LDFLAGS := $(rpath_command) -L$(ildir) +# We want the download to happen on a single thread. So we need to define a +# lock, and call a special script we have written for this job. These are +# placed here because we want them both in the `dependencies.mk' and +# `dependencies-python.mk'. +$(lockdir): | $(BDIR); mkdir $@ +downloader="wget --no-use-server-timestamps -O"; +downloadwrapper = ./reproduce/src/bash/download-multi-try + + # Python packages include reproduce/src/make/dependencies-python.mk @@ -125,7 +136,7 @@ tarballs = $(foreach t, cfitsio-$(cfitsio-version).tar.gz \ wcslib-$(wcslib-version).tar.bz2 \ zip-$(zip-version).tar.gz \ , $(tdir)/$(t) ) -$(tarballs): $(tdir)/%: +$(tarballs): $(tdir)/%: | $(lockdir) if [ -f $(DEPENDENCIES-DIR)/$* ]; then cp $(DEPENDENCIES-DIR)/$* $@ else @@ -203,14 +214,10 @@ $(tarballs): $(tdir)/%: else tarballurl=$$w fi - # If the download fails, Wget will write the error message in the - # target file, so Make will think that its done! To avoid this - # problem, we'll rename the output. - echo "Downloading $$tarballurl" - if ! wget --no-use-server-timestamps -O$@ $$tarballurl; then - rm -f $@ - echo; echo "DOWNLOAD FAILED: $$tarballurl"; echo; exit 1 - fi + # Download using the script specially defined for this job. + touch $(lockdir)/download + $(downloadwrapper) "$$downloader" $(lockdir)/download \ + $$tarballurl $@ fi @@ -232,7 +239,7 @@ $(tarballs): $(tdir)/%: # simple plain text file in it with the basic library name (an no prefix) # and create/write into it when the library is successfully built. $(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ - $(ibdir)/curl + $(ibidir)/curl # CFITSIO hard-codes the absolute address of cURL's `curl-config' # program (which gives the necessary header and linking @@ -252,22 +259,22 @@ $(ilidir)/cfitsio: $(tdir)/cfitsio-$(cfitsio-version).tar.gz \ $(call gbuild, $$customtar, cfitsio, static, \ --enable-sse2 --enable-reentrant) \ && rm $$customtar \ - && echo "CFITSIO is built" > $@ + && echo "CFITSIO $(cfitsio-version)" > $@ $(ilidir)/gsl: $(tdir)/gsl-$(gsl-version).tar.gz $(call gbuild, $<, gsl-$(gsl-version), static) \ - && echo "GNU Scientific Library is built" > $@ + && echo "GNU Scientific Library $(gsl-version)" > $@ $(ilidir)/fftw: $(tdir)/fftw-$(fftw-version).tar.gz $(call gbuild, $<, fftw-$(fftw-version), static, \ --enable-shared) \ - && echo "FFTW is built" > $@ + && echo "FFTW $(fftw-version)" > $@ # Freetype is necessary to install matplotlib $(ilidir)/freetype: $(tdir)/freetype-$(freetype-version).tar.gz \ $(ilidir)/libpng $(call gbuild, $<, freetype-$(freetype-version), static) \ - && echo "freetype is built" > $@ + && echo "FreeType $(freetype-version)" > $@ $(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ $(ilidir)/openmpi @@ -276,28 +283,29 @@ $(ilidir)/hdf5: $(tdir)/hdf5-$(hdf5-version).tar.gz \ $(call gbuild, $<, hdf5-$(hdf5-version), static, \ --enable-parallel \ --enable-fortran, V=1) \ - && echo "HDF5 library is built" > $@ + && echo "HDF5 library $(hdf5-version)" > $@ $(ilidir)/libbsd: $(tdir)/libbsd-$(libbsd-version).tar.xz $(call gbuild, $<, libbsd-$(libbsd-version), static,,V=1) \ - && echo "libbsd is built" > $@ + && echo "Libbsd $(libbsd-version)" > $@ $(ilidir)/libjpeg: $(tdir)/jpegsrc.$(libjpeg-version).tar.gz - $(call gbuild, $<, jpeg-9b, static) && echo "Libjpeg is built" > $@ + $(call gbuild, $<, jpeg-9b, static) \ + && echo "Libjpeg $(libjpeg-version)" > $@ $(ilidir)/libpng: $(tdir)/libpng-$(libpng-version).tar.xz $(call gbuild, $<, libpng-$(libpng-version), static) \ - && echo "Libpng is built" > $@ + && echo "Libpng $(libpng-version)" > $@ $(ilidir)/libtiff: $(tdir)/tiff-$(libtiff-version).tar.gz \ $(ilidir)/libjpeg $(call gbuild, $<, tiff-$(libtiff-version), static, \ --disable-webp --disable-zstd) \ - && echo "Libtiff is built" > $@ + && echo "Libtiff $(libtiff-version)" > $@ $(ilidir)/openmpi: $(tdir)/openmpi-$(openmpi-version).tar.gz $(call gbuild, $<, openmpi-$(openmpi-version), static, , V=1) \ - && echo "OpenMPI is built" > $@ + && echo "OpenMPI $(openmpi-version)" > $@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ $(tdir)/lapack-$(lapack-version).tar.gz @@ -375,7 +383,9 @@ $(ilidir)/atlas: $(tdir)/atlas-$(atlas-version).tar.bz2 \ # We'll check the full installation with the static library (not # currently building shared library on Mac. - if [ -f $(ildir)/libatlas.a ]; then echo "Atlas is built" > $@; fi + if [ -f $(ildir)/libatlas.a ]; then \ + echo "ATLAS $(atlas-version)" > $@; \ + fi $(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz if [ x$(on_mac_os) = xyes ]; then \ @@ -388,7 +398,7 @@ $(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz && make PREFIX=$(idir) install \ && cd .. \ && rm -rf OpenBLAS-$(openblas-version) \ - && echo "Libtiff is built" > $@ + && echo "OpenBLAS $(openblas-version)" > $@ @@ -411,8 +421,8 @@ $(ilidir)/openblas: $(tdir)/openblas-$(openblas-version).tar.gz # problem we have been having so far with Mac systems: # https://libgit2.org/docs/guides/build-and-link $(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \ - $(ibdir)/cmake \ - $(ibdir)/curl + $(ibidir)/cmake \ + $(ibidir)/curl # Build and install the library. $(call cbuild, $<, libgit2-$(libgit2-version), static, \ -DUSE_SSH=OFF -DBUILD_CLAR=OFF \ @@ -425,7 +435,7 @@ $(ilidir)/libgit2: $(tdir)/libgit2-$(libgit2-version).tar.gz \ fi # Write the target file. - echo "Libgit2 is built" > $@ + echo "Libgit2 $(libgit2-version)" > $@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ $(ilidir)/cfitsio @@ -443,7 +453,7 @@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ fi # Write the target file. - echo "WCSLIB is built" > $@ + echo "WCSLIB $(wcslib-version)" > $@ @@ -453,8 +463,8 @@ $(ilidir)/wcslib: $(tdir)/wcslib-$(wcslib-version).tar.bz2 \ # -------- # # CMake can be built with its custom `./bootstrap' script. -$(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ - $(ibdir)/curl +$(ibidir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ + $(ibidir)/curl # After searching in `bootstrap', I couldn't find `LIBS', only # `LDFLAGS'. So the extra libraries are being added to `LDFLAGS', # not `LIBS'. @@ -465,13 +475,16 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ export CC=clang; \ export CXX=clang++; \ fi; \ - cd $(ddir) && rm -rf cmake-$(cmake-version) && \ - tar xf $< && cd cmake-$(cmake-version) && \ - ./bootstrap --prefix=$(idir) --system-curl --system-zlib \ - --system-bzip2 --system-liblzma --no-qt-gui && \ - make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 && \ - make install && \ - cd ..&& rm -rf cmake-$(cmake-version) + cd $(ddir) \ + && rm -rf cmake-$(cmake-version) \ + && tar xf $< && cd cmake-$(cmake-version) \ + && ./bootstrap --prefix=$(idir) --system-curl --system-zlib\ + --system-bzip2 --system-liblzma --no-qt-gui \ + && make LIBS="$$LIBS -lssl -lcrypto -lz" VERBOSE=1 \ + && make install \ + && cd .. \ + && rm -rf cmake-$(cmake-version) \ + && echo "CMake $(cmake-version)" > $@ # cURL (and its library, which is needed by several programs here) can # optionally link with many different network-related libraries on the host @@ -481,7 +494,7 @@ $(ibdir)/cmake: $(tdir)/cmake-$(cmake-version).tar.gz \ # that if it does link with them, the pipeline will crash when the library # is updated/changed by the host, and the whole purpose of this pipeline is # avoid dependency on the host as much as possible. -$(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz +$(ibidir)/curl: $(tdir)/curl-$(curl-version).tar.gz $(call gbuild, $<, curl-$(curl-version), , \ LIBS="-pthread" \ --with-zlib=$(ildir) \ @@ -498,22 +511,26 @@ $(ibdir)/curl: $(tdir)/curl-$(curl-version).tar.gz --without-axtls \ --disable-ldaps \ --disable-ldap \ - --without-nss, V=1) + --without-nss, V=1) \ + && echo "cURL $(curl-version)" > $@ # On Mac OS, libtool does different things, so to avoid confusion, we'll # prefix GNU's libtool executables with `glibtool'. -$(ibdir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz +$(ibidir)/glibtool: $(tdir)/libtool-$(libtool-version).tar.xz $(call gbuild, $<, libtool-$(libtool-version), static, \ - --program-prefix=g) + --program-prefix=g) \ + && echo "GNU Libtool $(libtool-version)" > $@ -$(ibdir)/gs: $(tdir)/ghostscript-$(ghostscript-version).tar.gz - $(call gbuild, $<, ghostscript-$(ghostscript-version)) +$(ibidir)/ghostscript: $(tdir)/ghostscript-$(ghostscript-version).tar.gz + $(call gbuild, $<, ghostscript-$(ghostscript-version)) \ + && echo "GPL Ghostscript $(ghostscript-version)" > $@ -$(ibdir)/git: $(tdir)/git-$(git-version).tar.xz \ - $(ibdir)/curl +$(ibidir)/git: $(tdir)/git-$(git-version).tar.xz \ + $(ibidir)/curl $(call gbuild, $<, git-$(git-version), static, \ --without-tcltk --with-shell=$(ibdir)/bash, \ - V=1) + V=1) \ + && echo "Git $(git-version)" > $@ # Metastore is used (through a Git hook) to restore the source modification # dates of files after a Git checkout. Another Git hook saves all file @@ -533,9 +550,9 @@ needlibbsd = else needlibbsd = $(ilidir)/libbsd endif -$(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ - $(needlibbsd) \ - $(ibdir)/git +$(ibidir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ + $(needlibbsd) \ + $(ibidir)/git # The build command below will change the current directory of this # build, so we'll fix its value here. @@ -559,7 +576,7 @@ $(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ user=$$(whoami) group=$$(groups | awk '{print $$1}') cd $$current_dir - if [ -f $@ ]; then + if [ -f $(ibdir)/metastore ]; then for f in pre-commit post-checkout; do sed -e's|@USER[@]|'$$user'|g' \ -e's|@GROUP[@]|'$$group'|g' \ @@ -567,6 +584,7 @@ $(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ -e's|@TOP_PROJECT_DIR[@]|'$$current_dir'|g' \ reproduce/src/bash/git-$$f > .git/hooks/$$f chmod +x .git/hooks/$$f + echo "Metastore (forked) $(metastore-version)" > $@ done else echo; echo; echo; @@ -584,36 +602,39 @@ $(ibdir)/metastore: $(tdir)/metastore-$(metastore-version).tar.gz \ # build. Also, Ghostscript and GSL are relatively large packages. So when # building in parallel, its better to have these packages start building # early. -$(ibdir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \ - $(ilidir)/libgit2 \ - $(ibdir)/gs \ - $(ilidir)/gsl \ - $(ibdir)/glibtool \ - $(ilidir)/libjpeg \ - $(ilidir)/libtiff \ - $(ilidir)/wcslib +$(ibidir)/astnoisechisel: $(tdir)/gnuastro-$(gnuastro-version).tar.lz \ + $(ilidir)/gsl \ + $(ilidir)/wcslib \ + $(ilidir)/libjpeg \ + $(ilidir)/libtiff \ + $(ilidir)/libgit2 \ + $(ibidir)/glibtool \ + $(ibidir)/ghostscript ifeq ($(static_build),yes) staticopts="--enable-static=yes --enable-shared=no"; endif $(call gbuild, $<, gnuastro-$(gnuastro-version), static, \ $$staticopts, -j$(numthreads), \ - make check -j$(numthreads)) + make check -j$(numthreads)) \ + && echo "GNU Astronomy Utilities $(gnuastro-version)" > $@ -$(ibdir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz +$(ibidir)/unzip: $(tdir)/unzip-$(unzip-version).tar.gz v=$$(echo $(unzip-version) | sed -e's/\.//') - $(call gbuild, $<, unzip$$v, static,, \ - -f unix/Makefile generic_gcc \ - CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ - -f unix/Makefile \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) - -$(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz + $(call gbuild, $<, unzip$$v, static,, \ + -f unix/Makefile generic_gcc \ + CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ + -f unix/Makefile \ + BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ + && echo "Unzip $(unzip-version)" > $@ + +$(ibidir)/zip: $(tdir)/zip-$(zip-version).tar.gz v=$$(echo $(zip-version) | sed -e's/\.//') - $(call gbuild, $<, zip$$v, static,, \ - -f unix/Makefile generic_gcc \ - CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ - -f unix/Makefile \ - BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) + $(call gbuild, $<, zip$$v, static,, \ + -f unix/Makefile generic_gcc \ + CFLAGS="-DBIG_MEM -DMMAP",,pwd, \ + -f unix/Makefile \ + BINDIR=$(ibdir) MANDIR=$(idir)/man/man1 ) \ + && echo "Zip $(zip-version)" > $@ @@ -626,8 +647,8 @@ $(ibdir)/zip: $(tdir)/zip-$(zip-version).tar.gz # of the final PDF). So we'll make a simple ASCII file called # `texlive-ready-tlmgr' and use its contents to mark if we can use it or # not. -$(ibdir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \ - reproduce/config/pipeline/texlive.conf +$(itidir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \ + reproduce/config/pipeline/texlive.conf # Unpack, enter the directory, and install based on the given # configuration (prerequisite of this rule). @@ -668,11 +689,11 @@ $(ibdir)/texlive-ready-tlmgr: $(tdir)/install-tl-unx.tar.gz \ # To keep things modular and simple, we'll break up the installation of TeX # Live itself (only very basic TeX and LaTeX) and the installation of its # necessary packages into two packages. -$(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \ - $(ibdir)/texlive-ready-tlmgr +$(itidir)/texlive: reproduce/config/pipeline/dependency-texlive.mk \ + $(itidir)/texlive-ready-tlmgr # To work with TeX live installation, we'll need the internet. - @res=$$(cat $(ibdir)/texlive-ready-tlmgr) + @res=$$(cat $(itidir)/texlive-ready-tlmgr) if [ x"$$res" = x"NOT!" ]; then echo "" > $@ else @@ -688,7 +709,6 @@ $(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \ # # We are putting a notice, because if there is no internet, # `tlmgr' just hangs waiting. - echo; echo; echo "Downloading necessary TeX packages..."; echo; tlmgr install $(texlive-packages) # Make a symbolic link of all the TeX Live executables in the bin @@ -698,15 +718,13 @@ $(ddir)/texlive-versions.tex: reproduce/config/pipeline/dependency-texlive.mk \ # Get all the necessary versions. texlive=$$(pdflatex --version | awk 'NR==1' | sed 's/.*(\(.*\))/\1/' \ | awk '{print $$NF}'); - echo "\newcommand{\\texliveversion}{$$texlive}" > $@ - # LaTeX Package versions. + # Package names and versions. tlmgr info $(texlive-packages) --only-installed | awk \ '$$1=="package:" {version=0; \ if($$NF=="tex-gyre") name="texgyre"; \ else name=$$NF} \ $$1=="cat-version:" {version=$$NF} \ $$1=="cat-date:" {if(version==0) version=$$2; \ - printf("\\newcommand{\\tex%sversion}{%s}\n", \ - name, version)}' >> $@ + printf("%s %s\n", name, version)}' >> $@ fi diff --git a/reproduce/src/make/initialize.mk b/reproduce/src/make/initialize.mk index 66e7c77..3c272d8 100644 --- a/reproduce/src/make/initialize.mk +++ b/reproduce/src/make/initialize.mk @@ -160,8 +160,8 @@ export MPI_PYTHON3_SITEARCH := # option: they add too many extra checks that make it hard to find what you # are looking for in this pipeline. .SUFFIXES: -$(texdir) $(lockdir): | $(BDIR); mkdir $@ -$(mtexdir) $(texbdir): | $(texdir); mkdir $@ +$(lockdir): | $(BDIR); mkdir $@ +$(texbdir): | $(texdir); mkdir $@ $(tikzdir): | $(texbdir); mkdir $@ && ln -s $(tikzdir) tex/tikz @@ -347,97 +347,4 @@ $(mtexdir)/initialize.tex: | $(mtexdir) # Version of the pipeline and build directory (for LaTeX inputs). @v=$$(git describe --dirty --always); - echo "\newcommand{\pipelineversion}{$$v}" > $@ - - # Versions of programs (same order as 'dependency-versions.mk'), - # ordered alphabetically (by their executable name). - echo "\newcommand{\\bashversion}{$(bash-version)}" >> $@ -# echo "\newcommand{\\bashversion}{$(binutils-version)}" >> $@ - echo "\newcommand{\\bziptwoversion}{$(bzip2-version)}" >> $@ - echo "\newcommand{\\cmakeversion}{$(cmake-version)}" >> $@ - echo "\newcommand{\\coreutilsversion}{$(coreutils-version)}" >> $@ - echo "\newcommand{\\diffutilsversion}{$(diffutils-version)}" >> $@ - echo "\newcommand{\\findutilsversion}{$(findutils-version)}" >> $@ - echo "\newcommand{\\flockversion}{$(flock-version)}" >> $@ - echo "\newcommand{\\freetypeversion}{$(freetype-version)}" >> $@ - echo "\newcommand{\\gawkversion}{$(gawk-version)}" >> $@ - echo "\newcommand{\\gccversion}{$(gcc-version)}" >> $@ - echo "\newcommand{\\ghostscriptversion}{$(ghostscript-version)}" >> $@ - echo "\newcommand{\\gitversion}{$(git-version)}" >> $@ - echo "\newcommand{\\gmpversion}{$(gmp-version)}" >> $@ - echo "\newcommand{\\gnuastroversion}{$(gnuastro-version)}" >> $@ - echo "\newcommand{\\grepversion}{$(grep-version)}" >> $@ - echo "\newcommand{\\gzipversion}{$(gzip-version)}" >> $@ - echo "\newcommand{\\islversion}{$(isl-version)}" >> $@ - echo "\newcommand{\\lzipversion}{$(lzip-version)}" >> $@ - echo "\newcommand{\\makeversion}{$(make-version)}" >> $@ - echo "\newcommand{\\metastoreversion}{$(metastore-version)}" >> $@ - echo "\newcommand{\\mpfrversion}{$(mpfr-version)}" >> $@ - echo "\newcommand{\\mpcversion}{$(mpc-version)}" >> $@ - echo "\newcommand{\\ncursesversion}{$(ncurses-version)}" >> $@ - echo "\newcommand{\\opensslversion}{$(openssl-version)}" >> $@ - echo "\newcommand{\\patchelfversion}{$(patchelf-version)}" >> $@ - echo "\newcommand{\\pkgconfigversion}{$(pkgconfig-version)}" >> $@ - echo "\newcommand{\\pythonversion}{$(python-version)}" >> $@ - echo "\newcommand{\\readlineversion}{$(readline-version)}" >> $@ - echo "\newcommand{\\sedversion}{$(sed-version)}" >> $@ - echo "\newcommand{\\swarpversion}{$(swarp-version)}" >> $@ - echo "\newcommand{\\tarversion}{$(tar-version)}" >> $@ - echo "\newcommand{\\unzipversion}{$(unzip-version)}" >> $@ - echo "\newcommand{\\wgetversion}{$(wget-version)}" >> $@ - echo "\newcommand{\\whichversion}{$(which-version)}" >> $@ - echo "\newcommand{\\xzversion}{$(xz-version)}" >> $@ - echo "\newcommand{\\zipversion}{$(zip-version)}" >> $@ - - # Libraries. - echo "\newcommand{\\atlasversion}{$(cfitsio-version)}" >> $@ - echo "\newcommand{\\cfitsioversion}{$(cfitsio-version)}" >> $@ - echo "\newcommand{\\curlversion}{$(curl-version)}" >> $@ - echo "\newcommand{\\gslversion}{$(gsl-version)}" >> $@ - echo "\newcommand{\\lapack}{$(lapack-version)}" >> $@ - echo "\newcommand{\\libbsdversion}{$(libbsd-version)}" >> $@ - echo "\newcommand{\\libffiversion}{$(libffi-version)}" >> $@ - echo "\newcommand{\\libgittwoversion}{$(libgit2-version)}" >> $@ - echo "\newcommand{\\libjpegversion}{$(libjpeg-version)}" >> $@ - echo "\newcommand{\\libpngversion}{$(libpng-version)}" >> $@ - echo "\newcommand{\\libtiffversion}{$(libtiff-version)}" >> $@ - echo "\newcommand{\\libtoolversion}{$(libtool-version)}" >> $@ - echo "\newcommand{\\wcslibversion}{$(wcslib-version)}" >> $@ - echo "\newcommand{\\zlibversion}{$(zlib-version)}" >> $@ - - # Python modules. - echo "\newcommand{\\asncryptoversion}{$(asn1crypto-version)}" >> $@ - echo "\newcommand{\\astroqueryversion}{$(astroquery-version)}" >> $@ - echo "\newcommand{\\astropyversion}{$(astropy-version)}" >> $@ - echo "\newcommand{\\beautifulsoupversion}{$(beautifulsoup4-version)}" >> $@ - echo "\newcommand{\\certifiversion}{$(certifi-version)}" >> $@ - echo "\newcommand{\\cffiversion}{$(cffi-version)}" >> $@ - echo "\newcommand{\\chardetversion}{$(chardet-version)}" >> $@ - echo "\newcommand{\\cryptographyversion}{$(cryptography-version)}" >> $@ - echo "\newcommand{\\cyclerversion}{$(cycler-version)}" >> $@ - echo "\newcommand{\\entrypointsversion}{$(entrypoints-version)}" >> $@ - echo "\newcommand{\\hpyversion}{$(h5py-version)}" >> $@ - echo "\newcommand{\\htmlfivelibversion}{$(html5lib-version)}" >> $@ - echo "\newcommand{\\idaversion}{$(idna-version)}" >> $@ - echo "\newcommand{\\jeepneyversion}{$(jeepney-version)}" >> $@ - echo "\newcommand{\\kiwisolverversion}{$(kiwisolver-version)}" >> $@ - echo "\newcommand{\\keyringversion}{$(keyring-version)}" >> $@ - echo "\newcommand{\\matplotlibversion}{$(matplotlib-version)}" >> $@ - echo "\newcommand{\\numpyversion}{$(numpy-version)}" >> $@ -# echo "\newcommand{\\pipversion}{$(pip-version)}" >> $@ - echo "\newcommand{\\pycparserversion}{$(pycparser-version)}" >> $@ - echo "\newcommand{\\pyparsingversion}{$(pyparsing-version)}" >> $@ - echo "\newcommand{\\pythondateutilversion}{$(python-dateutil-version)}" >> $@ - echo "\newcommand{\\requestsversion}{$(requests-version)}" >> $@ - echo "\newcommand{\\scipyversion}{$(scipy-version)}" >> $@ - echo "\newcommand{\\screenstorageversion}{$(secretstorage-version)}" >> $@ - echo "\newcommand{\\setuptoolsversion}{$(setuptools-version)}" >> $@ - echo "\newcommand{\\setuptoolsscmversion}{$(setuptools_scm-version)}" >> $@ - echo "\newcommand{\\sixversion}{$(six-version)}" >> $@ - echo "\newcommand{\\soupsieveversion}{$(soupsieve-version)}" >> $@ - echo "\newcommand{\\urllibthreeversion}{$(urllib3-version)}" >> $@ -# echo "\newcommand{\\virtualenvversion}{$(virtualenv-version)}" >> $@ - echo "\newcommand{\\webencodingsversion}{$(webencodings-version)}" >> $@ - - # TeX package versions - cat $(BDIR)/dependencies/texlive-versions.tex >> $@ + echo "\newcommand{\pipelineversion}{$$v}" > $@ |