replacement_specs now can replace requires / build_requires / track_env

alibuild_helpers/utilities.py

@@ -630,7 +630,30 @@ def getPackageList(packages, specs, configDir, preferSystem, noSystem,
             # Allows generalising the version based on the actual key provided
             spec["version"] = spec["version"].replace("%(key)s", key)
             # We need the key to inject the version into the replacement recipe later.
-            spec["key"] = key 
+            spec["key"] = key
+            # The check can also print, possibly multiple times each,
+            # alibuild_system_replace_requires/_build_requires/_track_env to
+            # append to the replacement spec. This lets a single replacement be
+            # parametrised by what the check actually found on the host. Do this
+            # before rendering fullRecipe below, so the injected values are part
+            # of the replacement's hash.
+            for what in ("requires", "build_requires"):
+              extras = [dep.strip() for dep in
+                        re.findall(r"^alibuild_system_replace_%s:(?P<dep>.*)$" % what,
+                                   output, re.MULTILINE)]
+              if extras:
+                deps = list(spec.get(what) or [])
+                deps += [dep for dep in extras if dep and dep not in deps]
+                spec[what] = deps
+            for extra in re.findall(r"^alibuild_system_replace_track_env:(?P<var>.*)$",
+                                    output, re.MULTILINE):
+              name, sep, value = extra.strip().partition("=")
+              dieOnError(not sep or not name.strip(),
+                         "Malformed alibuild_system_replace_track_env for {}: {} "
+                         "(expected <NAME>=<value>)".format(spec["package"], extra.strip()))
+              # Unlike the recipe's own track_env, the value is not shell code to
+              # be run: the check has already computed it for us.
+              spec.setdefault("track_env", OrderedDict())[name.strip()] = value
             recipe = replacement.get("recipe", "")
             # If there's an explicitly-specified recipe, we're still building
             # the package. If not, aliBuild will still "build" it, but it's

docs/docs/reference.md

@@ -178,7 +178,12 @@ The following entries are optional in the header:
 
     If the check exits successfully, it can also print
     `alibuild_system_replace: <key>` to request a replacement spec (see
-    `prefer_system_replacement_specs`).
+    `prefer_system_replacement_specs`), and, together with it,
+    `alibuild_system_replace_requires: <package>`,
+    `alibuild_system_replace_build_requires: <package>` and
+    `alibuild_system_replace_track_env: <NAME>=<value>` to extend that
+    replacement spec with what the check actually found on the host. Each of
+    these can be printed several times, to append more than one entry.
 
   - `prefer_system`: a regular expression for architectures which should
     use the `prefer_system_check` by default to determine if the system version
@@ -211,12 +216,33 @@ The following entries are optional in the header:
     `%(key)s` can be used in the replacement `version` and will be replaced by
     the matched `<key>`.
 
+    Besides the key itself, the check can print the following lines, each of
+    them as many times as needed. They only take effect when a replacement spec
+    is actually selected, and are applied before the package's hash is computed:
+
+      - `alibuild_system_replace_requires: <package>`: append `<package>` to the
+        replacement's `requires`;
+      - `alibuild_system_replace_build_requires: <package>`: append `<package>`
+        to the replacement's `build_requires`;
+      - `alibuild_system_replace_track_env: <NAME>=<value>`: add `<NAME>` to the
+        replacement's `track_env`. Unlike in a recipe, `<value>` is not shell
+        code to be run, but the value itself, as computed by the check. This is
+        the only way to track a variable for a replacement: a `track_env` block
+        written inside a `prefer_system_replacement_specs` entry is *not*
+        evaluated, since replacements are selected after the recipe's own
+        `track_env` has been resolved.
+
+    Entries already present in the replacement's `requires` / `build_requires`
+    are not duplicated.
+
     Example:
 
     ```yaml
     prefer_system_check: |
       python3 -m pip --help >/dev/null || exit 1
       echo 'alibuild_system_replace: python-brew3.12'
+      echo "alibuild_system_replace_requires: OpenSSL"
+      echo "alibuild_system_replace_track_env: PYTHON_EXECUTABLE=$(command -v python3)"
       exit 0
     prefer_system_replacement_specs:
       "python-brew3.*":
@@ -225,6 +251,11 @@ The following entries are optional in the header:
           PYTHON_ROOT: $(python3 -c 'import sysconfig; print(sysconfig.get_config_var("exec_prefix"))')
     ```
 
+    The above ends up with `requires: [OpenSSL]` and
+    `track_env: {PYTHON_EXECUTABLE: /opt/homebrew/bin/python3}` in the
+    replacement spec, so that the package is rebuilt if the system Python
+    moves.
+
   - `relocate_paths`: a list of toplevel paths scanned recursively to perform
     relocation of executables and dynamic libraries **on macOS only**. If not
     specified, defaults to `bin`, `lib` and `lib64`.

tests/test_packagelist.py

@@ -1,3 +1,4 @@
+from collections import OrderedDict
 from textwrap import dedent
 import unittest
 from unittest import mock
@@ -46,6 +47,43 @@
             recipe: 'true'
     ---
     """),
+    "CONFIG_DIR/with-replacement-extras.sh": dedent("""\
+    package: with-replacement-extras
+    version: v1
+    prefer_system: '.*'
+    prefer_system_check: |
+        echo 'alibuild_system_replace: replacement'
+        echo 'alibuild_system_replace_requires: extra-dep'
+        echo 'alibuild_system_replace_requires: other-dep'
+        echo 'alibuild_system_replace_build_requires: extra-build-dep'
+        echo 'alibuild_system_replace_track_env: SENTINEL_ONE=magic one'
+        echo 'alibuild_system_replace_track_env: SENTINEL_TWO=magic two'
+    prefer_system_replacement_specs:
+        replacement:
+            requires:
+                - preexisting-dep
+    ---
+    """),
+    "CONFIG_DIR/extra-dep.sh": dedent("""\
+    package: extra-dep
+    version: v1
+    ---
+    """),
+    "CONFIG_DIR/other-dep.sh": dedent("""\
+    package: other-dep
+    version: v1
+    ---
+    """),
+    "CONFIG_DIR/extra-build-dep.sh": dedent("""\
+    package: extra-build-dep
+    version: v1
+    ---
+    """),
+    "CONFIG_DIR/preexisting-dep.sh": dedent("""\
+    package: preexisting-dep
+    version: v1
+    ---
+    """),
     "CONFIG_DIR/missing-spec.sh": dedent("""\
     package: missing-spec
     version: v1
@@ -204,6 +242,37 @@ def fake_exists(n):
             self.assertNotIn("with-replacement-recipe", systemPkgs)
             self.assertIn("with-replacement-recipe", ownPkgs)
 
+    def test_replacement_extras_given(self) -> None:
+        """Check that the check script can append to the replacement spec.
+
+        alibuild_system_replace_{requires,build_requires,track_env} can each be
+        printed several times, and are appended to what the replacement spec
+        already declares.
+        """
+        def fake_exists(n):
+            return n in RECIPES.keys()
+        with patch.object(os.path, "exists", fake_exists):
+            specs, systemPkgs, ownPkgs, failedReqs, validDefaults, systemSpecs = \
+                getPackageListWithDefaults(["with-replacement-extras"])
+            spec = specs["with-replacement-extras"]
+            # runtime_requires keeps what the replacement spec declared, plus the
+            # injected ones; requires is runtime_requires + build_requires.
+            self.assertEqual(spec["runtime_requires"],
+                             ["preexisting-dep", "extra-dep", "other-dep"])
+            self.assertEqual(spec["build_requires"],
+                             ["extra-build-dep", "defaults-release"])
+            self.assertIn("extra-dep", spec["requires"])
+            self.assertIn("extra-build-dep", spec["requires"])
+            # The injected dependencies must actually be resolved.
+            for dep in ("preexisting-dep", "extra-dep", "other-dep", "extra-build-dep"):
+                self.assertIn(dep, specs)
+            # Values are taken verbatim from the check output, not run as code.
+            # build.py asserts the type, so it must be an OrderedDict.
+            self.assertIsInstance(spec["track_env"], OrderedDict)
+            self.assertEqual(list(spec["track_env"].items()),
+                             [("SENTINEL_ONE", "magic one"),
+                              ("SENTINEL_TWO", "magic two")])
+
     @mock.patch("alibuild_helpers.utilities.warning")
     def test_missing_replacement_spec(self, mock_warning) -> None:
         """Check a warning is displayed when the replacement spec is not found."""