Typed criterion verdicts on grading results

docs/v6/reference/graders.mdx

@@ -9,7 +9,7 @@ Graders turn an agent's answer into a reward. HUD ships reusable ones so you don
 ```python
 from hud.graders import (
     BashGrader, LLMJudgeGrader, Grader,
-    SubScore, EvaluationResult,
+    SubScore, EvaluationResult, CriterionResult,
     combine, combine_any, combine_all,
     exact_match, contains, contains_any, contains_all,
     numeric_match, f1_score, normalize,
@@ -96,6 +96,9 @@ result = await LLMJudgeGrader.grade(
 `criteria` items are strings, or `(requirement, weight)` tuples. A negative weight marks an error to
 avoid: the criterion scores as a penalty, MET when the answer actually makes the mistake.
 
+Each judgment is reported as a [`CriterionResult`](#criterionresult) on the returned
+`SubScore.criteria`, so the per-criterion verdicts and reasons survive into the trace.
+
 ## `combine` - compose multiple graders
 
 `combine` resolves `SubScore`s and grader coroutines in parallel and combines them into a weighted `EvaluationResult`. Positive weights are normalized to sum to `1.0`; negative weights are penalties.
@@ -117,11 +120,11 @@ async def composed(answer: str = ""):
 | `combine_any(weight, subscores)` | Boolean OR: a `SubScore` that passes if any input passes (max). |
 | `combine_all(weight, subscores)` | Boolean AND: a `SubScore` that passes only if all inputs pass (min). |
 
-The subscores appear in the trace, so a partial reward is legible. `combine_any`/`combine_all` collapse alternatives into a single component you can feed to `combine` - e.g. "tests pass via `pytest` OR via `make test`" as one 0/1 subscore.
+The subscores appear in the trace, so a partial reward is legible. `combine_any`/`combine_all` fold alternatives into a single component you can feed to `combine` - e.g. "tests pass via `pytest` OR via `make test`" as one 0/1 subscore. The combined `SubScore` keeps its inputs on `children` and records the operator on `aggregation`, so the full grading tree - every grader run, its criteria, and how they were combined - survives into the trace. Combinators nest: a `combine_all` over `combine_any` nodes produces a deeper tree.
 
 ## Custom graders
 
-Subclass `Grader` and implement async `compute_score` (return a float, or `(float, metadata)`):
+Subclass `Grader` and implement async `compute_score`, returning a float or a `SubScore`:
 
 ```python
 class LengthGrader(Grader):
@@ -134,9 +137,23 @@ class LengthGrader(Grader):
 result = await LengthGrader.grade(weight=1.0, answer=answer, target=200)
 ```
 
+Return a `SubScore` to attach `criteria` (rubric verdicts) or `metadata` (diagnostics);
+`grade` stamps the caller's `name` and `weight` onto it and records the call parameters
+under the `_parameters` metadata key.
+
 ## `SubScore` and `EvaluationResult`
 
-A `SubScore` is one component of a grade: `name`, `value` (0-1), `weight` (default `1.0`; negative = penalty), optional `metadata`.
+A `SubScore` is one node in the grade breakdown tree:
+
+| Field | Default | Description |
+|-------|---------|-------------|
+| `name` | - | Name of this component. |
+| `value` | - | Measurement, 0-1. |
+| `weight` | `1.0` | Weight in the parent's combination; negative = penalty. |
+| `aggregation` | `None` | `"any"` / `"all"` when the value was derived from `children`. |
+| `children` | `None` | Input subscores this node was combined from. |
+| `criteria` | `None` | Itemized [`CriterionResult`](#criterionresult) verdicts. |
+| `metadata` | `None` | Grader-specific details (parameters, stdout, judge model, ...). |
 
 An `EvaluationResult` is the combined grade payload you can yield from a task:
 
@@ -151,5 +168,20 @@ An `EvaluationResult` is the combined grade payload you can yield from a task:
 
 `EvaluationResult.from_float(value)` wraps a bare reward.
 
+## `CriterionResult`
+
+Where a `SubScore` decomposes the reward numerically, criteria justify it: each
+`CriterionResult` is one checked requirement with a pass/fail verdict. Criteria
+ride on the `SubScore` of the grader that checked them: `LLMJudgeGrader` emits
+one per criterion on its returned subscore, and `combine` carries them through
+into `EvaluationResult.subscores`.
+
+| Field | Default | Description |
+|-------|---------|-------------|
+| `criterion` | - | The requirement that was checked. |
+| `passed` | - | Whether it was met. For negative-weight criteria, `True` means the error is present. |
+| `weight` | `1.0` | Rubric weight; negative marks an error to avoid. |
+| `reason` | `None` | The grader's justification for the verdict. |
+
 Graders run as the second yield of a [task](/v6/reference/tasks); for choosing a reward that separates
 good work from bad, see [designing tasks](/v6/reference/advice).

hud/graders/__init__.py

@@ -27,7 +27,7 @@
 from .bash import BashGrader
 from .combine import _combine_subscores, combine, combine_all, combine_any
 from .judge import LLMJudgeGrader
-from .results import EvaluationResult, SubScore
+from .results import CriterionResult, EvaluationResult, SubScore
 from .text import (
     contains,
     contains_all,
@@ -40,6 +40,7 @@
 
 __all__ = [
     "BashGrader",
+    "CriterionResult",
     "EvaluationResult",
     "Grader",
     "LLMJudgeGrader",

hud/graders/base.py

@@ -13,8 +13,8 @@ class Grader:
     """Async base class for reusable graders.
 
     Subclasses implement ``compute_score`` (async). The ``grade`` classmethod
-    calls it, wraps the result as a ``SubScore``, and records parameters
-    in metadata for reproducibility.
+    calls it, stamps the caller's ``name`` and ``weight`` onto the resulting
+    ``SubScore``, and records parameters in metadata for reproducibility.
     """
 
     name: str = "BaseGrader"
@@ -23,25 +23,23 @@ class Grader:
     async def grade(cls, weight: float, name: str | None = None, **kwargs: Any) -> SubScore:
         """Run the grader and package the result as a ``SubScore``."""
         result = await cls.compute_score(**kwargs)
-
-        if isinstance(result, tuple):
-            score, metadata = result
-        else:
-            score = result
-            metadata = {}
-
-        return SubScore(
-            name=name or cls.name,
-            weight=weight,
-            value=float(score),
-            metadata={**metadata, "_parameters": json_safe_dict(kwargs)},
+        if not isinstance(result, SubScore):
+            result = SubScore(name=cls.name, value=float(result))
+
+        return result.model_copy(
+            update={
+                "name": name or cls.name,
+                "weight": weight,
+                "metadata": {**(result.metadata or {}), "_parameters": json_safe_dict(kwargs)},
+            }
         )
 
     @classmethod
-    async def compute_score(cls, **kwargs: Any) -> float | tuple[float, dict[str, Any]]:
+    async def compute_score(cls, **kwargs: Any) -> float | SubScore:
         """Compute a score between ``0.0`` and ``1.0``.
 
-        Return a float, or ``(float, metadata_dict)`` to attach extra info.
+        Return a float, or a ``SubScore`` to attach criteria or metadata
+        (its ``name`` and ``weight`` are overwritten by ``grade``).
         """
         raise NotImplementedError("Subclasses must implement compute_score")
 

hud/graders/bash.py

@@ -9,6 +9,7 @@
 from hud.utils.process import create_process_group_exec
 
 from .base import Grader
+from .results import SubScore
 
 logger = logging.getLogger(__name__)
 
@@ -27,8 +28,8 @@ async def compute_score(
         cwd: str | None = None,
         timeout_seconds: int | None = None,
         **kwargs: Any,
-    ) -> tuple[float, dict[str, Any]]:
-        """Run ``command`` via ``bash -lc`` and return score + metadata."""
+    ) -> SubScore:
+        """Run ``command`` via ``bash -lc`` and score by exit code."""
         if timeout_seconds is None:
             timeout_seconds = cls.default_timeout
         del kwargs
@@ -49,9 +50,10 @@ async def compute_score(
             stderr = stderr_bytes.decode(errors="replace")
             returncode = proc.returncode if proc.returncode is not None else 1
         except TimeoutError:
-            return (
-                0.0,
-                {
+            return SubScore(
+                name=cls.name,
+                value=0.0,
+                metadata={
                     "exit_code": None,
                     "stdout": "",
                     "stderr": "",
@@ -60,18 +62,22 @@ async def compute_score(
                 },
             )
         except FileNotFoundError:
-            return (
-                0.0,
-                {
+            return SubScore(
+                name=cls.name,
+                value=0.0,
+                metadata={
                     "exit_code": None,
                     "stdout": "",
                     "stderr": "/bin/bash not found",
                     "timed_out": False,
                 },
             )
 
-        score = 1.0 if returncode == 0 else 0.0
-        return (score, {"exit_code": returncode, "stdout": stdout, "stderr": stderr})
+        return SubScore(
+            name=cls.name,
+            value=1.0 if returncode == 0 else 0.0,
+            metadata={"exit_code": returncode, "stdout": stdout, "stderr": stderr},
+        )
 
 
 __all__ = ["BashGrader"]

hud/graders/combine.py

@@ -4,7 +4,7 @@
 
 import asyncio
 import warnings
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Literal
 
 from .results import EvaluationResult, SubScore
 
@@ -71,28 +71,19 @@ def _combine_subscores(subscores: list[SubScore]) -> EvaluationResult:
         )
 
     normalized_subscores: list[SubScore] = []
-    metadata: dict[str, Any] = {}
 
     for item, final_name in zip(subscores, _dedupe_subscore_names(subscores), strict=True):
         normalized_weight = item.weight / positive_weight_sum if item.weight > 0 else item.weight
         normalized_subscores.append(
-            SubScore(
-                name=final_name,
-                weight=normalized_weight,
-                value=item.value,
-                metadata=item.metadata,
-            )
+            item.model_copy(update={"name": final_name, "weight": normalized_weight})
         )
-        if item.metadata is not None:
-            metadata[final_name] = item.metadata
 
     reward = float(sum(item.value * item.weight for item in normalized_subscores))
 
     return EvaluationResult(
         reward=reward,
         done=True,
         subscores=normalized_subscores,
-        info=metadata,
     )
 
 
@@ -137,36 +128,37 @@ async def combine(*items: SubScore | Awaitable[SubScore]) -> EvaluationResult:
 
 
 def _boolean_subscore(
-    name: str, weight: float, subscores: list[SubScore], value: float
+    name: str,
+    weight: float,
+    aggregation: Literal["any", "all"],
+    subscores: list[SubScore],
+    value: float,
 ) -> SubScore:
-    unique_names = _dedupe_subscore_names(subscores)
+    children = [
+        subscore.model_copy(update={"name": unique_name})
+        for unique_name, subscore in zip(_dedupe_subscore_names(subscores), subscores, strict=True)
+    ]
     return SubScore(
         name=name,
         value=value,
         weight=weight,
-        metadata={
-            "subscores": unique_names,
-            "subscore_metadata": {
-                unique_name: subscore.metadata
-                for unique_name, subscore in zip(unique_names, subscores, strict=True)
-                if subscore.metadata is not None
-            },
-        },
+        aggregation=aggregation,
+        children=children,
     )
 
 
 def combine_any(weight: float, subscores: list[SubScore], *, name: str = "any") -> SubScore:
-    """Subscore that passes if any input passes (max)."""
+    """Subscore that passes if any input passes (max); inputs kept as children."""
     if not subscores:
         raise ValueError("subscores must not be empty")
-    return _boolean_subscore(name, weight, subscores, max(s.value for s in subscores))
+    return _boolean_subscore(name, weight, "any", subscores, max(s.value for s in subscores))
 
 
 def combine_all(weight: float, subscores: list[SubScore], *, name: str = "all") -> SubScore:
-    """Subscore that passes only if all inputs pass (min)."""
+    """Subscore that passes only if all inputs pass (min); inputs kept as children."""
     if not subscores:
         raise ValueError("subscores must not be empty")
-    return _boolean_subscore(name, weight, subscores, min(s.value for s in subscores))
+    return _boolean_subscore(name, weight, "all", subscores, min(s.value for s in subscores))
 
 
 __all__ = ["_combine_subscores", "combine", "combine_all", "combine_any"]

hud/graders/judge.py

@@ -15,6 +15,7 @@
 from typing import TYPE_CHECKING, Any, cast
 
 from .base import Grader
+from .results import CriterionResult, SubScore
 
 if TYPE_CHECKING:
     from openai import AsyncOpenAI
@@ -50,13 +51,6 @@ class _Criterion:
     weight: float
 
 
-@dataclass(slots=True)
-class _Verdict:
-    criterion: _Criterion
-    met: bool
-    reason: str
-
-
 class LLMJudgeGrader(Grader):
     """Grade an answer against weighted criteria using an LLM judge.
 
@@ -85,19 +79,19 @@ async def compute_score(
         question: str = "",
         model: str = "claude-haiku-4-5",
         **kwargs: Any,
-    ) -> tuple[float, dict[str, Any]]:
+    ) -> SubScore:
         """Evaluate ``answer`` against ``criteria`` via parallel LLM judgments."""
         del kwargs
         parsed = _parse_criteria(criteria)
         if not parsed:
-            return (0.0, {"error": "no criteria provided"})
+            return SubScore(name=cls.name, value=0.0, metadata={"error": "no criteria provided"})
 
         from hud.utils.gateway import build_gateway_client
 
         client = cast("AsyncOpenAI", build_gateway_client("openai"))
         answer_text = str(answer)
 
-        async def _judge(criterion: _Criterion) -> _Verdict:
+        async def _judge(criterion: _Criterion) -> CriterionResult:
             response = await client.chat.completions.create(
                 model=model,
                 max_tokens=1024,
@@ -107,19 +101,20 @@ async def _judge(criterion: _Criterion) -> _Verdict:
                 ],
             )
             met, reason = _parse_verdict(response.choices[0].message.content or "")
-            return _Verdict(criterion, met, reason)
+            return CriterionResult(
+                criterion=criterion.requirement,
+                passed=met,
+                weight=criterion.weight,
+                reason=reason,
+            )
 
         verdicts = list(await asyncio.gather(*(_judge(c) for c in parsed)))
-        score = _aggregate(verdicts)
-        report = {
-            v.criterion.requirement[:80]: {
-                "verdict": "MET" if v.met else "UNMET",
-                "reason": v.reason,
-                "weight": v.criterion.weight,
-            }
-            for v in verdicts
-        }
-        return (score, {"criteria": report, "model": model})
+        return SubScore(
+            name=cls.name,
+            value=_aggregate(verdicts),
+            criteria=verdicts,
+            metadata={"model": model},
+        )
 
 
 def _parse_criteria(criteria: list[str | tuple[str, float]] | None) -> list[_Criterion]:
@@ -160,11 +155,11 @@ def _parse_verdict(content: str) -> tuple[bool, str]:
     return ("UNMET" not in upper and "MET" in upper), text[:200]
 
 
-def _aggregate(verdicts: list[_Verdict]) -> float:
+def _aggregate(verdicts: list[CriterionResult]) -> float:
     """Weighted MET-sum normalized by positive (or all-negative) weight, clamped 0-1."""
-    total_positive = sum(max(0.0, v.criterion.weight) for v in verdicts)
-    total_negative = sum(abs(v.criterion.weight) for v in verdicts if v.criterion.weight < 0)
-    weighted_sum = sum((1.0 if v.met else 0.0) * v.criterion.weight for v in verdicts)
+    total_positive = sum(max(0.0, v.weight) for v in verdicts)
+    total_negative = sum(abs(v.weight) for v in verdicts if v.weight < 0)
+    weighted_sum = sum((1.0 if v.passed else 0.0) * v.weight for v in verdicts)
     if total_positive > 0:
         return max(0.0, min(1.0, weighted_sum / total_positive))
     if total_negative > 0: