diff --git a/docs/v6/reference/graders.mdx b/docs/v6/reference/graders.mdx index c1da269c..2ee9d365 100644 --- a/docs/v6/reference/graders.mdx +++ b/docs/v6/reference/graders.mdx @@ -9,7 +9,7 @@ Graders turn an agent's answer into a reward. HUD ships reusable ones so you don ```python from hud.graders import ( BashGrader, LLMJudgeGrader, Grader, - SubScore, EvaluationResult, + SubScore, EvaluationResult, CriterionResult, combine, combine_any, combine_all, exact_match, contains, contains_any, contains_all, numeric_match, f1_score, normalize, @@ -96,6 +96,9 @@ result = await LLMJudgeGrader.grade( `criteria` items are strings, or `(requirement, weight)` tuples. A negative weight marks an error to avoid: the criterion scores as a penalty, MET when the answer actually makes the mistake. +Each judgment is reported as a [`CriterionResult`](#criterionresult) on the returned +`SubScore.criteria`, so the per-criterion verdicts and reasons survive into the trace. + ## `combine` - compose multiple graders `combine` resolves `SubScore`s and grader coroutines in parallel and combines them into a weighted `EvaluationResult`. Positive weights are normalized to sum to `1.0`; negative weights are penalties. @@ -117,11 +120,11 @@ async def composed(answer: str = ""): | `combine_any(weight, subscores)` | Boolean OR: a `SubScore` that passes if any input passes (max). | | `combine_all(weight, subscores)` | Boolean AND: a `SubScore` that passes only if all inputs pass (min). | -The subscores appear in the trace, so a partial reward is legible. `combine_any`/`combine_all` collapse alternatives into a single component you can feed to `combine` - e.g. "tests pass via `pytest` OR via `make test`" as one 0/1 subscore. +The subscores appear in the trace, so a partial reward is legible. `combine_any`/`combine_all` fold alternatives into a single component you can feed to `combine` - e.g. "tests pass via `pytest` OR via `make test`" as one 0/1 subscore. The combined `SubScore` keeps its inputs on `children` and records the operator on `aggregation`, so the full grading tree - every grader run, its criteria, and how they were combined - survives into the trace. Combinators nest: a `combine_all` over `combine_any` nodes produces a deeper tree. ## Custom graders -Subclass `Grader` and implement async `compute_score` (return a float, or `(float, metadata)`): +Subclass `Grader` and implement async `compute_score`, returning a float or a `SubScore`: ```python class LengthGrader(Grader): @@ -134,9 +137,23 @@ class LengthGrader(Grader): result = await LengthGrader.grade(weight=1.0, answer=answer, target=200) ``` +Return a `SubScore` to attach `criteria` (rubric verdicts) or `metadata` (diagnostics); +`grade` stamps the caller's `name` and `weight` onto it and records the call parameters +under the `_parameters` metadata key. + ## `SubScore` and `EvaluationResult` -A `SubScore` is one component of a grade: `name`, `value` (0-1), `weight` (default `1.0`; negative = penalty), optional `metadata`. +A `SubScore` is one node in the grade breakdown tree: + +| Field | Default | Description | +|-------|---------|-------------| +| `name` | - | Name of this component. | +| `value` | - | Measurement, 0-1. | +| `weight` | `1.0` | Weight in the parent's combination; negative = penalty. | +| `aggregation` | `None` | `"any"` / `"all"` when the value was derived from `children`. | +| `children` | `None` | Input subscores this node was combined from. | +| `criteria` | `None` | Itemized [`CriterionResult`](#criterionresult) verdicts. | +| `metadata` | `None` | Grader-specific details (parameters, stdout, judge model, ...). | An `EvaluationResult` is the combined grade payload you can yield from a task: @@ -151,5 +168,20 @@ An `EvaluationResult` is the combined grade payload you can yield from a task: `EvaluationResult.from_float(value)` wraps a bare reward. +## `CriterionResult` + +Where a `SubScore` decomposes the reward numerically, criteria justify it: each +`CriterionResult` is one checked requirement with a pass/fail verdict. Criteria +ride on the `SubScore` of the grader that checked them: `LLMJudgeGrader` emits +one per criterion on its returned subscore, and `combine` carries them through +into `EvaluationResult.subscores`. + +| Field | Default | Description | +|-------|---------|-------------| +| `criterion` | - | The requirement that was checked. | +| `passed` | - | Whether it was met. For negative-weight criteria, `True` means the error is present. | +| `weight` | `1.0` | Rubric weight; negative marks an error to avoid. | +| `reason` | `None` | The grader's justification for the verdict. | + Graders run as the second yield of a [task](/v6/reference/tasks); for choosing a reward that separates good work from bad, see [designing tasks](/v6/reference/advice). diff --git a/hud/graders/__init__.py b/hud/graders/__init__.py index 8df731d9..ab114471 100644 --- a/hud/graders/__init__.py +++ b/hud/graders/__init__.py @@ -27,7 +27,7 @@ from .bash import BashGrader from .combine import _combine_subscores, combine, combine_all, combine_any from .judge import LLMJudgeGrader -from .results import EvaluationResult, SubScore +from .results import CriterionResult, EvaluationResult, SubScore from .text import ( contains, contains_all, @@ -40,6 +40,7 @@ __all__ = [ "BashGrader", + "CriterionResult", "EvaluationResult", "Grader", "LLMJudgeGrader", diff --git a/hud/graders/base.py b/hud/graders/base.py index b6171690..5c5c0b97 100644 --- a/hud/graders/base.py +++ b/hud/graders/base.py @@ -13,8 +13,8 @@ class Grader: """Async base class for reusable graders. Subclasses implement ``compute_score`` (async). The ``grade`` classmethod - calls it, wraps the result as a ``SubScore``, and records parameters - in metadata for reproducibility. + calls it, stamps the caller's ``name`` and ``weight`` onto the resulting + ``SubScore``, and records parameters in metadata for reproducibility. """ name: str = "BaseGrader" @@ -23,25 +23,23 @@ class Grader: async def grade(cls, weight: float, name: str | None = None, **kwargs: Any) -> SubScore: """Run the grader and package the result as a ``SubScore``.""" result = await cls.compute_score(**kwargs) - - if isinstance(result, tuple): - score, metadata = result - else: - score = result - metadata = {} - - return SubScore( - name=name or cls.name, - weight=weight, - value=float(score), - metadata={**metadata, "_parameters": json_safe_dict(kwargs)}, + if not isinstance(result, SubScore): + result = SubScore(name=cls.name, value=float(result)) + + return result.model_copy( + update={ + "name": name or cls.name, + "weight": weight, + "metadata": {**(result.metadata or {}), "_parameters": json_safe_dict(kwargs)}, + } ) @classmethod - async def compute_score(cls, **kwargs: Any) -> float | tuple[float, dict[str, Any]]: + async def compute_score(cls, **kwargs: Any) -> float | SubScore: """Compute a score between ``0.0`` and ``1.0``. - Return a float, or ``(float, metadata_dict)`` to attach extra info. + Return a float, or a ``SubScore`` to attach criteria or metadata + (its ``name`` and ``weight`` are overwritten by ``grade``). """ raise NotImplementedError("Subclasses must implement compute_score") diff --git a/hud/graders/bash.py b/hud/graders/bash.py index 21c28774..9fd36b17 100644 --- a/hud/graders/bash.py +++ b/hud/graders/bash.py @@ -9,6 +9,7 @@ from hud.utils.process import create_process_group_exec from .base import Grader +from .results import SubScore logger = logging.getLogger(__name__) @@ -27,8 +28,8 @@ async def compute_score( cwd: str | None = None, timeout_seconds: int | None = None, **kwargs: Any, - ) -> tuple[float, dict[str, Any]]: - """Run ``command`` via ``bash -lc`` and return score + metadata.""" + ) -> SubScore: + """Run ``command`` via ``bash -lc`` and score by exit code.""" if timeout_seconds is None: timeout_seconds = cls.default_timeout del kwargs @@ -49,9 +50,10 @@ async def compute_score( stderr = stderr_bytes.decode(errors="replace") returncode = proc.returncode if proc.returncode is not None else 1 except TimeoutError: - return ( - 0.0, - { + return SubScore( + name=cls.name, + value=0.0, + metadata={ "exit_code": None, "stdout": "", "stderr": "", @@ -60,9 +62,10 @@ async def compute_score( }, ) except FileNotFoundError: - return ( - 0.0, - { + return SubScore( + name=cls.name, + value=0.0, + metadata={ "exit_code": None, "stdout": "", "stderr": "/bin/bash not found", @@ -70,8 +73,11 @@ async def compute_score( }, ) - score = 1.0 if returncode == 0 else 0.0 - return (score, {"exit_code": returncode, "stdout": stdout, "stderr": stderr}) + return SubScore( + name=cls.name, + value=1.0 if returncode == 0 else 0.0, + metadata={"exit_code": returncode, "stdout": stdout, "stderr": stderr}, + ) __all__ = ["BashGrader"] diff --git a/hud/graders/combine.py b/hud/graders/combine.py index 797bf0ff..0149a127 100644 --- a/hud/graders/combine.py +++ b/hud/graders/combine.py @@ -4,7 +4,7 @@ import asyncio import warnings -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Literal from .results import EvaluationResult, SubScore @@ -71,20 +71,12 @@ def _combine_subscores(subscores: list[SubScore]) -> EvaluationResult: ) normalized_subscores: list[SubScore] = [] - metadata: dict[str, Any] = {} for item, final_name in zip(subscores, _dedupe_subscore_names(subscores), strict=True): normalized_weight = item.weight / positive_weight_sum if item.weight > 0 else item.weight normalized_subscores.append( - SubScore( - name=final_name, - weight=normalized_weight, - value=item.value, - metadata=item.metadata, - ) + item.model_copy(update={"name": final_name, "weight": normalized_weight}) ) - if item.metadata is not None: - metadata[final_name] = item.metadata reward = float(sum(item.value * item.weight for item in normalized_subscores)) @@ -92,7 +84,6 @@ def _combine_subscores(subscores: list[SubScore]) -> EvaluationResult: reward=reward, done=True, subscores=normalized_subscores, - info=metadata, ) @@ -137,36 +128,37 @@ async def combine(*items: SubScore | Awaitable[SubScore]) -> EvaluationResult: def _boolean_subscore( - name: str, weight: float, subscores: list[SubScore], value: float + name: str, + weight: float, + aggregation: Literal["any", "all"], + subscores: list[SubScore], + value: float, ) -> SubScore: - unique_names = _dedupe_subscore_names(subscores) + children = [ + subscore.model_copy(update={"name": unique_name}) + for unique_name, subscore in zip(_dedupe_subscore_names(subscores), subscores, strict=True) + ] return SubScore( name=name, value=value, weight=weight, - metadata={ - "subscores": unique_names, - "subscore_metadata": { - unique_name: subscore.metadata - for unique_name, subscore in zip(unique_names, subscores, strict=True) - if subscore.metadata is not None - }, - }, + aggregation=aggregation, + children=children, ) def combine_any(weight: float, subscores: list[SubScore], *, name: str = "any") -> SubScore: - """Subscore that passes if any input passes (max).""" + """Subscore that passes if any input passes (max); inputs kept as children.""" if not subscores: raise ValueError("subscores must not be empty") - return _boolean_subscore(name, weight, subscores, max(s.value for s in subscores)) + return _boolean_subscore(name, weight, "any", subscores, max(s.value for s in subscores)) def combine_all(weight: float, subscores: list[SubScore], *, name: str = "all") -> SubScore: - """Subscore that passes only if all inputs pass (min).""" + """Subscore that passes only if all inputs pass (min); inputs kept as children.""" if not subscores: raise ValueError("subscores must not be empty") - return _boolean_subscore(name, weight, subscores, min(s.value for s in subscores)) + return _boolean_subscore(name, weight, "all", subscores, min(s.value for s in subscores)) __all__ = ["_combine_subscores", "combine", "combine_all", "combine_any"] diff --git a/hud/graders/judge.py b/hud/graders/judge.py index 88f95a25..f66775d1 100644 --- a/hud/graders/judge.py +++ b/hud/graders/judge.py @@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any, cast from .base import Grader +from .results import CriterionResult, SubScore if TYPE_CHECKING: from openai import AsyncOpenAI @@ -50,13 +51,6 @@ class _Criterion: weight: float -@dataclass(slots=True) -class _Verdict: - criterion: _Criterion - met: bool - reason: str - - class LLMJudgeGrader(Grader): """Grade an answer against weighted criteria using an LLM judge. @@ -85,19 +79,19 @@ async def compute_score( question: str = "", model: str = "claude-haiku-4-5", **kwargs: Any, - ) -> tuple[float, dict[str, Any]]: + ) -> SubScore: """Evaluate ``answer`` against ``criteria`` via parallel LLM judgments.""" del kwargs parsed = _parse_criteria(criteria) if not parsed: - return (0.0, {"error": "no criteria provided"}) + return SubScore(name=cls.name, value=0.0, metadata={"error": "no criteria provided"}) from hud.utils.gateway import build_gateway_client client = cast("AsyncOpenAI", build_gateway_client("openai")) answer_text = str(answer) - async def _judge(criterion: _Criterion) -> _Verdict: + async def _judge(criterion: _Criterion) -> CriterionResult: response = await client.chat.completions.create( model=model, max_tokens=1024, @@ -107,19 +101,20 @@ async def _judge(criterion: _Criterion) -> _Verdict: ], ) met, reason = _parse_verdict(response.choices[0].message.content or "") - return _Verdict(criterion, met, reason) + return CriterionResult( + criterion=criterion.requirement, + passed=met, + weight=criterion.weight, + reason=reason, + ) verdicts = list(await asyncio.gather(*(_judge(c) for c in parsed))) - score = _aggregate(verdicts) - report = { - v.criterion.requirement[:80]: { - "verdict": "MET" if v.met else "UNMET", - "reason": v.reason, - "weight": v.criterion.weight, - } - for v in verdicts - } - return (score, {"criteria": report, "model": model}) + return SubScore( + name=cls.name, + value=_aggregate(verdicts), + criteria=verdicts, + metadata={"model": model}, + ) def _parse_criteria(criteria: list[str | tuple[str, float]] | None) -> list[_Criterion]: @@ -160,11 +155,11 @@ def _parse_verdict(content: str) -> tuple[bool, str]: return ("UNMET" not in upper and "MET" in upper), text[:200] -def _aggregate(verdicts: list[_Verdict]) -> float: +def _aggregate(verdicts: list[CriterionResult]) -> float: """Weighted MET-sum normalized by positive (or all-negative) weight, clamped 0-1.""" - total_positive = sum(max(0.0, v.criterion.weight) for v in verdicts) - total_negative = sum(abs(v.criterion.weight) for v in verdicts if v.criterion.weight < 0) - weighted_sum = sum((1.0 if v.met else 0.0) * v.criterion.weight for v in verdicts) + total_positive = sum(max(0.0, v.weight) for v in verdicts) + total_negative = sum(abs(v.weight) for v in verdicts if v.weight < 0) + weighted_sum = sum((1.0 if v.passed else 0.0) * v.weight for v in verdicts) if total_positive > 0: return max(0.0, min(1.0, weighted_sum / total_positive)) if total_negative > 0: diff --git a/hud/graders/results.py b/hud/graders/results.py index 6c7aa9cb..43c2aded 100644 --- a/hud/graders/results.py +++ b/hud/graders/results.py @@ -1,18 +1,39 @@ -"""Grading result shapes: ``SubScore`` and ``EvaluationResult``.""" +"""Grading result shapes: ``CriterionResult``, ``SubScore``, and ``EvaluationResult``.""" from __future__ import annotations import warnings -from typing import Any +from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field, model_validator +class CriterionResult(BaseModel): + """One graded rubric criterion: a checked requirement with its verdict.""" + + model_config = ConfigDict(extra="forbid") + + criterion: str = Field(..., description="The requirement that was checked") + passed: bool = Field( + ..., + description="Whether the criterion was met. For negative-weight " + "(error-to-avoid) criteria, True means the error is present.", + ) + weight: float = Field( + default=1.0, + description="Rubric weight of this criterion. Negative marks an error to avoid.", + ) + reason: str | None = Field( + default=None, description="The grader's justification for the verdict" + ) + + class SubScore(BaseModel): - """Individual subscore for debugging and transparency. + """One node in the grade breakdown tree. - SubScores allow breaking down the final reward into component parts, - making it easier to understand what contributed to the evaluation. + A leaf subscore is a single measurement, usually one grader run. Combinators + (``combine_any``/``combine_all``) return a node that keeps the subscores it + combined as ``children``, so the full grading history stays inspectable. """ model_config = ConfigDict(extra="forbid") @@ -24,13 +45,44 @@ class SubScore(BaseModel): "Negative weights represent penalties.", ) value: float = Field(..., ge=0.0, le=1.0, description="Value of this subscore, 0.0 to 1.0") - metadata: dict[str, Any] | None = Field(default=None, exclude=True) + aggregation: Literal["any", "all"] | None = Field( + default=None, + description="How value was derived from children: 'any' (max) or 'all' (min). " + "None for leaf subscores.", + ) + children: list[SubScore] | None = Field( + default=None, + description="Input subscores this node was combined from", + ) + criteria: list[CriterionResult] | None = Field( + default=None, + description="Itemized criterion verdicts behind this subscore's value", + ) + metadata: dict[str, Any] | None = Field( + default=None, + description="Grader-specific details (parameters, stdout, judge model, ...)", + ) @property def score(self) -> float: """Alias for value. Deprecated — use .value instead.""" return self.value + @model_validator(mode="after") + def _check_aggregation(self) -> SubScore: + if self.aggregation is None: + return self + if not self.children: + raise ValueError(f"aggregation={self.aggregation!r} requires children") + expected = (max if self.aggregation == "any" else min)(c.value for c in self.children) + if abs(expected - self.value) > 0.01: + warnings.warn( + f"SubScore {self.name!r}: value={self.value:.4f} disagrees with " + f"{self.aggregation}(children)={expected:.4f}", + stacklevel=2, + ) + return self + class EvaluationResult(BaseModel): """Result of a task's evaluate phase. @@ -81,4 +133,4 @@ def from_float(cls, value: float) -> EvaluationResult: return cls(reward=value, done=True) -__all__ = ["EvaluationResult", "SubScore"] +__all__ = ["CriterionResult", "EvaluationResult", "SubScore"] diff --git a/hud/tests/test_graders.py b/hud/tests/test_graders.py index f48e4af0..24f7443c 100644 --- a/hud/tests/test_graders.py +++ b/hud/tests/test_graders.py @@ -2,15 +2,20 @@ from __future__ import annotations +import json import os import warnings +from types import SimpleNamespace +from typing import Any import pytest from hud.graders import ( BashGrader, + CriterionResult, EvaluationResult, Grader, + LLMJudgeGrader, SubScore, combine, combine_all, @@ -39,6 +44,53 @@ def test_evaluation_result_warns_when_subscores_disagree_with_reward(self) -> No with pytest.warns(UserWarning): EvaluationResult(reward=1.0, subscores=[SubScore(name="a", value=0.5, weight=1.0)]) + def test_subscore_serializes_criteria(self) -> None: + result = EvaluationResult( + reward=1.0, + subscores=[ + SubScore( + name="judge", + value=1.0, + weight=1.0, + criteria=[CriterionResult(criterion="Mentions Paris", passed=True)], + ) + ], + ) + dumped = result.model_dump(mode="json") + assert dumped["subscores"][0]["criteria"] == [ + { + "criterion": "Mentions Paris", + "passed": True, + "weight": 1.0, + "reason": None, + } + ] + + def test_subscore_serializes_metadata_and_children(self) -> None: + node = SubScore( + name="any", + value=1.0, + aggregation="any", + children=[SubScore(name="tests", value=1.0, metadata={"exit_code": 0})], + ) + dumped = node.model_dump(mode="json") + assert dumped["aggregation"] == "any" + assert dumped["children"][0]["name"] == "tests" + assert dumped["children"][0]["metadata"] == {"exit_code": 0} + + def test_subscore_aggregation_requires_children(self) -> None: + with pytest.raises(ValueError, match="requires children"): + SubScore(name="any", value=1.0, aggregation="any") + + def test_subscore_warns_when_value_disagrees_with_aggregation(self) -> None: + with pytest.warns(UserWarning, match="disagrees"): + SubScore( + name="all", + value=1.0, + aggregation="all", + children=[SubScore(name="tests", value=0.0)], + ) + class TestNormalize: def test_lowercases(self) -> None: @@ -283,9 +335,43 @@ async def test_combine_duplicate_names_avoid_existing_suffix_collisions(self) -> async def test_combine_propagates_metadata(self) -> None: metadata = {"stdout": "ok"} result = await combine(SubScore(name="grader", value=1.0, weight=1.0, metadata=metadata)) - assert result.info["grader"] == metadata + assert result.info == {} assert result.subscores is not None assert result.subscores[0].metadata == metadata + assert result.model_dump(mode="json")["subscores"][0]["metadata"] == metadata + + async def test_combine_keeps_criteria_on_subscores(self) -> None: + judge_criteria = [ + CriterionResult(criterion="Mentions Paris", passed=True), + CriterionResult(criterion="Names the river", passed=False), + ] + result = await combine( + SubScore(name="judge", value=1.0, weight=0.5, criteria=judge_criteria), + SubScore(name="tests", value=0.0, weight=0.5), + ) + assert result.subscores is not None + by_name = {subscore.name: subscore for subscore in result.subscores} + assert by_name["judge"].criteria == judge_criteria + assert by_name["tests"].criteria is None + + async def test_combine_preserves_combinator_children(self) -> None: + result = await combine( + combine_any( + weight=0.5, + subscores=[ + SubScore(name="pytest", value=1.0), + SubScore(name="make", value=0.0), + ], + ), + SubScore(name="format", value=1.0, weight=0.5), + ) + assert result.subscores is not None + by_name = {subscore.name: subscore for subscore in result.subscores} + any_node = by_name["any"] + assert any_node.aggregation == "any" + assert any_node.children is not None + assert [child.name for child in any_node.children] == ["pytest", "make"] + assert by_name["format"].children is None async def test_combine_preserves_negative_reward_without_validator_warning(self) -> None: with warnings.catch_warnings(record=True) as caught: @@ -323,13 +409,13 @@ def test_from_subscores_is_sync(self) -> None: class TestGrader: - async def test_grade_returns_subscore_and_stores_parameters(self) -> None: + async def test_grade_wraps_float_and_stores_parameters(self) -> None: class DummyGrader(Grader): name = "DummyGrader" @classmethod - async def compute_score(cls, **kwargs: object) -> tuple[float, dict[str, object]]: - return 0.75, {"source": "dummy", "kwargs_seen": sorted(kwargs)} + async def compute_score(cls, **kwargs: object) -> float: + return 0.75 subscore = await DummyGrader.grade(weight=0.4, marker="ok", payload=object()) assert isinstance(subscore, SubScore) @@ -337,10 +423,30 @@ async def compute_score(cls, **kwargs: object) -> tuple[float, dict[str, object] assert subscore.value == pytest.approx(0.75) assert subscore.weight == pytest.approx(0.4) assert subscore.metadata is not None - assert subscore.metadata["source"] == "dummy" assert subscore.metadata["_parameters"]["marker"] == "ok" assert subscore.metadata["_parameters"]["payload"] == "" + async def test_grade_stamps_name_and_weight_on_returned_subscore(self) -> None: + class RubricGrader(Grader): + name = "rubric" + + @classmethod + async def compute_score(cls, **kwargs: object) -> SubScore: + return SubScore( + name="ignored", + value=1.0, + criteria=[CriterionResult(criterion="did the thing", passed=True)], + metadata={"extra": "kept"}, + ) + + subscore = await RubricGrader.grade(weight=0.7, name="my-rubric") + assert subscore.name == "my-rubric" + assert subscore.weight == pytest.approx(0.7) + assert subscore.criteria == [CriterionResult(criterion="did the thing", passed=True)] + assert subscore.metadata is not None + assert subscore.metadata["extra"] == "kept" + assert "_parameters" in subscore.metadata + class TestBooleanCombinators: def test_combine_any_picks_max(self) -> None: @@ -354,7 +460,26 @@ def test_combine_any_picks_max(self) -> None: assert combined.name == "any" assert combined.value == 1.0 - def test_combine_any_preserves_metadata_for_duplicate_named_subscores(self) -> None: + def test_combine_any_keeps_inputs_as_children(self) -> None: + combined = combine_any( + weight=1.0, + subscores=[ + SubScore( + name="judge", + value=1.0, + weight=0.5, + criteria=[CriterionResult(criterion="c1", passed=True)], + ), + SubScore(name="tests", value=0.0, weight=0.5), + ], + ) + assert combined.aggregation == "any" + assert combined.criteria is None + assert combined.children is not None + assert combined.children[0].criteria == [CriterionResult(criterion="c1", passed=True)] + assert combined.children[1].criteria is None + + def test_combine_any_dedupes_child_names(self) -> None: combined = combine_any( weight=1.0, subscores=[ @@ -362,13 +487,13 @@ def test_combine_any_preserves_metadata_for_duplicate_named_subscores(self) -> N SubScore(name="BashGrader", value=0.0, weight=0.5, metadata={"exit_code": 1}), ], ) - assert combined.metadata == { - "subscores": ["BashGrader-1", "BashGrader-2"], - "subscore_metadata": { - "BashGrader-1": {"exit_code": 0}, - "BashGrader-2": {"exit_code": 1}, - }, - } + assert combined.children is not None + assert [child.name for child in combined.children] == ["BashGrader-1", "BashGrader-2"] + assert [child.metadata for child in combined.children] == [ + {"exit_code": 0}, + {"exit_code": 1}, + ] + assert combined.metadata is None def test_combine_all_picks_min(self) -> None: combined = combine_all( @@ -382,42 +507,96 @@ def test_combine_all_picks_min(self) -> None: assert combined.name == "tests_all" assert combined.value == 0.0 - def test_combine_all_preserves_metadata_for_duplicate_named_subscores(self) -> None: + def test_nested_combinators_build_a_tree(self) -> None: combined = combine_all( weight=1.0, subscores=[ - SubScore(name="BashGrader", value=1.0, weight=0.5, metadata={"exit_code": 0}), - SubScore(name="BashGrader", value=0.0, weight=0.5, metadata={"exit_code": 1}), + combine_any( + weight=1.0, + subscores=[ + SubScore(name="pytest", value=0.0), + SubScore(name="make", value=1.0), + ], + ), + SubScore(name="lint", value=1.0), ], ) - assert combined.metadata == { - "subscores": ["BashGrader-1", "BashGrader-2"], - "subscore_metadata": { - "BashGrader-1": {"exit_code": 0}, - "BashGrader-2": {"exit_code": 1}, - }, - } + assert combined.value == 1.0 + assert combined.aggregation == "all" + assert combined.children is not None + inner = combined.children[0] + assert inner.aggregation == "any" + assert inner.children is not None + assert [child.name for child in inner.children] == ["pytest", "make"] + + +class _FakeGatewayClient: + """Judges MET when the criterion text (inside the user prompt) contains 'met:'.""" + + def __init__(self) -> None: + self.chat = SimpleNamespace(completions=SimpleNamespace(create=self._create)) + + async def _create(self, **kwargs: Any) -> Any: + user_prompt: str = kwargs["messages"][1]["content"] + status = "MET" if "met:" in user_prompt else "UNMET" + content = json.dumps({"criterion_status": status, "explanation": "because"}) + return SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=content))]) + + +class TestLLMJudgeGrader: + async def test_compute_score_returns_subscore_with_typed_criteria( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr( + "hud.utils.gateway.build_gateway_client", lambda provider: _FakeGatewayClient() + ) + subscore = await LLMJudgeGrader.compute_score( + answer="some answer", + criteria=["met: mentions Paris", ("names the river", 3.0)], + ) + assert subscore.value == pytest.approx(0.25) + assert subscore.criteria is not None + assert [(c.criterion, c.passed, c.weight, c.reason) for c in subscore.criteria] == [ + ("met: mentions Paris", True, 1.0, "because"), + ("names the river", False, 3.0, "because"), + ] + + async def test_grade_puts_criteria_on_subscore(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "hud.utils.gateway.build_gateway_client", lambda provider: _FakeGatewayClient() + ) + subscore = await LLMJudgeGrader.grade( + weight=1.0, answer="some answer", criteria=["met: correct"] + ) + assert subscore.criteria == [ + CriterionResult(criterion="met: correct", passed=True, weight=1.0, reason="because") + ] + assert subscore.metadata is not None + assert subscore.metadata["model"] == "claude-haiku-4-5" @pytest.mark.skipif(not _HAS_BASH, reason="/bin/bash not available (e.g. Windows)") class TestBashGrader: async def test_compute_score_for_passing_command(self) -> None: - score, metadata = await BashGrader.compute_score(command="echo hello") - assert score == 1.0 - assert metadata["exit_code"] == 0 - assert "hello" in metadata["stdout"] + subscore = await BashGrader.compute_score(command="echo hello") + assert subscore.value == 1.0 + assert subscore.metadata is not None + assert subscore.metadata["exit_code"] == 0 + assert "hello" in subscore.metadata["stdout"] async def test_compute_score_for_failing_command(self) -> None: - score, metadata = await BashGrader.compute_score(command="echo oops >&2 && false") - assert score == 0.0 - assert metadata["exit_code"] != 0 - assert "oops" in metadata["stderr"] + subscore = await BashGrader.compute_score(command="echo oops >&2 && false") + assert subscore.value == 0.0 + assert subscore.metadata is not None + assert subscore.metadata["exit_code"] != 0 + assert "oops" in subscore.metadata["stderr"] async def test_compute_score_timeout(self) -> None: - score, metadata = await BashGrader.compute_score(command="sleep 2", timeout_seconds=1) - assert score == 0.0 - assert metadata["timed_out"] is True - assert metadata["timeout"] == 1 + subscore = await BashGrader.compute_score(command="sleep 2", timeout_seconds=1) + assert subscore.value == 0.0 + assert subscore.metadata is not None + assert subscore.metadata["timed_out"] is True + assert subscore.metadata["timeout"] == 1 async def test_grade_and_combine_compose(self) -> None: result = await combine( @@ -425,5 +604,9 @@ async def test_grade_and_combine_compose(self) -> None: BashGrader.grade(weight=0.5, command="false"), ) assert result.reward == pytest.approx(0.5) - assert result.info["BashGrader-1"]["exit_code"] == 0 - assert result.info["BashGrader-2"]["exit_code"] != 0 + assert result.subscores is not None + by_name = {subscore.name: subscore for subscore in result.subscores} + assert by_name["BashGrader-1"].metadata is not None + assert by_name["BashGrader-1"].metadata["exit_code"] == 0 + assert by_name["BashGrader-2"].metadata is not None + assert by_name["BashGrader-2"].metadata["exit_code"] != 0