diff --git a/python/fast_mlsirm/llm_judge.py b/python/fast_mlsirm/llm_judge.py index 61d2934f7..cc02e2395 100644 --- a/python/fast_mlsirm/llm_judge.py +++ b/python/fast_mlsirm/llm_judge.py @@ -263,7 +263,7 @@ def _criteria(values: Iterable[JudgeCriterion | Mapping[str, Any]]) -> tuple[Jud weight=value.get("weight", 1.0), ) else: - raise TypeError("criteria must contain JudgeCriterion or mapping values") + raise ValueError("criteria must contain JudgeCriterion or mapping values") normalized.append(criterion) if not 1 <= len(normalized) <= MAX_JUDGE_CRITERIA: raise ValueError(f"criteria must contain 1..{MAX_JUDGE_CRITERIA} values") diff --git a/tests/test_llm_judge.py b/tests/test_llm_judge.py index 8e1244a47..b7359442e 100644 --- a/tests/test_llm_judge.py +++ b/tests/test_llm_judge.py @@ -269,6 +269,16 @@ def __float__(self): assert _HookedFloat.invoked is False +def test_judge_criteria_reject_non_contract_values_with_value_error() -> None: + """Arbitrary criterion elements must fail through the stable benign error contract.""" + with pytest.raises(ValueError, match="JudgeCriterion or mapping"): + ContextualOrchestratorJudge(_FakeOrchestrator(_payload())).judge( + task="task", + answer="answer", + criteria=[object()], + ) + + if __name__ == "__main__": test_judge_uses_contextual_orchestrator_route_and_reports_usage() test_judge_rejects_malformed_decisions_and_derives_acceptance() @@ -279,4 +289,5 @@ def __float__(self): test_category_judgment_rejects_non_integral_categories() test_judge_rejects_missing_or_malformed_model_fields() test_judge_criteria_reject_invalid_runtime_types() + test_judge_criteria_reject_non_contract_values_with_value_error() print("ok")