From f258f3ce696e4faaa5e2e989761af6770895d4e5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 12 Aug 2026 01:13:11 +0900 Subject: [PATCH 1/2] test(judge): require stable invalid-criteria error contract --- tests/test_llm_judge.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_llm_judge.py b/tests/test_llm_judge.py index 8e1244a47..b7359442e 100644 --- a/tests/test_llm_judge.py +++ b/tests/test_llm_judge.py @@ -269,6 +269,16 @@ def __float__(self): assert _HookedFloat.invoked is False +def test_judge_criteria_reject_non_contract_values_with_value_error() -> None: + """Arbitrary criterion elements must fail through the stable benign error contract.""" + with pytest.raises(ValueError, match="JudgeCriterion or mapping"): + ContextualOrchestratorJudge(_FakeOrchestrator(_payload())).judge( + task="task", + answer="answer", + criteria=[object()], + ) + + if __name__ == "__main__": test_judge_uses_contextual_orchestrator_route_and_reports_usage() test_judge_rejects_malformed_decisions_and_derives_acceptance() @@ -279,4 +289,5 @@ def __float__(self): test_category_judgment_rejects_non_integral_categories() test_judge_rejects_missing_or_malformed_model_fields() test_judge_criteria_reject_invalid_runtime_types() + test_judge_criteria_reject_non_contract_values_with_value_error() print("ok") From 8a38917ceee545c55405d8c2033178c4b8efa6a8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 12 Aug 2026 01:25:52 +0900 Subject: [PATCH 2/2] fix(judge): normalize invalid criterion container errors --- python/fast_mlsirm/llm_judge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/fast_mlsirm/llm_judge.py b/python/fast_mlsirm/llm_judge.py index 61d2934f7..cc02e2395 100644 --- a/python/fast_mlsirm/llm_judge.py +++ b/python/fast_mlsirm/llm_judge.py @@ -263,7 +263,7 @@ def _criteria(values: Iterable[JudgeCriterion | Mapping[str, Any]]) -> tuple[Jud weight=value.get("weight", 1.0), ) else: - raise TypeError("criteria must contain JudgeCriterion or mapping values") + raise ValueError("criteria must contain JudgeCriterion or mapping values") normalized.append(criterion) if not 1 <= len(normalized) <= MAX_JUDGE_CRITERIA: raise ValueError(f"criteria must contain 1..{MAX_JUDGE_CRITERIA} values")