From 17201bdc44f814a137d10efe36c9bf4104deae8d Mon Sep 17 00:00:00 2001 From: Rudy Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:15:50 -0500 Subject: [PATCH] fix(eval): leave unreadable Juice Shop outcomes ungraded --- eval/run_eval.py | 31 +++++++++++++------ tests/test_eval_juice_oracle.py | 55 +++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 10 deletions(-) create mode 100644 tests/test_eval_juice_oracle.py diff --git a/eval/run_eval.py b/eval/run_eval.py index 94af2967..c673725e 100644 --- a/eval/run_eval.py +++ b/eval/run_eval.py @@ -46,9 +46,12 @@ def oracle(base, name): data = json.load(r)["data"] except Exception: return None, False - for c in data: - if c["name"] == name: - return bool(c["solved"]), True + if not isinstance(data, list): + return None, False + for challenge in data: + if isinstance(challenge, dict) and challenge.get("name") == name: + solved = challenge.get("solved") + return (solved, True) if isinstance(solved, bool) else (None, False) return None, False @@ -155,7 +158,8 @@ def main(): ap.add_argument("--out", default=os.path.join(EVAL, "results", "run.jsonl")) a = ap.parse_args() - challenges = json.load(open(a.challenges)) + with open(a.challenges, encoding="utf-8") as config: + challenges = json.load(config) if a.limit: challenges = challenges[:a.limit] conditions = [c.strip() for c in a.conditions.split(",") if c.strip()] @@ -178,11 +182,10 @@ def main(): print(f" running agent (max_turns={a.max_turns}, timeout={a.timeout}s)...", flush=True) r = run_agent(build_task(ch["objective"], a.base, JUICE_DESC), skills_on, a.model, a.max_turns, a.timeout) time.sleep(2) - post, _ = oracle(a.base, ch["oracle_name"]) - solved = bool(post) + solved, oracle_ok = oracle(a.base, ch["oracle_name"]) rec = {"ts": datetime.now(timezone.utc).isoformat(), "challenge": ch["key"], "category": ch["category"], "skill_hint": ch.get("skill_hint"), - "condition": cond, "solved": solved, **r} + "condition": cond, "solved": solved, "oracle_ok": oracle_ok, **r} rows.append(rec) out.write(json.dumps(rec) + "\n"); out.flush() print(f" -> solved={solved} turns={r.get('num_turns')} " @@ -196,16 +199,24 @@ def main(): for r in rows: by_cond.setdefault(r["condition"], []).append(r) for cond, rs in by_cond.items(): - s = sum(1 for r in rs if r["solved"]) + scored = [r for r in rs if r["oracle_ok"]] + s = sum(1 for r in scored if r["solved"]) cost = sum((r.get("cost_usd") or 0) for r in rs) - print(f" {cond:9s}: solved {s}/{len(rs)} total ${cost:.2f}") + unreadable = len(rs) - len(scored) + note = f" ({unreadable} oracle-unreadable, excluded)" if unreadable else "" + print(f" {cond:9s}: solved {s}/{len(scored)} total ${cost:.2f}{note}") print("\n per-challenge:") print(f" {'challenge':24s} " + " ".join(f"{c:>9s}" for c in conditions)) for ch in challenges: cells = [] for cond in conditions: m = [r for r in rows if r["challenge"] == ch["key"] and r["condition"] == cond] - cells.append("✓" if (m and m[0]["solved"]) else ("✗" if m else "-")) + if not m: + cells.append("-") + elif not m[0]["oracle_ok"]: + cells.append("?") + else: + cells.append("✓" if m[0]["solved"] else "✗") print(f" {ch['key']:24s} " + " ".join(f"{c:>9s}" for c in cells)) print(f"\n raw: {a.out}") diff --git a/tests/test_eval_juice_oracle.py b/tests/test_eval_juice_oracle.py new file mode 100644 index 00000000..c796f224 --- /dev/null +++ b/tests/test_eval_juice_oracle.py @@ -0,0 +1,55 @@ +import contextlib +import io +import json +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from eval import run_eval + + +class JuiceShopOracleTests(unittest.TestCase): + def test_unknown_or_malformed_challenges_are_unreadable(self): + for payload in ( + {"data": None}, + {"data": [{"name": "fixture"}]}, + {"data": [{"name": "fixture", "solved": "false"}]}, + {"data": [{"name": "other", "solved": True}]}, + ): + with self.subTest(payload=payload), patch.object( + run_eval.urllib.request, "urlopen", return_value=contextlib.nullcontext( + io.StringIO(json.dumps(payload)) + ) + ): + self.assertEqual(run_eval.oracle("http://localhost", "fixture"), (None, False)) + + def test_oracle_loss_preserves_attempt_without_counting_a_miss(self): + challenge = {"key": "fixture", "category": "test", "objective": "fixture", + "oracle_name": "fixture"} + with tempfile.TemporaryDirectory() as tmp: + config = Path(tmp, "challenges.json") + output = Path(tmp, "results.jsonl") + config.write_text(json.dumps([challenge]), encoding="utf-8") + args = ["run_eval.py", "--challenges", str(config), "--out", str(output), + "--conditions", "baseline"] + with patch.object(sys, "argv", args), patch.object( + run_eval, "reset_target", return_value=True + ), patch.object( + run_eval, "oracle", side_effect=[(False, True), (None, False)] + ), patch.object( + run_eval, "run_agent", return_value={"cost_usd": 0.25} + ), patch.object(run_eval.time, "sleep"), contextlib.redirect_stdout(io.StringIO()) as stdout: + run_eval.main() + + row = json.loads(output.read_text(encoding="utf-8")) + self.assertIsNone(row["solved"]) + self.assertFalse(row["oracle_ok"]) + self.assertIn("solved 0/0", stdout.getvalue()) + self.assertIn("total $0.25", stdout.getvalue()) + self.assertIn("1 oracle-unreadable", stdout.getvalue()) + + +if __name__ == "__main__": + unittest.main()