diff --git a/eval/run_eval_ps_auto.py b/eval/run_eval_ps_auto.py index 20842d3e..4a56abd6 100644 --- a/eval/run_eval_ps_auto.py +++ b/eval/run_eval_ps_auto.py @@ -108,13 +108,13 @@ def run_one(lab, url, conditions, model, max_turns, timeout): print(f" {cond}: running agent...", flush=True) r = RE.run_agent(base_task, cond == "skills", model, max_turns, timeout) time.sleep(3) - post, _ = ps_solved(url) - solved = bool(post) + post, post_ok = ps_solved(url) + solved = post if post_ok else None if solved: solved_already = True print(f" -> solved={solved} turns={r.get('num_turns')} cost=${r.get('cost_usd')} {r.get('duration_s')}s" f"{' [' + r['agent_error'] + ']' if r.get('agent_error') else ''}") - recs.append({"condition": cond, "solved": solved, **r}) + recs.append({"condition": cond, "solved": solved, "oracle_ok": post_ok, **r}) return recs @@ -166,10 +166,13 @@ def main(): runs = [r for r in rows if r.get("solved") is not None and not r.get("skipped")] for cond in conditions: rs = [r for r in runs if r["condition"] == cond] - if rs: + attempts = [r for r in rows if r["condition"] == cond and not r.get("skipped")] + if attempts: s = sum(1 for r in rs if r["solved"]) - cost = sum((r.get("cost_usd") or 0) for r in rs) - print(f" {cond:9s}: solved {s}/{len(rs)} ${cost:.2f}") + cost = sum((r.get("cost_usd") or 0) for r in attempts) + unreadable = len(attempts) - len(rs) + print(f" {cond:9s}: solved {s}/{len(rs)} ${cost:.2f} " + f"{unreadable} oracle-unreadable") # the money table: per lab, baseline vs skills print("\n per-lab (baseline → skills):") for lab in labs: @@ -178,6 +181,7 @@ def cell(c): r = lr.get(c) if not r: return "-" if r.get("skipped"): return "(skip:base-solved)" + if r["solved"] is None: return "?" return "✓" if r["solved"] else "✗" print(f" {lab['key']:24s} {lab.get('class',''):14s} base={cell('baseline'):>4s} skills={cell('skills')}") # skill delta = labs where baseline failed but skills solved diff --git a/eval/run_eval_ps_par.py b/eval/run_eval_ps_par.py index b28fad56..4ca6cd61 100644 --- a/eval/run_eval_ps_par.py +++ b/eval/run_eval_ps_par.py @@ -30,8 +30,8 @@ def do_run(lab, url, cond, model, max_turns, timeout): r = RE.run_agent(RE.build_task(lab["objective"], url, AUTO.TARGET_DESC), cond == "skills", model, max_turns, timeout) time.sleep(2) - post, _ = ps_solved(url) - return {**base, "solved": bool(post), **r} + post, post_ok = ps_solved(url) + return {**base, "solved": post if post_ok else None, "oracle_ok": post_ok, **r} def main(): @@ -106,15 +106,23 @@ def record(rec): runs = [r for r in rows if r.get("solved") is not None] for c in conds: rs = [r for r in runs if r["condition"] == c] - if rs: + attempts = [r for r in rows if r["condition"] == c and not r.get("skipped")] + if attempts: print(f" {c:9s}: solved {sum(1 for r in rs if r['solved'])}/{len(rs)} " - f"${sum((r.get('cost_usd') or 0) for r in rs):.2f}") + f"${sum((r.get('cost_usd') or 0) for r in attempts):.2f} " + f"{len(attempts) - len(rs)} oracle-unreadable") print("\n per-lab (baseline → skills):") for lab, _ in launched: lr = {r["condition"]: r for r in rows if r["lab"] == lab["key"]} def cell(c): r = lr.get(c) - return "-" if not r else ("(skip)" if r.get("skipped") else ("✓" if r["solved"] else "✗")) + if not r: + return "-" + if r.get("skipped"): + return "(skip)" + if r["solved"] is None: + return "?" + return "✓" if r["solved"] else "✗" print(f" {lab['key']:26s} {lab.get('class',''):14s} base={cell('baseline'):>6s} skills={cell('skills')}") delta = [k for k in base if base[k].get("solved") is False and any(r["lab"] == k and r["condition"] == "skills" and r.get("solved") for r in rows)] diff --git a/tests/test_eval_ps_oracle_loss.py b/tests/test_eval_ps_oracle_loss.py new file mode 100644 index 00000000..1e1943a2 --- /dev/null +++ b/tests/test_eval_ps_oracle_loss.py @@ -0,0 +1,23 @@ +import unittest +from unittest.mock import patch + +from eval import run_eval_ps_auto as auto, run_eval_ps_par as parallel + + +class OracleLossTests(unittest.TestCase): + def test_post_run_oracle_loss_is_unscored(self): + lab = {"key": "synthetic", "objective": "fixture"} + url = "https://example.invalid" + for runner in (auto, parallel): + with self.subTest(runner=runner.__name__), patch.object( + runner, "ps_solved", side_effect=[(False, True), (None, False)] + ), patch.object(runner.RE, "run_agent", return_value={"cost_usd": 0.1}), patch.object( + runner.time, "sleep" + ): + if runner is auto: + record, = runner.run_one(lab, url, ["baseline"], "model", 1, 1) + else: + record = runner.do_run(lab, url, "baseline", "model", 1, 1) + self.assertIsNone(record["solved"]) + self.assertFalse(record["oracle_ok"]) + self.assertEqual(record["cost_usd"], 0.1)