Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions eval/run_eval_ps_auto.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,13 +108,13 @@ def run_one(lab, url, conditions, model, max_turns, timeout):
print(f" {cond}: running agent...", flush=True)
r = RE.run_agent(base_task, cond == "skills", model, max_turns, timeout)
time.sleep(3)
post, _ = ps_solved(url)
solved = bool(post)
post, post_ok = ps_solved(url)
solved = post if post_ok else None
if solved:
solved_already = True
print(f" -> solved={solved} turns={r.get('num_turns')} cost=${r.get('cost_usd')} {r.get('duration_s')}s"
f"{' [' + r['agent_error'] + ']' if r.get('agent_error') else ''}")
recs.append({"condition": cond, "solved": solved, **r})
recs.append({"condition": cond, "solved": solved, "oracle_ok": post_ok, **r})
return recs


Expand Down Expand Up @@ -166,10 +166,13 @@ def main():
runs = [r for r in rows if r.get("solved") is not None and not r.get("skipped")]
for cond in conditions:
rs = [r for r in runs if r["condition"] == cond]
if rs:
attempts = [r for r in rows if r["condition"] == cond and not r.get("skipped")]
if attempts:
s = sum(1 for r in rs if r["solved"])
cost = sum((r.get("cost_usd") or 0) for r in rs)
print(f" {cond:9s}: solved {s}/{len(rs)} ${cost:.2f}")
cost = sum((r.get("cost_usd") or 0) for r in attempts)
unreadable = len(attempts) - len(rs)
print(f" {cond:9s}: solved {s}/{len(rs)} ${cost:.2f} "
f"{unreadable} oracle-unreadable")
# the money table: per lab, baseline vs skills
print("\n per-lab (baseline → skills):")
for lab in labs:
Expand All @@ -178,6 +181,7 @@ def cell(c):
r = lr.get(c)
if not r: return "-"
if r.get("skipped"): return "(skip:base-solved)"
if r["solved"] is None: return "?"
return "✓" if r["solved"] else "✗"
print(f" {lab['key']:24s} {lab.get('class',''):14s} base={cell('baseline'):>4s} skills={cell('skills')}")
# skill delta = labs where baseline failed but skills solved
Expand Down
18 changes: 13 additions & 5 deletions eval/run_eval_ps_par.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@ def do_run(lab, url, cond, model, max_turns, timeout):
r = RE.run_agent(RE.build_task(lab["objective"], url, AUTO.TARGET_DESC),
cond == "skills", model, max_turns, timeout)
time.sleep(2)
post, _ = ps_solved(url)
return {**base, "solved": bool(post), **r}
post, post_ok = ps_solved(url)
return {**base, "solved": post if post_ok else None, "oracle_ok": post_ok, **r}


def main():
Expand Down Expand Up @@ -106,15 +106,23 @@ def record(rec):
runs = [r for r in rows if r.get("solved") is not None]
for c in conds:
rs = [r for r in runs if r["condition"] == c]
if rs:
attempts = [r for r in rows if r["condition"] == c and not r.get("skipped")]
if attempts:
print(f" {c:9s}: solved {sum(1 for r in rs if r['solved'])}/{len(rs)} "
f"${sum((r.get('cost_usd') or 0) for r in rs):.2f}")
f"${sum((r.get('cost_usd') or 0) for r in attempts):.2f} "
f"{len(attempts) - len(rs)} oracle-unreadable")
print("\n per-lab (baseline → skills):")
for lab, _ in launched:
lr = {r["condition"]: r for r in rows if r["lab"] == lab["key"]}
def cell(c):
r = lr.get(c)
return "-" if not r else ("(skip)" if r.get("skipped") else ("✓" if r["solved"] else "✗"))
if not r:
return "-"
if r.get("skipped"):
return "(skip)"
if r["solved"] is None:
return "?"
return "✓" if r["solved"] else "✗"
print(f" {lab['key']:26s} {lab.get('class',''):14s} base={cell('baseline'):>6s} skills={cell('skills')}")
delta = [k for k in base if base[k].get("solved") is False
and any(r["lab"] == k and r["condition"] == "skills" and r.get("solved") for r in rows)]
Expand Down
23 changes: 23 additions & 0 deletions tests/test_eval_ps_oracle_loss.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
import unittest
from unittest.mock import patch

from eval import run_eval_ps_auto as auto, run_eval_ps_par as parallel


class OracleLossTests(unittest.TestCase):
def test_post_run_oracle_loss_is_unscored(self):
lab = {"key": "synthetic", "objective": "fixture"}
url = "https://example.invalid"
for runner in (auto, parallel):
with self.subTest(runner=runner.__name__), patch.object(
runner, "ps_solved", side_effect=[(False, True), (None, False)]
), patch.object(runner.RE, "run_agent", return_value={"cost_usd": 0.1}), patch.object(
runner.time, "sleep"
):
if runner is auto:
record, = runner.run_one(lab, url, ["baseline"], "model", 1, 1)
else:
record = runner.do_run(lab, url, "baseline", "model", 1, 1)
self.assertIsNone(record["solved"])
self.assertFalse(record["oracle_ok"])
self.assertEqual(record["cost_usd"], 0.1)