Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 21 additions & 10 deletions eval/run_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,12 @@ def oracle(base, name):
data = json.load(r)["data"]
except Exception:
return None, False
for c in data:
if c["name"] == name:
return bool(c["solved"]), True
if not isinstance(data, list):
return None, False
for challenge in data:
if isinstance(challenge, dict) and challenge.get("name") == name:
solved = challenge.get("solved")
return (solved, True) if isinstance(solved, bool) else (None, False)
return None, False


Expand Down Expand Up @@ -155,7 +158,8 @@ def main():
ap.add_argument("--out", default=os.path.join(EVAL, "results", "run.jsonl"))
a = ap.parse_args()

challenges = json.load(open(a.challenges))
with open(a.challenges, encoding="utf-8") as config:
challenges = json.load(config)
if a.limit:
challenges = challenges[:a.limit]
conditions = [c.strip() for c in a.conditions.split(",") if c.strip()]
Expand All @@ -178,11 +182,10 @@ def main():
print(f" running agent (max_turns={a.max_turns}, timeout={a.timeout}s)...", flush=True)
r = run_agent(build_task(ch["objective"], a.base, JUICE_DESC), skills_on, a.model, a.max_turns, a.timeout)
time.sleep(2)
post, _ = oracle(a.base, ch["oracle_name"])
solved = bool(post)
solved, oracle_ok = oracle(a.base, ch["oracle_name"])
rec = {"ts": datetime.now(timezone.utc).isoformat(), "challenge": ch["key"],
"category": ch["category"], "skill_hint": ch.get("skill_hint"),
"condition": cond, "solved": solved, **r}
"condition": cond, "solved": solved, "oracle_ok": oracle_ok, **r}
rows.append(rec)
out.write(json.dumps(rec) + "\n"); out.flush()
print(f" -> solved={solved} turns={r.get('num_turns')} "
Expand All @@ -196,16 +199,24 @@ def main():
for r in rows:
by_cond.setdefault(r["condition"], []).append(r)
for cond, rs in by_cond.items():
s = sum(1 for r in rs if r["solved"])
scored = [r for r in rs if r["oracle_ok"]]
s = sum(1 for r in scored if r["solved"])
cost = sum((r.get("cost_usd") or 0) for r in rs)
print(f" {cond:9s}: solved {s}/{len(rs)} total ${cost:.2f}")
unreadable = len(rs) - len(scored)
note = f" ({unreadable} oracle-unreadable, excluded)" if unreadable else ""
print(f" {cond:9s}: solved {s}/{len(scored)} total ${cost:.2f}{note}")
print("\n per-challenge:")
print(f" {'challenge':24s} " + " ".join(f"{c:>9s}" for c in conditions))
for ch in challenges:
cells = []
for cond in conditions:
m = [r for r in rows if r["challenge"] == ch["key"] and r["condition"] == cond]
cells.append("✓" if (m and m[0]["solved"]) else ("✗" if m else "-"))
if not m:
cells.append("-")
elif not m[0]["oracle_ok"]:
cells.append("?")
else:
cells.append("✓" if m[0]["solved"] else "✗")
print(f" {ch['key']:24s} " + " ".join(f"{c:>9s}" for c in cells))
print(f"\n raw: {a.out}")

Expand Down
55 changes: 55 additions & 0 deletions tests/test_eval_juice_oracle.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import contextlib
import io
import json
import sys
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch

from eval import run_eval


class JuiceShopOracleTests(unittest.TestCase):
def test_unknown_or_malformed_challenges_are_unreadable(self):
for payload in (
{"data": None},
{"data": [{"name": "fixture"}]},
{"data": [{"name": "fixture", "solved": "false"}]},
{"data": [{"name": "other", "solved": True}]},
):
with self.subTest(payload=payload), patch.object(
run_eval.urllib.request, "urlopen", return_value=contextlib.nullcontext(
io.StringIO(json.dumps(payload))
)
):
self.assertEqual(run_eval.oracle("http://localhost", "fixture"), (None, False))

def test_oracle_loss_preserves_attempt_without_counting_a_miss(self):
challenge = {"key": "fixture", "category": "test", "objective": "fixture",
"oracle_name": "fixture"}
with tempfile.TemporaryDirectory() as tmp:
config = Path(tmp, "challenges.json")
output = Path(tmp, "results.jsonl")
config.write_text(json.dumps([challenge]), encoding="utf-8")
args = ["run_eval.py", "--challenges", str(config), "--out", str(output),
"--conditions", "baseline"]
with patch.object(sys, "argv", args), patch.object(
run_eval, "reset_target", return_value=True
), patch.object(
run_eval, "oracle", side_effect=[(False, True), (None, False)]
), patch.object(
run_eval, "run_agent", return_value={"cost_usd": 0.25}
), patch.object(run_eval.time, "sleep"), contextlib.redirect_stdout(io.StringIO()) as stdout:
run_eval.main()

row = json.loads(output.read_text(encoding="utf-8"))
self.assertIsNone(row["solved"])
self.assertFalse(row["oracle_ok"])
self.assertIn("solved 0/0", stdout.getvalue())
self.assertIn("total $0.25", stdout.getvalue())
self.assertIn("1 oracle-unreadable", stdout.getvalue())


if __name__ == "__main__":
unittest.main()
Loading