Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 35 additions & 0 deletions .github/workflows/eval-harness.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
name: eval-harness

# Sanity-checks the eval/ benchmark WITHOUT running the full agent eval (which
# needs Docker + Burp + an authed claude CLI and can't run on a hosted runner).
# Keeps the oracle parser, JSON configs, and the FP-trap app from rotting.

on:
pull_request:
paths:
- "eval/**"
- ".github/workflows/eval-harness.yml"
push:
branches: [main]
paths:
- "eval/**"
workflow_dispatch:

permissions:
contents: read

jobs:
harness:
name: eval harness sanity (network-free)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: py_compile all eval scripts
run: python3 -m py_compile eval/*.py
- name: oracle parser self-test
run: python3 eval/oracle_portswigger.py
- name: harness sanity (configs + oracle + fp_app)
run: python3 eval/test_harness.py
40 changes: 40 additions & 0 deletions eval/BASELINE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# Eval baseline — skills-on vs skills-off

The harness under `eval/` measures whether the `hunt-*` skills actually help an
autonomous agent find bugs, via a skills-on / skills-off ablation on a self-grading
target. The full run needs **Docker + Burp + an authed `claude` CLI**, so it runs
locally, not in CI. CI only sanity-checks the harness (`eval-harness.yml`); the
numbers below are produced by running the eval yourself and committing the table.

## How to produce the numbers

See `eval/README.md` for full setup. Short version:

```bash
# v0 — OWASP Juice Shop (memorized; weak delta, quick proof)
docker run -d -p 3001:3000 --name juiceshop bkimminich/juice-shop
python3 eval/run_eval.py # both conditions, full set

# v1 — PortSwigger Web Security Academy (stronger, less memorized)
cp eval/burp-mcp.json.example eval/burp-mcp.json # set your mcp-proxy jar path
python3 eval/run_eval_ps_auto.py # baseline,skills (needs playwright + PS creds)
```

Results stream to `eval/results/*.jsonl` (gitignored). Summarize into the table
below and commit **this file** (not the raw run artifacts).

## Latest baseline

- Date: _YYYY-MM-DD_
- Model (held constant across conditions): _e.g. claude-sonnet-4-6_
- Oracle: _juice-shop v0 / portswigger v1_
- Labs/challenges attempted: _N_

| Condition | Solved | Solve-rate | Median turns | Median $ |
|-------------|-------:|-----------:|-------------:|---------:|
| skills-off | | | | |
| skills-on | | | | |
| **delta** | | | | |

Notes: _one or two lines — which classes the skills helped most on, any caveats
(memorized labs, flaky oracle, etc.)._
141 changes: 141 additions & 0 deletions eval/test_harness.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""
Harness sanity checks for the eval/ benchmark — network-free, CI-able.
The real ablation eval (run_eval*.py) needs Docker + Burp + an authed claude CLI,
so it can't run on a hosted CI runner. But the harness around it can still rot:
the oracle parser, the JSON config shapes, and the FP-trap app drifting from its
ground-truth cases. This checks all of that without any external dependency:
1. every eval/*.json config parses and has the keys the runners expect
2. oracle_portswigger classifies the widget markup correctly
3. fp_app.py starts and serves every path listed in fp_cases.json
(keeps the benchmark's cases and the app in sync)
Run: python3 eval/test_harness.py (exit 0 = pass). Stdlib only.
"""
import json
import os
import socket
import subprocess
import sys
import time
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import oracle_portswigger as ORACLE # noqa: E402


def check_configs():
errs = []
for name in ("challenges.json", "ps_labs.json", "ps_labs_hard.json", "fp_cases.json"):
p = os.path.join(HERE, name)
if not os.path.exists(p):
errs.append(f"{name}: missing")
continue
try:
data = json.load(open(p))
except Exception as e:
errs.append(f"{name}: invalid JSON ({e})")
continue
if not data:
errs.append(f"{name}: empty")
# fp_cases requires a specific shape the FP runner depends on
cases = json.load(open(os.path.join(HERE, "fp_cases.json")))
for c in cases:
for k in ("key", "path", "class", "ground"):
if k not in c:
errs.append(f"fp_cases.json: case {c.get('key','?')} missing '{k}'")
if c.get("ground") not in ("safe", "vulnerable"):
errs.append(f"fp_cases.json: case {c.get('key','?')} bad ground '{c.get('ground')}'")
return errs


class _NoRedirect(urllib.request.HTTPRedirectHandler):
"""Keep 3xx as-is — a served open-redirect trap must not be auto-followed to a 404."""
def redirect_request(self, *a, **k):
return None


def check_oracle():
errs = []
cases = [
("<div class='widgetcontainer-lab-status is-solved'><p>Solved</p></div>", True),
("<div class='widgetcontainer-lab-status is-notsolved'><p>Not solved</p></div>", False),
('<div class="widgetcontainer-lab-status is-solved"><p>Solved</p></div>', True),
("<html>nothing here</html>", None),
]
for html, want in cases:
got, _ = ORACLE._classify(html)
if got != want:
errs.append(f"oracle: classified {want!r} case as {got!r}")
return errs


def _free_port():
s = socket.socket()
s.bind(("127.0.0.1", 0))
port = s.getsockname()[1]
s.close()
return port


def check_fp_app():
errs = []
port = _free_port()
proc = subprocess.Popen([sys.executable, os.path.join(HERE, "fp_app.py"), str(port)],
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
base = f"http://127.0.0.1:{port}"
try:
# wait for it to come up
up = False
for _ in range(50):
try:
urllib.request.urlopen(base + "/", timeout=1)
up = True
break
except Exception:
time.sleep(0.1)
if not up:
return ["fp_app: did not start within 5s"]
# every case path must be served (not 404) — keeps cases <-> app aligned.
# A path ending in '=' is a query param → append a value; otherwise use as-is
# (appending would corrupt an exact-match route like /api/public-config).
# Don't follow redirects, so the /redirect trap's 302 counts as served.
opener = urllib.request.build_opener(_NoRedirect)
cases = json.load(open(os.path.join(HERE, "fp_cases.json")))
for c in cases:
url = base + c["path"] + ("x" if c["path"].endswith("=") else "")
try:
code = opener.open(url, timeout=3).getcode()
except urllib.error.HTTPError as e:
code = e.code # 3xx handled here because redirects are disabled
except Exception as e:
errs.append(f"fp_app: {c['path']} errored ({e})")
continue
if code == 404:
errs.append(f"fp_app: case '{c['key']}' path {c['path']} returns 404 "
f"— fp_cases.json and fp_app.py are out of sync")
finally:
proc.terminate()
try:
proc.wait(timeout=5)
except Exception:
proc.kill()
return errs


def main():
errs = check_configs() + check_oracle() + check_fp_app()
for e in errs:
print(f"ERROR {e}")
if errs:
print(f"\nFAIL: {len(errs)} harness issue(s).")
return 1
print("PASS: eval harness sanity (configs · oracle parser · fp_app serves all cases).")
return 0


if __name__ == "__main__":
sys.exit(main())
Loading