From 3a662c2152f9eab4a29aca6054407333d575666c Mon Sep 17 00:00:00 2001 From: Sachin Sharma Date: Mon, 24 Aug 2026 21:06:10 +0530 Subject: [PATCH] ci(eval): harness sanity gate + baseline scaffold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3 (BugHunter). The full ablation eval (run_eval*.py — skills-on vs skills-off on a self-grading target) needs Docker + Burp + an authed claude CLI, so it runs locally, not in CI. But the harness around it can silently rot. This adds the CI-able half plus a place to record the real numbers: - .github/workflows/eval-harness.yml — on eval/ changes: py_compile every eval/*.py, run the oracle parser self-test, run the new harness sanity test. Network-free; contents: read. - eval/test_harness.py — asserts (1) every eval/*.json config parses with the keys the runners expect, (2) oracle_portswigger classifies the lab-status widget correctly, (3) fp_app.py starts and serves every path in fp_cases.json (keeps the FP-trap app and its ground-truth cases in sync). Stdlib, localhost. - eval/BASELINE.md — the local run procedure + a results table to fill in (skills-on/off solve-rate delta) and commit after a real run. Building test_harness caught two bugs in my own probe (not the app): appending a canary to an exact-match route corrupted it, and urllib auto-followed the open-redirect trap's 302 to a 404. Fixed (query-aware URL + no-follow opener); fp_app and fp_cases are in sync. The real skills-on/off baseline is produced by running the eval locally and committing BASELINE.md — CI can't run it. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/eval-harness.yml | 35 +++++++ eval/BASELINE.md | 40 ++++++++ eval/test_harness.py | 141 +++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+) create mode 100644 .github/workflows/eval-harness.yml create mode 100644 eval/BASELINE.md create mode 100644 eval/test_harness.py diff --git a/.github/workflows/eval-harness.yml b/.github/workflows/eval-harness.yml new file mode 100644 index 00000000..f1aff4bc --- /dev/null +++ b/.github/workflows/eval-harness.yml @@ -0,0 +1,35 @@ +name: eval-harness + +# Sanity-checks the eval/ benchmark WITHOUT running the full agent eval (which +# needs Docker + Burp + an authed claude CLI and can't run on a hosted runner). +# Keeps the oracle parser, JSON configs, and the FP-trap app from rotting. + +on: + pull_request: + paths: + - "eval/**" + - ".github/workflows/eval-harness.yml" + push: + branches: [main] + paths: + - "eval/**" + workflow_dispatch: + +permissions: + contents: read + +jobs: + harness: + name: eval harness sanity (network-free) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: py_compile all eval scripts + run: python3 -m py_compile eval/*.py + - name: oracle parser self-test + run: python3 eval/oracle_portswigger.py + - name: harness sanity (configs + oracle + fp_app) + run: python3 eval/test_harness.py diff --git a/eval/BASELINE.md b/eval/BASELINE.md new file mode 100644 index 00000000..bb84cf5d --- /dev/null +++ b/eval/BASELINE.md @@ -0,0 +1,40 @@ +# Eval baseline — skills-on vs skills-off + +The harness under `eval/` measures whether the `hunt-*` skills actually help an +autonomous agent find bugs, via a skills-on / skills-off ablation on a self-grading +target. The full run needs **Docker + Burp + an authed `claude` CLI**, so it runs +locally, not in CI. CI only sanity-checks the harness (`eval-harness.yml`); the +numbers below are produced by running the eval yourself and committing the table. + +## How to produce the numbers + +See `eval/README.md` for full setup. Short version: + +```bash +# v0 — OWASP Juice Shop (memorized; weak delta, quick proof) +docker run -d -p 3001:3000 --name juiceshop bkimminich/juice-shop +python3 eval/run_eval.py # both conditions, full set + +# v1 — PortSwigger Web Security Academy (stronger, less memorized) +cp eval/burp-mcp.json.example eval/burp-mcp.json # set your mcp-proxy jar path +python3 eval/run_eval_ps_auto.py # baseline,skills (needs playwright + PS creds) +``` + +Results stream to `eval/results/*.jsonl` (gitignored). Summarize into the table +below and commit **this file** (not the raw run artifacts). + +## Latest baseline + +- Date: _YYYY-MM-DD_ +- Model (held constant across conditions): _e.g. claude-sonnet-4-6_ +- Oracle: _juice-shop v0 / portswigger v1_ +- Labs/challenges attempted: _N_ + +| Condition | Solved | Solve-rate | Median turns | Median $ | +|-------------|-------:|-----------:|-------------:|---------:| +| skills-off | | | | | +| skills-on | | | | | +| **delta** | | | | | + +Notes: _one or two lines — which classes the skills helped most on, any caveats +(memorized labs, flaky oracle, etc.)._ diff --git a/eval/test_harness.py b/eval/test_harness.py new file mode 100644 index 00000000..a33c81c9 --- /dev/null +++ b/eval/test_harness.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +""" +Harness sanity checks for the eval/ benchmark — network-free, CI-able. + +The real ablation eval (run_eval*.py) needs Docker + Burp + an authed claude CLI, +so it can't run on a hosted CI runner. But the harness around it can still rot: +the oracle parser, the JSON config shapes, and the FP-trap app drifting from its +ground-truth cases. This checks all of that without any external dependency: + + 1. every eval/*.json config parses and has the keys the runners expect + 2. oracle_portswigger classifies the widget markup correctly + 3. fp_app.py starts and serves every path listed in fp_cases.json + (keeps the benchmark's cases and the app in sync) + +Run: python3 eval/test_harness.py (exit 0 = pass). Stdlib only. +""" +import json +import os +import socket +import subprocess +import sys +import time +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import oracle_portswigger as ORACLE # noqa: E402 + + +def check_configs(): + errs = [] + for name in ("challenges.json", "ps_labs.json", "ps_labs_hard.json", "fp_cases.json"): + p = os.path.join(HERE, name) + if not os.path.exists(p): + errs.append(f"{name}: missing") + continue + try: + data = json.load(open(p)) + except Exception as e: + errs.append(f"{name}: invalid JSON ({e})") + continue + if not data: + errs.append(f"{name}: empty") + # fp_cases requires a specific shape the FP runner depends on + cases = json.load(open(os.path.join(HERE, "fp_cases.json"))) + for c in cases: + for k in ("key", "path", "class", "ground"): + if k not in c: + errs.append(f"fp_cases.json: case {c.get('key','?')} missing '{k}'") + if c.get("ground") not in ("safe", "vulnerable"): + errs.append(f"fp_cases.json: case {c.get('key','?')} bad ground '{c.get('ground')}'") + return errs + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Keep 3xx as-is — a served open-redirect trap must not be auto-followed to a 404.""" + def redirect_request(self, *a, **k): + return None + + +def check_oracle(): + errs = [] + cases = [ + ("

Solved

", True), + ("

Not solved

", False), + ('

Solved

', True), + ("nothing here", None), + ] + for html, want in cases: + got, _ = ORACLE._classify(html) + if got != want: + errs.append(f"oracle: classified {want!r} case as {got!r}") + return errs + + +def _free_port(): + s = socket.socket() + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + s.close() + return port + + +def check_fp_app(): + errs = [] + port = _free_port() + proc = subprocess.Popen([sys.executable, os.path.join(HERE, "fp_app.py"), str(port)], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + base = f"http://127.0.0.1:{port}" + try: + # wait for it to come up + up = False + for _ in range(50): + try: + urllib.request.urlopen(base + "/", timeout=1) + up = True + break + except Exception: + time.sleep(0.1) + if not up: + return ["fp_app: did not start within 5s"] + # every case path must be served (not 404) — keeps cases <-> app aligned. + # A path ending in '=' is a query param → append a value; otherwise use as-is + # (appending would corrupt an exact-match route like /api/public-config). + # Don't follow redirects, so the /redirect trap's 302 counts as served. + opener = urllib.request.build_opener(_NoRedirect) + cases = json.load(open(os.path.join(HERE, "fp_cases.json"))) + for c in cases: + url = base + c["path"] + ("x" if c["path"].endswith("=") else "") + try: + code = opener.open(url, timeout=3).getcode() + except urllib.error.HTTPError as e: + code = e.code # 3xx handled here because redirects are disabled + except Exception as e: + errs.append(f"fp_app: {c['path']} errored ({e})") + continue + if code == 404: + errs.append(f"fp_app: case '{c['key']}' path {c['path']} returns 404 " + f"— fp_cases.json and fp_app.py are out of sync") + finally: + proc.terminate() + try: + proc.wait(timeout=5) + except Exception: + proc.kill() + return errs + + +def main(): + errs = check_configs() + check_oracle() + check_fp_app() + for e in errs: + print(f"ERROR {e}") + if errs: + print(f"\nFAIL: {len(errs)} harness issue(s).") + return 1 + print("PASS: eval harness sanity (configs · oracle parser · fp_app serves all cases).") + return 0 + + +if __name__ == "__main__": + sys.exit(main())