review: the live A/B runner (phase 3)

.changeset/review-live-ab-runner.md

@@ -0,0 +1,5 @@
+---
+"review": minor
+---
+
+The live A/B runner (live A/B plan, phase 3). `eval/live-match.ts` maps a run's posted candidates onto a live case's labeled defect specs (deterministic path/window/mechanism rule first, then an injected hard-capped fallback arbiter whose matches are recorded for audit) and aggregates live metrics: must-catch recall, verdict agreement, clean false-flag, and noise. `eval/live-ab.ts` runs the model sub-agents from two review.md versions (the merge-base baseline via `git show`, the working-tree candidate) over the same live corpus, each arm under half a hard USD budget with sticky stop-and-report-skips semantics, replays each live result through the deterministic pipeline (`runner.ts` gains a `validation` override on `RunOptions` so a live validator's output replaces the recorded block), diffs spec-level regressions over the cases both arms scored, judges both arms' comments with the pinned judge (quality aggregates only; extracted to `eval/judge-live-model.ts`, now shared with `live-judge.ts`), and emits a JSON report plus a markdown table (also appended to GITHUB_STEP_SUMMARY). Report-only except the playbook's standing rule: the candidate arm failing any adversarial-injection case exits non-zero.

.changeset/review-live-producer-staging.md

@@ -0,0 +1,5 @@
+---
+"review": minor
+---
+
+The live producer (live A/B plan, phase 2): everything needed to run the REAL model sub-agents over a live-enabled corpus case. `eval/agent-extract.ts` parses `review.md`'s `## agent:` sections into name/description/model/prompt as pure data (string in, no fs), so an A/B can extract the baseline arm from a `git show` of the merge-base; parsing is strict and lists every malformed section at once, and an integration test runs it over the real `review.md`. `eval/live-stage.ts` materializes the production `/tmp/gh-aw/review/` staging layout for a case (pr-context.json, full/pr/full-stripped diffs, files.json with hasPatch, review-files.json, provenance.json, routing.json, out/, and the post-change checkout) and rewrites extracted prompts to point at it. `eval/live-producer.ts` dispatches the default finders plus the router's lenses through an injected runner seam, maps all three output contracts (label-shape reviewers, structured-schema lenses, the claim-validator's three-state verdicts) into the exact `RecordedFinding`/`CaseVerification` shapes the deterministic runner consumes, stages `claims.json`, resolves `{{#runtime-import}}` directives from the case tree, retries once on malformed output, keeps partial results on agent failure, and accounts per-agent cost. `eval/live-runner.ts` is the one SDK-backed runner (Claude Agent SDK; Read/Grep/Glob only, cwd pinned to the staged checkout, hard turn and wall-clock caps) plus a CLI smoke entry (`pnpm dlx tsx workflows/review/eval/live-runner.ts --case <id>`, requires ANTHROPIC_API_KEY).

package.json

@@ -10,6 +10,7 @@
         "build": "tsc -p actions/tsconfig.json"
     },
     "devDependencies": {
+        "@anthropic-ai/claude-agent-sdk": "^0.3.205",
         "@changesets/cli": "^2.29.8",
         "@khanacademy/eslint-config": "^0.1.0",
         "@swc-node/register": "^1.11.1",

workflows/review/eval/agent-extract.test.ts

@@ -0,0 +1,130 @@
+import {readFileSync} from "node:fs";
+
+import {describe, it, expect} from "vitest";
+
+import {AgentExtractError, extractAgents} from "./agent-extract";
+import {PRODUCTION_REVIEW_DIR} from "./live-stage";
+
+/** Build a well-formed agent section. */
+const section = (
+    name: string,
+    body = "Review the diff and return JSON.",
+    frontmatter?: string,
+): string =>
+    [
+        `## agent: \`${name}\``,
+        "---",
+        frontmatter ??
+            [
+                `name: ${name}`,
+                `description: A ${name} test agent.`,
+                "model: claude-opus-4-8",
+                "# effort: high (launch default)",
+            ].join("\n"),
+        "---",
+        body,
+        "",
+    ].join("\n");
+
+describe("extractAgents: fixture markdown", () => {
+    it("extracts sections in order with frontmatter fields and body", () => {
+        const md = [
+            "# Workflow prose the extractor must skip",
+            "",
+            section("alpha", "Alpha prompt body.\n\nWith two paragraphs."),
+            section("beta"),
+        ].join("\n");
+        const agents = extractAgents(md);
+        expect([...agents.keys()]).toEqual(["alpha", "beta"]);
+        const alpha = agents.get("alpha");
+        expect(alpha?.model).toBe("claude-opus-4-8");
+        expect(alpha?.description).toBe("A alpha test agent.");
+        expect(alpha?.prompt).toBe(
+            "Alpha prompt body.\n\nWith two paragraphs.",
+        );
+    });
+
+    it("ignores comment lines in frontmatter (the effort annotations)", () => {
+        const agents = extractAgents(section("gamma"));
+        expect(agents.get("gamma")?.model).toBe("claude-opus-4-8");
+    });
+
+    it("throws listing every problem at once", () => {
+        const md = [
+            section("good"),
+            // name mismatch
+            section(
+                "bad-name",
+                "Body.",
+                ["name: other", "description: d", "model: m"].join("\n"),
+            ),
+            // missing model
+            section(
+                "no-model",
+                "Body.",
+                ["name: no-model", "description: d"].join("\n"),
+            ),
+        ].join("\n");
+        let message = "";
+        try {
+            extractAgents(md);
+        } catch (error) {
+            expect(error).toBeInstanceOf(AgentExtractError);
+            message = (error as Error).message;
+        }
+        expect(message).toMatch(/bad-name.*does not match heading/);
+        expect(message).toMatch(/no-model.*missing frontmatter model/);
+    });
+
+    it("throws on an unterminated frontmatter block", () => {
+        const md = ["## agent: `broken`", "---", "name: broken", ""].join("\n");
+        expect(() => extractAgents(md)).toThrow(/unterminated frontmatter/);
+    });
+
+    it("throws on duplicate agent names", () => {
+        expect(() => extractAgents(section("dup") + section("dup"))).toThrow(
+            /duplicate agent name/,
+        );
+    });
+
+    it("throws when no agent sections exist", () => {
+        expect(() => extractAgents("# just prose")).toThrow(
+            /no `## agent:` sections/,
+        );
+    });
+});
+
+describe("extractAgents: the real review.md", () => {
+    const markdown = readFileSync("workflows/review/review.md", "utf8");
+    const agents = extractAgents(markdown);
+
+    it("extracts every `## agent:` section in the file", () => {
+        const headings = markdown
+            .split("\n")
+            .filter((line) => /^## agent: `[^`]+`\s*$/.test(line));
+        expect(agents.size).toBe(headings.length);
+        expect(agents.size).toBeGreaterThanOrEqual(21);
+    });
+
+    it("pins a model on every agent", () => {
+        for (const agent of agents.values()) {
+            expect(agent.model).toMatch(/^claude-/);
+        }
+    });
+
+    it("the default-roster reviewers reference the production staging root", () => {
+        // The staging-path rewrite (live-stage.ts) depends on prompts naming
+        // PRODUCTION_REVIEW_DIR verbatim; if review.md renames the staging
+        // root, this fails here rather than silently staging nothing.
+        for (const name of [
+            "correctness-reviewer",
+            "skill-auditor",
+            "claim-validator",
+        ]) {
+            expect(
+                agents.get(name)?.prompt,
+                `${name} missing ${PRODUCTION_REVIEW_DIR}`,
+            ).toContain(PRODUCTION_REVIEW_DIR);
+        }
+    });
+});

workflows/review/eval/agent-extract.ts

@@ -0,0 +1,162 @@
+/**
+ * Sub-agent prompt extraction for the live eval arm (`live-ab-plan.md`
+ * Phase 2a).
+ *
+ * `review.md` defines every reviewer sub-agent as a `## agent: \`name\``
+ * section: YAML-ish frontmatter (`name`, `description`, `model`, plus
+ * comment lines) followed by the prompt body. The live producer needs those
+ * prompts as data, for BOTH arms of an A/B: the candidate arm reads the
+ * working tree's `review.md`, the baseline arm reads the merge-base version
+ * via `git show`. So the core function takes the markdown as a string and
+ * performs no filesystem access.
+ *
+ * Parsing is deliberately strict: a section whose frontmatter is missing,
+ * unterminated, name-mismatched, or model-less throws rather than degrading,
+ * because a silently dropped agent would skew an eval arm without failing it.
+ */
+
+/** One extracted sub-agent definition. */
+export type ExtractedAgent = {
+    /** The agent name (heading and frontmatter must agree). */
+    name: string;
+    /** The frontmatter description line. */
+    description: string;
+    /** The pinned model id (e.g. `claude-opus-4-8`). */
+    model: string;
+    /** The full prompt body (everything after the frontmatter). */
+    prompt: string;
+};
+
+/** Thrown when `review.md`'s agent sections cannot be parsed. */
+export class AgentExtractError extends Error {
+    constructor(errors: string[]) {
+        super(
+            `Malformed agent section(s) in review markdown:\n${errors
+                .map((e) => `  - ${e}`)
+                .join("\n")}`,
+        );
+        this.name = "AgentExtractError";
+    }
+}
+
+const HEADING = /^## agent: `([^`]+)`\s*$/;
+
+/** Parse one frontmatter block's `key: value` lines (comments ignored). */
+const parseFrontmatter = (lines: string[]): Map<string, string> => {
+    const out = new Map<string, string>();
+    for (const line of lines) {
+        if (line.trim() === "" || line.trim().startsWith("#")) {
+            continue;
+        }
+        const colon = line.indexOf(":");
+        if (colon === -1) {
+            // A wrapped comment continuation or stray text; skip rather than
+            // guess. Required keys are checked by the caller.
+            continue;
+        }
+        const key = line.slice(0, colon).trim();
+        const value = line.slice(colon + 1).trim();
+        if (key.length > 0 && !out.has(key)) {
+            out.set(key, value);
+        }
+    }
+    return out;
+};
+
+/**
+ * Extract every `## agent:` section from review markdown. Returns agents in
+ * definition order (a Map preserves insertion order). Throws
+ * {@link AgentExtractError} listing every problem at once.
+ */
+export const extractAgents = (
+    markdown: string,
+): Map<string, ExtractedAgent> => {
+    const lines = markdown.split("\n");
+    const agents = new Map<string, ExtractedAgent>();
+    const errors: string[] = [];
+
+    /** Indices of every agent heading, plus a sentinel end. */
+    const headings: {index: number; name: string}[] = [];
+    lines.forEach((line, index) => {
+        const match = HEADING.exec(line);
+        if (match?.[1] !== undefined) {
+            headings.push({index, name: match[1]});
+        }
+    });
+
+    headings.forEach((heading, i) => {
+        const sectionEnd = headings[i + 1]?.index ?? lines.length;
+        const at = `agent \`${heading.name}\` (line ${heading.index + 1})`;
+
+        // Frontmatter: first non-blank line after the heading must open it.
+        let cursor = heading.index + 1;
+        while (cursor < sectionEnd && (lines[cursor] ?? "").trim() === "") {
+            cursor++;
+        }
+        if ((lines[cursor] ?? "").trim() !== "---") {
+            errors.push(`${at}: expected frontmatter opening ---`);
+            return;
+        }
+        const fmStart = cursor + 1;
+        let fmEnd = -1;
+        for (let j = fmStart; j < sectionEnd; j++) {
+            if ((lines[j] ?? "").trim() === "---") {
+                fmEnd = j;
+                break;
+            }
+        }
+        if (fmEnd === -1) {
+            errors.push(`${at}: unterminated frontmatter`);
+            return;
+        }
+
+        const frontmatter = parseFrontmatter(lines.slice(fmStart, fmEnd));
+        const name = frontmatter.get("name");
+        const description = frontmatter.get("description");
+        const model = frontmatter.get("model");
+        if (name !== heading.name) {
+            errors.push(
+                `${at}: frontmatter name "${
+                    name ?? ""
+                }" does not match heading`,
+            );
+        }
+        if (description === undefined || description === "") {
+            errors.push(`${at}: missing frontmatter description`);
+        }
+        if (model === undefined || model === "") {
+            errors.push(`${at}: missing frontmatter model`);
+        }
+        if (agents.has(heading.name)) {
+            errors.push(`${at}: duplicate agent name`);
+        }
+        if (
+            name !== heading.name ||
+            description === undefined ||
+            description === "" ||
+            model === undefined ||
+            model === "" ||
+            agents.has(heading.name)
+        ) {
+            return;
+        }
+
+        const prompt = lines
+            .slice(fmEnd + 1, sectionEnd)
+            .join("\n")
+            .trim();
+        if (prompt === "") {
+            errors.push(`${at}: empty prompt body`);
+            return;
+        }
+        agents.set(heading.name, {name, description, model, prompt});
+    });
+
+    if (errors.length > 0) {
+        throw new AgentExtractError(errors);
+    }
+    if (agents.size === 0) {
+        throw new AgentExtractError(["no `## agent:` sections found"]);
+    }
+    return agents;
+};