review: the live producer (live A/B phase 2)

.changeset/review-live-corpus-format.md

@@ -0,0 +1,5 @@
+---
+"review": minor
+---
+
+Live-enabled corpus cases (live A/B eval plan, phase 1). The eval corpus gains an opt-in `live` block carrying what a REAL model run needs and the deterministic replay does not: the PR context (`prContext`, with the description treated as untrusted author text), a post-change file tree on disk next to the case (`<id>/case.json` + `<id>/tree/`, coexisting with the flat `<id>.json` layout), and labeled defect specs (`mustCatchSpecs` / `mustNotFlagSpecs`: path, line window, mechanism keyword alternates) so a live run's model-chosen finding ids can be matched to ground truth. A live case must carry the `live` tag, a cleanly-parseable diff, and every non-removed changed file in its tree; the loader enforces all of it and `loadLiveCorpus()` returns the subset. Ten cases are converted with hand-authored real diffs and trees (five smoke incidents, both clean cases, one adversarial injection whose payload lives in the diff, one golden holdout, one synthetic mutation); their recorded line anchors now point at the authored defect lines, and the deterministic suite runs them unchanged, provenance gate included.

.changeset/review-live-producer-staging.md

@@ -0,0 +1,5 @@
+---
+"review": minor
+---
+
+The live producer (live A/B plan, phase 2): everything needed to run the REAL model sub-agents over a live-enabled corpus case. `eval/agent-extract.ts` parses `review.md`'s `## agent:` sections into name/description/model/prompt as pure data (string in, no fs), so an A/B can extract the baseline arm from a `git show` of the merge-base; parsing is strict and lists every malformed section at once, and an integration test runs it over the real `review.md`. `eval/live-stage.ts` materializes the production `/tmp/gh-aw/review/` staging layout for a case (pr-context.json, full/pr/full-stripped diffs, files.json with hasPatch, review-files.json, provenance.json, routing.json, out/, and the post-change checkout) and rewrites extracted prompts to point at it. `eval/live-producer.ts` dispatches the default finders plus the router's lenses through an injected runner seam, maps all three output contracts (label-shape reviewers, structured-schema lenses, the claim-validator's three-state verdicts) into the exact `RecordedFinding`/`CaseVerification` shapes the deterministic runner consumes, stages `claims.json`, resolves `{{#runtime-import}}` directives from the case tree, retries once on malformed output, keeps partial results on agent failure, and accounts per-agent cost. `eval/live-runner.ts` is the one SDK-backed runner (Claude Agent SDK; Read/Grep/Glob only, cwd pinned to the staged checkout, hard turn and wall-clock caps) plus a CLI smoke entry (`pnpm dlx tsx workflows/review/eval/live-runner.ts --case <id>`, requires ANTHROPIC_API_KEY).

.eslintrc.js

@@ -26,7 +26,14 @@ module.exports = {
         sourceType: "module",
         ecmaVersion: 2022,
     },
-    ignorePatterns: ["**/dist/", "actions/**/index.js"],
+    ignorePatterns: [
+        "**/dist/",
+        "actions/**/index.js",
+        // Live eval-corpus trees are byte-exact fixtures paired with each
+        // case's diff: linting (and especially prettier auto-formatting)
+        // would desync them from the diff the provenance gate parses.
+        "workflows/review/eval/corpus/**/tree/",
+    ],
     overrides: [
         {
             files: "**/*.ts",

package.json

@@ -10,6 +10,7 @@
         "build": "tsc -p actions/tsconfig.json"
     },
     "devDependencies": {
+        "@anthropic-ai/claude-agent-sdk": "^0.3.205",
         "@changesets/cli": "^2.29.8",
         "@khanacademy/eslint-config": "^0.1.0",
         "@swc-node/register": "^1.11.1",

vitest.config.ts

@@ -1,9 +1,16 @@
-import {defineConfig} from "vitest/config";
+import {configDefaults, defineConfig} from "vitest/config";
 
 export default defineConfig({
     test: {
         watch: false,
         clearMocks: true,
         setupFiles: ["./config/tests/setup.ts"],
+        // Live eval-corpus trees are case fixtures, not suite code: a tree
+        // may legitimately carry a *.test.ts whose tests fail by design
+        // (the test-adequacy cases), so vitest must never execute them.
+        exclude: [
+            ...configDefaults.exclude,
+            "workflows/review/eval/corpus/**/tree/**",
+        ],
     },
 });

workflows/review/eval/agent-extract.test.ts

@@ -0,0 +1,130 @@
+import {readFileSync} from "node:fs";
+
+import {describe, it, expect} from "vitest";
+
+import {AgentExtractError, extractAgents} from "./agent-extract";
+import {PRODUCTION_REVIEW_DIR} from "./live-stage";
+
+/** Build a well-formed agent section. */
+const section = (
+    name: string,
+    body = "Review the diff and return JSON.",
+    frontmatter?: string,
+): string =>
+    [
+        `## agent: \`${name}\``,
+        "---",
+        frontmatter ??
+            [
+                `name: ${name}`,
+                `description: A ${name} test agent.`,
+                "model: claude-opus-4-8",
+                "# effort: high (launch default)",
+            ].join("\n"),
+        "---",
+        body,
+        "",
+    ].join("\n");
+
+describe("extractAgents: fixture markdown", () => {
+    it("extracts sections in order with frontmatter fields and body", () => {
+        const md = [
+            "# Workflow prose the extractor must skip",
+            "",
+            section("alpha", "Alpha prompt body.\n\nWith two paragraphs."),
+            section("beta"),
+        ].join("\n");
+        const agents = extractAgents(md);
+        expect([...agents.keys()]).toEqual(["alpha", "beta"]);
+        const alpha = agents.get("alpha");
+        expect(alpha?.model).toBe("claude-opus-4-8");
+        expect(alpha?.description).toBe("A alpha test agent.");
+        expect(alpha?.prompt).toBe(
+            "Alpha prompt body.\n\nWith two paragraphs.",
+        );
+    });
+
+    it("ignores comment lines in frontmatter (the effort annotations)", () => {
+        const agents = extractAgents(section("gamma"));
+        expect(agents.get("gamma")?.model).toBe("claude-opus-4-8");
+    });
+
+    it("throws listing every problem at once", () => {
+        const md = [
+            section("good"),
+            // name mismatch
+            section(
+                "bad-name",
+                "Body.",
+                ["name: other", "description: d", "model: m"].join("\n"),
+            ),
+            // missing model
+            section(
+                "no-model",
+                "Body.",
+                ["name: no-model", "description: d"].join("\n"),
+            ),
+        ].join("\n");
+        let message = "";
+        try {
+            extractAgents(md);
+        } catch (error) {
+            expect(error).toBeInstanceOf(AgentExtractError);
+            message = (error as Error).message;
+        }
+        expect(message).toMatch(/bad-name.*does not match heading/);
+        expect(message).toMatch(/no-model.*missing frontmatter model/);
+    });
+
+    it("throws on an unterminated frontmatter block", () => {
+        const md = ["## agent: `broken`", "---", "name: broken", ""].join("\n");
+        expect(() => extractAgents(md)).toThrow(/unterminated frontmatter/);
+    });
+
+    it("throws on duplicate agent names", () => {
+        expect(() => extractAgents(section("dup") + section("dup"))).toThrow(
+            /duplicate agent name/,
+        );
+    });
+
+    it("throws when no agent sections exist", () => {
+        expect(() => extractAgents("# just prose")).toThrow(
+            /no `## agent:` sections/,
+        );
+    });
+});
+
+describe("extractAgents: the real review.md", () => {
+    const markdown = readFileSync("workflows/review/review.md", "utf8");
+    const agents = extractAgents(markdown);
+
+    it("extracts every `## agent:` section in the file", () => {
+        const headings = markdown
+            .split("\n")
+            .filter((line) => /^## agent: `[^`]+`\s*$/.test(line));
+        expect(agents.size).toBe(headings.length);
+        expect(agents.size).toBeGreaterThanOrEqual(21);
+    });
+
+    it("pins a model on every agent", () => {
+        for (const agent of agents.values()) {
+            expect(agent.model).toMatch(/^claude-/);
+        }
+    });
+
+    it("the default-roster reviewers reference the production staging root", () => {
+        // The staging-path rewrite (live-stage.ts) depends on prompts naming
+        // PRODUCTION_REVIEW_DIR verbatim; if review.md renames the staging
+        // root, this fails here rather than silently staging nothing.
+        for (const name of [
+            "correctness-reviewer",
+            "skill-auditor",
+            "claim-validator",
+        ]) {
+            expect(
+                agents.get(name)?.prompt,
+                `${name} missing ${PRODUCTION_REVIEW_DIR}`,
+            ).toContain(PRODUCTION_REVIEW_DIR);
+        }
+    });
+});

workflows/review/eval/agent-extract.ts

@@ -0,0 +1,162 @@
+/**
+ * Sub-agent prompt extraction for the live eval arm (`live-ab-plan.md`
+ * Phase 2a).
+ *
+ * `review.md` defines every reviewer sub-agent as a `## agent: \`name\``
+ * section: YAML-ish frontmatter (`name`, `description`, `model`, plus
+ * comment lines) followed by the prompt body. The live producer needs those
+ * prompts as data, for BOTH arms of an A/B: the candidate arm reads the
+ * working tree's `review.md`, the baseline arm reads the merge-base version
+ * via `git show`. So the core function takes the markdown as a string and
+ * performs no filesystem access.
+ *
+ * Parsing is deliberately strict: a section whose frontmatter is missing,
+ * unterminated, name-mismatched, or model-less throws rather than degrading,
+ * because a silently dropped agent would skew an eval arm without failing it.
+ */
+
+/** One extracted sub-agent definition. */
+export type ExtractedAgent = {
+    /** The agent name (heading and frontmatter must agree). */
+    name: string;
+    /** The frontmatter description line. */
+    description: string;
+    /** The pinned model id (e.g. `claude-opus-4-8`). */
+    model: string;
+    /** The full prompt body (everything after the frontmatter). */
+    prompt: string;
+};
+
+/** Thrown when `review.md`'s agent sections cannot be parsed. */
+export class AgentExtractError extends Error {
+    constructor(errors: string[]) {
+        super(
+            `Malformed agent section(s) in review markdown:\n${errors
+                .map((e) => `  - ${e}`)
+                .join("\n")}`,
+        );
+        this.name = "AgentExtractError";
+    }
+}
+
+const HEADING = /^## agent: `([^`]+)`\s*$/;
+
+/** Parse one frontmatter block's `key: value` lines (comments ignored). */
+const parseFrontmatter = (lines: string[]): Map<string, string> => {
+    const out = new Map<string, string>();
+    for (const line of lines) {
+        if (line.trim() === "" || line.trim().startsWith("#")) {
+            continue;
+        }
+        const colon = line.indexOf(":");
+        if (colon === -1) {
+            // A wrapped comment continuation or stray text; skip rather than
+            // guess. Required keys are checked by the caller.
+            continue;
+        }
+        const key = line.slice(0, colon).trim();
+        const value = line.slice(colon + 1).trim();
+        if (key.length > 0 && !out.has(key)) {
+            out.set(key, value);
+        }
+    }
+    return out;
+};
+
+/**
+ * Extract every `## agent:` section from review markdown. Returns agents in
+ * definition order (a Map preserves insertion order). Throws
+ * {@link AgentExtractError} listing every problem at once.
+ */
+export const extractAgents = (
+    markdown: string,
+): Map<string, ExtractedAgent> => {
+    const lines = markdown.split("\n");
+    const agents = new Map<string, ExtractedAgent>();
+    const errors: string[] = [];
+
+    /** Indices of every agent heading, plus a sentinel end. */
+    const headings: {index: number; name: string}[] = [];
+    lines.forEach((line, index) => {
+        const match = HEADING.exec(line);
+        if (match?.[1] !== undefined) {
+            headings.push({index, name: match[1]});
+        }
+    });
+
+    headings.forEach((heading, i) => {
+        const sectionEnd = headings[i + 1]?.index ?? lines.length;
+        const at = `agent \`${heading.name}\` (line ${heading.index + 1})`;
+
+        // Frontmatter: first non-blank line after the heading must open it.
+        let cursor = heading.index + 1;
+        while (cursor < sectionEnd && (lines[cursor] ?? "").trim() === "") {
+            cursor++;
+        }
+        if ((lines[cursor] ?? "").trim() !== "---") {
+            errors.push(`${at}: expected frontmatter opening ---`);
+            return;
+        }
+        const fmStart = cursor + 1;
+        let fmEnd = -1;
+        for (let j = fmStart; j < sectionEnd; j++) {
+            if ((lines[j] ?? "").trim() === "---") {
+                fmEnd = j;
+                break;
+            }
+        }
+        if (fmEnd === -1) {
+            errors.push(`${at}: unterminated frontmatter`);
+            return;
+        }
+
+        const frontmatter = parseFrontmatter(lines.slice(fmStart, fmEnd));
+        const name = frontmatter.get("name");
+        const description = frontmatter.get("description");
+        const model = frontmatter.get("model");
+        if (name !== heading.name) {
+            errors.push(
+                `${at}: frontmatter name "${
+                    name ?? ""
+                }" does not match heading`,
+            );
+        }
+        if (description === undefined || description === "") {
+            errors.push(`${at}: missing frontmatter description`);
+        }
+        if (model === undefined || model === "") {
+            errors.push(`${at}: missing frontmatter model`);
+        }
+        if (agents.has(heading.name)) {
+            errors.push(`${at}: duplicate agent name`);
+        }
+        if (
+            name !== heading.name ||
+            description === undefined ||
+            description === "" ||
+            model === undefined ||
+            model === "" ||
+            agents.has(heading.name)
+        ) {
+            return;
+        }
+
+        const prompt = lines
+            .slice(fmEnd + 1, sectionEnd)
+            .join("\n")
+            .trim();
+        if (prompt === "") {
+            errors.push(`${at}: empty prompt body`);
+            return;
+        }
+        agents.set(heading.name, {name, description, model, prompt});
+    });
+
+    if (errors.length > 0) {
+        throw new AgentExtractError(errors);
+    }
+    if (agents.size === 0) {
+        throw new AgentExtractError(["no `## agent:` sections found"]);
+    }
+    return agents;
+};

workflows/review/eval/corpus/clean/clean-typed-refactor/case.json

@@ -0,0 +1,33 @@
+{
+    "id": "clean-typed-refactor",
+    "tags": [
+        "clean",
+        "live"
+    ],
+    "category": "clean",
+    "description": "Known-clean PR: a mechanical rename of a helper across a few files with no behaviour change. No lens produces a finding; the run must approve without posting.",
+    "changedFiles": [
+        {
+            "path": "src/util/format.ts",
+            "status": "modified"
+        },
+        {
+            "path": "src/util/format.test.ts",
+            "status": "modified"
+        }
+    ],
+    "findings": [],
+    "expected": {
+        "verdict": "APPROVE",
+        "postedCommentCount": 0
+    },
+    "diff": "diff --git a/src/util/format.ts b/src/util/format.ts\n--- a/src/util/format.ts\n+++ b/src/util/format.ts\n@@ -1,7 +1,7 @@\n /** Shared display formatting helpers. */\n \n /** Format a fractional amount as a fixed two-decimal string. */\n-export const fmt = (amount: number): string => amount.toFixed(2);\n+export const formatAmount = (amount: number): string => amount.toFixed(2);\n \n /** Format a byte count using binary units. */\n export const formatBytes = (bytes: number): string => {\ndiff --git a/src/util/format.test.ts b/src/util/format.test.ts\n--- a/src/util/format.test.ts\n+++ b/src/util/format.test.ts\n@@ -1,11 +1,11 @@\n import {describe, expect, it} from \"vitest\";\n \n-import {fmt, formatBytes} from \"./format\";\n+import {formatAmount, formatBytes} from \"./format\";\n \n-describe(\"fmt\", () => {\n+describe(\"formatAmount\", () => {\n     it(\"renders two decimals\", () => {\n-        expect(fmt(3)).toBe(\"3.00\");\n-        expect(fmt(3.456)).toBe(\"3.46\");\n+        expect(formatAmount(3)).toBe(\"3.00\");\n+        expect(formatAmount(3.456)).toBe(\"3.46\");\n     });\n });\n \n",
+    "live": {
+        "prContext": {
+            "title": "util: rename fmt to formatAmount",
+            "description": "Mechanical rename; fmt was too terse to grep for. No behavior change.",
+            "author": "dev-web",
+            "baseBranch": "main"
+        }
+    }
+}