diff --git a/docs/testing.md b/docs/testing.md index 81fd862..716f6b1 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -16,6 +16,7 @@ land, never lower them to get a build green. | Fuzz | `test/fuzz.test.ts` | A parser crashing, hanging, or corrupting source it did not write | | Contract | `test/contract.test.ts` | A provider implementation that satisfies the types but not the contract | | Chaos | `test/chaos.test.ts` | A dependency failing, hanging, or racing — and the damage that leaves behind | +| Behavior | `test/behavior.test.ts` | A documented promise that stopped being true even though every unit still passes | | Characterization | `test/characterization*.test.ts` | A change to anything this library *generates* — rewritten source, emitted instrumentation, prompts, CLI output, the export surface | ## Running diff --git a/test/behavior.test.ts b/test/behavior.test.ts new file mode 100644 index 0000000..42d8a3e --- /dev/null +++ b/test/behavior.test.ts @@ -0,0 +1,259 @@ +import { describe, expect, it } from "vitest"; + +import { writeFile } from "node:fs/promises"; + +import { + defineTrainable, + InsufficientTracesError, + isTsAutocodeError, + PromotionRejectedError, +} from "../src/index.js"; +import { given } from "./support/scenario.js"; + +// Behavior specs, written in the vocabulary the README uses. +// +// Every other suite is organized around units and failure modes. These are +// organized around what a user is trying to do, so they are the suite that +// fails when a documented promise stops being true even though every unit +// still passes. Each `it` is one sentence of the contract this library offers. + +describe("Marking a method for training", () => { + it("captures calls without changing what the method returns", async () => { + const scenario = await given({ name: "capture-transparent" }); + + scenario.whenTheApplicationCalls("invoice", "password"); + await scenario.training.flush(); + + // The promise: marking is observation, not interception. + const records = await scenario.training.records(); + expect(records).toHaveLength(2); + expect(records.every((record) => record.succeeded)).toBe(true); + }); + + it("records a failed call as a failure without swallowing the error", async () => { + const scenario = await given({ name: "capture-failure" }); + + expect(() => scenario.training.capture( + defineTrainable("Router.route"), + "route", + undefined, + (_input: string) => { throw new Error("downstream is down"); }, + ["x"] as [string], + )).toThrow("downstream is down"); + await scenario.training.flush(); + + expect((await scenario.training.records()).map((record) => record.succeeded)).toEqual([false]); + }); + + it("leaves the source file untouched until something is activated", async () => { + const scenario = await given({ name: "capture-no-write" }); + + scenario.whenTheApplicationCalls("a", "b", "c"); + await scenario.training.flush(); + + expect(await scenario.currentSource()).toBe(scenario.originalSource); + }); +}); + +describe("Training a marked method against eval cases", () => { + it("promotes a candidate that passes every case", async () => { + const scenario = await given({ name: "train-promotes" }); + + await scenario.whenTrainedAgainst([["abc", "ABC"], ["xyz", "XYZ"]]); + + expect(scenario.run.outcome).toBe("ready"); + expect(scenario.run.canActivate()).toEqual({ ready: true }); + }, 30_000); + + it("refuses a candidate that fails a case, and says which gate refused", async () => { + const scenario = await given({ name: "train-refuses" }); + scenario.givenAnEngineThatProposes('return "always-wrong";'); + + await scenario.whenTrainedAgainst([["abc", "ABC"]]); + + const readiness = scenario.run.canActivate(); + expect(readiness.ready).toBe(false); + if (!readiness.ready) { + expect(readiness.failures.length).toBeGreaterThan(0); + expect(readiness.outcome).not.toBe("ready"); + } + }, 30_000); + + it("reports an engine failure as a typed error rather than a stalled run", async () => { + const scenario = await given({ name: "train-engine-fails" }); + scenario.givenAnEngineThatFails("model unavailable"); + + await scenario.whenTrainedAgainst([["abc", "ABC"]]); + + expect(scenario.failure).toBeInstanceOf(Error); + expect((scenario.failure as Error).message).toContain("model unavailable"); + }, 30_000); +}); + +describe("Training from captured traffic", () => { + it("turns successful calls into eval cases and trains against them", async () => { + const scenario = await given({ name: "live-trains" }); + scenario.givenAnEngineThatProposes("return input;"); + + scenario.whenTheApplicationCalls("alpha", "beta", "gamma"); + await scenario.whenTrainedFromCapturedTraffic(3); + + expect(scenario.failure).toBeUndefined(); + expect(scenario.run.baseline.evaluations.length).toBeGreaterThan(0); + }, 30_000); + + it("refuses to train on too little traffic, and says how much it needed", async () => { + const scenario = await given({ name: "live-insufficient" }); + + scenario.whenTheApplicationCalls("only-one"); + await scenario.whenTrainedFromCapturedTraffic(5); + + expect(scenario.failure).toBeInstanceOf(InsufficientTracesError); + expect(scenario.failure).toMatchObject({ required: 5, found: 1 }); + }, 30_000); + + it("counts distinct inputs, not repeated ones", async () => { + const scenario = await given({ name: "live-distinct" }); + + // The README promises repeated inputs use the latest output rather than + // producing contradictory replay cases. + scenario.whenTheApplicationCalls("same", "same", "same"); + await scenario.whenTrainedFromCapturedTraffic(3); + + expect(scenario.failure).toBeInstanceOf(InsufficientTracesError); + expect(scenario.failure).toMatchObject({ found: 1 }); + }, 30_000); +}); + +describe("Activating a training run", () => { + it("rewrites only the marked body and leaves the rest of the file alone", async () => { + const scenario = await given({ name: "activate-writes" }); + + await scenario.whenTrainedAgainst([["abc", "ABC"]]); + await scenario.whenActivated(); + + const rewritten = await scenario.currentSource(); + expect(rewritten).toContain("toUpperCase"); + expect(rewritten).toContain('"use training";'); + expect(rewritten).toContain("class Router {"); + expect(rewritten.split("\n").length).toBe(scenario.originalSource.split("\n").length); + }, 30_000); + + it("refuses to activate a run that did not pass the gate", async () => { + const scenario = await given({ name: "activate-refused" }); + scenario.givenAnEngineThatProposes('return "always-wrong";'); + + await scenario.whenTrainedAgainst([["abc", "ABC"]]); + await scenario.whenActivated(); + + expect(scenario.failure).toBeInstanceOf(PromotionRejectedError); + // And the promise that matters: the file is untouched. + expect(await scenario.currentSource()).toBe(scenario.originalSource); + }, 30_000); + + it("restores the file exactly on rollback", async () => { + const scenario = await given({ name: "activate-rollback" }); + + await scenario.whenTrainedAgainst([["abc", "ABC"]]); + await scenario.whenActivated(); + expect(await scenario.currentSource()).not.toBe(scenario.originalSource); + + await scenario.whenRolledBack(); + + expect(scenario.failure).toBeUndefined(); + expect(await scenario.currentSource()).toBe(scenario.originalSource); + }, 30_000); + + it("refuses to overwrite an edit made after activation", async () => { + const scenario = await given({ name: "activate-conflict" }); + + await scenario.whenTrainedAgainst([["abc", "ABC"]]); + await scenario.whenActivated(); + // A developer edits the rewritten method before rolling back. + const rewritten = await scenario.currentSource(); + await scenario.whenTheFileIsEditedTo("return input;"); + expect(await scenario.currentSource()).toBe(rewritten); + + const edited = rewritten.replace("toUpperCase()", "toLowerCase()"); + await writeFile(scenario.artifact, edited, "utf8"); + await scenario.whenRolledBack(); + + // The README's promise: rollback refuses to overwrite later changes. + expect(scenario.failure).toBeDefined(); + expect(await scenario.currentSource()).toBe(edited); + }, 30_000); +}); + +describe("Zero-config evolution", () => { + it("trains and rewrites on its own once enough traffic accumulates", async () => { + const scenario = await given({ name: "evolve-applies" }); + scenario.givenEvolutionIsOn(2); + // Evolution replays captured traffic as equality cases, so a candidate + // only promotes if it reproduces the behavior that was observed. + scenario.givenAnEngineThatProposes("return input.trim();"); + + scenario.whenTheApplicationCalls("alpha", "beta"); + await scenario.whenBackgroundWorkSettles(() => scenario.eventTypes().includes("evolution.applied")); + + expect(scenario.eventTypes()).toContain("evolution.started"); + expect(scenario.eventTypes()).toContain("evolution.applied"); + expect(await scenario.currentSource()).toContain("trim()"); + }, 60_000); + + it("refuses a candidate that changes the behavior the traffic demonstrated", async () => { + // The whole safety story for unattended rewriting: evolution trains to + // preserve what production actually does, so a candidate that alters it + // cannot pass the gate no matter how confident the model was. + const scenario = await given({ name: "evolve-behavior-change" }); + scenario.givenEvolutionIsOn(2); + scenario.givenAnEngineThatProposes("return input.toUpperCase();"); + + scenario.whenTheApplicationCalls("alpha", "beta"); + await scenario.whenBackgroundWorkSettles(() => scenario.eventTypes().includes("evolution.failed")); + + expect(scenario.eventTypes()).toContain("evolution.started"); + expect(scenario.eventTypes()).not.toContain("evolution.applied"); + expect(await scenario.currentSource()).toBe(scenario.originalSource); + }, 60_000); + + it("waits rather than training on too little traffic", async () => { + const scenario = await given({ name: "evolve-waits" }); + scenario.givenEvolutionIsOn(10); + + scenario.whenTheApplicationCalls("alpha"); + await scenario.whenBackgroundWorkSettles(() => scenario.eventTypes().includes("evolution.skipped")); + + expect(scenario.eventTypes()).toContain("evolution.skipped"); + expect(scenario.eventTypes()).not.toContain("evolution.applied"); + expect(await scenario.currentSource()).toBe(scenario.originalSource); + }, 60_000); + + it("never breaks an application call when evolution fails", async () => { + const scenario = await given({ name: "evolve-fails" }); + scenario.givenEvolutionIsOn(1); + scenario.givenAnEngineThatFails("model unavailable"); + + // The calls still return normally. + scenario.whenTheApplicationCalls("alpha", "beta"); + await scenario.whenBackgroundWorkSettles(() => scenario.eventTypes().includes("evolution.failed")); + + expect(scenario.eventTypes()).toContain("evolution.failed"); + expect(await scenario.currentSource()).toBe(scenario.originalSource); + }, 60_000); +}); + +describe("The errors a user meets", () => { + it("always carries a code they can branch on", async () => { + const scenario = await given({ name: "errors-coded" }); + scenario.givenAnEngineThatFails("model unavailable"); + await scenario.whenTrainedAgainst([["abc", "ABC"]]); + + // Not every failure originates here -- an engine's own error propagates + // unchanged, by design -- but library failures are always typed. + const scenario2 = await given({ name: "errors-coded-2" }); + scenario2.whenTheApplicationCalls("one"); + await scenario2.whenTrainedFromCapturedTraffic(9); + expect(isTsAutocodeError(scenario2.failure)).toBe(true); + expect((scenario2.failure as { code: string }).code).toBe("insufficient_traces"); + }, 30_000); +}); diff --git a/test/support/scenario.ts b/test/support/scenario.ts new file mode 100644 index 0000000..fdb569f --- /dev/null +++ b/test/support/scenario.ts @@ -0,0 +1,247 @@ +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { + createTrainingRuntime, + defineTrainable, + type ImplementationExecutor, + type Training, + type TrainingEngine, + type TrainingEvent, + type TrainingRun, +} from "../../src/index.js"; + +// A given/when/then harness for behavior specs. +// +// The other suites are organized around units and failure modes. These are +// organized around what a *user* is trying to do, in the vocabulary the README +// uses -- mark a method, capture traffic, train, gate, activate, roll back. +// That makes them the suite that fails when the documented promise stops being +// true, even if every unit still passes. +// +// No BDD framework: Gherkin's parser buys little when the steps are TypeScript +// anyway, and a plain builder keeps the spec and its assertions in one file. + +export interface ScenarioOptions { + /** Directory for the fixture module and run artifacts. */ + readonly name: string; + /** The marked source the scenario starts from. */ + readonly source?: string; + /** What a proposing engine returns. */ + readonly proposal?: string; +} + +const defaultSource = `class Router { + route(input: string): string { + "use training"; + return input; + } +} +`; + +/** Runs candidate bodies directly; the sandbox is exercised in the contract + * suite, and a spec about user-visible behavior should not depend on it. */ +export const directExecutor: ImplementationExecutor = async (target, implementation, args) => + new Function(...target.parameters.map((parameter) => parameter.name), implementation)(...args) as unknown; + +export class Scenario { + readonly events: TrainingEvent[] = []; + readonly directory: string; + readonly artifact: string; + + #source: string; + #proposal: string; + #training: Training | undefined; + #run: TrainingRun | undefined; + #activation: Awaited> | undefined; + #failure: unknown; + #engine: TrainingEngine | undefined; + #minTraces: number | undefined; + #autoEvolve = false; + + constructor(options: ScenarioOptions) { + this.directory = join("test/output/specs", options.name); + this.artifact = join(this.directory, "router.ts"); + this.#source = options.source ?? defaultSource; + this.#proposal = options.proposal ?? "return input.toUpperCase();"; + } + + // ------------------------------------------------------------------ given + + /** The marked module exists on disk, as a developer's project would. */ + async givenAMarkedModule(): Promise { + await rm(this.directory, { recursive: true, force: true }); + await mkdir(this.directory, { recursive: true }); + await writeFile(this.artifact, this.#source, "utf8"); + return this; + } + + /** An engine that proposes the configured replacement body. */ + givenAnEngineThatProposes(implementation: string): this { + this.#proposal = implementation; + return this; + } + + /** An engine that fails every time it is asked. */ + givenAnEngineThatFails(message: string): this { + this.#engine = { id: "spec/failing", optimize: async () => { throw new Error(message); } }; + return this; + } + + /** Background evolution is switched on, as `ts-autocode/register` does. */ + givenEvolutionIsOn(minTraces: number): this { + this.#autoEvolve = true; + this.#minTraces = minTraces; + return this; + } + + /** The runtime a user would have after configuring the library. */ + get training(): Training { + this.#training ??= createTrainingRuntime({ + engine: this.#engine ?? { id: "spec/engine", optimize: async () => ({ implementation: this.#proposal }) }, + executor: directExecutor, + source: { files: [this.artifact] }, + tracing: { enabled: false }, + onEvent: (event) => this.events.push(event), + ...(this.#autoEvolve + ? { + evolution: { + auto: true, + minTraces: this.#minTraces ?? 1, + evaluation: { outputDir: join(this.directory, "agentv-evolve") }, + }, + } + : {}), + }); + return this.#training; + } + + // ------------------------------------------------------------------- when + + /** The application calls the marked method. */ + whenTheApplicationCalls(...inputs: readonly string[]): this { + for (const input of inputs) { + this.training.capture( + defineTrainable("Router.route"), + "route", + undefined, + (value: string) => value, + [input], + ); + } + return this; + } + + /** The user trains against explicit eval cases. */ + async whenTrainedAgainst(cases: ReadonlyArray): Promise { + this.#failure = undefined; + try { + this.#run = await this.training.train({ + trainable: defineTrainable("Router.route").symbol, + evaluation: { + tests: cases.map(([input, expected], index) => ({ + id: `case-${index}`, + input, + assert: [{ type: "equals" as const, value: expected }], + })), + task: (value: string) => value.toUpperCase(), + outputDir: join(this.directory, "agentv"), + }, + rounds: { max: 1 }, + }); + } catch (error) { + this.#failure = error; + } + return this; + } + + /** The user trains from captured traffic instead of explicit cases. */ + async whenTrainedFromCapturedTraffic(minTraces: number): Promise { + this.#failure = undefined; + try { + this.#run = await this.training.train({ + trainable: defineTrainable("Router.route").symbol, + minTraces, + evaluation: { outputDir: join(this.directory, "agentv-live") }, + rounds: { max: 1 }, + }); + } catch (error) { + this.#failure = error; + } + return this; + } + + /** The user applies the result. */ + async whenActivated(): Promise { + this.#failure = undefined; + try { + this.#activation = await this.run.activate(); + } catch (error) { + this.#failure = error; + } + return this; + } + + /** The user undoes it. */ + async whenRolledBack(): Promise { + this.#failure = undefined; + try { + await this.activation.rollback(); + } catch (error) { + this.#failure = error; + } + return this; + } + + /** Someone edits the file out from under the run. */ + async whenTheFileIsEditedTo(replacement: string): Promise { + const current = await readFile(this.artifact, "utf8"); + await writeFile(this.artifact, current.replace("return input;", replacement), "utf8"); + return this; + } + + /** Background work settles, or the attempts run out. */ + async whenBackgroundWorkSettles(done: () => boolean, attempts = 60): Promise { + await this.training.flush(); + for (let attempt = 0; attempt < attempts && !done(); attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 25)); + } + return this; + } + + // ------------------------------------------------------------------- then + + get run(): TrainingRun { + if (this.#run === undefined) throw new Error("no training run: the scenario never trained, or training threw"); + return this.#run; + } + + get activation(): Awaited> { + if (this.#activation === undefined) throw new Error("no activation: the scenario never activated, or it threw"); + return this.#activation; + } + + /** Whatever the last `when` step threw, if anything. */ + get failure(): unknown { + return this.#failure; + } + + /** The current contents of the marked module. */ + async currentSource(): Promise { + return readFile(this.artifact, "utf8"); + } + + /** The source the scenario started from. */ + get originalSource(): string { + return this.#source; + } + + eventTypes(): readonly string[] { + return this.events.map((event) => event.type); + } +} + +/** Starts a scenario with its fixture already on disk. */ +export async function given(options: ScenarioOptions): Promise { + return new Scenario(options).givenAMarkedModule(); +}