diff --git a/README.md b/README.md index ed85c8d..afc51af 100644 --- a/README.md +++ b/README.md @@ -502,6 +502,10 @@ const loop: TrainingLoop = async (input) => { `createEvalRun` and `createPromotionDecision` build the parts individually when you have real evidence to carry. +[docs/authoring-providers.md](docs/authoring-providers.md) covers all five +injected seams — engine, executor, loop, promotion applier and store — with the +rules each one must satisfy and the conformance suites that check them. + ## Errors Every failure this library raises is a `TsAutocodeError` carrying a `code` you diff --git a/docs/architecture.md b/docs/architecture.md index b354170..91c667a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -177,6 +177,7 @@ root, so the split is organizational rather than a break. Extension points are constructible: a custom `TrainingLoop` returns a `CandidateReview` containing a `TrainableEvalRun`, and `createCandidateReview`, `createEvalRun`, and `createPromotionDecision` build those without a cast. +[Authoring providers](authoring-providers.md) is the how-to for each seam. Every failure the library raises carries a `code` and is recognized by `isTsAutocodeError`. Errors that have always been `TypeError`s or diff --git a/docs/authoring-providers.md b/docs/authoring-providers.md new file mode 100644 index 0000000..ab2f94c --- /dev/null +++ b/docs/authoring-providers.md @@ -0,0 +1,290 @@ +# Authoring providers + +`ts-autocode` ships working implementations of everything it needs, but none of +them are the interface. Five seams are injected, and any structurally compatible +implementation works: the runtime never imports a provider, and providers never +import each other. + +This guide is for writing one. Every snippet here is compiled by +`test/docs.test.ts`, so it cannot drift from the API. + +## Which primitive do you want? + +| You want to | Write | +|---|---| +| Use a different model or provider | **Nothing** — set `model`, see below | +| Add a rule a candidate must clear | A [`PromotionGate`](#promotiongate) | +| Call your own optimizer instead of Ax | A [`TrainingEngine`](#trainingengine) | +| Run candidate code somewhere safer | An [`ImplementationExecutor`](#implementationexecutor) | +| Change how rounds are explored | A [`TrainingLoop`](#trainingloop) | +| Write the result somewhere other than the source file | A [`PromotionApplier`](#promotionapplier) | +| Persist captured traces | A [`TrainingStore`](#trainingstore) | + +The first row is the common wrong turn. Choosing a model is a **setting**, not a +reason to replace the engine: + +```ts +import { configureTraining } from "ts-autocode"; + +configureTraining({ + model: { provider: "anthropic", name: "claude-sonnet-5" }, +}); +``` + +`apiKey` is optional there — it falls back to the configured `SecretProvider`, +then the environment. `teacher` names an optional stronger model for the +optimizer's teacher role. + +## PromotionGate + +The smallest thing most people write. A gate reads the decision context and +returns a failure reason — or `undefined` to allow. Returning a string is what +refuses; the strings land in `decision.failures` and in `PromotionRejectedError`. + +```ts +import { defaultPromotionGates, type PromotionGate } from "ts-autocode"; + +const noNetwork: PromotionGate = ({ candidate }) => + /\bfetch\s*\(/.test(candidate.implementation) + ? "candidate makes a network call" + : undefined; + +const gates = [...defaultPromotionGates, noNetwork]; +``` + +**`promotion.gates` replaces the standard set rather than extending it**, so +spread `defaultPromotionGates` unless you genuinely mean to drop them. Rules +never mutate and never see each other; the context carries `candidate`, +`evaluations`, `results`, `conformance`, `meanScore`, `passRate`, and the +resolved `minScore` / `minPassRate` thresholds. + +A gate may be async. `policy` is the deprecated spelling of the same idea — it +was always a gate that returned a boolean. + +## TrainingEngine + +Proposes a replacement body. The only required output is a string. + +The root README has [a complete worked example](../README.md#custom-engines). +Two things it does not cover: + +```ts +import type { TrainingEngine } from "ts-autocode"; + +declare function callMyOptimizer(prompt: string, model: string | undefined): Promise; + +const engine: TrainingEngine = { + id: "acme/optimizer", + async optimize(request, context) { + // Honor cancellation: the runtime aborts a round when the caller does, and + // retrying work nobody wants costs real money. + context.signal?.throwIfAborted(); + // `model` is the user's `TrainingSettings.model`, carried through + // unmodified. The runtime knows nothing about any provider. + const implementation = await callMyOptimizer(request.objective, context.model?.name); + return { implementation }; + }, +}; +``` + +The core validates identity, source digests, and the final candidate regardless +of engine, so an engine that returns nonsense is refused rather than applied. + +## ImplementationExecutor + +Runs a proposed body against arguments, in isolation you own. + +```ts +import type { ImplementationExecutor } from "ts-autocode"; + +const executor: ImplementationExecutor = async (target, implementation, args, options) => { + const parameters = target.parameters.map((parameter) => parameter.name); + const candidate = new Function(...parameters, implementation) as (...values: unknown[]) => unknown; + return candidate.apply(options?.receiver, [...args]); +}; +``` + +Rules the conformance suite enforces: + +- **A throwing body must surface as a rejected promise**, not a synchronous + throw. Callers `await` you; a synchronous throw escapes their `catch`. +- `options.timeoutMs` and `options.signal` are yours to honor. The shipped + sandbox executor applies `execution.timeoutMs`, defaulting to 5s. +- `options.receiver` is the live `this` when a hot-swapped instance method is + invoked. A sandboxed executor may ignore it. + +The example above is deliberately the *unsafe* one — it is what a test double +looks like. A real executor runs the body in a worker, a VM context, or a +container. + +## TrainingLoop + +Orchestrates propose/review rounds. The runtime owns proposing and reviewing; +the loop owns iteration and stopping. + +The root README has [a complete worked example](../README.md#extending-the-library) +using `createCandidateReview`, which builds the `CandidateReview` a loop must +return without any casts. The shapes you are handed: + +```ts +import type { ProposalTurn, ReviewContext, TrainingLoopInput } from "ts-autocode"; + +declare const input: TrainingLoopInput; +// `slot` is the 1-based fan-out slot, always 1 without fan-out; `feedback` +// carries failure strings from earlier reviews of rejected candidates. +declare const turn: ProposalTurn; // { round, slot, feedback, signal? } +declare const review: ReviewContext; // { label, signal? } +``` + +The one rule no type expresses: + +> **The winning round must be last.** When a loop returns `outcome: "ready"`, +> the runtime activates `rounds.at(-1)`. A loop that finds a winner in round 2, +> keeps exploring, and returns all four rounds in order will activate round 4's +> candidate instead. + +Also honor `input.signal`, and treat `maxRounds` and `fanOut` as budgets rather +than suggestions — `sequentialLoop` supports fan-out; the default governed +harness loop reviews one candidate per round and refuses more. + +## PromotionApplier + +Applies a gate-approved candidate, undoably. How it applies is the provider's +concern — the shipped one rewrites the source file; yours could open a pull +request or patch a running process. Training requires only that it be reversible. + +```ts +import { readFile, writeFile } from "node:fs/promises"; +import type { PromotionApplier } from "ts-autocode"; + +declare function patch(source: string, implementation: string): string; + +const applier: PromotionApplier = async (candidate, decision) => { + // A decision names the candidate it was made about. Applying it to a + // different one would write code that never passed a gate. + if (!decision.promote || decision.candidateId !== candidate.id) { + throw new Error(`candidate has not passed the promotion gate: ${candidate.id}`); + } + const artifact = candidate.target.artifactRef; + const before = await readFile(artifact, "utf8"); + await writeFile(artifact, patch(before, candidate.implementation), "utf8"); + return { + rollback: async () => { await writeFile(artifact, before, "utf8"); }, + }; +}; +``` + +The returned `rollback` is what `Activation.rollback()` calls. The shipped +applier refuses to roll back over an edit made after activation, by comparing +body digests; if yours writes to something a human can also edit, do the same. + +## TrainingStore + +Two methods. The default is in-memory and volatile. + +```ts +import type { TrainingRecord, TrainingStore } from "ts-autocode"; + +class ArrayStore implements TrainingStore { + readonly #records: TrainingRecord[] = []; + + async append(record: TrainingRecord): Promise { + this.#records.push(structuredClone(record)); + } + + async list(trainableId?: string): Promise { + // Never hand back live internal state: one caller's mutation would + // corrupt every other reader. + const all = this.#records.map((record) => structuredClone(record)); + return trainableId === undefined + ? all + : all.filter((record) => record.trainableId === trainableId); + } +} +``` + +Rules the conformance suite enforces: **append order is preserved**, `list()` +does not alias internal state, and an omitted `trainableId` returns everything +while a given one filters. Training reads captured traces back as eval cases, +so a store that reorders or drops records silently changes what a candidate is +trained to reproduce. + +## Registering what you wrote + +Three ways in, for three different situations: + +```ts +import { configureTraining, createTrainingRuntime } from "ts-autocode"; +import type { ImplementationExecutor, PromotionApplier, TrainingEngine } from "ts-autocode"; + +declare const engine: TrainingEngine; +declare const executor: ImplementationExecutor; +declare const promote: PromotionApplier; + +// Isolated: owns its settings, store and evolution state, registers nothing +// globally. Use this in tests and multi-tenant hosts. +const runtime = createTrainingRuntime({ + engine, + executor, + promote, + execution: { timeoutMs: 10_000 }, + source: { files: ["src/router.ts"] }, + onEvent: (event) => console.log(event.type), +}); + +// Process-wide: what an application does once at startup. Replaces the +// previous settings unless you pass `{ merge: true }`. +configureTraining({ engine }); +``` + +The third is `provideTrainingDefaults`, which supplies *lazy fallbacks* rather +than settings. It is for provider packages — `ts-autocode` itself calls it to +wire the Ax engine, its sandbox executor, the harness loop, and the rewrite +applier — not for applications. Explicit settings always win over it. + +`resetTraining()` discards the process-wide runtime and its settings, restoring +the state of a fresh import. Without it, one test's `configureTraining` call is +visible to every later one. + +## Proving it conforms + +Types cannot state "the winning round must be last", or "append order is +preserved". Those rules are carried by a conformance kit that ships with the +package, so you can check your implementation against the same suite the +built-in providers are checked against: + +```ts +import { trainingStoreContract, type TrainingStore } from "ts-autocode"; + +declare function it(name: string, body: () => Promise): void; +declare function makeMyStore(): TrainingStore; + +for (const check of trainingStoreContract) { + it(check.name, () => check.run(() => makeMyStore())); +} +``` + +Deliberately framework-agnostic: a check is `{ name, run(subject) }` and throws +on violation, so it works under Vitest, Jest, `node:test`, or a bare loop. + +One suite per seam — `trainingEngineContract`, +`implementationExecutorContract`, `trainingLoopContract`, +`promotionApplierContract`, `trainingStoreContract` — plus `conformanceSuites`, +which bundles all five. Fixtures let you build a subject without a checkout of +this repo: + +```ts +import { conformanceCandidate, conformanceTarget } from "ts-autocode"; + +// A discovered target and a candidate patch for it, ready to hand to an +// executor or an applier under test. +const target = conformanceTarget; +const candidate = conformanceCandidate("return input.toUpperCase();"); +``` + +`conformanceAsyncTarget` is the same for a method returning a promise. + +These suites are also how this repo checks its own providers: `test/contract.test.ts` +runs every shipped implementation through them, alongside a deliberately +different second store — a suite that only ever sees one shape is describing +that shape rather than a contract. diff --git a/docs/testing.md b/docs/testing.md index a05a21e..a91948a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -8,7 +8,7 @@ land, never lower them to get a build green. |---|---|---| | Atomic unit | `packages/*/test/*.test.ts` | Behavior of one function or class, including every branch of its defaulting and rejection rules | | Functional | `test/*.test.ts` | A whole path through the runtime: mark, capture, train, gate, activate, roll back | -| Documentation | `test/docs.test.ts` | README and architecture snippets that no longer compile | +| Documentation | `test/docs.test.ts` | README, architecture and provider-authoring snippets that no longer compile | | Surface | `test/surface.test.ts` | Re-export drift between the root package and its siblings | | Protocol | `test/digest-protocol.test.ts` | Two packages that must agree without importing each other | | Compatibility | `test/deprecated.test.ts` | A deprecated spelling that silently stopped working | diff --git a/packages/training/README.md b/packages/training/README.md index d67b5b4..907cd5f 100644 --- a/packages/training/README.md +++ b/packages/training/README.md @@ -8,11 +8,12 @@ and a bounded sequential propose/review loop. It depends on **no sibling package and no provider**, and it has no knowledge of weaving, AOP, or source rewriting: `TrainingEngine` (the candidate optimization strategy, composed into the internal engine), `ImplementationExecutor` -(running proposed bodies), `TrainingLoop` (driving training rounds), and -`PromotionApplier` (applying a gate-approved candidate undoably) are all +(running proposed bodies), `TrainingLoop` (driving training rounds), +`PromotionApplier` (applying a gate-approved candidate undoably), and +`TrainingStore` (persisting captured traces) are all injected boundaries, and `captureTrainable(...)` is the entry any external instrumentation calls to route a marked call through runtime capture. Supply -engine, executor, and loop per runtime through `TrainingSettings`, or register +any of them per runtime through `TrainingSettings`, or register lazy defaults once with `provideTrainingDefaults(...)` — that is how the `ts-autocode` package wires Ax as the default engine and executor, the governed `ts-autocode-harness` loop as the default orchestrator, and @@ -41,6 +42,10 @@ Extension points are constructible: a custom `TrainingLoop` must return a Every failure carries a `code` and is recognized by `isTsAutocodeError`; the ones that have always been `TypeError`s still are. +Implementing any of these seams is documented in +[docs/authoring-providers.md](../../docs/authoring-providers.md), including the +conformance suites this package publishes for checking one. + Most applications should depend on [`ts-autocode`](../../README.md), which re-exports this package's API with Ax defaults already registered. diff --git a/packages/training/src/training.ts b/packages/training/src/training.ts index b12a34b..1f456af 100644 --- a/packages/training/src/training.ts +++ b/packages/training/src/training.ts @@ -117,6 +117,12 @@ export interface TrainingSettings { /** Options handed to the executor on every candidate run. */ readonly execution?: ExecutionSettings; readonly loop?: TrainingLoop; + /** Applies a gate-approved candidate. Every other seam resolves from these + * settings before falling back to {@link provideTrainingDefaults}; this one + * did not exist, so an applier could only ever be registered process-wide. + * That left {@link createTrainingRuntime} sharing one applier between + * runtimes -- the component that writes generated code into a source file. */ + readonly promote?: PromotionApplier; readonly evolution?: EvolutionSettings; /** Default directory for run artifacts and eval output; a run's * `EvalConfig.outputDir` still overrides it. */ @@ -548,7 +554,7 @@ class TrainingRuntime implements Training { if (!decision.promote) { throw new PromotionRejectedError(candidate.id, decision); } - const promote = defaultProviders.promote; + const promote = this.#settings.promote ?? defaultProviders.promote; if (!promote) { throw new PromotionApplierNotConfiguredError(); } diff --git a/packages/training/test/training.test.ts b/packages/training/test/training.test.ts index cbb4ac3..90979a1 100644 --- a/packages/training/test/training.test.ts +++ b/packages/training/test/training.test.ts @@ -16,6 +16,7 @@ import { training as defaultTraining, type Activation, type ImplementationExecutor, + type PromotionApplier, type TrainingEngine, type TrainingStore, } from "../src/index.js"; @@ -379,3 +380,65 @@ describe("capture on an isolated runtime", () => { expect((await isolated.records(token))[0]?.succeeded).toBe(true); }); }); + +describe("promotion applier on an isolated runtime", () => { + // Every other seam resolved `settings.X ?? defaultProviders.X`; `promote` + // alone read the process-wide provider, so an applier could not be injected + // per runtime. `createTrainingRuntime` therefore left one seam global -- the + // one that writes generated code into a source file. Found while writing the + // provider authoring guide: the wiring example would not compile. + async function trained(promote: PromotionApplier, name: string) { + const directory = await mkdtemp(join(tmpdir(), `ts-autocode-${name}-`)); + const artifact = join(directory, "echo.ts"); + await writeFile(artifact, `export function ${name}(input: string): string { + "use training"; + return input; +}\n`); + const runtime = createTrainingRuntime({ + engine: { id: "applier-test", optimize: async () => ({ implementation: "return input.toUpperCase();" }) }, + executor: functionExecutor, + promote, + source: { files: [artifact] }, + tracing: { enabled: false }, + }); + const run = await runtime.train({ + trainable: defineTrainable(name).symbol, + evaluation: { + tests: [{ id: "upper", input: "abc", assert: [{ type: "equals", value: "ABC" }] }], + task: (input) => input.toUpperCase(), + outputDir: join(directory, "agentv"), + }, + rounds: { max: 1 }, + }); + expect(run.outcome).toBe("ready"); + return run; + } + + it("prefers the runtime's applier over the process-wide one", async () => { + const applied: string[] = []; + const run = await trained(async (candidate) => { + applied.push(candidate.id); + return { rollback: async () => { applied.push("rolled-back"); } }; + }, "applierPreferred"); + + const activation = await run.activate(); + expect(applied).toHaveLength(1); + await activation.rollback(); + expect(applied.at(-1)).toBe("rolled-back"); + }); + + it("keeps two runtimes' appliers apart", async () => { + const first: string[] = []; + const second: string[] = []; + const runs = await Promise.all([ + trained(async () => { first.push("applied"); return { rollback: async () => {} }; }, "applierFirst"), + trained(async () => { second.push("applied"); return { rollback: async () => {} }; }, "applierSecond"), + ]); + + await runs[0]?.activate(); + + expect(first).toEqual(["applied"]); + expect(second).toEqual([]); + }); + +}); diff --git a/test/docs.test.ts b/test/docs.test.ts index 931f842..de848cf 100644 --- a/test/docs.test.ts +++ b/test/docs.test.ts @@ -20,6 +20,7 @@ const docs = [ "packages/rewrite/README.md", "packages/grounding/README.md", "docs/architecture.md", + "docs/authoring-providers.md", ]; interface Snippet { @@ -50,8 +51,18 @@ export function typescriptSnippets(doc: string, markdown: string): readonly Snip return snippets; } -const snippets = docs.flatMap((doc) => - typescriptSnippets(doc, readFileSync(join(repoRoot, doc), "utf8"))); +/** Reads a listed doc, naming it when it is missing. An unguarded read throws + * at module load for a renamed or misspelled entry, which takes down the whole + * suite instead of failing one case that says which file it wanted. */ +function readDoc(doc: string): string { + try { + return readFileSync(join(repoRoot, doc), "utf8"); + } catch (error) { + throw new Error(`documentation file listed for typechecking is unreadable: ${doc}`, { cause: error }); + } +} + +const snippets = docs.flatMap((doc) => typescriptSnippets(doc, readDoc(doc))); // Snippets compile inside the repo (under the git-ignored test output tree) so // NodeNext resolution sees the real node_modules and the root package's