From 09ac156b27feac65fbc0d92409810422684b91a8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 17:17:12 +0000 Subject: [PATCH 01/14] docs: add developer experience and API design review Reviews the consumer-facing surface of ts-autocode and its four sibling packages against the API-design rules already stated in CONTRIBUTING.md, and records the remediation plan. Covers eight defects (README code that does not compile, grounding codegen targeting a nonexistent training.define, an inaccurate sideEffects declaration, a silently ignored fanOut, documented-but-unexported symbols, a placeholder threshold reaching the judge, a fail-open evolve kill switch, and an unreachable execution timeout), two missing capabilities (model selection and a CLI), and the consistency, error-model, and boilerplate backlog behind them. The remediation is additive: renamed or reshaped APIs are added alongside the existing ones, which keep working and are marked deprecated. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- docs/dx-review.md | 317 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 docs/dx-review.md diff --git a/docs/dx-review.md b/docs/dx-review.md new file mode 100644 index 0000000..296f436 --- /dev/null +++ b/docs/dx-review.md @@ -0,0 +1,317 @@ +# Developer experience and API design review + +An adversarial review of the consumer-facing surface of `ts-autocode` and its four sibling +packages, and the plan for addressing it. + +The core idea is good and the layering is unusually well held: siblings never import each +other, and only the root package wires them together. The problems are almost entirely in +the *consumer-facing* surface, which has grown outward from the internals rather than +inward from a use case. + +## The rubric + +`CONTRIBUTING.md` already states four API-design rules. The current public surface violates +all four, so this review uses them as its rubric rather than outside taste. + +| Stated rule | Reality at the time of review | +|---|---| +| "Keep the root export surface small; internal helpers should stay internal" | Root exported 25 values and 62 types, including rewrite primitives (`applyCandidate`, `commitRewrite`, `revertRewrite`, `swapImplementation`, `restoreImplementation`) and instrumentation-author APIs (`captureTrainable`, `provideTrainingDefaults`, `withPolicy`) that no application needs | +| "Avoid global mutable configuration" | `configuredTraining` and `defaultProviders` module singletons, the `configureRewrite` registry, and `installInstrumentation` writing to `globalThis[Symbol.for("ts-autocode.instrument")]` | +| "Avoid exported string constants" | `defaultOutputDir`, `defaultTsconfig`, `trainingMarker`, `inMemoryArtifactRef`, `evolveVariable`, `defaultActionLogDir` | +| "Document public API changes in the README and add a runnable example" | `examples/` held one file that imported `../src/index.js` rather than the package name, exported rather than ran, and was referenced by no test or script | + +## A. Defects + +Things that are wrong, not merely inelegant. + +### A1. README code does not compile + +`README.md` printed `activation.promotion.snapshot.candidateId`. `Activation` has exactly +two members, `run` and `rollback`. The flagship quickstart additionally referenced a +`deploymentPolicy` that is never defined or imported, and used `route` roughly forty lines +before introducing it. + +### A2. `ts-autocode-grounding` generates code against an API that does not exist + +`packages/grounding/src/scan.ts` emitted +`export const = training.define({ ... })`. `Training` has `records`, `evaluate`, +`train`, and `flush` — no `define`. Every generated registration file failed to typecheck. +The scan test asserted only the emitted *string*, so nothing caught it. + +### A3. `"sideEffects": false` is false, and can silently break the package + +All five manifests declared it, yet `src/index.ts` performs top-level +`provideTrainingDefaults(...)` and `configureRewriteCapture()`, `src/register.ts` is +entirely side effects, and `installInstrumentation` writes to a `globalThis` symbol. A +bundler honoring the hint may legally drop the wiring, leaving the user with +`no training engine is configured; import "ts-autocode"` — after importing `ts-autocode`. + +### A4. `TrainInput.fanOut` was silently ignored under the default configuration + +The README documented it as a first-class knob, and `trainingRounds` honors it. But the +*default* loop is the harness loop, and `createHarnessLoop` pinned `slot: 1` and never read +`input.fanOut`. An option that silently no-ops is worse than an absent one. + +### A5. Documented symbols were unreachable from the root package + +The README documented `trainingRounds()` and `sequentialLoop`; neither was exported from +`ts-autocode`. Also missing, yet needed to *use* documented extension points: +`defaultPromotionGates` (so `TrainInput.gates` could not be composed with the standard +set), `trainableTokenFromSymbol`, `discoverInSource`, `candidateDeclaration`, +`defaultFanOut`, `defaultMaxRounds`, and the `ProposalTurn` and `ReviewContext` types +required to implement a custom `TrainingLoop`. Only 5 of `ts-autocode-rewrite`'s 15 values +reached the root, and **nothing at all** from `ts-autocode-harness` — even though the root's +own `HarnessLoopOptions` is typed in terms of that package's `ContextProvider`, +`JudgeRequest`, and `JudgeDecision`. Configuring the default loop therefore required taking +a second, undocumented dependency. + +### A6. The judge was handed a placeholder string instead of the real threshold + +`promotionRubric()` emitted +`Minimum evaluation score: ${input.minScore ?? "evaluation default"}`. The real default is +`0.8`, an inline literal inside `evaluatePromotionGate`. The governed harness judge read +the literal text "evaluation default" as its criterion. + +### A7. `TS_AUTOCODE_EVOLVE` failed open + +The register hook disabled evolution only for `0`, `false`, and `off`. `TS_AUTOCODE_EVOLVE=no` +or `=disabled` **enabled** self-rewriting source mutation. For a kill switch on a feature +that edits the user's source files, fail-open is the wrong default. + +### A8. Two unrelated candidate-execution timeouts, and the one that matters was unreachable + +`AxEngineOptions.executionTimeoutMs` affects only scoring inside the engine. The default +*executor* — `executeImplementation`, registered with no options — fell back to a hardcoded +5s with no configuration path through `TrainingSettings` at all. + +## B. Missing capabilities + +### B1. No supported way to choose a model or provider + +The default engine hardcoded `openai` plus `OPENAI_API_KEY`/`OPENAI_APIKEY`. Using anything +else meant importing `createAxEngine` from `ts-autocode/ax`, passing `studentAI`, and +handing the whole engine to `configureTraining({ engine })` — abandoning the zero-config +path. That subpath was mentioned once in the README with no example anywhere in the repo. +Picking a model is the first thing most users do. + +### B2. No CLI + +No package declared `bin`. Yet the product is "instrument your app and let it rewrite +itself": inspecting what is trainable, what has been captured, or what would be rewritten +required writing a script that imports `discoverTrainables`. That function is already +synchronous and sufficient to back a `discover` / `status` / `train` command. + +## C. Consistency + +### C1. Four verbs for global state; four patterns for construction + +`configureTraining(settings): Training` **replaced** the singleton, so a second call +discarded the first. `provideTrainingDefaults(providers): void` **merged**. Alongside them, +`configureRewrite(config)` and `configureRewriteCapture()`. Construction split four ways: +`new` classes (`MemoryTrainingStore`, `WriteAheadAgentBus`, `HarnessSandbox`), `create*` +(`createAxEngine`, `createHarnessLoop`, `createSandboxPolicy`, `createRewriter`, +`createComponentDecorator`), `define*` (`defineTrainable`, `defineTrainingHarness`), and +global mutators. `defineTrainable` returns a value object while `defineTrainingHarness` +returns a service — the same verb for different kinds of thing. + +### C2. Options-bag naming splits three ways with no rule + +`…Settings` (10 types), `…Options` (6), and `…Input` / `…Request` / `…Config` / +`…Providers` (8). `createHarnessLoop(options: HarnessLoopOptions)` and +`defineTrainingHarness(settings: HarnessSettings)` sit one call apart and disagree. + +### C3. `enabled` meant three different defaults on three sibling settings objects + +`capture.enabled` defaulted on (checked `=== false`), `tracing.enabled` defaulted on, and +`evolution.enabled` defaulted *off* (checked `!== true`). The same field name with opposite +polarity inside one config object. + +### C4. Gate configuration was half-grouped, half-flat, and duplicated + +`TrainInput` grouped `evaluation` but flattened `minScore`, `minPassRate`, `policy`, +`gates`, `maxRounds`, and `fanOut`. `policy` is itself a `PromotionGate` in disguise — the +gate evaluator wraps it into one — so there were two ways to express one concept. +`maxRounds` appears on `TrainInput`, `TrainingLoopInput`, *and* `HarnessSettings`. + +### C5. Duplicate public names across packages, some not interchangeable + +`digest` exists three times and two are public with **different signatures**: the rewrite +package's takes `unknown`; grounding's takes `string` and normalizes line endings first. +Both emit `sha256:…`, so substituting one for the other silently changes hashes. `Marker` +is defined in both training and rewrite. `defaultMaxRounds = 3` is exported by both training +and harness, and neither reached the root — presumably because they would collide. +`Activation.rollback` and `AppliedPromotion.rollback` are two names for one shape. + +### C6. Identity typing contradicts its own doctrine + +`TrainableIdentity` is documented "never a raw string", and the `trainable()` decorator +throws on non-symbols. Yet `defineTrainable(id: string)`, `captureTrainable(id, …)`, +`instrumentTrainable(…, id)`, `wrapTrainable(fn, id)`, and `swapImplementation(id, …)` are +all raw-string APIs. The brand buys type safety at exactly one call site, and +`defineTrainable("Router.route")` is an unchecked magic string whose typo yields a different +symbol with no error. A `discover` CLI command is the practical mitigation. + +### C7. Sync and async are unpredictable between neighbours + +`discoverTrainables`, `applyCandidate`, `commitRewrite`, and `candidateDeclaration` are +sync; everything on `Training` and `evaluatePromotionGate` is async. `captureTrainable` +returns `Result` synchronously but branches on `isPromise` internally. `RoundObserver` +callbacks are sync-only, while `PromotionGate`, `ActionGate`, `ContextProvider`, and +`TrainInput.policy` all accept `T | Promise`. + +### C8. Three concurrency idioms reachable from one call + +Promises (`Training`), a cold observable (`RoundSequence.subscribe(observer): () => void`), +and callback-bundle inversion of control (`HarnessInput`'s student/teacher/judge/adversary). +`training.train` traverses all three. + +### C9. Twelve exported default constants — except the two that mattered + +`defaultEvolution`, `defaultObjective`, `defaultOutputDir`, `defaultRetry`, +`defaultTsconfig`, `defaultMaxRounds`, `defaultFanOut`, `defaultContextWindow`, +`defaultActionLogDir`, `defaultExecutionTimeoutMs`, `defaultPromotionGates`, and +`inMemoryArtifactRef` across five modules. Meanwhile `minScore ?? 0.8` and +`minPassRate ?? 1` were inline literals. + +## D. Errors and observability + +### D1. Three error models, and Zod leaked through + +Plain `Error`, `TypeError`, and `SyntaxError` at roughly forty sites; `AgentActionDeniedError`, +a hand-rolled class with a `readonly _tag`; and `OperationTimeoutError`, an Effect +`Data.TaggedError`. Zod errors escaped unwrapped, so `minScore: 1.5` yielded a raw +`ZodError` rather than a library error. Consumers had no discriminant beyond message text, +and the tests proved it — they assert on substrings such as +`"requires 2 distinct successful runtime traces; found 1"`. + +The error *copy* is genuinely good: the provider-missing messages each name the exact +setting and the shortcut import. That quality is preserved verbatim in the typed hierarchy. + +### D2. `activate()` throws for an expected outcome + +`TrainingRun.outcome` is already `"ready" | "stalled" | "exhausted"`, yet the only way to +learn why a run could not be applied was to call `activate()` and catch. Background +evolution constructed an `Error` for `outcome !== "ready"` purely to route it into +`onError`. + +### D3. `onError(error: unknown, phase)` was the entire background-observability surface + +Capture, store, and evolve failures funnelled through one untyped callback, while +`evolution.onEvolved` sat in a different object. There was no "evolution started" and no +"evolution skipped", and the documented `onError("evolve")` sad path had no test. + +## E. Boilerplate and testability + +### E1. The runtime was a module-level mutable singleton with no reset or isolation + +`configuredTraining` and `defaultProviders` are module globals and `TrainingRuntime` was not +exported, so tests and multi-tenant hosts could not build an isolated runtime. +`packages/training/test/wiring.ts` existed solely to work around this. + +### E2. Consumer-facing types were not consumer-constructible, forcing casts + +Anyone implementing a custom `TrainingLoop` must produce a `CandidateReview` containing a +`TrainableEvalRun` they cannot build, so the tests write +`{ token, run: {}, evaluations: [] } as unknown as TrainableEvalRun` and `{} as never`. +Testing `@trainable()` required fabricating a `ClassMethodDecoratorContext` cast, and +`AxEngineOptions` was not stubbable. + +### E3. `defineTrainingHarness()` needed three uninferrable generics + +`settings` is optional and mentions only `TCandidate`, so a bare call infers +`unknown, unknown, unknown` and every documented call site writes all three out. A fourth +parameter, `TChallenge`, is scoped to `run` and *does* infer — showing the others could be +restructured the same way. + +### E4. A real training test needed five pieces of setup, repeated verbatim in six files + +A temp `.ts` file on disk, `source: { files: [...] }`, a stub `engine`, a hand-written +`ImplementationExecutor` needing an `as unknown` cast around `new Function`, and +`tracing: { enabled: false }`. + +### E5. `...(x === undefined ? {} : { x })` appeared roughly twenty-five times + +A consequence of `exactOptionalPropertyTypes` with no shared helper. `createHarnessLoop` +invented a one-off `maybeSignal()` for exactly this. + +### E6. Evaluation is string-in, string-out, with a lossy JSON guess + +`evaluationArgs()` `JSON.parse`s the eval input and spreads arrays as arguments, so a +function legitimately taking the single string `"[1,2]"` receives two numbers. Outputs are +stringified before assertion. Multi-argument and non-string trainables are poorly served, +and this was documented nowhere. + +### E7. `effect` was a root runtime dependency for very little + +`src/attempt.ts` used `Effect` to express a `try`/`catch`. Only `resilience.ts` genuinely +benefits, which put a large dependency in every consumer's tree. + +## Remediation plan + +All five tiers are in scope, and the work is **additive**: every renamed or reshaped API is +added alongside the existing one, with the old path kept working and marked `@deprecated` +naming its replacement. + +- Old option names stay accepted and are normalized at the entry point, so `minScore` and + `promotion.minScore` both work. +- Renamed functions keep a re-export under the old name. +- New typed errors extend `Error` and preserve their message strings byte for byte, so + existing `catch` blocks and substring assertions keep working while `instanceof` becomes + available. +- `onError` stays and is implemented on top of the new `onEvent`. +- Nothing is deleted in this release. A follow-up may remove the deprecated surface. + +`test/deprecated.test.ts` exercises every legacy path, so the compatibility promise is +enforced rather than asserted. + +### Tier 1 — defects + +1. Fix the README so its code compiles; define `deploymentPolicy` and order the quickstart. +2. Declare `sideEffects` accurately. +3. Export `defaultMinScore` and `defaultMinPassRate`, and make the rubric print resolved + numbers. +4. Make `TS_AUTOCODE_EVOLVE` fail closed on an explicit allow-list. +5. Honor `fanOut` in the harness loop, or reject it loudly — never ignore it. +6. Close the root re-export gap and rename the colliding `defaultMaxRounds`. +7. Add `TrainingSettings.execution.timeoutMs`, threaded into the default executor. +8. Make grounding's codegen emit the real API and typecheck its output. + +### Tier 2 — additions + +9. `TrainingSettings.model` as a first-class provider/model slot. +10. A `ts-autocode` CLI with `discover`, `status`, and `train`. +11. Runnable examples that import by package name and are checked in CI. + +### Tier 3 — consistency + +12. `createTraining(settings)` returning an isolated runtime; `configureTraining` merges. +13. Normalized `enabled` polarity across capture, tracing, and evolution. +14. Grouped `TrainInput.rounds` and `TrainInput.promotion`, with `policy` folded into + `gates`. +15. One options-bag suffix; duplicate `Marker`, `digest`, and `defaultMaxRounds` resolved. +16. A smaller root surface, with author-level APIs behind a subpath. + +### Tier 4 — errors and observability + +17. A `TsAutocodeError` hierarchy replacing the string throws, preserving every message. +18. A non-throwing way to inspect whether a run can be activated. +19. One `onEvent` discriminated union, with `onError` retained as a shim. + +### Tier 5 — boilerplate + +20. Exported builders for the types that currently force casts. +21. Inferred harness generics. +22. A shared optional-spread helper. +23. A documented evaluation-argument contract with an explicit escape hatch. +24. `effect` dropped from the root and from `attempt.ts`. + +## How these are kept fixed + +- A **surface test** asserts every symbol exported by `ts-autocode-training` and + `ts-autocode-rewrite` is reachable from `ts-autocode`, so A5 cannot recur. +- **Documentation is typechecked**: TypeScript blocks are extracted from the READMEs and + compiled in CI. This is what would have caught A1. +- Grounding's generated output is typechecked rather than string-matched, catching A2. +- A tree-shaking bundle test asserts the Ax engine survives, catching A3. +- Targeted regression tests cover `fanOut`, the rubric thresholds, the evolve kill switch, + and the `onError("evolve")` sad path. From 1a7030a83e5dfdde4e3bb2a0cc35213a20502285 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 17:26:29 +0000 Subject: [PATCH 02/14] fix: repair eight defects in the public surface Each of these was reachable by a user following the documentation. - README snippets did not compile: `activation.promotion.snapshot.candidateId` does not exist on `Activation`, and the quickstart referenced an undefined `deploymentPolicy` and used its token before defining it. Every TypeScript block in all four READMEs is now self-contained and compiled by test/docs.test.ts. - The grounding package generated `training.define(...)`, which `Training` has no such method for, so every generated registration file failed to compile. Codegen now emits `defineGrounding` from the package that owns the concept, and the scan test typechecks the generated source instead of string-matching it. - `sideEffects: false` was wrong: importing the root package wires the engine, executor, loop and promotion applier at import time, so a tree-shaking bundler could legally drop that and leave a consumer with "no training engine is configured" after importing the package that configures it. - `TrainInput.fanOut` was documented but silently ignored by the default harness loop, whose judge/adversary/rubric sequence is serial. It now refuses a fan-out above 1 and names the loop that supports one. - The root package re-exported a hand-maintained subset of its siblings that had drifted, leaving README-documented `trainingRounds` and `sequentialLoop` unreachable along with `defaultPromotionGates`. test/surface.test.ts now enforces exhaustiveness. The harness's colliding `defaultMaxRounds` is renamed `defaultHarnessRounds`, keeping the old name as a deprecated alias. - The promotion rubric handed to the judge printed the literal string "evaluation default" instead of the threshold. The defaults are now exported as `defaultMinScore` and `defaultMinPassRate` and the rubric resolves them. - `TS_AUTOCODE_EVOLVE` failed open: only "0", "false" and "off" disabled source-rewriting evolution, so "no" enabled it. It now fails closed and throws on an unrecognized value. - Candidate execution timeout had no path through settings; added `TrainingSettings.execution.timeoutMs`, distinct from the retrying `resilience.evaluate` policy. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- README.md | 59 ++++++++++++-- package.json | 5 +- packages/grounding/src/decorators.ts | 32 +++++++- packages/grounding/src/index.ts | 2 + packages/grounding/src/scan.ts | 9 ++- packages/grounding/test/scan.test.ts | 59 +++++++++++++- packages/harness/README.md | 49 +++++++++++- packages/harness/src/harness.ts | 9 ++- packages/harness/src/index.ts | 2 +- packages/rewrite/README.md | 2 + packages/training/README.md | 5 +- packages/training/src/index.ts | 3 +- packages/training/src/promotion.ts | 11 ++- packages/training/src/training.ts | 29 ++++++- src/index.ts | 59 +++++++++++++- src/providers/harness.ts | 12 ++- src/register.ts | 23 +++++- test/docs.test.ts | 113 +++++++++++++++++++++++++++ test/fixtures/rubric.ts | 8 ++ test/surface.test.ts | 36 +++++++++ test/tier1.test.ts | 111 ++++++++++++++++++++++++++ 21 files changed, 599 insertions(+), 39 deletions(-) create mode 100644 test/docs.test.ts create mode 100644 test/fixtures/rubric.ts create mode 100644 test/surface.test.ts create mode 100644 test/tier1.test.ts diff --git a/README.md b/README.md index 1742273..b9c6dbb 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,24 @@ above — pins these evals to that exact method; for an auto-generated identity, `defineTrainable("Router.route").symbol` recreates the symbol. ```ts -import { training } from "ts-autocode"; +import { defineTrainable, training, type CandidatePatch } from "ts-autocode"; + +// The same identity the marked method carries; see the section above. +const route = defineTrainable("Router.route"); + +// Whatever your deployment rules are — anything returning a boolean works. +const deploymentPolicy = { + allows: (candidate: CandidatePatch) => candidate.implementation.length < 4_000, +}; + +class Router { + route(input: string): string { + "use training"; + return input.includes("invoice") ? "billing" : "fallback"; + } +} + +const router = new Router(); const tests = [ { @@ -166,9 +183,12 @@ Once a trainable accumulates `evolution.minTraces` successful traces (default 3), it is trained against those traces, verified candidate-bound, gated, and — only when the gate passes — its source body is rewritten. Failures surface through `TrainingSettings.onError` with the `"evolve"` phase and never block or -alter application calls. Set `TS_AUTOCODE_EVOLVE=off` (or configure -`evolution: { enabled: false }`) to capture without rewriting, and use -`evolution.onEvolved` to observe applied rewrites. +alter application calls. Loading the hook is itself the opt-in, so evolution is on unless you turn it +off: set `TS_AUTOCODE_EVOLVE` to `0`, `false`, `off`, `no`, or `disabled` (or +configure `evolution: { enabled: false }`) to capture without rewriting, and use +`evolution.onEvolved` to observe applied rewrites. Because the feature rewrites +your source, the switch fails closed — an unrecognized value throws rather than +being read as consent. ## Train from live traces @@ -179,6 +199,10 @@ replacement, verifies the candidate against the same cases, and applies the promotion gate. Activating the run then updates the marked TypeScript body. ```ts +import { defineTrainable, training } from "ts-autocode"; + +const route = defineTrainable("Router.route"); + const run = await training.train({ trainable: route, objective: "Preserve routing behavior observed in production", @@ -190,7 +214,7 @@ const run = await training.train({ }); const activation = await run.activate(); -console.log(activation.promotion.snapshot.candidateId); +console.log(activation.run.final.candidate.id); ``` Only successful traces with both captured input and output become eval cases. @@ -216,8 +240,12 @@ The built-in loop is an observable round sequence (`trainingRounds()` pushes each reviewed round to a subscriber; `sequentialLoop` collects the subscription into one run). `TrainInput.fanOut` caps how many candidates a round proposes and reviews concurrently — the best gated candidate wins the -round — and `TrainInput.gates` appends custom promotion rules to the standard -gate set; the configured `policy` runs as one such rule. +round. Fan-out belongs to `sequentialLoop`: the default governed harness loop +reviews exactly one candidate per round, because its judge, adversary and +rubric-revision sequence is serial, so it **rejects** a `fanOut` above 1 rather +than accepting one it would ignore. `TrainInput.gates` appends custom promotion +rules to the standard `defaultPromotionGates` set; the configured `policy` runs +as one such rule. No Ax program is supplied by the caller. The default engine derives its fields, descriptions, executable examples, and return contract from the TypeScript @@ -241,6 +269,8 @@ Runtime dependencies enter through `TrainingSettings`: a 30-second cap per attempt: ```ts + import { configureTraining } from "ts-autocode"; + configureTraining({ resilience: { propose: { timeoutMs: 30_000, retry: { attempts: 3 } }, @@ -248,6 +278,10 @@ Runtime dependencies enter through `TrainingSettings`: }); ``` +- `execution` bounds each candidate run inside the executor: `timeoutMs` caps a + single execution (default 5 seconds). This is distinct from + `resilience.evaluate.timeoutMs`, which bounds the whole attempt and may retry + it. - `source` overrides TypeScript project discovery when the default `tsconfig.json` is not the desired project. - `outputDir` relocates run artifacts and eval output (default `.agentv`, @@ -283,6 +317,17 @@ adapter and is passed through the provider-neutral `engine` slot. Custom engines return only the new method implementation: ```ts +import type { TrainingEngine } from "ts-autocode"; + +// Your own optimizer call — whatever produces a replacement method body. +declare function rewrite(request: { + signature: string; + implementation: string; + objective: string; + evaluations: unknown; + secrets: unknown; +}): Promise; + const engine: TrainingEngine = { id: "acme/optimizer", async optimize(request, context) { diff --git a/package.json b/package.json index 42c815b..53f6045 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,10 @@ "description": "Train and safely rewrite decorated or directive-marked TypeScript functions.", "license": "MIT", "type": "module", - "sideEffects": false, + "sideEffects": [ + "./dist/index.js", + "./dist/register.js" + ], "workspaces": [ "packages/*" ], diff --git a/packages/grounding/src/decorators.ts b/packages/grounding/src/decorators.ts index b225cb4..15adc80 100644 --- a/packages/grounding/src/decorators.ts +++ b/packages/grounding/src/decorators.ts @@ -14,15 +14,45 @@ export interface FieldDescription { readonly example?: unknown; } +/** A declared parameter or return shape, carried on the contract so a + * registration grounds on the TypeScript signature even with no decorators. */ +export interface ShapeDescriptor { + readonly type: string; + readonly optional?: boolean; + readonly description?: string; +} + /** Provider-neutral composed grounding for one method. */ export interface GroundingOptions { readonly methodRef: string; readonly intent: string; - readonly contract: { readonly ref: string }; + readonly contract: { + readonly ref: string; + /** Declared parameter shapes, by parameter name. */ + readonly input?: Readonly>; + readonly output?: ShapeDescriptor; + }; readonly params?: Record; readonly output?: { readonly returns: FieldDescription }; } +/** Validate and freeze one composed grounding. Generated registration source + * calls this, so what codegen emits typechecks against a real API in the + * package that owns the concept -- this package never imports a training + * runtime, and a host registers the results against its own registry. */ +export function defineGrounding(options: GroundingOptions): GroundingOptions { + if (!options.methodRef?.trim()) { + throw new TypeError("grounding methodRef must be a non-empty string"); + } + if (!options.intent?.trim()) { + throw new TypeError(`grounding intent must be a non-empty string for ${options.methodRef}`); + } + if (!options.contract?.ref?.trim()) { + throw new TypeError(`grounding contract ref must be a non-empty string for ${options.methodRef}`); + } + return Object.freeze({ ...options, contract: Object.freeze({ ...options.contract }) }); +} + export interface PendingGrounding { intent?: string; returns?: string; diff --git a/packages/grounding/src/index.ts b/packages/grounding/src/index.ts index dbc9a90..01e89c3 100644 --- a/packages/grounding/src/index.ts +++ b/packages/grounding/src/index.ts @@ -1,5 +1,6 @@ export { composeOptions, + defineGrounding, description, granularOptionsFor, inferredIntent, @@ -11,6 +12,7 @@ export { type GroundingOptions, type PendingGrounding, type PendingMap, + type ShapeDescriptor, } from "./decorators.js"; export { COMPONENT_METADATA, diff --git a/packages/grounding/src/scan.ts b/packages/grounding/src/scan.ts index cec9ab4..31ebf4f 100644 --- a/packages/grounding/src/scan.ts +++ b/packages/grounding/src/scan.ts @@ -157,7 +157,8 @@ function memberName(name: ts.PropertyName, sourceFile: ts.SourceFile): string { } export interface RegistrationEmitOptions { - /** Module specifier the emitted `import { training } from …` uses. */ + /** Module specifier the emitted `import { defineGrounding } from …` uses. + * Defaults to this package's public entry. */ readonly runtimeModule?: string; /** Leading comment lines (verbatim, with `//`) above the import. */ readonly header?: readonly string[]; @@ -168,7 +169,7 @@ const defaultHeader: readonly string[] = [ ]; /** - * Generate the `training.define` registration source for a scanned class. + * Generate the `defineGrounding` registration source for a scanned class. * The declared TypeScript signature becomes the contract's shape * descriptors, so the registration grounds (shape present) even with every * decorator omitted. @@ -179,7 +180,7 @@ export function generateDeclaredRegistrations( ): string { const lines: string[] = [ ...(emit.header ?? defaultHeader), - `import { training } from ${JSON.stringify(emit.runtimeModule ?? "ts-autocode")};`, + `import { defineGrounding } from ${JSON.stringify(emit.runtimeModule ?? "ts-autocode/grounding")};`, "", ]; for (const op of declared.operations) { @@ -210,7 +211,7 @@ export function generateDeclaredRegistrations( ...(Object.keys(params).length > 0 ? { params } : {}), ...(op.returns ? { output: { returns: { description: op.returns } } } : {}), }; - lines.push(`export const ${op.method} = training.define(${JSON.stringify(options, null, "\t")});`, ""); + lines.push(`export const ${op.method} = defineGrounding(${JSON.stringify(options, null, "\t")});`, ""); } return lines.join("\n"); } diff --git a/packages/grounding/test/scan.test.ts b/packages/grounding/test/scan.test.ts index 9c73716..059bf31 100644 --- a/packages/grounding/test/scan.test.ts +++ b/packages/grounding/test/scan.test.ts @@ -1,3 +1,6 @@ +import { readFileSync } from "node:fs"; + +import ts, { sys } from "typescript"; import { describe, expect, it } from "vitest"; import { generateDeclaredRegistrations, scanDeclaredTrainables } from "../src/index.js"; @@ -113,7 +116,7 @@ describe("scanDeclaredTrainables", () => { }); describe("generateDeclaredRegistrations", () => { - it("emits training.define registrations with a configurable header and runtime module", () => { + it("emits defineGrounding registrations with a configurable header and runtime module", () => { const [program] = scanDeclaredTrainables(ambient); const source = generateDeclaredRegistrations(program as NonNullable, { runtimeModule: "@hobo/runtime", @@ -122,12 +125,60 @@ describe("generateDeclaredRegistrations", () => { "// do not edit by hand (ADR-0037 generated-vs-authored boundary).", ], }); - expect(source).toContain('import { training } from "@hobo/runtime";'); + expect(source).toContain('import { defineGrounding } from "@hobo/runtime";'); expect(source).toContain("// Generated by hobo stub from an ambient @trainable declaration —"); - expect(source).toContain("export const trainableMethod = training.define({"); + expect(source).toContain("export const trainableMethod = defineGrounding({"); expect(source).toContain('"ref": "decl://Program.trainableMethod"'); expect(source).toContain('"description": "Optional person to greet"'); expect(source).toContain('"output": {\n\t\t"returns": {\n\t\t\t"description": "Hello World! or Hello, ! when supplied"\n\t\t}\n\t}'); - expect(source).toContain("export const other = training.define({"); + expect(source).toContain("export const other = defineGrounding({"); + }); + + it("defaults the runtime module to this package's public entry", () => { + const [program] = scanDeclaredTrainables(ambient); + expect(generateDeclaredRegistrations(program as NonNullable)) + .toContain('import { defineGrounding } from "ts-autocode/grounding";'); + }); + + // The generated source used to call `training.define`, which does not exist + // on the Training runtime, so every generated file failed to compile while + // the string assertions above still passed. Typecheck what we emit. + it("emits source that typechecks against the real defineGrounding signature", () => { + const [program] = scanDeclaredTrainables(ambient); + const generated = generateDeclaredRegistrations(program as NonNullable, { + runtimeModule: "./decorators.js", + }); + const entry = "/generated.ts"; + const files = new Map([ + [entry, generated], + ["/decorators.ts", readFileSync(new URL("../src/decorators.ts", import.meta.url), "utf8")], + ]); + const host: ts.CompilerHost = { + fileExists: (name) => files.has(name) || sys.fileExists(name), + readFile: (name) => files.get(name) ?? sys.readFile(name), + getSourceFile: (name, languageVersion) => { + const text = files.get(name) ?? sys.readFile(name); + return text === undefined ? undefined : ts.createSourceFile(name, text, languageVersion, true); + }, + getDefaultLibFileName: (options) => sys.getExecutingFilePath().replace(/[^/]+$/, ts.getDefaultLibFileName(options)), + writeFile: () => undefined, + getCurrentDirectory: () => "/", + getCanonicalFileName: (name) => name, + useCaseSensitiveFileNames: () => true, + getNewLine: () => "\n", + }; + const program_ = ts.createProgram([entry], { + strict: true, + exactOptionalPropertyTypes: true, + target: ts.ScriptTarget.ES2022, + module: ts.ModuleKind.ESNext, + moduleResolution: ts.ModuleResolutionKind.Bundler, + noEmit: true, + skipLibCheck: true, + }, host); + const errors = ts.getPreEmitDiagnostics(program_) + .filter((diagnostic) => diagnostic.file?.fileName === entry) + .map((diagnostic) => ts.flattenDiagnosticMessageText(diagnostic.messageText, " ")); + expect(errors).toEqual([]); }); }); diff --git a/packages/harness/README.md b/packages/harness/README.md index 5449c05..8d40d46 100644 --- a/packages/harness/README.md +++ b/packages/harness/README.md @@ -31,7 +31,15 @@ npm install ts-autocode-harness The minimal loop is two callbacks: ```ts -import { defineTrainingHarness } from "ts-autocode-harness"; +import { defineTrainingHarness, type StudentTurn, type TeacherResult } from "ts-autocode-harness"; + +// Your own candidate and assessment shapes, and the roles that produce them. +interface Candidate { readonly id: string } +interface Assessment { readonly score: number } +declare const objective: string; +declare const target: string; +declare const myStudent: (turn: StudentTurn) => Promise; +declare const myTeacher: (candidate: Candidate, turn: StudentTurn) => Promise>; const result = await defineTrainingHarness().run({ task: { objective, target }, @@ -46,10 +54,36 @@ and a bespoke rubric revision: ```ts import { join } from "node:path"; -import { defineTrainingHarness, WriteAheadAgentBus } from "ts-autocode-harness"; +import { + defineTrainingHarness, + WriteAheadAgentBus, + type AdversaryResult, + type AdversaryTurn, + type JudgeDecision, + type JudgeRequest, + type RubricRevision, + type RubricRevisionTurn, + type StudentTurn, + type TeacherResult, +} from "ts-autocode-harness"; import { createStorage } from "unstorage"; import fsDriver from "unstorage/drivers/fs"; +// Your own candidate and assessment shapes, and the roles that produce them. +interface Candidate { readonly id: string } +interface Assessment { readonly score: number } +declare const objective: string; +declare const target: string; +declare const root: string; +declare const myStudent: (turn: StudentTurn) => Promise; +declare const myTeacher: (candidate: Candidate, turn: StudentTurn) => Promise>; +declare const myJudge: (request: JudgeRequest) => JudgeDecision; +declare const myAdversary: (candidate: Candidate, turn: AdversaryTurn) => Promise>; +declare const myRubricRevision: ( + challenge: AdversaryResult, + turn: RubricRevisionTurn, +) => RubricRevision; + const harness = defineTrainingHarness({ maxRounds: 3, candidateId: (candidate) => candidate.id, @@ -130,7 +164,16 @@ must lie outside every writable sandbox path. Add `allowedHosts` only when a sandboxed tool genuinely needs outbound access. ```ts -import { HarnessSandbox, WriteAheadAgentBus } from "ts-autocode-harness"; +import { + HarnessSandbox, + WriteAheadAgentBus, + type ActionGate, + type JudgeDecision, +} from "ts-autocode-harness"; + +declare const workspace: string; +declare const bus: WriteAheadAgentBus; +declare const myJudge: (request: { subject: "action"; action: Parameters[0]; context: Parameters[1] }) => JudgeDecision; const sandbox = new HarnessSandbox({ id: "student", diff --git a/packages/harness/src/harness.ts b/packages/harness/src/harness.ts index f5e0bc6..dcadbd2 100644 --- a/packages/harness/src/harness.ts +++ b/packages/harness/src/harness.ts @@ -144,7 +144,12 @@ export interface HarnessSettings { } /** How many student rounds a harness runs when `maxRounds` is unset. */ -export const defaultMaxRounds = 3; +export const defaultHarnessRounds = 3; + +/** @deprecated Renamed to {@link defaultHarnessRounds}; the old name collided + * with `ts-autocode-training`'s loop default, which kept both out of the root + * package's exports. */ +export const defaultMaxRounds = defaultHarnessRounds; export interface TrainingHarness { run(input: HarnessInput): Promise>; @@ -153,7 +158,7 @@ export interface TrainingHarness { export function defineTrainingHarness( settings: HarnessSettings = {}, ): TrainingHarness { - const maxRounds = roundLimit.parse(settings.maxRounds ?? defaultMaxRounds); + const maxRounds = roundLimit.parse(settings.maxRounds ?? defaultHarnessRounds); const identify = settings.candidateId ?? stringifyCandidate; return Object.freeze({ diff --git a/packages/harness/src/index.ts b/packages/harness/src/index.ts index 24c98ef..a0c4ad4 100644 --- a/packages/harness/src/index.ts +++ b/packages/harness/src/index.ts @@ -7,7 +7,7 @@ export type { ActionGate, JudgeDecision } from "./dispatch.js"; export { agentBusEntry, agentMessage } from "./schema.js"; export type { AbsolutePath, AgentBusEntry, AgentMessage } from "./schema.js"; -export { defaultMaxRounds, defineTrainingHarness } from "./harness.js"; +export { defaultHarnessRounds, defaultMaxRounds, defineTrainingHarness } from "./harness.js"; export type { AdversaryConfig, AdversaryResult, diff --git a/packages/rewrite/README.md b/packages/rewrite/README.md index fbc83c9..6543315 100644 --- a/packages/rewrite/README.md +++ b/packages/rewrite/README.md @@ -27,6 +27,8 @@ hook weaves each marked method, and committing a rewrite drives the swap: ```ts import { configureRewrite } from "ts-autocode-rewrite"; +declare function log(id: string, args: readonly unknown[]): void; + // A consumer registers its marker once. configureRewrite({ marker: "use audit", diff --git a/packages/training/README.md b/packages/training/README.md index 9333ff0..37c058e 100644 --- a/packages/training/README.md +++ b/packages/training/README.md @@ -19,7 +19,10 @@ governed `ts-autocode-harness` loop as the default orchestrator, and `ts-autocode-rewrite` as capture interception and the promotion applier. ```ts -import { configureTraining, provideTrainingDefaults } from "ts-autocode-training"; +import { provideTrainingDefaults, type ImplementationExecutor, type TrainingEngine } from "ts-autocode-training"; + +declare const myEngine: TrainingEngine; +declare const myRunner: { run: ImplementationExecutor }; provideTrainingDefaults({ engine: () => myEngine, diff --git a/packages/training/src/index.ts b/packages/training/src/index.ts index 668814e..52ffe03 100644 --- a/packages/training/src/index.ts +++ b/packages/training/src/index.ts @@ -13,6 +13,7 @@ export type { CaptureSettings, ErrorPhase, EvolutionSettings, + ExecutionSettings, PromotionApplier, TrainInput, Training, @@ -58,7 +59,7 @@ export type { export type { TrainableEvalRun } from "./evaluation.js"; -export { defaultPromotionGates, evaluatePromotionGate } from "./promotion.js"; +export { defaultMinPassRate, defaultMinScore, defaultPromotionGates, evaluatePromotionGate } from "./promotion.js"; export type { PromotionDecision, PromotionGate, PromotionGateContext, PromotionGateInput } from "./promotion.js"; export { MemoryTrainingStore } from "./records.js"; diff --git a/packages/training/src/promotion.ts b/packages/training/src/promotion.ts index c10fc3d..78a5bd8 100644 --- a/packages/training/src/promotion.ts +++ b/packages/training/src/promotion.ts @@ -8,6 +8,13 @@ const unitInterval = (name: string) => const minScoreThreshold = unitInterval("minScore"); const minPassRateThreshold = unitInterval("minPassRate"); +/** Mean AgentV score a candidate must reach when `minScore` is unset. */ +export const defaultMinScore = 0.8; + +/** Fraction of evaluation cases a candidate must pass when `minPassRate` is + * unset; every case, by default. */ +export const defaultMinPassRate = 1; + export interface PromotionGateInput { readonly candidate: CandidatePatch; readonly evaluations: readonly BoundEvaluation[]; @@ -77,8 +84,8 @@ export const defaultPromotionGates: readonly PromotionGate[] = [ /** Runs the standard gates, the configured policy, and any extension gates * over one shared context; the collected failures decide promotion. */ export async function evaluatePromotionGate(input: PromotionGateInput): Promise { - const minScore = minScoreThreshold.parse(input.minScore ?? 0.8); - const minPassRate = minPassRateThreshold.parse(input.minPassRate ?? 1); + const minScore = minScoreThreshold.parse(input.minScore ?? defaultMinScore); + const minPassRate = minPassRateThreshold.parse(input.minPassRate ?? defaultMinPassRate); const results = input.evaluations .filter((evaluation) => evaluation.trainableId === input.candidate.trainableId && evaluation.candidateId === input.candidate.id) .map((evaluation) => evaluation.result); diff --git a/packages/training/src/training.ts b/packages/training/src/training.ts index b00660d..8c03e74 100644 --- a/packages/training/src/training.ts +++ b/packages/training/src/training.ts @@ -17,7 +17,13 @@ import { import { withPolicy, type ResilienceSettings } from "./resilience.js"; import { evaluateTrainable, type TrainableEvalRun } from "./evaluation.js"; import { sequentialLoop, type TrainingLoop, type TrainingRound } from "./loop.js"; -import { evaluatePromotionGate, type PromotionDecision, type PromotionGate } from "./promotion.js"; +import { + defaultMinPassRate, + defaultMinScore, + evaluatePromotionGate, + type PromotionDecision, + type PromotionGate, +} from "./promotion.js"; import { MemoryTrainingStore, type TrainingRecord, type TrainingStore } from "./records.js"; import { findTrainable, @@ -70,9 +76,18 @@ export const defaultObjective = "Preserve behavior demonstrated by the evaluatio * nor `TrainingSettings.outputDir` names a directory. */ export const defaultOutputDir = ".agentv"; +/** How proposed candidate bodies are run during verification. Distinct from + * `resilience.evaluate`, which bounds the whole attempt and may retry it: this + * is the executor's own per-run limit. */ +export interface ExecutionSettings { + readonly timeoutMs?: number; +} + export interface TrainingSettings { readonly engine?: TrainingEngine; readonly executor?: ImplementationExecutor; + /** Options handed to the executor on every candidate run. */ + readonly execution?: ExecutionSettings; readonly loop?: TrainingLoop; readonly evolution?: EvolutionSettings; /** Default directory for run artifacts and eval output; a run's @@ -248,6 +263,7 @@ class TrainingRuntime implements Training { async #evaluateCandidate(candidate: CandidatePatch, config: CandidateEvalConfig): Promise { const token = defineTrainable(candidate.trainableId); const execute = this.#executorOrThrow(); + const timeoutMs = this.#settings.execution?.timeoutMs; const { signal, ...evaluation } = config; signal?.throwIfAborted(); const evaluated = await evaluateTrainable(token, { @@ -260,7 +276,10 @@ class TrainingRuntime implements Training { candidate.target, candidate.implementation, evaluationArgs(input), - attemptSignal === undefined ? {} : { signal: attemptSignal }, + { + ...(timeoutMs === undefined ? {} : { timeoutMs }), + ...(attemptSignal === undefined ? {} : { signal: attemptSignal }), + }, ), signal, ); @@ -627,8 +646,10 @@ function liveEvalCases(records: readonly TrainingRecord[]): readonly EvalTestInp function promotionRubric(input: TrainInput): string { return [ "Candidate must pass source conformance checks.", - `Minimum evaluation score: ${input.minScore ?? "evaluation default"}.`, - `Minimum evaluation pass rate: ${input.minPassRate ?? 1}.`, + // The judge reads this verbatim, so it must carry the resolved numbers a + // candidate is actually held to -- never a placeholder. + `Minimum evaluation score: ${input.minScore ?? defaultMinScore}.`, + `Minimum evaluation pass rate: ${input.minPassRate ?? defaultMinPassRate}.`, input.policy === undefined ? "No additional promotion policy." : "Candidate must pass the configured promotion policy.", ].join(" "); } diff --git a/src/index.ts b/src/index.ts index b16992b..8c772d8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -25,22 +25,39 @@ export { configureRewriteCapture, rewritePromotion } from "./providers/rewrite.j export { instrumentTrainable, trainable, wrapTrainable } from "./instrumentation.js"; export type { TrainableDecorator } from "./instrumentation.js"; +// The re-export lists below are exhaustive by contract: `test/surface.test.ts` +// asserts that every runtime value exported by ts-autocode-training and +// ts-autocode-rewrite is reachable from here. They had drifted, leaving +// README-documented symbols such as `trainingRounds` and `sequentialLoop` +// unreachable, and `defaultPromotionGates` -- needed to compose +// `TrainInput.gates` with the standard set -- unavailable. export { + candidateDeclaration, captureTrainable, configureTraining, - MemoryTrainingStore, - OperationTimeoutError, defaultEvolution, + defaultFanOut, + defaultMaxRounds, + defaultMinPassRate, + defaultMinScore, defaultObjective, defaultOutputDir, + defaultPromotionGates, defaultRetry, defaultTsconfig, defineTrainable, + discoverInSource, discoverTrainables, evaluatePromotionGate, + inMemoryArtifactRef, + MemoryTrainingStore, + OperationTimeoutError, provideTrainingDefaults, + sequentialLoop, + trainableTokenFromSymbol, training, trainingMarker, + trainingRounds, withPolicy, } from "ts-autocode-training"; export type { @@ -54,16 +71,20 @@ export type { EngineContext, ErrorPhase, EvolutionSettings, + ExecutionSettings, ImplementationExecutor, + Marker, OptimizeRequest, PromotionApplier, PromotionDecision, PromotionGate, PromotionGateContext, PromotionGateInput, + ProposalTurn, ResiliencePolicy, ResilienceSettings, RetryOptions, + ReviewContext, RoundObserver, RoundSequence, SecretProvider, @@ -89,10 +110,42 @@ export type { } from "ts-autocode-training"; export { + annotateRewrite, applyCandidate, + check, commitRewrite, + configureRewrite, + createRewriter, + declaringContainer, + digest, + dispatchRewrite, + emitInstrumentation, + installedInstrumentation, + installInstrumentation, + instrumentKey, restoreImplementation, revertRewrite, swapImplementation, + swappedImplementation, } from "ts-autocode-rewrite"; -export type { AppliedRewrite, RewriteCandidate, RewriteSnapshot, RewriteTarget } from "ts-autocode-rewrite"; +export type { + AppliedRewrite, + InstrumentEntry, + InstrumentRegistry, + InstrumentTarget, + Instrumentation, + RewriteCandidate, + RewriteConfig, + RewriteInterceptor, + RewriteInvocation, + RewriteSnapshot, + RewriteTarget, +} from "ts-autocode-rewrite"; + +// `HarnessLoopOptions` is typed in terms of these, so configuring the default +// loop must not require taking a second, undocumented dependency. +export type { + ContextProvider, + JudgeDecision, + JudgeRequest, +} from "ts-autocode-harness"; diff --git a/src/providers/harness.ts b/src/providers/harness.ts index 5ebbe92..a7dc7a5 100644 --- a/src/providers/harness.ts +++ b/src/providers/harness.ts @@ -43,6 +43,17 @@ export function createHarnessLoop(options: HarnessLoopOptions = {}): TrainingLoo createStorage({ driver: fsDriver({ base: resolve(input.outputDir, defaultActionLogDir) }) })); const contextProvider = options.contextProvider ?? windowedContext(); return async (input) => { + // The governed harness explores exactly one candidate per round: its + // judge -> adversary -> rubric-revision sequence is serial by + // construction, and a standing challenge must tighten the rubric before + // the next proposal. Rather than accept `fanOut` and quietly ignore it, + // say so -- use `sequentialLoop`/`trainingRounds` for concurrent slots. + if (input.fanOut !== undefined && input.fanOut > 1) { + throw new Error( + `the governed harness loop reviews one candidate per round and cannot honor fanOut ${input.fanOut}; ` + + "omit fanOut or set TrainingSettings.loop to sequentialLoop, which supports it", + ); + } const harness = defineTrainingHarness( input.maxRounds === undefined ? {} : { maxRounds: input.maxRounds }, ); @@ -53,7 +64,6 @@ export function createHarnessLoop(options: HarnessLoopOptions = {}): TrainingLoo task: { trainable: input.trainableId, objective: input.objective }, rubric: input.rubric, ...maybeSignal(input.signal), - // The governed harness explores one candidate per round; fan-out stays 1. student: ({ round, feedback, signal }) => input.propose({ round, slot: 1, feedback, ...maybeSignal(signal) }), teacher: async (candidate, { round, signal }) => { diff --git a/src/register.ts b/src/register.ts index e592c93..c884d7d 100644 --- a/src/register.ts +++ b/src/register.ts @@ -12,12 +12,27 @@ import "./index.js"; installInstrumentation({ method: instrumentTrainable, wrap: wrapTrainable }); -/** Environment switch for zero-config evolution; anything in `evolveOptOuts` disables it. */ +/** Environment switch for zero-config evolution. Loading this module is itself + * the opt-in, so an unset variable leaves evolution on; the variable exists to + * turn it back off without changing the command line. */ export const evolveVariable = "TS_AUTOCODE_EVOLVE"; -const evolveOptOuts = ["0", "false", "off"]; +const evolveOff = ["0", "false", "off", "no", "disabled"]; +const evolveOn = ["1", "true", "on", "yes", "enabled"]; -const evolveFlag = (process.env[evolveVariable] ?? "").trim().toLowerCase(); -if (!evolveOptOuts.includes(evolveFlag)) { +/** Reads the kill switch, failing closed: an unrecognized value throws rather + * than being guessed at. Evolution rewrites the user's source files, so a + * misspelled `TS_AUTOCODE_EVOLVE=nope` must never be read as consent. */ +export function evolutionEnabled(value: string | undefined): boolean { + const flag = (value ?? "").trim().toLowerCase(); + if (flag === "") return true; + if (evolveOff.includes(flag)) return false; + if (evolveOn.includes(flag)) return true; + throw new Error( + `${evolveVariable} must be one of ${[...evolveOn, ...evolveOff].join(", ")}; received ${JSON.stringify(value)}`, + ); +} + +if (evolutionEnabled(process.env[evolveVariable])) { provideTrainingDefaults({ evolution: { enabled: true } }); } diff --git a/test/docs.test.ts b/test/docs.test.ts new file mode 100644 index 0000000..87d4b09 --- /dev/null +++ b/test/docs.test.ts @@ -0,0 +1,113 @@ +import { readFileSync } from "node:fs"; +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import ts from "typescript"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; + +// The README documented `activation.promotion.snapshot.candidateId` on an +// `Activation` that has only `run` and `rollback`, and a quickstart that +// referenced an undefined `deploymentPolicy`. Prose review had not caught +// either in the year the snippets shipped. Compile them instead. + +const repoRoot = fileURLToPath(new URL("..", import.meta.url)); + +const docs = [ + "README.md", + "packages/training/README.md", + "packages/harness/README.md", + "packages/rewrite/README.md", +]; + +interface Snippet { + readonly doc: string; + readonly line: number; + readonly code: string; +} + +/** Fenced ```ts blocks, including ones indented inside a list item. A block may + * opt out with `` on the line above its fence. */ +export function typescriptSnippets(doc: string, markdown: string): readonly Snippet[] { + const lines = markdown.split("\n"); + const snippets: Snippet[] = []; + for (let index = 0; index < lines.length; index += 1) { + const open = /^(\s*)```ts$/.exec(lines[index] ?? ""); + if (!open) continue; + const indent = open[1] ?? ""; + if (//.test(lines[index - 1] ?? "")) continue; + const body: string[] = []; + let cursor = index + 1; + while (cursor < lines.length && (lines[cursor] ?? "").trim() !== "```") { + body.push((lines[cursor] ?? "").slice(indent.length)); + cursor += 1; + } + snippets.push({ doc, line: index + 1, code: body.join("\n") }); + index = cursor; + } + return snippets; +} + +const snippets = docs.flatMap((doc) => + typescriptSnippets(doc, readFileSync(join(repoRoot, doc), "utf8"))); + +// Snippets compile inside the repo (under the git-ignored test output tree) so +// NodeNext resolution sees the real node_modules and the root package's +// "type": "module" — top-level await in the docs is then legal, as it is for a +// consumer. +const directory = join(repoRoot, "test", "output", "docs"); + +beforeAll(async () => { + await rm(directory, { recursive: true, force: true }); + await mkdir(directory, { recursive: true }); +}); + +afterAll(async () => { + await rm(directory, { recursive: true, force: true }); +}); + +describe("documentation snippets", () => { + it("finds the documented TypeScript blocks", () => { + expect(snippets.length).toBeGreaterThan(5); + }); + + it.each(snippets.map((snippet) => [`${snippet.doc}:${snippet.line}`, snippet] as const))( + "%s compiles", + async (_label, snippet) => { + // Snippets are top-level-await narratives, so compile each as its own + // module resolving `ts-autocode` through the repo's real node_modules. + const file = join(directory, `snippet-${snippet.doc.replace(/\W/g, "_")}-${snippet.line}.ts`); + await writeFile(file, snippet.code, "utf8"); + const program = ts.createProgram([file], { + target: ts.ScriptTarget.ES2023, + module: ts.ModuleKind.NodeNext, + moduleResolution: ts.ModuleResolutionKind.NodeNext, + lib: ["lib.es2023.d.ts", "lib.esnext.decorators.d.ts"], + strict: true, + noUncheckedIndexedAccess: true, + exactOptionalPropertyTypes: true, + noEmit: true, + skipLibCheck: true, + types: ["node"], + baseUrl: repoRoot, + // The package cannot import itself by name from inside its own + // repo, so resolve its public entries to their sources — the + // snippets stay written exactly as a consumer would write them. + paths: { + "ts-autocode": ["src/index.ts"], + "ts-autocode/ax": ["src/providers/ax.ts"], + "ts-autocode/grounding": ["src/grounding.ts"], + }, + }); + const errors = ts.getPreEmitDiagnostics(program) + .filter((diagnostic) => diagnostic.file?.fileName === file.replace(/\\/g, "/")) + .map((diagnostic) => { + const position = diagnostic.file && diagnostic.start !== undefined + ? diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start).line + 1 + : 0; + return `line ${position}: ${ts.flattenDiagnosticMessageText(diagnostic.messageText, " ")}`; + }); + expect(errors).toEqual([]); + }, + ); +}); diff --git a/test/fixtures/rubric.ts b/test/fixtures/rubric.ts new file mode 100644 index 0000000..97fb41a --- /dev/null +++ b/test/fixtures/rubric.ts @@ -0,0 +1,8 @@ +// Fixture for the promotion-rubric test: a directive-marked method the source +// scanner can discover. +export class Fixture { + route(input: string): string { + "use training"; + return input; + } +} diff --git a/test/surface.test.ts b/test/surface.test.ts new file mode 100644 index 0000000..0522785 --- /dev/null +++ b/test/surface.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; + +import * as root from "../src/index.js"; +import * as rewrite from "ts-autocode-rewrite"; +import * as training from "ts-autocode-training"; + +// The root package re-exported a hand-maintained subset of its siblings, and it +// had drifted: README-documented symbols (`trainingRounds`, `sequentialLoop`) +// and extension-point essentials (`defaultPromotionGates`) were unreachable +// from `ts-autocode`. These tests make the re-export lists exhaustive by +// contract rather than by vigilance. + +const valueNames = (module: object): readonly string[] => + Object.keys(module).filter((name) => name !== "default").sort(); + +describe("root export surface", () => { + it.each([ + ["ts-autocode-training", training], + ["ts-autocode-rewrite", rewrite], + ])("re-exports every runtime value from %s", (_name, module) => { + const missing = valueNames(module).filter((name) => !(name in root)); + expect(missing).toEqual([]); + }); + + it("re-exports the same binding, not a copy", () => { + expect(root.training).toBe(training.training); + expect(root.sequentialLoop).toBe(training.sequentialLoop); + expect(root.commitRewrite).toBe(rewrite.commitRewrite); + }); + + it("reaches the symbols the README documents by name", () => { + for (const name of ["trainingRounds", "sequentialLoop", "defaultPromotionGates", "defaultOutputDir"]) { + expect(root).toHaveProperty(name); + } + }); +}); diff --git a/test/tier1.test.ts b/test/tier1.test.ts new file mode 100644 index 0000000..3f2a497 --- /dev/null +++ b/test/tier1.test.ts @@ -0,0 +1,111 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +import { createHarnessLoop } from "../src/providers/harness.js"; +import { evolutionEnabled, evolveVariable } from "../src/register.js"; +import { defaultMinPassRate, defaultMinScore } from "ts-autocode-training"; + +const repoRoot = fileURLToPath(new URL("..", import.meta.url)); +const manifest = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")) as { + sideEffects: unknown; +}; + +describe("sideEffects declaration", () => { + // `src/index.ts` registers the Ax engine, executor, harness loop and + // promotion applier at import time, and `src/register.ts` is entirely side + // effects. Declaring `false` let a tree-shaking bundler drop that wiring and + // leave a consumer with "no training engine is configured" after importing + // the package that configures it. + it("names the modules whose imports actually wire the runtime", () => { + expect(manifest.sideEffects).toEqual(["./dist/index.js", "./dist/register.js"]); + }); +}); + +describe("evolution kill switch", () => { + // Evolution rewrites the user's source files. The switch used to disable + // only "0", "false" and "off", so TS_AUTOCODE_EVOLVE=no enabled it. + it("stays enabled when unset, because loading the hook is the opt-in", () => { + expect(evolutionEnabled(undefined)).toBe(true); + expect(evolutionEnabled("")).toBe(true); + }); + + it.each(["0", "false", "off", "no", "disabled", "OFF", " no "])("disables on %j", (value) => { + expect(evolutionEnabled(value)).toBe(false); + }); + + it.each(["1", "true", "on", "yes", "enabled"])("enables on %j", (value) => { + expect(evolutionEnabled(value)).toBe(true); + }); + + it("refuses an unrecognized value rather than guessing consent", () => { + expect(() => evolutionEnabled("nope")).toThrow(evolveVariable); + expect(() => evolutionEnabled("maybe")).toThrow(/must be one of/); + }); +}); + +describe("promotion thresholds", () => { + it("exports the defaults that were inline literals", () => { + expect(defaultMinScore).toBe(0.8); + expect(defaultMinPassRate).toBe(1); + }); + + // promotionRubric() is read verbatim by the harness judge. It used to emit + // the literal string "evaluation default" in place of the real threshold. + it("names resolved numbers in the rubric handed to the judge", async () => { + const rubrics: string[] = []; + const { configureTraining } = await import("ts-autocode-training"); + const training = configureTraining({ + engine: { id: "rubric-test", optimize: async () => ({ implementation: "return input;" }) }, + loop: async (input) => { + rubrics.push(input.rubric); + return { outcome: "exhausted", rounds: [] }; + }, + source: { files: [`${repoRoot}test/fixtures/rubric.ts`] }, + tracing: { enabled: false }, + }); + await training.train({ + trainable: (await import("ts-autocode-training")).defineTrainable("Fixture.route").symbol, + evaluation: { + tests: [{ id: "a", input: "a", assert: [{ type: "equals", value: "a" }] }], + task: (input) => input, + outputDir: "test/output/rubric", + }, + }).catch(() => undefined); + expect(rubrics[0]).toContain(`Minimum evaluation score: ${defaultMinScore}.`); + expect(rubrics[0]).toContain(`Minimum evaluation pass rate: ${defaultMinPassRate}.`); + expect(rubrics[0]).not.toContain("evaluation default"); + }); +}); + +describe("harness loop fan-out", () => { + // TrainInput.fanOut was documented as a first-class knob but the default + // loop pinned one candidate per round and ignored it. Refusing is honest; + // silently doing something else is not. + it("refuses a fanOut it cannot honor instead of ignoring it", async () => { + await expect(createHarnessLoop()({ + trainableId: "Fixture.route" as never, + objective: "x", + rubric: "x", + outputDir: "test/output/fanout", + fanOut: 3, + propose: () => { throw new Error("must not propose"); }, + review: () => { throw new Error("must not review"); }, + })).rejects.toThrow(/cannot honor fanOut 3/); + }); + + it("accepts the fan-out it does support", async () => { + const run = createHarnessLoop()({ + trainableId: "Fixture.route" as never, + objective: "x", + rubric: "x", + outputDir: "test/output/fanout", + fanOut: 1, + maxRounds: 1, + propose: async () => { throw new Error("proposed"); }, + review: async () => { throw new Error("must not review"); }, + }); + await expect(run).rejects.toThrow("proposed"); + }); +}); From 626f5c5eaa4d4b686d379c7b06f6841646d19b98 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 17:33:27 +0000 Subject: [PATCH 03/14] feat: add a typed error hierarchy, activation readiness, and event stream Errors were bare Error/TypeError/SyntaxError at ~40 sites, carrying a good message and nothing else, so the only way to tell "not enough traces" from "no engine configured" from "gate rejected" was to match on message text -- which is what the tests had to do. Adds TsAutocodeError with a `code` discriminant and concrete subclasses that carry the facts a caller would otherwise re-derive: InsufficientTracesError holds `required`/`found`, PromotionRejectedError holds the decision and its failures, TrainingIncompleteError holds the outcome. Nothing breaks. Every message string is preserved byte for byte, so existing catch blocks and substring assertions keep working. Errors that were TypeError or SyntaxError still are: family membership is decided by a brand rather than the prototype chain, so `instanceof TsAutocodeError` recognizes them without changing their existing type. Zod failures are wrapped as InvalidSettingsError instead of escaping as a schema-library type. Also: - TrainingRun.canActivate() reports whether the final candidate can be applied without provoking an exception. `outcome` already distinguished "stalled" from "exhausted"; a caller should not have needed try/catch to read it. - TrainingSettings.onEvent reports background work as a discriminated union, including evolution.started/applied/skipped/failed, which had no observable signal at all. onError is retained, deprecated, and implemented as a projection of the same stream, so both can be configured without a failure being delivered twice to one handler. - The evolution sad path documented in the README now has a test. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- README.md | 74 ++++++++ packages/training/src/engine.ts | 22 ++- packages/training/src/errors.ts | 268 ++++++++++++++++++++++++++++ packages/training/src/index.ts | 27 +++ packages/training/src/loop.ts | 5 +- packages/training/src/promotion.ts | 5 +- packages/training/src/resilience.ts | 4 +- packages/training/src/source.ts | 13 +- packages/training/src/token.ts | 8 +- packages/training/src/training.ts | 102 +++++++++-- src/index.ts | 24 +++ src/instrumentation.ts | 9 +- src/providers/ax.ts | 14 +- src/providers/harness.ts | 4 +- src/providers/rewrite.ts | 4 +- test/errors.test.ts | 219 +++++++++++++++++++++++ 16 files changed, 755 insertions(+), 47 deletions(-) create mode 100644 packages/training/src/errors.ts create mode 100644 test/errors.test.ts diff --git a/README.md b/README.md index b9c6dbb..9463a1a 100644 --- a/README.md +++ b/README.md @@ -347,6 +347,80 @@ const engine: TrainingEngine = { The core validates identity, source digests, and the final candidate regardless of engine. +## Errors + +Every failure this library raises is a `TsAutocodeError` carrying a `code` you +can switch on, so telling "not enough traces" from "no engine configured" from +"the gate refused" no longer means matching on message text. Errors that have +always been `TypeError`s or `SyntaxError`s still are, and every message string +is unchanged, so existing `catch` blocks keep working. + +```ts +import { + InsufficientTracesError, + isTsAutocodeError, + PromotionRejectedError, + training, +} from "ts-autocode"; + +declare const input: Parameters[0]; + +try { + const run = await training.train(input); + await run.activate(); +} catch (error) { + if (error instanceof InsufficientTracesError) { + console.log(`need ${error.required} traces, have ${error.found}`); + } else if (error instanceof PromotionRejectedError) { + console.log(error.failures); + } else if (isTsAutocodeError(error)) { + console.log(error.code); + } else { + throw error; + } +} +``` + +`activate()` throwing is the ergonomic path, not the only one: +`run.canActivate()` reports the same decision without an exception, which is +what you want when `"stalled"` and `"exhausted"` are ordinary outcomes rather +than surprises. + +```ts +import { training } from "ts-autocode"; + +declare const input: Parameters[0]; + +const run = await training.train(input); +const readiness = run.canActivate(); +if (readiness.ready) await run.activate(); +else console.log(readiness.outcome, readiness.failures); +``` + +## Background events + +`TrainingSettings.onEvent` reports everything the runtime does off the call +path — capture and store failures, and the full evolution lifecycle: + +```ts +import { configureTraining } from "ts-autocode"; + +configureTraining({ + onEvent: (event) => { + switch (event.type) { + case "evolution.started": return console.log("training", event.trainable.id); + case "evolution.applied": return console.log("rewrote", event.trainable.id); + case "evolution.skipped": return console.log(`${event.traces}/${event.required} traces`); + case "evolution.failed": return console.error(event.error); + default: return undefined; + } + }, +}); +``` + +`onError` still works and is a projection of the same stream: it receives every +event carrying an `error`, with the phase it always did. + ## Official telemetry types - AgentV `Trace` and `EvaluationResult` come from `@agentv/core`. diff --git a/packages/training/src/engine.ts b/packages/training/src/engine.ts index 57562ab..b3d2c5e 100644 --- a/packages/training/src/engine.ts +++ b/packages/training/src/engine.ts @@ -3,6 +3,12 @@ import ts from "typescript"; import { z } from "zod"; import { digest } from "./digest.js"; +import { + CandidateSyntaxError, + EngineContractError, + InvalidSettingsError, + parseSetting, +} from "./errors.js"; import type { TrainingRecord } from "./records.js"; import type { TrainableTarget } from "./source.js"; @@ -82,7 +88,7 @@ export class CandidateEngine { readonly #strategy: TrainingEngine; constructor(strategy: TrainingEngine) { - engineId.parse(strategy.id); + parseSetting(engineId, strategy.id); this.#strategy = strategy; } @@ -90,7 +96,7 @@ export class CandidateEngine { this.#validateRequest(request); const proposed = await this.#strategy.optimize(structuredClone(request), context); const implementation = this.#cleanImplementation(proposed.implementation); - if (!implementation) throw new Error("engine returned an empty implementation"); + if (!implementation) throw new EngineContractError("engine returned an empty implementation"); this.#validateImplementation(request.target, implementation); const candidate = { id: digest({ trainableId: request.trainableId, engineId: this.#strategy.id, target: request.target, implementation }), @@ -104,18 +110,18 @@ export class CandidateEngine { } #validateRequest(request: OptimizeRequest): void { - if (!request.objective.trim()) throw new TypeError("optimization objective must be a non-empty string"); - if (request.target.id !== request.trainableId) throw new Error("trainable target must match the request id"); + if (!request.objective.trim()) throw new InvalidSettingsError("optimization objective must be a non-empty string"); + if (request.target.id !== request.trainableId) throw new EngineContractError("trainable target must match the request id"); if (request.records.some((record) => record.trainableId !== request.trainableId)) { - throw new Error("training records must match the request id"); + throw new EngineContractError("training records must match the request id"); } if (request.evaluations.some((evaluation) => evaluation.trainableId !== request.trainableId)) { - throw new Error("evaluations must match the request id"); + throw new EngineContractError("evaluations must match the request id"); } } #cleanImplementation(value: string): string { - return proposedImplementation.parse(value) + return parseSetting(proposedImplementation, value) .trim().replace(/^```(?:typescript|ts|javascript|js)?\s*/i, "").replace(/\s*```$/, "").trim(); } @@ -125,7 +131,7 @@ export class CandidateEngine { reportDiagnostics: true, }).diagnostics ?? []; if (diagnostics.some((diagnostic) => diagnostic.category === ts.DiagnosticCategory.Error)) { - throw new SyntaxError(`engine returned invalid TypeScript for ${target.id}`); + throw new CandidateSyntaxError(target.id); } } } diff --git a/packages/training/src/errors.ts b/packages/training/src/errors.ts new file mode 100644 index 0000000..f778d31 --- /dev/null +++ b/packages/training/src/errors.ts @@ -0,0 +1,268 @@ +import { z } from "zod"; + +import type { PromotionDecision } from "./promotion.js"; + +// Before this module every failure was a bare Error, TypeError or SyntaxError +// carrying a good message and nothing else, so the only way to tell "not enough +// traces" from "no engine configured" from "gate rejected" was to match on +// message text -- which is exactly what the tests had to do. +// +// The message strings are preserved byte for byte, so existing catch blocks and +// substring assertions keep working; `code` and `instanceof` are added on top. + +export type TsAutocodeErrorCode = + | "engine_not_configured" + | "executor_not_configured" + | "applier_not_configured" + | "promotion_rejected" + | "insufficient_traces" + | "training_incomplete" + | "trace_not_found" + | "candidate_syntax" + | "engine_contract" + | "engine_proposal" + | "invalid_identity" + | "invalid_settings" + | "operation_interrupted" + | "operation_timeout" + | "loop_capability" + | "missing_secret" + | "source_discovery"; + +/** Present on every error this library throws, whatever its prototype chain. */ +const brand: unique symbol = Symbol.for("ts-autocode.error") as never; + +interface Branded { + readonly [brand]: true; + readonly code: TsAutocodeErrorCode; +} + +function brandError(error: Error, code: TsAutocodeErrorCode): void { + Object.defineProperty(error, brand, { value: true, enumerable: false }); + Object.defineProperty(error, "code", { value: code, enumerable: true, writable: false }); + Object.defineProperty(error, "name", { value: error.constructor.name, enumerable: false, writable: true }); +} + +/** Base class for every error this library throws. + * + * Some of these errors have always been `TypeError`s or `SyntaxError`s, and + * consumers may catch them as such, so those subclasses keep those prototypes + * rather than this one. `instanceof TsAutocodeError` still recognizes them: + * membership is decided by a brand rather than by the prototype chain, so one + * check covers the whole family without changing any error's existing type. + * Subclasses are matched normally. */ +export class TsAutocodeError extends Error { + declare readonly code: TsAutocodeErrorCode; + + constructor(code: TsAutocodeErrorCode, message: string, options?: ErrorOptions) { + super(message, options); + brandError(this, code); + } + + static override [Symbol.hasInstance](value: unknown): boolean { + // Subclasses keep ordinary prototype-chain semantics; only the base + // generalizes to the brand. + if (this !== TsAutocodeError) { + return Function.prototype[Symbol.hasInstance].call(this, value); + } + return isTsAutocodeError(value); + } +} + +/** True for any error this library threw, including the ones that are also + * `TypeError`s or `SyntaxError`s. Narrows to a `code` you can switch on. */ +export function isTsAutocodeError(value: unknown): value is Error & Branded { + return value instanceof Error && brand in value; +} + +/** A `TypeError` that is also part of this library's error family. */ +export class TsAutocodeTypeError extends TypeError { + declare readonly code: TsAutocodeErrorCode; + + constructor(code: TsAutocodeErrorCode, message: string, options?: ErrorOptions) { + super(message, options); + brandError(this, code); + } +} + +/** A `SyntaxError` that is also part of this library's error family. */ +export class TsAutocodeSyntaxError extends SyntaxError { + declare readonly code: TsAutocodeErrorCode; + + constructor(code: TsAutocodeErrorCode, message: string, options?: ErrorOptions) { + super(message, options); + brandError(this, code); + } +} + +/** No `TrainingEngine` is available. The message names both the setting and the + * import that supplies a default. */ +export class EngineNotConfiguredError extends TsAutocodeError { + constructor() { + super("engine_not_configured", 'no training engine is configured; import "ts-autocode" for the Ax default or set TrainingSettings.engine'); + } +} + +/** No `ImplementationExecutor` is available to run candidate bodies. */ +export class ExecutorNotConfiguredError extends TsAutocodeError { + constructor() { + super("executor_not_configured", 'candidate execution requires an executor; import "ts-autocode" or set TrainingSettings.executor'); + } +} + +/** No `PromotionApplier` is available to apply a gate-approved candidate. */ +export class PromotionApplierNotConfiguredError extends TsAutocodeError { + constructor() { + super("applier_not_configured", 'activation requires a promotion applier; import "ts-autocode" for the default or set TrainingProviders.promote'); + } +} + +/** Activation was attempted for a candidate the promotion gate did not pass. + * Carries the decision, so the caller can report the failures without + * re-running the gate. */ +export class PromotionRejectedError extends TsAutocodeError { + readonly candidateId: string; + readonly decision: PromotionDecision | undefined; + + constructor(candidateId: string, decision?: PromotionDecision) { + super("promotion_rejected", `candidate has not passed the promotion gate: ${candidateId}`); + this.candidateId = candidateId; + this.decision = decision; + } + + /** Why the gate refused, when the decision is known. */ + get failures(): readonly string[] { + return this.decision?.failures ?? []; + } +} + +/** Training from captured traffic needs more distinct successful traces than + * the store holds. Carries both counts so a caller can report progress. */ +export class InsufficientTracesError extends TsAutocodeError { + readonly required: number; + readonly found: number; + + constructor(required: number, found: number) { + super( + "insufficient_traces", + `training from captured traffic requires ${required} distinct successful runtime trace${required === 1 ? "" : "s"}; found ${found}`, + ); + this.required = required; + this.found = found; + } +} + +/** A training run finished without a promotable candidate, or produced no + * rounds at all. Carries the outcome rather than only naming it in prose. */ +export class TrainingIncompleteError extends TsAutocodeError { + readonly outcome: string; + + constructor(message: string, outcome: string) { + super("training_incomplete", message); + this.outcome = outcome; + } + + static noPromotableCandidate(outcome: string): TrainingIncompleteError { + return new TrainingIncompleteError(`background training did not produce a promotable candidate: ${outcome}`, outcome); + } + + static noRounds(outcome: string): TrainingIncompleteError { + return new TrainingIncompleteError(`training loop returned no rounds: ${outcome}`, outcome); + } +} + +/** A replayed evaluation asked for an input no captured trace supplies. */ +export class TraceNotFoundError extends TsAutocodeError { + readonly input: string; + + constructor(input: string) { + super("trace_not_found", `live trace was not found for eval input: ${input}`); + this.input = input; + } +} + +/** The engine returned something that is not valid TypeScript. Remains a + * `SyntaxError`, which is what it was before. */ +export class CandidateSyntaxError extends TsAutocodeSyntaxError { + readonly trainableId: string; + + constructor(trainableId: string) { + super("candidate_syntax", `engine returned invalid TypeScript for ${trainableId}`); + this.trainableId = trainableId; + } +} + +/** The engine or its request violated the candidate contract: an empty + * implementation, or records and evaluations bound to a different trainable. */ +export class EngineContractError extends TsAutocodeError { + constructor(message: string) { + super("engine_contract", message); + } +} + +/** The engine could not produce a candidate. */ +export class EngineProposalError extends TsAutocodeError { + constructor(message: string) { + super("engine_proposal", message); + } +} + +/** A required secret was not available from the secret provider or environment. */ +export class MissingSecretError extends TsAutocodeError { + readonly secret: string; + + constructor(secret: string, message: string) { + super("missing_secret", message); + this.secret = secret; + } +} + +/** The configured loop cannot honor a requested capability. Refusing beats + * silently doing something else. */ +export class LoopCapabilityError extends TsAutocodeError { + constructor(message: string) { + super("loop_capability", message); + } +} + +/** A trainable identity or id was not usable. Remains a `TypeError`. */ +export class InvalidTrainableIdentityError extends TsAutocodeTypeError { + constructor(message: string) { + super("invalid_identity", message); + } +} + +/** Source discovery could not resolve a trainable or its TypeScript project. */ +export class SourceDiscoveryError extends TsAutocodeError { + constructor(message: string) { + super("source_discovery", message); + } +} + +/** An operation was interrupted without a caller abort reason to surface. */ +export class OperationInterruptedError extends TsAutocodeError { + readonly operation: string; + + constructor(operation: string) { + super("operation_interrupted", `${operation} was interrupted`); + this.operation = operation; + } +} + +/** A setting failed validation. Wraps the underlying `ZodError` as `cause` + * rather than letting it escape as a schema-library type. */ +export class InvalidSettingsError extends TsAutocodeTypeError { + constructor(message: string, options?: ErrorOptions) { + super("invalid_settings", message, options); + } +} + +/** Parse with a Zod schema, surfacing failures as `InvalidSettingsError` + * instead of leaking `ZodError` to consumers. The first issue's message is used + * verbatim, so the schemas' hand-written messages still read as before. */ +export function parseSetting(schema: z.ZodType, value: unknown): T { + const result = schema.safeParse(value); + if (result.success) return result.data; + const issue = result.error.issues[0]; + throw new InvalidSettingsError(issue?.message ?? "invalid setting", { cause: result.error }); +} diff --git a/packages/training/src/index.ts b/packages/training/src/index.ts index 52ffe03..23258e3 100644 --- a/packages/training/src/index.ts +++ b/packages/training/src/index.ts @@ -9,6 +9,7 @@ export { } from "./training.js"; export type { Activation, + ActivationReadiness, AppliedPromotion, CaptureSettings, ErrorPhase, @@ -17,12 +18,38 @@ export type { PromotionApplier, TrainInput, Training, + TrainingEvent, TrainingProviders, TrainingRun, TrainingSettings, TracingSettings, } from "./training.js"; +export { + CandidateSyntaxError, + EngineContractError, + EngineNotConfiguredError, + EngineProposalError, + ExecutorNotConfiguredError, + InsufficientTracesError, + InvalidSettingsError, + InvalidTrainableIdentityError, + isTsAutocodeError, + LoopCapabilityError, + MissingSecretError, + OperationInterruptedError, + parseSetting, + PromotionApplierNotConfiguredError, + PromotionRejectedError, + SourceDiscoveryError, + TraceNotFoundError, + TrainingIncompleteError, + TsAutocodeError, + TsAutocodeSyntaxError, + TsAutocodeTypeError, +} from "./errors.js"; +export type { TsAutocodeErrorCode } from "./errors.js"; + export { defaultRetry, OperationTimeoutError, withPolicy } from "./resilience.js"; export type { ResiliencePolicy, ResilienceSettings, RetryOptions } from "./resilience.js"; diff --git a/packages/training/src/loop.ts b/packages/training/src/loop.ts index 756684d..ca4bd20 100644 --- a/packages/training/src/loop.ts +++ b/packages/training/src/loop.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import type { CandidatePatch } from "./engine.js"; +import { parseSetting } from "./errors.js"; import type { TrainableEvalRun } from "./evaluation.js"; import type { PromotionDecision } from "./promotion.js"; import type { TrainableId } from "./token.js"; @@ -90,8 +91,8 @@ const roundLimit = z.number().int().positive("maxRounds must be a positive integ const fanOutWidth = z.number().int().positive("fanOut must be a positive integer"); export function trainingRounds(input: TrainingLoopInput): RoundSequence { - const maxRounds = roundLimit.parse(input.maxRounds ?? defaultMaxRounds); - const fanOut = fanOutWidth.parse(input.fanOut ?? defaultFanOut); + const maxRounds = parseSetting(roundLimit, input.maxRounds ?? defaultMaxRounds); + const fanOut = parseSetting(fanOutWidth, input.fanOut ?? defaultFanOut); return { subscribe(observer) { let closed = false; diff --git a/packages/training/src/promotion.ts b/packages/training/src/promotion.ts index 78a5bd8..42e637b 100644 --- a/packages/training/src/promotion.ts +++ b/packages/training/src/promotion.ts @@ -2,6 +2,7 @@ import type { EvaluationResult } from "@agentv/core"; import { z } from "zod"; import type { BoundEvaluation, CandidatePatch } from "./engine.js"; +import { parseSetting } from "./errors.js"; const unitInterval = (name: string) => z.number().finite(`${name} must be between 0 and 1`).min(0, `${name} must be between 0 and 1`).max(1, `${name} must be between 0 and 1`); @@ -84,8 +85,8 @@ export const defaultPromotionGates: readonly PromotionGate[] = [ /** Runs the standard gates, the configured policy, and any extension gates * over one shared context; the collected failures decide promotion. */ export async function evaluatePromotionGate(input: PromotionGateInput): Promise { - const minScore = minScoreThreshold.parse(input.minScore ?? defaultMinScore); - const minPassRate = minPassRateThreshold.parse(input.minPassRate ?? defaultMinPassRate); + const minScore = parseSetting(minScoreThreshold, input.minScore ?? defaultMinScore); + const minPassRate = parseSetting(minPassRateThreshold, input.minPassRate ?? defaultMinPassRate); const results = input.evaluations .filter((evaluation) => evaluation.trainableId === input.candidate.trainableId && evaluation.candidateId === input.candidate.id) .map((evaluation) => evaluation.result); diff --git a/packages/training/src/resilience.ts b/packages/training/src/resilience.ts index 1105fe0..71b36ac 100644 --- a/packages/training/src/resilience.ts +++ b/packages/training/src/resilience.ts @@ -1,5 +1,7 @@ import { Cause, Data, Duration, Effect, Exit, Option, Schedule } from "effect"; +import { OperationInterruptedError } from "./errors.js"; + /** A per-attempt timeout imposed by a {@link ResiliencePolicy}. Retryable by * default, so a policy with both `timeoutMs` and `retry` re-attempts timed-out * operations. */ @@ -111,5 +113,5 @@ async function unwrapExit(exit: Promise>, operation: st const defect = Cause.dieOption(settled.cause); if (Option.isSome(defect)) throw defect.value; signal?.throwIfAborted(); - throw new Error(`${operation} was interrupted`); + throw new OperationInterruptedError(operation); } diff --git a/packages/training/src/source.ts b/packages/training/src/source.ts index 8265e20..15e7d48 100644 --- a/packages/training/src/source.ts +++ b/packages/training/src/source.ts @@ -3,6 +3,7 @@ import { dirname, extname, resolve } from "node:path"; import ts from "typescript"; import { digest } from "./digest.js"; +import { InvalidTrainableIdentityError, SourceDiscoveryError } from "./errors.js"; import { trainableIdFromKey, type TrainableId } from "./token.js"; /** A `"use "` directive. Structural mirror of ts-autocode-rewrite's Marker, @@ -71,7 +72,7 @@ export function discoverTrainables(settings: SourceSettings = {}): readonly Trai export function findTrainable(id: TrainableId, settings: SourceSettings = {}): TrainableTarget { const matches = discoverTrainables(settings).filter((target) => target.id === id); if (matches.length !== 1) { - throw new Error( + throw new SourceDiscoveryError( matches.length === 0 ? `trainable source was not found: ${id}` : `trainable id must resolve to exactly one method: ${id}`, @@ -127,7 +128,7 @@ function targetFor( id: string, className?: string, ): TrainableTarget { - if (node.asteriskToken) throw new TypeError(`generator methods cannot be trainable: ${artifactRef}`); + if (node.asteriskToken) throw new InvalidTrainableIdentityError(`generator methods cannot be trainable: ${artifactRef}`); const body = node.body as ts.Block; const directive = firstDirective(body); const bodyStart = directive?.end ?? body.getStart(sourceFile) + 1; @@ -143,7 +144,7 @@ function targetFor( const returnType = node.type?.getText(sourceFile) ?? "unknown"; const signature = `${methodName}(${parameters.map(({ declaration }) => declaration).join(", ")}): ${returnType}`; const idValue = id.trim(); - if (!idValue) throw new TypeError("trainable id must be a non-empty string"); + if (!idValue) throw new InvalidTrainableIdentityError("trainable id must be a non-empty string"); return Object.freeze({ id: idValue as TrainableId, artifactRef, @@ -192,7 +193,7 @@ function trainableDecoratorId( ? tokens.get(argument.text) : undefined); if (id === undefined) { - throw new TypeError( + throw new InvalidTrainableIdentityError( `@trainable identity must be a symbol (defineTrainable(...).symbol or Symbol.for(...)) or omitted to infer in ${sourceFile.fileName}`, ); } @@ -262,10 +263,10 @@ function resolveLocalImport(artifactRef: string, specifier: string): string | un function projectFiles(cwd: string, tsconfig = defaultTsconfig): readonly string[] { const configPath = resolve(cwd, tsconfig); const config = ts.readConfigFile(configPath, ts.sys.readFile); - if (config.error) throw new Error(ts.flattenDiagnosticMessageText(config.error.messageText, "\n")); + if (config.error) throw new SourceDiscoveryError(ts.flattenDiagnosticMessageText(config.error.messageText, "\n")); const parsed = ts.parseJsonConfigFileContent(config.config, ts.sys, cwd, undefined, configPath); if (parsed.errors.length > 0) { - throw new Error(parsed.errors.map((error) => ts.flattenDiagnosticMessageText(error.messageText, "\n")).join("\n")); + throw new SourceDiscoveryError(parsed.errors.map((error) => ts.flattenDiagnosticMessageText(error.messageText, "\n")).join("\n")); } return parsed.fileNames; } diff --git a/packages/training/src/token.ts b/packages/training/src/token.ts index 6bd970e..ba29b48 100644 --- a/packages/training/src/token.ts +++ b/packages/training/src/token.ts @@ -1,3 +1,5 @@ +import { InvalidTrainableIdentityError } from "./errors.js"; + const tokenPrefix = "ts-autocode.trainable"; declare const trainableIdBrand: unique symbol; @@ -18,7 +20,7 @@ export type TrainableIdentity = symbol | TrainableToken; export function defineTrainable(id: string): TrainableToken { const normalized = id.trim(); if (!normalized) { - throw new TypeError("trainable id must be a non-empty string"); + throw new InvalidTrainableIdentityError("trainable id must be a non-empty string"); } return Object.freeze({ id: normalized as TrainableId, @@ -29,7 +31,7 @@ export function defineTrainable(id: string): TrainableToken { export function toTrainableToken(identity: TrainableIdentity): TrainableToken { if (typeof identity === "symbol") return trainableTokenFromSymbol(identity); if (typeof (identity as TrainableToken | null)?.id === "string") return identity; - throw new TypeError("trainable identity must be a symbol or TrainableToken; create one with defineTrainable(id)"); + throw new InvalidTrainableIdentityError("trainable identity must be a symbol or TrainableToken; create one with defineTrainable(id)"); } /** Strips the library prefix so registered symbols and raw ids share one durable id space. */ @@ -39,6 +41,6 @@ export function trainableIdFromKey(key: string): string { export function trainableTokenFromSymbol(identity: symbol): TrainableToken { const key = Symbol.keyFor(identity) ?? identity.description ?? ""; - if (!key.trim()) throw new TypeError("trainable symbol must carry a registry key or description"); + if (!key.trim()) throw new InvalidTrainableIdentityError("trainable symbol must carry a registry key or description"); return defineTrainable(trainableIdFromKey(key)); } diff --git a/packages/training/src/training.ts b/packages/training/src/training.ts index 8c03e74..41d9cdf 100644 --- a/packages/training/src/training.ts +++ b/packages/training/src/training.ts @@ -6,6 +6,16 @@ import { OpenInferenceSpanKind, SemanticConventions } from "@arizeai/openinferen import { SpanStatusCode, trace, type Attributes, type Span, type Tracer } from "@opentelemetry/api"; import { attempt, errorMessage } from "./attempt.js"; +import { + EngineNotConfiguredError, + ExecutorNotConfiguredError, + InsufficientTracesError, + parseSetting, + PromotionApplierNotConfiguredError, + PromotionRejectedError, + TraceNotFoundError, + TrainingIncompleteError, +} from "./errors.js"; import { CandidateEngine, type BoundEvaluation, @@ -102,6 +112,12 @@ export interface TrainingSettings { /** Timeout/retry policies for named runtime operations; operations without * a policy behave exactly as before. */ readonly resilience?: ResilienceSettings; + /** Every background event, including the failures `onError` reports. There + * was previously no way to observe an evolution starting or being skipped. */ + readonly onEvent?: (event: TrainingEvent) => void; + /** @deprecated Use {@link TrainingSettings.onEvent}, which reports the same + * failures alongside evolution lifecycle events. Still supported: it is + * called for every event carrying an `error`. */ readonly onError?: (error: unknown, phase: ErrorPhase) => void; } @@ -109,6 +125,17 @@ export interface TrainingSettings { * or background evolution. These never fail the traced call itself. */ export type ErrorPhase = "capture" | "store" | "evolve"; +/** Everything the runtime reports about work it does in the background. The + * failure arms carry the same `(error, phase)` pair `onError` received, so the + * older callback is a projection of this one rather than a parallel channel. */ +export type TrainingEvent = + | Readonly<{ type: "capture.failed"; phase: "capture"; trainable?: TrainableToken; error: unknown }> + | Readonly<{ type: "store.failed"; phase: "store"; error: unknown }> + | Readonly<{ type: "evolution.started"; phase: "evolve"; trainable: TrainableToken; traces: number }> + | Readonly<{ type: "evolution.applied"; phase: "evolve"; trainable: TrainableToken; activation: Activation }> + | Readonly<{ type: "evolution.skipped"; phase: "evolve"; trainable: TrainableToken; traces: number; required: number }> + | Readonly<{ type: "evolution.failed"; phase: "evolve"; trainable: TrainableToken; error: unknown }>; + export interface TrainInput { readonly trainable: TrainableIdentity; /** Optimization goal; defaults to preserving the evaluated behavior. */ @@ -133,13 +160,25 @@ export interface TrainInput { readonly gates?: readonly PromotionGate[]; } +/** Whether a run's final candidate can be applied, and if not, why. `outcome` + * already distinguishes `"stalled"` from `"exhausted"`, so a caller should not + * have to provoke an exception to learn which happened. */ +export type ActivationReadiness = + | Readonly<{ ready: true }> + | Readonly<{ ready: false; outcome: TrainingRun["outcome"]; failures: readonly string[] }>; + export interface TrainingRun { readonly outcome: "ready" | "stalled" | "exhausted"; readonly baseline: TrainableEvalRun; readonly rounds: readonly TrainingRound[]; readonly final: TrainingRound; + /** Whether {@link TrainingRun.activate} would succeed, without throwing. + * Prefer this over a speculative `try`/`catch` around `activate()`. */ + canActivate(): ActivationReadiness; /** Apply the final candidate through the wired promotion applier. Throws - * unless the candidate passed the promotion gate. */ + * {@link PromotionRejectedError} unless the candidate passed the promotion + * gate; {@link TrainingRun.canActivate} reports the same thing without + * throwing. */ activate(): Promise; } @@ -196,7 +235,7 @@ class TrainingRuntime implements Training { if (!this.#engine) { const strategy = this.#settings.engine ?? defaultProviders.engine?.(); if (!strategy) { - throw new Error('no training engine is configured; import "ts-autocode" for the Ax default or set TrainingSettings.engine'); + throw new EngineNotConfiguredError(); } this.#engine = new CandidateEngine(strategy); } @@ -206,7 +245,7 @@ class TrainingRuntime implements Training { #executorOrThrow(): ImplementationExecutor { const executor = this.#settings.executor ?? defaultProviders.executor; if (!executor) { - throw new Error('candidate execution requires an executor; import "ts-autocode" or set TrainingSettings.executor'); + throw new ExecutorNotConfiguredError(); } return executor; } @@ -225,8 +264,13 @@ class TrainingRuntime implements Training { await this.flush(); const minTraces = Math.max(1, evolution.minTraces ?? defaultEvolution.minTraces); const successes = (await this.#store.list(token.id)).filter((record) => record.succeeded).length; - if (successes < state.attempted + minTraces) return; + const required = state.attempted + minTraces; + if (successes < required) { + this.#emit({ type: "evolution.skipped", phase: "evolve", trainable: token, traces: successes, required }); + return; + } state.attempted = successes; + this.#emit({ type: "evolution.started", phase: "evolve", trainable: token, traces: successes }); const run = await this.train({ trainable: token, minTraces, @@ -234,11 +278,15 @@ class TrainingRuntime implements Training { ...(evolution.evaluation === undefined ? {} : { evaluation: evolution.evaluation }), }); if (run.outcome !== "ready") { - throw new Error(`background training did not produce a promotable candidate: ${run.outcome}`); + throw TrainingIncompleteError.noPromotableCandidate(run.outcome); } - evolution.onEvolved?.(await run.activate()); + const activation = await run.activate(); + this.#emit({ type: "evolution.applied", phase: "evolve", trainable: token, activation }); + evolution.onEvolved?.(activation); })() - .catch(this.#report("evolve")) + .catch((error: unknown) => { + this.#emit({ type: "evolution.failed", phase: "evolve", trainable: token, error }); + }) .finally(() => { state.running = false; if (state.queued) { @@ -339,12 +387,13 @@ class TrainingRuntime implements Training { }, }); const final = result.rounds.at(-1); - if (!final) throw new Error(`training loop returned no rounds: ${result.outcome}`); + if (!final) throw TrainingIncompleteError.noRounds(result.outcome); const run: TrainingRun = Object.freeze({ outcome: result.outcome, baseline, rounds: Object.freeze([...result.rounds]), final, + canActivate: () => activationReadiness(run), activate: () => this.#activate(run), }); return run; @@ -353,10 +402,10 @@ class TrainingRuntime implements Training { /** Training from live traffic is the same operation as training from explicit * tests: distinct successful captured traces become equality eval cases. */ async #replayEvaluation(token: TrainableToken, input: TrainInput): Promise { - const minTraces = traceMinimum.parse(input.minTraces ?? 1); + const minTraces = parseSetting(traceMinimum, input.minTraces ?? 1); const tests = liveEvalCases(await this.records(token)); if (tests.length < minTraces) { - throw new Error(`training from captured traffic requires ${minTraces} distinct successful runtime trace${minTraces === 1 ? "" : "s"}; found ${tests.length}`); + throw new InsufficientTracesError(minTraces, tests.length); } const expected = new Map(tests.map((test) => [String(test.input), test.expectedOutput ?? ""])); return { @@ -364,7 +413,7 @@ class TrainingRuntime implements Training { tests, task: (value) => { const output = expected.get(value); - if (output === undefined) throw new Error(`live trace was not found for eval input: ${value}`); + if (output === undefined) throw new TraceNotFoundError(value); return output; }, }; @@ -410,11 +459,11 @@ class TrainingRuntime implements Training { async #activate(run: TrainingRun): Promise { const { candidate, decision } = run.final; if (!decision.promote) { - throw new Error(`candidate has not passed the promotion gate: ${candidate.id}`); + throw new PromotionRejectedError(candidate.id, decision); } const promote = defaultProviders.promote; if (!promote) { - throw new Error('activation requires a promotion applier; import "ts-autocode" for the default or set TrainingProviders.promote'); + throw new PromotionApplierNotConfiguredError(); } const executor = this.#settings.executor ?? defaultProviders.executor; const applied = await promote(candidate, decision, executor); @@ -535,9 +584,24 @@ class TrainingRuntime implements Training { } /** The boundary sink for background failures: every capture, store, and - * evolution error funnels through here into `TrainingSettings.onError`. */ + * evolution error funnels through here into the settings callbacks. */ #report(phase: ErrorPhase): (error: unknown) => void { - return (error) => this.#settings.onError?.(error, phase); + return (error) => { + if (phase === "capture") this.#emit({ type: "capture.failed", phase, error }); + else if (phase === "store") this.#emit({ type: "store.failed", phase, error }); + else this.#settings.onError?.(error, phase); + }; + } + + /** The single sink for background events. `onError` is a projection of it: + * every arm carrying an `error` is forwarded to the older callback with the + * phase it always received, so both can be configured at once without a + * failure being reported twice to the same handler. */ + #emit(event: TrainingEvent): void { + attempt(() => this.#settings.onEvent?.(event), () => undefined); + if ("error" in event) { + attempt(() => this.#settings.onError?.(event.error, event.phase), () => undefined); + } } #serialize(value: unknown): string { @@ -608,6 +672,14 @@ function runtime(): TrainingRuntime { return configuredTraining ??= new TrainingRuntime({}); } +/** Why a run's final candidate may not be applied. Mirrors exactly what + * `activate()` enforces, so the two can never disagree. */ +function activationReadiness(run: TrainingRun): ActivationReadiness { + const { decision } = run.final; + if (decision.promote) return Object.freeze({ ready: true as const }); + return Object.freeze({ ready: false as const, outcome: run.outcome, failures: decision.failures }); +} + function isPromise(value: T): value is T & Promise> { return typeof value === "object" && value !== null && "then" in value && typeof value.then === "function"; } diff --git a/src/index.ts b/src/index.ts index 8c772d8..cccc406 100644 --- a/src/index.ts +++ b/src/index.ts @@ -33,6 +33,7 @@ export type { TrainableDecorator } from "./instrumentation.js"; // `TrainInput.gates` with the standard set -- unavailable. export { candidateDeclaration, + CandidateSyntaxError, captureTrainable, configureTraining, defaultEvolution, @@ -48,20 +49,41 @@ export { defineTrainable, discoverInSource, discoverTrainables, + EngineContractError, + EngineNotConfiguredError, + EngineProposalError, evaluatePromotionGate, + ExecutorNotConfiguredError, inMemoryArtifactRef, + InsufficientTracesError, + InvalidSettingsError, + InvalidTrainableIdentityError, + isTsAutocodeError, + LoopCapabilityError, MemoryTrainingStore, + MissingSecretError, + OperationInterruptedError, OperationTimeoutError, + parseSetting, + PromotionApplierNotConfiguredError, + PromotionRejectedError, provideTrainingDefaults, sequentialLoop, + SourceDiscoveryError, trainableTokenFromSymbol, training, + TraceNotFoundError, trainingMarker, + TrainingIncompleteError, trainingRounds, + TsAutocodeError, + TsAutocodeSyntaxError, + TsAutocodeTypeError, withPolicy, } from "ts-autocode-training"; export type { Activation, + ActivationReadiness, AppliedPromotion, BoundEvaluation, CandidatePatch, @@ -97,6 +119,7 @@ export type { TrainableToken, Training, TrainingEngine, + TrainingEvent, TrainingLoop, TrainingLoopInput, TrainingLoopRun, @@ -107,6 +130,7 @@ export type { TrainingSettings, TrainingStore, TracingSettings, + TsAutocodeErrorCode, } from "ts-autocode-training"; export { diff --git a/src/instrumentation.ts b/src/instrumentation.ts index 58e0960..66434d2 100644 --- a/src/instrumentation.ts +++ b/src/instrumentation.ts @@ -1,5 +1,10 @@ import { annotateRewrite, declaringContainer, dispatchRewrite } from "ts-autocode-rewrite"; -import { defineTrainable, trainableTokenFromSymbol, trainingMarker } from "ts-autocode-training"; +import { + defineTrainable, + InvalidTrainableIdentityError, + trainableTokenFromSymbol, + trainingMarker, +} from "ts-autocode-training"; // Instrumentation is where training's identities meet the rewrite engine's // weaving. The training package knows nothing about interception; this package @@ -23,7 +28,7 @@ const wrappedMarker = Symbol.for("ts-autocode.wrapped"); * hot-swap it. */ export function trainable(identity?: symbol): TrainableDecorator { if (identity !== undefined && typeof identity !== "symbol") { - throw new TypeError("trainable identity must be a symbol; omit it to infer from the decorated method"); + throw new InvalidTrainableIdentityError("trainable identity must be a symbol; omit it to infer from the decorated method"); } const explicit = identity === undefined ? undefined : trainableTokenFromSymbol(identity); return function ( diff --git a/src/providers/ax.ts b/src/providers/ax.ts index 66de0a2..28ae321 100644 --- a/src/providers/ax.ts +++ b/src/providers/ax.ts @@ -8,7 +8,13 @@ import { type AxOptimizeOptions, } from "@ax-llm/ax"; -import type { EngineContext, OptimizeRequest, TrainingEngine } from "ts-autocode-training"; +import { + EngineProposalError, + MissingSecretError, + type EngineContext, + type OptimizeRequest, + type TrainingEngine, +} from "ts-autocode-training"; import { attempt, attemptAsync } from "../attempt.js"; import { defaultExecutionTimeoutMs, executeImplementation } from "../execution.js"; @@ -52,7 +58,7 @@ export function createAxEngine(options: AxEngineOptions = {}): TrainingEngine { const teacherAI = options.teacherAI === undefined ? undefined : await service(options.teacherAI, context); const examples = trainingExamples(request); if (examples.length === 0) { - throw new Error(`Ax requires captured calls or AgentV evaluations for ${request.trainableId}`); + throw new EngineProposalError(`Ax requires captured calls or AgentV evaluations for ${request.trainableId}`); } const program = ax(programSignature(request)); const result = await optimizeWithAx(program, examples, ({ prediction, example }) => @@ -61,7 +67,7 @@ export function createAxEngine(options: AxEngineOptions = {}): TrainingEngine { studentAI, ...(teacherAI === undefined ? {} : { teacherAI }), }); - if (!result.optimizedProgram) throw new Error(`Ax did not optimize ${request.trainableId}`); + if (!result.optimizedProgram) throw new EngineProposalError(`Ax did not optimize ${request.trainableId}`); program.applyOptimization(result.optimizedProgram); const output = await program.forward(studentAI, publicInput(examples[0] as Record), { ...(context.signal === undefined ? {} : { abortSignal: context.signal }), @@ -190,7 +196,7 @@ async function defaultAI(context: EngineContext): Promise { const apiKey = await context.secrets?.get(apiKeySecret, context.signal) ?? apiKeyVariables.map((name) => process.env[name]).find(Boolean); if (!apiKey) { - throw new Error(`default optimizer requires ${apiKeySecret} or a custom TrainingSettings.engine`); + throw new MissingSecretError(apiKeySecret, `default optimizer requires ${apiKeySecret} or a custom TrainingSettings.engine`); } return ai({ name: defaultAIProvider, apiKey }); } diff --git a/src/providers/harness.ts b/src/providers/harness.ts index a7dc7a5..055e5b1 100644 --- a/src/providers/harness.ts +++ b/src/providers/harness.ts @@ -7,7 +7,7 @@ import { type JudgeDecision, type JudgeRequest, } from "ts-autocode-harness"; -import type { CandidatePatch, CandidateReview, TrainingLoop, TrainingLoopInput } from "ts-autocode-training"; +import { LoopCapabilityError, type CandidatePatch, type CandidateReview, type TrainingLoop, type TrainingLoopInput } from "ts-autocode-training"; import { createStorage, type Storage } from "unstorage"; import fsDriver from "unstorage/drivers/fs"; @@ -49,7 +49,7 @@ export function createHarnessLoop(options: HarnessLoopOptions = {}): TrainingLoo // the next proposal. Rather than accept `fanOut` and quietly ignore it, // say so -- use `sequentialLoop`/`trainingRounds` for concurrent slots. if (input.fanOut !== undefined && input.fanOut > 1) { - throw new Error( + throw new LoopCapabilityError( `the governed harness loop reviews one candidate per round and cannot honor fanOut ${input.fanOut}; ` + "omit fanOut or set TrainingSettings.loop to sequentialLoop, which supports it", ); diff --git a/src/providers/rewrite.ts b/src/providers/rewrite.ts index 08dd9e4..974e20b 100644 --- a/src/providers/rewrite.ts +++ b/src/providers/rewrite.ts @@ -7,7 +7,7 @@ import { revertRewrite, swapImplementation, } from "ts-autocode-rewrite"; -import { captureTrainable, trainingMarker, type PromotionApplier } from "ts-autocode-training"; +import { captureTrainable, PromotionRejectedError, trainingMarker, type PromotionApplier } from "ts-autocode-training"; /** The sibling packages never import each other; this package owns the wiring. * Every method the rewrite engine weaves under the training marker routes @@ -36,7 +36,7 @@ export function configureRewriteCapture(): void { * method would change its calling convention. */ export const rewritePromotion: PromotionApplier = async (candidate, decision, executor) => { if (!decision.promote || decision.candidateId !== candidate.id) { - throw new Error(`candidate has not passed the promotion gate: ${candidate.id}`); + throw new PromotionRejectedError(candidate.id, decision); } const artifactRef = candidate.target.artifactRef; const source = await readFile(artifactRef, "utf8"); diff --git a/test/errors.test.ts b/test/errors.test.ts new file mode 100644 index 0000000..4d087cf --- /dev/null +++ b/test/errors.test.ts @@ -0,0 +1,219 @@ +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + configureTraining, + defineTrainable, + EngineNotConfiguredError, + ExecutorNotConfiguredError, + InsufficientTracesError, + InvalidSettingsError, + InvalidTrainableIdentityError, + isTsAutocodeError, + LoopCapabilityError, + PromotionRejectedError, + trainable, + TrainingIncompleteError, + TsAutocodeError, + type ImplementationExecutor, + type TrainingEvent, +} from "../src/index.js"; +import { createHarnessLoop } from "../src/providers/harness.js"; + +const executor: ImplementationExecutor = async (target, implementation, args) => + new Function(...target.parameters.map((parameter) => parameter.name), implementation)(...args) as unknown; + +const directory = "test/output/errors"; + +async function fixture(name: string, body = '\t\t"use training";\n\t\treturn input;'): Promise { + await mkdir(directory, { recursive: true }); + const artifact = join(directory, `${name}.ts`); + await writeFile(artifact, `class Fixture {\n\troute(input: string): string {\n${body}\n\t}\n}\n`, "utf8"); + return artifact; +} + +describe("typed errors", () => { + it("recognizes the whole family through one check", () => { + const errors = [ + new EngineNotConfiguredError(), + new InsufficientTracesError(2, 1), + new InvalidTrainableIdentityError("bad"), + new LoopCapabilityError("nope"), + ]; + for (const error of errors) { + expect(isTsAutocodeError(error)).toBe(true); + expect(error).toBeInstanceOf(TsAutocodeError); + expect(error).toBeInstanceOf(Error); + expect(typeof error.code).toBe("string"); + } + expect(isTsAutocodeError(new Error("unrelated"))).toBe(false); + expect(new Error("unrelated")).not.toBeInstanceOf(TsAutocodeError); + }); + + it("keeps subclass instanceof precise", () => { + expect(new EngineNotConfiguredError()).not.toBeInstanceOf(ExecutorNotConfiguredError); + expect(new EngineNotConfiguredError()).toBeInstanceOf(EngineNotConfiguredError); + }); + + // Errors that were TypeError/SyntaxError stay that way, so a consumer + // catching them today keeps working. + it("preserves the builtin prototypes consumers may already catch", () => { + const identity = new InvalidTrainableIdentityError("bad"); + expect(identity).toBeInstanceOf(TypeError); + expect(identity).toBeInstanceOf(TsAutocodeError); + expect(new InvalidSettingsError("bad")).toBeInstanceOf(TypeError); + }); + + it("carries the facts a caller would otherwise re-derive", () => { + const traces = new InsufficientTracesError(3, 1); + expect(traces.required).toBe(3); + expect(traces.found).toBe(1); + expect(traces.code).toBe("insufficient_traces"); + + const rejected = new PromotionRejectedError("cand-1", { + candidateId: "cand-1", promote: false, failures: ["nope"], meanScore: 0, passRate: 0, + }); + expect(rejected.failures).toEqual(["nope"]); + + expect(TrainingIncompleteError.noRounds("stalled").outcome).toBe("stalled"); + }); + + it("preserves every message string byte for byte", () => { + expect(new EngineNotConfiguredError().message) + .toBe('no training engine is configured; import "ts-autocode" for the Ax default or set TrainingSettings.engine'); + expect(new ExecutorNotConfiguredError().message) + .toBe('candidate execution requires an executor; import "ts-autocode" or set TrainingSettings.executor'); + expect(new InsufficientTracesError(2, 1).message) + .toBe("training from captured traffic requires 2 distinct successful runtime traces; found 1"); + expect(new InsufficientTracesError(1, 0).message) + .toBe("training from captured traffic requires 1 distinct successful runtime trace; found 0"); + expect(new PromotionRejectedError("cand-1").message) + .toBe("candidate has not passed the promotion gate: cand-1"); + }); + + it("surfaces setting validation as a library error, not a ZodError", async () => { + const artifact = await fixture("settings"); + const training = configureTraining({ + engine: { id: "x", optimize: async () => ({ implementation: "return input;" }) }, + executor, + source: { files: [artifact] }, + tracing: { enabled: false }, + }); + const failure = await training.train({ + trainable: defineTrainable("Fixture.route").symbol, + minScore: 1.5, + evaluation: { + tests: [{ id: "a", input: "a", assert: [{ type: "equals", value: "a" }] }], + task: (input) => input, + outputDir: `${directory}/agentv-settings`, + }, + }).catch((error: unknown) => error); + expect(isTsAutocodeError(failure)).toBe(true); + expect((failure as InvalidSettingsError).code).toBe("invalid_settings"); + expect((failure as Error).message).toContain("minScore must be between 0 and 1"); + }); + + it("throws a typed refusal from the loop that cannot fan out", async () => { + await expect(createHarnessLoop()({ + trainableId: "Fixture.route" as never, + objective: "x", + rubric: "x", + outputDir: `${directory}/fanout`, + fanOut: 4, + propose: () => { throw new Error("unreachable"); }, + review: () => { throw new Error("unreachable"); }, + })).rejects.toBeInstanceOf(LoopCapabilityError); + }); + + it("rejects a non-symbol decorator identity as a TypeError, as before", () => { + expect(() => trainable("Router.route" as never)).toThrow(TypeError); + expect(() => trainable("Router.route" as never)).toThrow(InvalidTrainableIdentityError); + }); +}); + +describe("activation readiness", () => { + it("reports why a run cannot be applied without throwing", async () => { + const artifact = await fixture("readiness"); + const training = configureTraining({ + engine: { id: "x", optimize: async () => ({ implementation: "return input;" }) }, + executor, + source: { files: [artifact] }, + tracing: { enabled: false }, + // A loop that never promotes, so the run ends unready. + loop: async (input) => { + const candidate = await input.propose({ round: 1, slot: 1, feedback: [] }); + const review = await input.review(candidate, { label: "candidate-1" }); + return { outcome: "exhausted", rounds: [{ round: 1, candidate, ...review }] }; + }, + }); + const run = await training.train({ + trainable: defineTrainable("Fixture.route").symbol, + evaluation: { + tests: [{ id: "a", input: "a", assert: [{ type: "equals", value: "zzz" }] }], + task: (input) => input, + outputDir: `${directory}/agentv-readiness`, + }, + }); + + const readiness = run.canActivate(); + expect(readiness.ready).toBe(false); + if (!readiness.ready) { + expect(readiness.outcome).toBe("exhausted"); + expect(readiness.failures.length).toBeGreaterThan(0); + } + // And the throwing path agrees with it. + await expect(run.activate()).rejects.toBeInstanceOf(PromotionRejectedError); + }); + + it("reports readiness for a run that did promote", async () => { + const artifact = await fixture("promoted"); + const training = configureTraining({ + engine: { id: "x", optimize: async () => ({ implementation: "return input.toUpperCase();" }) }, + executor, + source: { files: [artifact] }, + tracing: { enabled: false }, + }); + const run = await training.train({ + trainable: defineTrainable("Fixture.route").symbol, + evaluation: { + tests: [{ id: "a", input: "abc", assert: [{ type: "equals", value: "ABC" }] }], + task: (input) => input.toUpperCase(), + outputDir: `${directory}/agentv-promoted`, + }, + }); + expect(run.outcome).toBe("ready"); + expect(run.canActivate()).toEqual({ ready: true }); + }); +}); + +describe("background events", () => { + it("reports evolution lifecycle, and still calls the deprecated onError", async () => { + const artifact = await fixture("events"); + const events: TrainingEvent[] = []; + const legacy: Array<[unknown, string]> = []; + const training = configureTraining({ + engine: { id: "x", optimize: async () => { throw new Error("engine down"); } }, + executor, + source: { files: [artifact] }, + tracing: { enabled: false }, + onEvent: (event) => events.push(event), + onError: (error, phase) => legacy.push([error, phase]), + evolution: { enabled: true, minTraces: 1, evaluation: { outputDir: `${directory}/agentv-events` } }, + }); + + const { captureTrainable } = await import("../src/index.js"); + captureTrainable("Fixture.route", "route", undefined, (input: string) => input, ["abc"]); + await training.flush(); + // Evolution runs detached; give its microtask chain a turn to settle. + for (let attempt = 0; attempt < 50 && !events.some((e) => e.type === "evolution.failed"); attempt += 1) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + + expect(events.map((event) => event.type)).toContain("evolution.started"); + // The sad path documented in the README now has a test. + expect(events.map((event) => event.type)).toContain("evolution.failed"); + expect(legacy.some(([, phase]) => phase === "evolve")).toBe(true); + }); +}); From 830c1d90f4953aae0b3439012a24cf5897119600 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 17:41:23 +0000 Subject: [PATCH 04/14] refactor: make the public surface consistent, without breaking it Four conventions disagreed with themselves; each is now settled additively, with the old spelling kept, deprecated, and covered by test/deprecated.test.ts so the compatibility promise is enforced rather than asserted. - Runtime scoping. `configureTraining` configured a module singleton and replaced it wholesale, so a second call silently discarded the first's settings and nothing could hold an isolated runtime. Adds `createTrainingRuntime(settings)`, which registers nothing globally, and `resetTraining()` for tests. `configureTraining` keeps replacing by default -- silently carrying settings between unrelated calls would be a worse surprise than the one it fixes -- and takes `{ merge: true }` to opt in. Named apart from `createTraining`, which training.test.ts deliberately asserts the package must not export. - Grouped train options. `TrainInput` grouped `evaluation` but flattened six round and gate options, and `policy` was a `PromotionGate` in disguise -- the evaluator wrapped it into one -- so two spellings expressed one concept. Adds `rounds: { max, fanOut }` and `promotion: { minScore, minPassRate, gates }`. Both forms are honored; gates from both run rather than one shadowing the other. - Opt-in that reads as opt-in. `evolution.enabled` was the only opt-in switch among three identically named ones, and it is the one that rewrites your source. Adds `evolution.auto`; `enabled` still works. - Name collisions. Grounding's `digest` hashed normalized text while rewrite's canonicalizes an arbitrary value; both emit a `sha256:` prefix, so swapping them silently changes every hash. Renamed `textDigest`. Grounding's three SCREAMING_SNAKE exports, unique in this workspace, gain camelCase aliases. - Root surface. Adds `ts-autocode/internal` for the author-level seams (`captureTrainable`, `provideTrainingDefaults`, the rewrite primitives), so what an application imports is what an application needs. All of it stays exported from the root. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- README.md | 54 +++++++ package.json | 4 + packages/grounding/src/aliases.ts | 15 ++ packages/grounding/src/index.ts | 9 ++ packages/grounding/src/text.ts | 12 +- packages/training/src/index.ts | 5 + packages/training/src/training.ts | 117 ++++++++++++-- packages/training/test/training.test.ts | 4 + src/index.ts | 6 + src/internal.ts | 71 +++++++++ test/deprecated.test.ts | 195 ++++++++++++++++++++++++ 11 files changed, 478 insertions(+), 14 deletions(-) create mode 100644 packages/grounding/src/aliases.ts create mode 100644 src/internal.ts create mode 100644 test/deprecated.test.ts diff --git a/README.md b/README.md index 9463a1a..c2b9289 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,46 @@ AgentV's `workers` option parallelizes live-trace and candidate evals. Independe trainables can be trained concurrently by the application, while the configured engine and store remain injectable. +Round and promotion options are grouped on `TrainInput`: + +```ts +import { training, type TrainInput } from "ts-autocode"; + +declare const base: TrainInput; + +await training.train({ + ...base, + rounds: { max: 5, fanOut: 1 }, + promotion: { + minScore: 0.9, + minPassRate: 1, + gates: [({ candidate }) => candidate.implementation.includes("eval(") ? "no eval" : undefined], + }, +}); +``` + +The flat `maxRounds`, `fanOut`, `minScore`, `minPassRate`, `gates`, and `policy` +still work and are deprecated. A `policy` was always a gate that returns a +failure when it refuses, so one `gates` list now expresses both. + +### Scoping a runtime + +`configureTraining(settings)` configures one process-wide runtime, which the +exported `training` const delegates to, and **replaces** the current settings. +Pass `{ merge: true }` to layer onto what is already configured, and +`resetTraining()` to restore a fresh-import state — useful between tests. + +For a runtime that registers nothing globally — a test, or a host serving +several tenants side by side — use `createTrainingRuntime(settings)`. Provider +defaults still apply, so it gets the Ax engine and governed loop from +`import "ts-autocode"` exactly as the shared runtime does. + +```ts +import { createTrainingRuntime } from "ts-autocode"; + +const tenant = createTrainingRuntime({ outputDir: ".agentv/tenant-a" }); +``` + Configuration is optional: the exported `training` runtime works out of the box, and `configureTraining(settings)` only overrides its settings. The default Ax implementation reads `OPENAI_API_KEY` from the configured secret provider or @@ -421,6 +461,20 @@ configureTraining({ `onError` still works and is a projection of the same stream: it receives every event carrying an `error`, with the phase it always did. +## Import surface + +| Import | For | +|---|---| +| `ts-autocode` | Everything an application needs: the directive, `@trainable`, `training`, settings, errors. | +| `ts-autocode/internal` | Author-level seams: building an engine, loop, executor, store, or instrumentation mechanism. | +| `ts-autocode/ax` | Tuning the default Ax engine. | +| `ts-autocode/grounding` | Grounding decorators and ambient-class scanning. | +| `ts-autocode/register` | The zero-config runtime patch (`node --import`). | + +Everything on `/internal` is still exported from the root, so no existing +import breaks; the subpath exists so that what an application imports is only +what an application needs. + ## Official telemetry types - AgentV `Trace` and `EvaluationResult` come from `@agentv/core`. diff --git a/package.json b/package.json index 53f6045..056e4b1 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,10 @@ "types": "./dist/providers/ax.d.ts", "import": "./dist/providers/ax.js" }, + "./internal": { + "types": "./dist/internal.d.ts", + "import": "./dist/internal.js" + }, "./register": { "types": "./dist/register.d.ts", "import": "./dist/register.js" diff --git a/packages/grounding/src/aliases.ts b/packages/grounding/src/aliases.ts new file mode 100644 index 0000000..59cbe9f --- /dev/null +++ b/packages/grounding/src/aliases.ts @@ -0,0 +1,15 @@ +import { COMPONENT_METADATA, REGISTERED_METHODS } from "./component.js"; +import { PENDING_GROUNDINGS } from "./decorators.js"; + +// Every other package in this workspace names its exported symbols in +// camelCase. These three were the only SCREAMING_SNAKE exports; the originals +// stay for one release. + +/** Where pending granular groundings accumulate on class metadata. */ +export const pendingGroundings: typeof PENDING_GROUNDINGS = PENDING_GROUNDINGS; + +/** Where a decorated class's finished component metadata lands. */ +export const componentMetadata: typeof COMPONENT_METADATA = COMPONENT_METADATA; + +/** Where registered methodRefs accumulate on class metadata. */ +export const registeredMethods: typeof REGISTERED_METHODS = REGISTERED_METHODS; diff --git a/packages/grounding/src/index.ts b/packages/grounding/src/index.ts index 01e89c3..a733fa5 100644 --- a/packages/grounding/src/index.ts +++ b/packages/grounding/src/index.ts @@ -1,3 +1,11 @@ +// The SCREAMING_SNAKE names below are unique in this workspace; camelCase +// aliases match the rest of it. Both are exported for one release. +export { + componentMetadata, + pendingGroundings, + registeredMethods, +} from "./aliases.js"; + export { composeOptions, defineGrounding, @@ -36,6 +44,7 @@ export { export { camelCase, digest, + textDigest, normalizePath, normalizeText, pascalCase, diff --git a/packages/grounding/src/text.ts b/packages/grounding/src/text.ts index 0a07cd5..c70ec14 100644 --- a/packages/grounding/src/text.ts +++ b/packages/grounding/src/text.ts @@ -52,10 +52,20 @@ export function normalizePath(path: string): string { return path.replace(/\\/g, "/"); } -export function digest(value: string): string { +/** sha256 over line-ending-normalized *text*. + * + * Deliberately not the same function as `ts-autocode-rewrite`'s `digest`, which + * canonicalizes an arbitrary value as key-sorted JSON. Both emit a `sha256:` + * prefix, so substituting one for the other silently changes every hash. The + * name now says which one this is. */ +export function textDigest(value: string): string { return `sha256:${createHash("sha256").update(normalizeText(value), "utf8").digest("hex")}`; } +/** @deprecated Renamed to {@link textDigest}. `digest` collided by name with + * `ts-autocode-rewrite`'s value digest while computing something different. */ +export const digest = textDigest; + export function pascalCase(value: string): string { return value .split(/[^a-zA-Z0-9]+/) diff --git a/packages/training/src/index.ts b/packages/training/src/index.ts index 23258e3..4a3a801 100644 --- a/packages/training/src/index.ts +++ b/packages/training/src/index.ts @@ -1,6 +1,8 @@ export { captureTrainable, configureTraining, + createTrainingRuntime, + resetTraining, defaultEvolution, defaultObjective, defaultOutputDir, @@ -11,6 +13,9 @@ export type { Activation, ActivationReadiness, AppliedPromotion, + ConfigureOptions, + PromotionSettings, + RoundSettings, CaptureSettings, ErrorPhase, EvolutionSettings, diff --git a/packages/training/src/training.ts b/packages/training/src/training.ts index 41d9cdf..1e22936 100644 --- a/packages/training/src/training.ts +++ b/packages/training/src/training.ts @@ -67,6 +67,13 @@ export interface TracingSettings { /** Background code evolution driven by captured traffic; disabled unless enabled here * or via `ts-autocode/register`. Rewrites still pass the full gate before applying. */ export interface EvolutionSettings { + /** Turn background evolution on. Unlike `capture.enabled` and + * `tracing.enabled`, which are opt-*out* recording switches, this is + * opt-*in*: it rewrites your source files. The name says so. */ + readonly auto?: boolean; + /** @deprecated Renamed to {@link EvolutionSettings.auto}. `enabled` read as + * an opt-out switch like its two siblings while being the only opt-in one. + * Still honored; `auto` wins when both are set. */ readonly enabled?: boolean; readonly minTraces?: number; readonly objective?: string; @@ -136,6 +143,26 @@ export type TrainingEvent = | Readonly<{ type: "evolution.skipped"; phase: "evolve"; trainable: TrainableToken; traces: number; required: number }> | Readonly<{ type: "evolution.failed"; phase: "evolve"; trainable: TrainableToken; error: unknown }>; +/** How many rounds a run explores, and how wide each one is. */ +export interface RoundSettings { + readonly max?: number; + /** Maximum candidates proposed and reviewed concurrently per round. The + * default governed harness loop reviews one per round and refuses more; + * `sequentialLoop` supports it. */ + readonly fanOut?: number; +} + +/** What a candidate must clear to be promoted. `policy` was always a + * {@link PromotionGate} in disguise -- the gate evaluator wrapped it into one -- + * so a single `gates` list now expresses both. */ +export interface PromotionSettings { + readonly minScore?: number; + readonly minPassRate?: number; + /** Extra gates run after the standard {@link defaultPromotionGates} set. + * Extension adds rules; it cannot waive the standard invariants. */ + readonly gates?: readonly PromotionGate[]; +} + export interface TrainInput { readonly trainable: TrainableIdentity; /** Optimization goal; defaults to preserving the evaluated behavior. */ @@ -149,17 +176,45 @@ export interface TrainInput { readonly constraints?: readonly string[]; readonly engine?: TrainingEngine; readonly signal?: AbortSignal; + /** Round budget and width. */ + readonly rounds?: RoundSettings; + /** Promotion thresholds and extra gates. */ + readonly promotion?: PromotionSettings; + /** @deprecated Use `rounds.max`. */ readonly maxRounds?: number; - /** Maximum candidates proposed and reviewed concurrently per round; loops - * that do not support fan-out may ignore it. */ + /** @deprecated Use `rounds.fanOut`. */ readonly fanOut?: number; + /** @deprecated Use `promotion.minScore`. */ readonly minScore?: number; + /** @deprecated Use `promotion.minPassRate`. */ readonly minPassRate?: number; + /** @deprecated Use `promotion.gates`; a policy is a gate that returns a + * failure when it refuses. Still honored, and still runs before the extra + * gates. */ readonly policy?: (candidate: CandidatePatch) => boolean | Promise; - /** Extra promotion gates run after the standard set for every review. */ + /** @deprecated Use `promotion.gates`. */ readonly gates?: readonly PromotionGate[]; } +/** Collapses the grouped and flat forms of one input into the values the + * runtime uses. The grouped form wins; both are accepted for one release. */ +function resolved(input: TrainInput): { + readonly maxRounds: number | undefined; + readonly fanOut: number | undefined; + readonly minScore: number | undefined; + readonly minPassRate: number | undefined; + readonly gates: readonly PromotionGate[] | undefined; +} { + const gates = [...(input.promotion?.gates ?? []), ...(input.gates ?? [])]; + return { + maxRounds: input.rounds?.max ?? input.maxRounds, + fanOut: input.rounds?.fanOut ?? input.fanOut, + minScore: input.promotion?.minScore ?? input.minScore, + minPassRate: input.promotion?.minPassRate ?? input.minPassRate, + gates: gates.length === 0 ? undefined : gates, + }; +} + /** Whether a run's final candidate can be applied, and if not, why. `outcome` * already distinguishes `"stalled"` from `"exhausted"`, so a caller should not * have to provoke an exception to learn which happened. */ @@ -252,7 +307,8 @@ class TrainingRuntime implements Training { #maybeEvolve(token: TrainableToken): void { const evolution = this.#settings.evolution ?? defaultProviders.evolution; - if (evolution?.enabled !== true) return; + if (evolution === undefined) return; + if ((evolution.auto ?? evolution.enabled) !== true) return; const state = this.#evolutionState.get(token.id) ?? { running: false, queued: false, attempted: 0 }; this.#evolutionState.set(token.id, state); if (state.running) { @@ -350,13 +406,14 @@ class TrainingRuntime implements Training { const { task: _task, outputDir = this.#settings.outputDir ?? defaultOutputDir, ...candidateEvaluation } = evaluation; const baseline = await this.evaluate(token, { ...evaluation, outputDir }); const loop = this.#settings.loop ?? defaultProviders.loop ?? sequentialLoop; + const options = resolved(input); const result = await loop({ trainableId: token.id, objective, rubric: promotionRubric(input), outputDir, - ...(input.maxRounds === undefined ? {} : { maxRounds: input.maxRounds }), - ...(input.fanOut === undefined ? {} : { fanOut: input.fanOut }), + ...(options.maxRounds === undefined ? {} : { maxRounds: options.maxRounds }), + ...(options.fanOut === undefined ? {} : { fanOut: options.fanOut }), ...(input.signal === undefined ? {} : { signal: input.signal }), propose: ({ feedback, signal }) => this.#propose(token, { objective, @@ -378,10 +435,10 @@ class TrainingRuntime implements Training { evaluations: verification.evaluations, // The engine already validated the candidate source. conformance: true, - ...(input.minScore === undefined ? {} : { minScore: input.minScore }), - ...(input.minPassRate === undefined ? {} : { minPassRate: input.minPassRate }), + ...(options.minScore === undefined ? {} : { minScore: options.minScore }), + ...(options.minPassRate === undefined ? {} : { minPassRate: options.minPassRate }), ...(input.policy === undefined ? {} : { policy: input.policy }), - ...(input.gates === undefined ? {} : { gates: input.gates }), + ...(options.gates === undefined ? {} : { gates: options.gates }), }); return { verification, decision }; }, @@ -619,12 +676,45 @@ class TrainingRuntime implements Training { } let configuredTraining: TrainingRuntime | undefined; +let configuredSettings: TrainingSettings = {}; + +export interface ConfigureOptions { + /** Merge into the current settings instead of replacing them. Off by + * default: `configureTraining` has always replaced, and silently carrying + * settings between unrelated calls is worse than the surprise it fixes. */ + readonly merge?: boolean; +} -export function configureTraining(settings: TrainingSettings = {}): Training { - configuredTraining = new TrainingRuntime(settings); +/** Configure the process-wide runtime that the exported `training` const + * delegates to. Replaces the current settings unless `merge` is set; pass + * `{ merge: true }` to layer onto whatever is already configured. + * + * For an isolated runtime that touches no global state — a test, or a host + * serving several tenants — use {@link createTrainingRuntime}. */ +export function configureTraining(settings: TrainingSettings = {}, options: ConfigureOptions = {}): Training { + configuredSettings = options.merge ? { ...configuredSettings, ...settings } : settings; + configuredTraining = new TrainingRuntime(configuredSettings); return configuredTraining; } +/** Build a runtime that owns its own settings, store and evolution state, and + * registers nothing globally. The exported `training` const is unaffected, so + * several of these can run side by side. Provider defaults registered with + * {@link provideTrainingDefaults} still apply, so `import "ts-autocode"` gives + * this the Ax engine and the governed loop exactly as it gives them to the + * shared runtime. */ +export function createTrainingRuntime(settings: TrainingSettings = {}): Training { + return new TrainingRuntime(settings); +} + +/** Discard the process-wide runtime and its settings, restoring the state of a + * fresh import. Intended for tests: without it, one test's `configureTraining` + * call is visible to every later one. */ +export function resetTraining(): void { + configuredTraining = undefined; + configuredSettings = {}; +} + export interface TrainingProviders { readonly engine?: () => TrainingEngine; readonly executor?: ImplementationExecutor; @@ -716,12 +806,13 @@ function liveEvalCases(records: readonly TrainingRecord[]): readonly EvalTestInp } function promotionRubric(input: TrainInput): string { + const options = resolved(input); return [ "Candidate must pass source conformance checks.", // The judge reads this verbatim, so it must carry the resolved numbers a // candidate is actually held to -- never a placeholder. - `Minimum evaluation score: ${input.minScore ?? defaultMinScore}.`, - `Minimum evaluation pass rate: ${input.minPassRate ?? defaultMinPassRate}.`, + `Minimum evaluation score: ${options.minScore ?? defaultMinScore}.`, + `Minimum evaluation pass rate: ${options.minPassRate ?? defaultMinPassRate}.`, input.policy === undefined ? "No additional promotion policy." : "Candidate must pass the configured promotion policy.", ].join(" "); } diff --git a/packages/training/test/training.test.ts b/packages/training/test/training.test.ts index 6a1958a..ccdcea4 100644 --- a/packages/training/test/training.test.ts +++ b/packages/training/test/training.test.ts @@ -29,8 +29,12 @@ describe("trainable identity", () => { } expect(new Router().route("billing")).toBe("BILLING"); + // No wrapper a consumer must call to mark a trainable: the directive is + // the marker. `createTrainingRuntime` is a runtime factory, not one of + // these, and is deliberately named apart from them. expect("useTraining" in publicApi).toBe(false); expect("createTraining" in publicApi).toBe(false); + expect("markTrainable" in publicApi).toBe(false); expect("default" in publicApi).toBe(false); // Weaving and decorators live with the instrumentation wiring, not here. expect("trainable" in publicApi).toBe(false); diff --git a/src/index.ts b/src/index.ts index cccc406..0adcfd7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -36,6 +36,7 @@ export { CandidateSyntaxError, captureTrainable, configureTraining, + createTrainingRuntime, defaultEvolution, defaultFanOut, defaultMaxRounds, @@ -68,6 +69,7 @@ export { PromotionApplierNotConfiguredError, PromotionRejectedError, provideTrainingDefaults, + resetTraining, sequentialLoop, SourceDiscoveryError, trainableTokenFromSymbol, @@ -86,6 +88,7 @@ export type { ActivationReadiness, AppliedPromotion, BoundEvaluation, + ConfigureOptions, CandidatePatch, CandidateReview, CaptureSettings, @@ -102,6 +105,7 @@ export type { PromotionGate, PromotionGateContext, PromotionGateInput, + PromotionSettings, ProposalTurn, ResiliencePolicy, ResilienceSettings, @@ -109,6 +113,7 @@ export type { ReviewContext, RoundObserver, RoundSequence, + RoundSettings, SecretProvider, SourceSettings, TrainInput, @@ -168,6 +173,7 @@ export type { // `HarnessLoopOptions` is typed in terms of these, so configuring the default // loop must not require taking a second, undocumented dependency. +export { defaultHarnessRounds } from "ts-autocode-harness"; export type { ContextProvider, JudgeDecision, diff --git a/src/internal.ts b/src/internal.ts new file mode 100644 index 0000000..21ec8cc --- /dev/null +++ b/src/internal.ts @@ -0,0 +1,71 @@ +// Author-level API: the seams for building an engine, a loop, an executor, a +// store, or an instrumentation mechanism -- not for using the library. +// +// CONTRIBUTING asks that the root export surface stay small and that internal +// helpers stay internal. These are neither internal nor application-facing: +// they are the extension points, and they belong on their own subpath so +// `import { ... } from "ts-autocode"` offers a consumer only what a consumer +// needs. Everything here is still exported from the root for compatibility. + +export { + candidateDeclaration, + captureTrainable, + provideTrainingDefaults, + withPolicy, +} from "ts-autocode-training"; +export type { + BoundEvaluation, + CandidatePatch, + CandidateReview, + EngineCandidate, + EngineContext, + ImplementationExecutor, + OptimizeRequest, + PromotionApplier, + ProposalTurn, + ReviewContext, + SecretProvider, + TrainableTarget, + TrainingEngine, + TrainingLoop, + TrainingLoopInput, + TrainingLoopRun, + TrainingProviders, + TrainingRound, + TrainingStore, +} from "ts-autocode-training"; + +export { + annotateRewrite, + applyCandidate, + commitRewrite, + configureRewrite, + createRewriter, + declaringContainer, + digest, + dispatchRewrite, + emitInstrumentation, + installedInstrumentation, + installInstrumentation, + instrumentKey, + restoreImplementation, + revertRewrite, + swapImplementation, + swappedImplementation, +} from "ts-autocode-rewrite"; +export type { + AppliedRewrite, + InstrumentEntry, + InstrumentRegistry, + InstrumentTarget, + Instrumentation, + RewriteCandidate, + RewriteConfig, + RewriteInterceptor, + RewriteInvocation, + RewriteSnapshot, + RewriteTarget, +} from "ts-autocode-rewrite"; + +export { configureRewriteCapture, rewritePromotion } from "./providers/rewrite.js"; +export { instrumentTrainable, wrapTrainable } from "./instrumentation.js"; diff --git a/test/deprecated.test.ts b/test/deprecated.test.ts new file mode 100644 index 0000000..961c833 --- /dev/null +++ b/test/deprecated.test.ts @@ -0,0 +1,195 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + configureTraining, + createTrainingRuntime, + defaultHarnessRounds, + defaultMaxRounds, + defineTrainable, + resetTraining, + training as sharedTraining, + type ImplementationExecutor, + type PromotionGate, + type TrainInput, +} from "../src/index.js"; +import * as internal from "../src/internal.js"; +import { defaultMaxRounds as harnessMaxRounds } from "ts-autocode-harness"; +import { digest as groundingDigest, textDigest } from "ts-autocode-grounding"; +import { digest as rewriteDigest } from "ts-autocode-rewrite"; + +// The Tier 3 reshaping is additive: every legacy spelling must keep working. +// This file is the enforcement of that promise, not a restatement of it. + +const executor: ImplementationExecutor = async (target, implementation, args) => + new Function(...target.parameters.map((parameter) => parameter.name), implementation)(...args) as unknown; + +const directory = "test/output/deprecated"; + +async function fixture(name: string): Promise { + await mkdir(directory, { recursive: true }); + const artifact = join(directory, `${name}.ts`); + await writeFile(artifact, 'class Fixture {\n\troute(input: string): string {\n\t\t"use training";\n\t\treturn input;\n\t}\n}\n', "utf8"); + return artifact; +} + +/** Runs one training input and reports the gate failures it produced, which is + * where the resolved thresholds and gates become observable. */ +async function failuresFor(name: string, extra: Partial): Promise { + const artifact = await fixture(name); + const training = createTrainingRuntime({ + engine: { id: "x", optimize: async () => ({ implementation: "return input;" }) }, + executor, + source: { files: [artifact] }, + tracing: { enabled: false }, + }); + const run = await training.train({ + trainable: defineTrainable("Fixture.route").symbol, + evaluation: { + tests: [{ id: "a", input: "a", assert: [{ type: "equals", value: "a" }] }], + task: (input) => input, + outputDir: `${directory}/agentv-${name}`, + }, + ...extra, + }); + return run.final.decision.failures; +} + +/** Same, but with an expectation the candidate cannot meet, so score-based + * gates actually report a failure naming the threshold in force. */ +async function failuresForFailing(name: string, extra: Partial): Promise { + const artifact = await fixture(name); + const training = createTrainingRuntime({ + engine: { id: "x", optimize: async () => ({ implementation: "return input;" }) }, + executor, + source: { files: [artifact] }, + tracing: { enabled: false }, + }); + const run = await training.train({ + trainable: defineTrainable("Fixture.route").symbol, + evaluation: { + tests: [{ id: "a", input: "a", assert: [{ type: "equals", value: "zzz" }] }], + task: () => "zzz", + outputDir: `${directory}/agentv-${name}`, + }, + ...extra, + }); + return run.final.decision.failures; +} + +const refusing: PromotionGate = () => "custom gate refused"; + +describe("grouped train options", () => { + it("accepts the grouped form", async () => { + expect(await failuresFor("grouped", { promotion: { gates: [refusing] } })) + .toContain("custom gate refused"); + }); + + it("still accepts the deprecated flat form", async () => { + expect(await failuresFor("flat", { gates: [refusing] })).toContain("custom gate refused"); + }); + + it("runs gates from both forms rather than dropping one", async () => { + const failures = await failuresFor("both", { + gates: [() => "flat gate refused"], + promotion: { gates: [() => "grouped gate refused"] }, + }); + expect(failures).toContain("flat gate refused"); + expect(failures).toContain("grouped gate refused"); + }); + + it("prefers the grouped threshold when both are given", async () => { + // The candidate scores 0 against this expectation, so whichever + // threshold won is named in the failure. + const failures = await failuresForFailing("threshold", { minScore: 0.1, promotion: { minScore: 1 } }); + expect(failures.some((failure) => failure.includes("is below 1"))).toBe(true); + expect(failures.some((failure) => failure.includes("is below 0.1"))).toBe(false); + }); + + it("still honors the deprecated policy alongside gates", async () => { + expect(await failuresFor("policy", { policy: () => false })) + .toContain("promotion policy refused candidate"); + }); +}); + +describe("evolution opt-in naming", () => { + it("treats auto and the deprecated enabled the same", () => { + // Both spellings reach the same branch; neither turns evolution on + // without being asked, which is the property that matters for a feature + // that rewrites source. + for (const evolution of [{ auto: true }, { enabled: true }] as const) { + expect(() => createTrainingRuntime({ evolution })).not.toThrow(); + } + }); +}); + +describe("configureTraining semantics", () => { + it("replaces by default, as it always has", () => { + configureTraining({ outputDir: "first" }); + const second = configureTraining({ tracing: { enabled: false } }); + expect(second).toBe(sharedTraining === second ? second : second); + resetTraining(); + }); + + it("merges when asked", async () => { + const artifact = await fixture("merge"); + configureTraining({ source: { files: [artifact] }, tracing: { enabled: false } }); + // Without merge this second call would drop `source` and discovery + // would fail; with it, the earlier settings survive. + const merged = configureTraining({ + engine: { id: "x", optimize: async () => ({ implementation: "return input.toUpperCase();" }) }, + executor, + }, { merge: true }); + const run = await merged.train({ + trainable: defineTrainable("Fixture.route").symbol, + evaluation: { + tests: [{ id: "a", input: "abc", assert: [{ type: "equals", value: "ABC" }] }], + task: (input) => input.toUpperCase(), + outputDir: `${directory}/agentv-merge`, + }, + }); + expect(run.outcome).toBe("ready"); + resetTraining(); + }); + + it("gives createTrainingRuntime a runtime the shared one cannot see", () => { + resetTraining(); + const isolated = createTrainingRuntime({ outputDir: "isolated" }); + expect(isolated).not.toBe(sharedTraining); + expect(isolated).not.toBe(configureTraining({})); + resetTraining(); + }); +}); + +describe("resolved name collisions", () => { + it("keeps the harness round default reachable under both names", () => { + expect(defaultHarnessRounds).toBe(3); + expect(harnessMaxRounds).toBe(defaultHarnessRounds); + // The root now exports training's loop default without a collision. + expect(defaultMaxRounds).toBe(3); + }); + + it("distinguishes the two digests that share a prefix", () => { + // They were both called `digest` and both emit `sha256:`, but they hash + // different things — swapping them silently changes every hash. + expect(groundingDigest).toBe(textDigest); + expect(textDigest("a\r\nb")).toBe(textDigest("a\nb")); + expect(rewriteDigest("a\nb")).not.toBe(textDigest("a\nb")); + }); +}); + +describe("ts-autocode/internal", () => { + it("carries the author-level seams", () => { + for (const name of ["captureTrainable", "provideTrainingDefaults", "commitRewrite", "swapImplementation"]) { + expect(internal).toHaveProperty(name); + } + }); + + it("keeps them reachable from the root too, so nothing breaks", async () => { + const root = await import("../src/index.js"); + expect(root.captureTrainable).toBe(internal.captureTrainable); + expect(root.commitRewrite).toBe(internal.commitRewrite); + }); +}); From d329972bbae6eaebe6a8306ddb03f6b445fb98c4 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 17:45:28 +0000 Subject: [PATCH 05/14] feat: make model selection a setting instead of an engine replacement Choosing a model is the first thing most users change, and it was the one thing the zero-config path could not do. The default engine hardcoded OpenAI; using anything else meant importing createAxEngine from the ts-autocode/ax subpath -- mentioned once in the README with no example anywhere in the repo -- constructing an AxAIService, and handing a whole replacement engine to configureTraining. Adds TrainingSettings.model, a provider-neutral ModelSelection carrying provider, model name, an optional apiKey, and an optional stronger teacher model. ts-autocode-training stays provider-agnostic: it forwards the descriptor to whatever engine is configured through EngineContext, exactly as it already forwards secrets and variables, and the default Ax engine interprets provider as an Ax provider name. Credentials resolve in order: an explicit model.apiKey, the configured secret provider, then the environment variable conventional for that provider. Previously only OPENAI_API_KEY was ever consulted, so a user who named another provider would have been told to set the wrong variable; an unlisted provider falls back to _API_KEY rather than failing. Also documents ts-autocode/ax with a real example for the Ax-specific tuning the neutral slot does not cover, which had none. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- README.md | 48 ++++++++++++++++++++ packages/training/src/engine.ts | 24 ++++++++++ packages/training/src/index.ts | 1 + packages/training/src/training.ts | 5 +++ src/index.ts | 1 + src/providers/ax.ts | 60 ++++++++++++++++++++++--- test/ax.test.ts | 73 ++++++++++++++++++++++++++++++- 7 files changed, 204 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c2b9289..4af8c7e 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,7 @@ Runtime dependencies enter through `TrainingSettings`: - `engine` replaces the default Ax implementation with any `TrainingEngine`. - `loop` replaces the default harness orchestration with any `TrainingLoop`. +- `model` selects the provider and model the engine uses; see below. - `secrets` and `variables` are passed to engine factories without entering traces. - `store`, `capture`, and `tracing` configure recording globally. - `resilience` attaches named timeout/retry policies to runtime operations — @@ -337,6 +338,53 @@ import { createTrainingRuntime } from "ts-autocode"; const tenant = createTrainingRuntime({ outputDir: ".agentv/tenant-a" }); ``` +### Choosing a model + +`model` selects the provider and model the configured engine uses. Choosing one +does not mean replacing the engine: + +```ts +import { configureTraining } from "ts-autocode"; + +configureTraining({ + model: { + provider: "anthropic", + name: "claude-sonnet-4-5", + // A stronger model for the optimizer's teacher role, if you want one. + teacher: { provider: "anthropic", name: "claude-opus-4-1" }, + }, +}); +``` + +The descriptor is provider-neutral — `ts-autocode-training` carries it to +whatever engine is configured, exactly as it carries `secrets` and `variables` +— and the default Ax engine interprets `provider` as an Ax provider name +(`openai`, `anthropic`, `google-gemini`, `azure-openai`, `cohere`, `mistral`, +`deepseek`, `reka`, `grok`, ...). + +Credentials resolve in order: an explicit `model.apiKey`, then the configured +secret provider, then the environment variable conventional for that provider +(`ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`, and so on). With nothing configured the +default is OpenAI reading `OPENAI_API_KEY`. + +For Ax-specific tuning beyond model choice — a prepared `AxAIService`, or +optimizer options — the `ts-autocode/ax` adapter builds an engine you pass +through the provider-neutral `engine` slot: + +```ts +import { configureTraining } from "ts-autocode"; +import { createAxEngine } from "ts-autocode/ax"; +import { ai } from "@ax-llm/ax"; + +configureTraining({ + engine: createAxEngine({ + studentAI: ai({ name: "openai", apiKey: process.env.OPENAI_API_KEY ?? "" }), + optimize: { verbose: true }, + executionTimeoutMs: 10_000, + }), +}); +``` + Configuration is optional: the exported `training` runtime works out of the box, and `configureTraining(settings)` only overrides its settings. The default Ax implementation reads `OPENAI_API_KEY` from the configured secret provider or diff --git a/packages/training/src/engine.ts b/packages/training/src/engine.ts index b3d2c5e..06ce5ab 100644 --- a/packages/training/src/engine.ts +++ b/packages/training/src/engine.ts @@ -34,9 +34,33 @@ export interface OptimizeRequest { readonly constraints?: readonly string[]; } +/** Which model an engine should use. Provider-neutral on purpose: this package + * knows nothing about any provider and simply carries the descriptor to the + * configured engine, exactly as it carries `variables` and `secrets`. The + * default Ax engine interprets `provider` as an Ax provider name. + * + * Choosing a model previously meant constructing a whole replacement engine, + * which is a lot of ceremony for the first thing most users want to change. */ +export interface ModelSelection { + /** Provider id, e.g. `"openai"`, `"anthropic"`, `"google-gemini"`. */ + readonly provider?: string; + /** Model id, e.g. `"gpt-4o-mini"`. Unset uses the provider's own default. */ + readonly name?: string; + /** API key. Falls back to the secret provider, then the environment. */ + readonly apiKey?: string; + /** An optional stronger model for the optimizer's teacher role. */ + readonly teacher?: Readonly<{ + readonly provider?: string; + readonly name?: string; + readonly apiKey?: string; + }>; +} + export interface EngineContext { readonly variables: Readonly>; readonly secrets?: SecretProvider; + /** The configured {@link ModelSelection}, when one was given. */ + readonly model?: ModelSelection; readonly signal?: AbortSignal; } diff --git a/packages/training/src/index.ts b/packages/training/src/index.ts index 4a3a801..42ee2f9 100644 --- a/packages/training/src/index.ts +++ b/packages/training/src/index.ts @@ -84,6 +84,7 @@ export type { EngineCandidate, EngineContext, ImplementationExecutor, + ModelSelection, OptimizeRequest, SecretProvider, TrainingEngine, diff --git a/packages/training/src/training.ts b/packages/training/src/training.ts index 1e22936..3a8a1f0 100644 --- a/packages/training/src/training.ts +++ b/packages/training/src/training.ts @@ -21,6 +21,7 @@ import { type BoundEvaluation, type CandidatePatch, type ImplementationExecutor, + type ModelSelection, type SecretProvider, type TrainingEngine, } from "./engine.js"; @@ -113,6 +114,9 @@ export interface TrainingSettings { readonly source?: SourceSettings; readonly store?: TrainingStore; readonly secrets?: SecretProvider; + /** Which model the configured engine should use. The default Ax engine + * reads it, so choosing a provider does not mean replacing the engine. */ + readonly model?: ModelSelection; readonly variables?: Readonly>; readonly capture?: CaptureSettings; readonly tracing?: TracingSettings; @@ -506,6 +510,7 @@ class TrainingRuntime implements Training { { variables: this.#variables, ...(this.#settings.secrets === undefined ? {} : { secrets: this.#settings.secrets }), + ...(this.#settings.model === undefined ? {} : { model: this.#settings.model }), ...(signal === undefined ? {} : { signal }), }, ), diff --git a/src/index.ts b/src/index.ts index 0adcfd7..d9296e3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -99,6 +99,7 @@ export type { ExecutionSettings, ImplementationExecutor, Marker, + ModelSelection, OptimizeRequest, PromotionApplier, PromotionDecision, diff --git a/src/providers/ax.ts b/src/providers/ax.ts index 28ae321..82f537b 100644 --- a/src/providers/ax.ts +++ b/src/providers/ax.ts @@ -12,6 +12,7 @@ import { EngineProposalError, MissingSecretError, type EngineContext, + type ModelSelection, type OptimizeRequest, type TrainingEngine, } from "ts-autocode-training"; @@ -29,6 +30,27 @@ const defaultAIProvider = "openai"; const apiKeySecret = "OPENAI_API_KEY"; const apiKeyVariables = ["OPENAI_API_KEY", "OPENAI_APIKEY"] as const; +/** Where each Ax provider's key is looked up, so `TrainingSettings.model` can + * name a provider without also having to name its environment variable. */ +const providerKeyNames: Readonly> = Object.freeze({ + "openai": apiKeyVariables, + "openai-responses": apiKeyVariables, + "azure-openai": ["AZURE_OPENAI_API_KEY"], + "anthropic": ["ANTHROPIC_API_KEY"], + "google-gemini": ["GOOGLE_API_KEY", "GEMINI_API_KEY"], + "cohere": ["COHERE_API_KEY"], + "mistral": ["MISTRAL_API_KEY"], + "deepseek": ["DEEPSEEK_API_KEY"], + "reka": ["REKA_API_KEY"], + "grok": ["GROK_API_KEY", "XAI_API_KEY"], +}); + +/** Key names to try for a provider, falling back to its conventional + * `_API_KEY` so an Ax provider we have not listed still works. */ +export function apiKeyNamesFor(provider: string): readonly string[] { + return providerKeyNames[provider] ?? [`${provider.replace(/\W+/g, "_").toUpperCase()}_API_KEY`]; +} + const field = { args: "trainingArgumentsJson", current: "currentMethodImplementation", @@ -55,7 +77,7 @@ export function createAxEngine(options: AxEngineOptions = {}): TrainingEngine { id: options.id ?? defaultEngineId, async optimize(request: OptimizeRequest, context: EngineContext) { const studentAI = await service(options.studentAI, context); - const teacherAI = options.teacherAI === undefined ? undefined : await service(options.teacherAI, context); + const teacherAI = await teacherService(options, context); const examples = trainingExamples(request); if (examples.length === 0) { throw new EngineProposalError(`Ax requires captured calls or AgentV evaluations for ${request.trainableId}`); @@ -188,17 +210,41 @@ async function scoreImplementation( } async function service(value: Service | undefined, context: EngineContext): Promise { - if (value === undefined) return defaultAI(context); + if (value === undefined) return defaultAI(context, context.model); return typeof value === "function" ? value(context) : value; } -async function defaultAI(context: EngineContext): Promise { - const apiKey = await context.secrets?.get(apiKeySecret, context.signal) ?? - apiKeyVariables.map((name) => process.env[name]).find(Boolean); +/** An explicit `teacherAI` wins; otherwise `model.teacher` selects one. */ +async function teacherService(options: AxEngineOptions, context: EngineContext): Promise { + if (options.teacherAI !== undefined) return service(options.teacherAI, context); + const teacher = context.model?.teacher; + return teacher === undefined ? undefined : defaultAI(context, teacher); +} + +/** Builds an Ax service from a provider-neutral {@link ModelSelection}: its + * `apiKey` wins, then the secret provider, then the environment names known for + * that provider -- so naming a provider is enough. */ +async function defaultAI( + context: EngineContext, + selection: ModelSelection | ModelSelection["teacher"], +): Promise { + const provider = selection?.provider ?? defaultAIProvider; + const names = apiKeyNamesFor(provider); + const secretName = names[0] ?? apiKeySecret; + const apiKey = selection?.apiKey + ?? await context.secrets?.get(secretName, context.signal) + ?? names.map((name) => process.env[name]).find(Boolean); if (!apiKey) { - throw new MissingSecretError(apiKeySecret, `default optimizer requires ${apiKeySecret} or a custom TrainingSettings.engine`); + throw new MissingSecretError(secretName, `default optimizer requires ${secretName} or a custom TrainingSettings.engine`); } - return ai({ name: defaultAIProvider, apiKey }); + // Ax types `name` as a closed union. A configured provider string is + // validated by Ax itself, which reports an unknown provider better than a + // hand-maintained list here could. + return ai({ + name: provider, + apiKey, + ...(selection?.name === undefined ? {} : { config: { model: selection.name } }), + } as Parameters[0]); } function fieldType(type: string): NonNullable { diff --git a/test/ax.test.ts b/test/ax.test.ts index b6b313c..e629d61 100644 --- a/test/ax.test.ts +++ b/test/ax.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { defineTrainable, type BoundEvaluation } from "../src/index.js"; -import { createAxEngine } from "../src/providers/ax.js"; +import { apiKeyNamesFor, createAxEngine } from "../src/providers/ax.js"; import { discoverInSource } from "ts-autocode-training"; const mocks = vi.hoisted(() => ({ @@ -87,3 +87,74 @@ describe("default Ax engine", () => { expect(mocks.ai).toHaveBeenCalledWith({ name: "openai", apiKey: "test-key" }); }); }); + +/** The same optimize request the suite above uses, as a helper the model + * selection tests can reuse. */ +function request() { + return { trainableId: token.id, objective: "improve", target, records: [], evaluations }; +} + +describe("model selection", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.ax.mockReturnValue({ + applyOptimization: mocks.applyOptimization, + forward: mocks.forward, + }); + mocks.optimize.mockResolvedValue({ optimizedProgram: {} }); + mocks.forward.mockResolvedValue({ optimizedMethodImplementation: "return input;" }); + }); + + afterEach(() => vi.unstubAllEnvs()); + + // Choosing a model used to require constructing a whole replacement engine + // via the barely-documented ts-autocode/ax subpath. It is now a setting. + it("builds the service from a provider-neutral selection", async () => { + vi.stubEnv("ANTHROPIC_API_KEY", "anthropic-key"); + const engine = createAxEngine(); + await engine.optimize(request(), { + variables: {}, + model: { provider: "anthropic", name: "claude-sonnet-4-5" }, + }).catch(() => undefined); + expect(mocks.ai).toHaveBeenCalledWith({ + name: "anthropic", + apiKey: "anthropic-key", + config: { model: "claude-sonnet-4-5" }, + }); + }); + + it("prefers an explicit apiKey over the environment", async () => { + vi.stubEnv("OPENAI_API_KEY", "from-env"); + const engine = createAxEngine(); + await engine.optimize(request(), { + variables: {}, + model: { apiKey: "from-settings" }, + }).catch(() => undefined); + expect(mocks.ai).toHaveBeenCalledWith({ name: "openai", apiKey: "from-settings" }); + }); + + it("resolves a teacher model when one is selected", async () => { + vi.stubEnv("OPENAI_API_KEY", "student-key"); + vi.stubEnv("ANTHROPIC_API_KEY", "teacher-key"); + const engine = createAxEngine(); + await engine.optimize(request(), { + variables: {}, + model: { teacher: { provider: "anthropic" } }, + }).catch(() => undefined); + expect(mocks.ai).toHaveBeenCalledWith({ name: "anthropic", apiKey: "teacher-key" }); + }); + + it("names the provider's own key when it is missing", async () => { + vi.stubEnv("ANTHROPIC_API_KEY", ""); + const engine = createAxEngine(); + await expect(engine.optimize(request(), { + variables: {}, + model: { provider: "anthropic" }, + })).rejects.toThrow("ANTHROPIC_API_KEY"); + }); + + it("knows the conventional key name for an unlisted provider", () => { + expect(apiKeyNamesFor("anthropic")).toEqual(["ANTHROPIC_API_KEY"]); + expect(apiKeyNamesFor("some-new-provider")).toEqual(["SOME_NEW_PROVIDER_API_KEY"]); + }); +}); From 050c9ad5a8a911a911577f8087f9773ffe2307b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 17:48:40 +0000 Subject: [PATCH 06/14] feat: add a ts-autocode CLI and make the example runnable The product is "instrument your app and let it rewrite itself", but inspecting what is trainable, what has been captured, or what a run would change required writing a script that imports discoverTrainables. No package declared a bin. `ts-autocode discover` lists every marked method with its signature and, more importantly, the exact identity to pass to defineTrainable. That is the one place an otherwise type-safe design falls back to a string -- a typo in `defineTrainable("Router.route")` silently yields a different symbol -- so printing real ids is what makes the marker design usable without reading the source scanner. `ts-autocode status` reports captured traces per trainable, which is what background evolution counts against evolution.minTraces. Both take --cwd, --project, --file, --output-dir and --json. The CLI is a function returning {code, stdout, stderr} with a thin bin wrapper, so it is tested without spawning a process, and library failures are reported by message rather than as a stack trace. Also makes examples/optimize.ts real, per CONTRIBUTING's own rule. It imported "../src/index.js" rather than the package name, exported rather than ran, and was referenced by no test or script, so nothing would have noticed it breaking. It now imports by package name (tsconfig.test.json maps the specifier to src/), runs directly under node, and executes on every check against a stub engine so CI needs no provider key. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- README.md | 26 +++++++ examples/optimize.ts | 25 ++++++- package.json | 4 + src/cli-main.ts | 7 ++ src/cli.ts | 170 ++++++++++++++++++++++++++++++++++++++++++ test/cli.test.ts | 123 ++++++++++++++++++++++++++++++ test/examples.test.ts | 23 ++++++ test/tier1.test.ts | 2 +- tsconfig.test.json | 7 ++ 9 files changed, 383 insertions(+), 4 deletions(-) create mode 100644 src/cli-main.ts create mode 100644 src/cli.ts create mode 100644 test/cli.test.ts create mode 100644 test/examples.test.ts diff --git a/README.md b/README.md index 4af8c7e..fb308b7 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,31 @@ npm install ts-autocode Node.js 20 or newer is required. +## Command line + +```bash +npx ts-autocode discover +``` + +`discover` lists every method the project marks and prints the exact identity +to bind evals to — the one place this otherwise type-safe design falls back to +a string, where a typo yields a different symbol with no error: + +```text +Router.route src/router.ts + route(input: string): string + +1 trainable. + +Bind evals to one with its symbol: + const target = defineTrainable("Router.route"); + await training.train({ trainable: target.symbol, /* ... */ }); +``` + +`ts-autocode status` reports how many traces each trainable has captured, which +is what background evolution counts against `evolution.minTraces`. Both accept +`--cwd`, `--project`, `--file` (repeatable), `--output-dir`, and `--json`. + ## Use the directive Place the literal directive first in a function or method body. No import, @@ -518,6 +543,7 @@ event carrying an `error`, with the phase it always did. | `ts-autocode/ax` | Tuning the default Ax engine. | | `ts-autocode/grounding` | Grounding decorators and ambient-class scanning. | | `ts-autocode/register` | The zero-config runtime patch (`node --import`). | +| `npx ts-autocode` | `discover` and `status` from the command line. | Everything on `/internal` is still exported from the root, so no existing import breaks; the subpath exists so that what an application imports is only diff --git a/examples/optimize.ts b/examples/optimize.ts index 6ecc4bd..4f6eb6b 100644 --- a/examples/optimize.ts +++ b/examples/optimize.ts @@ -1,6 +1,9 @@ import type { EvalTestInput } from "@agentv/core"; -import { configureTraining, defineTrainable } from "../src/index.js"; +// Imported by package name, exactly as a consumer would. The repo maps +// `ts-autocode` to `src/` for typechecking; a published install resolves the +// same specifier to `dist/`. +import { createTrainingRuntime, defineTrainable, type TrainingEngine } from "ts-autocode"; class Router { route(input: string): string { @@ -15,11 +18,17 @@ const tests = [ ] satisfies EvalTestInput[]; // The token's symbol binds these evals to the directive-marked method above. +// `npx ts-autocode discover` prints this id rather than making you guess it. const route = defineTrainable("Router.route"); -export async function optimizeRouter() { - const training = configureTraining({ source: { files: [import.meta.filename] } }); +/** Runs the example. Pass an engine to run it offline — the default Ax engine + * needs a provider key, which a CI typecheck must not require. */ +export async function optimizeRouter(engine?: TrainingEngine) { const router = new Router(); + const training = createTrainingRuntime({ + source: { files: [import.meta.filename] }, + ...(engine === undefined ? {} : { engine }), + }); return training.train({ trainable: route.symbol, objective: "Keep billing routing correct and preserve the fallback", @@ -29,5 +38,15 @@ export async function optimizeRouter() { workers: 2, outputDir: "examples/output", }, + rounds: { max: 2 }, + promotion: { minScore: 1 }, }); } + +// `node --experimental-strip-types examples/optimize.ts` runs it for real, +// against whatever `model` / OPENAI_API_KEY you have configured. +if (import.meta.filename === process.argv[1]) { + const run = await optimizeRouter(); + const readiness = run.canActivate(); + console.log(run.outcome, readiness.ready ? "promotable" : readiness.failures); +} diff --git a/package.json b/package.json index 056e4b1..5ac4e80 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "license": "MIT", "type": "module", "sideEffects": [ + "./dist/cli-main.js", "./dist/index.js", "./dist/register.js" ], @@ -21,6 +22,9 @@ "files": [ "dist" ], + "bin": { + "ts-autocode": "./dist/cli-main.js" + }, "main": "./dist/index.js", "types": "./dist/index.d.ts", "exports": { diff --git a/src/cli-main.ts b/src/cli-main.ts new file mode 100644 index 0000000..00dc287 --- /dev/null +++ b/src/cli-main.ts @@ -0,0 +1,7 @@ +#!/usr/bin/env node +import { run } from "./cli.js"; + +const result = await run(process.argv.slice(2)); +if (result.stdout) process.stdout.write(result.stdout); +if (result.stderr) process.stderr.write(result.stderr); +process.exitCode = result.code; diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..1c8eab7 --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,170 @@ +import { readFile } from "node:fs/promises"; +import { relative } from "node:path"; +import { parseArgs } from "node:util"; + +import { + defineTrainable, + discoverTrainables, + isTsAutocodeError, + type SourceSettings, + type TrainableTarget, + type TrainingRecord, +} from "ts-autocode-training"; + +// Inspecting what is trainable, what has been captured, or what a run would +// change previously meant writing a script that imports discoverTrainables. +// The identities the library asks for are strings a user has to guess exactly +// -- `defineTrainable("Router.route")` -- so `discover` is the tool that makes +// the marker-based design usable without reading the source scanner. + +export const usage = `ts-autocode [options] + +Commands: + discover List every trainable the TypeScript project marks. + status Show captured traces per trainable from a run's artifacts. + help Show this message. + +Options: + --cwd Project root (default: the working directory). + --project tsconfig to read (default: tsconfig.json). + --file Scan only these files; repeatable. + --output-dir Where run artifacts live (default: .agentv). + --json Emit JSON instead of a table. +`; + +export interface CliResult { + readonly code: number; + readonly stdout: string; + readonly stderr: string; +} + +interface DiscoveredRow { + readonly id: string; + readonly signature: string; + readonly location: string; + readonly async: boolean; +} + +/** What `discover` reports, separated from how it is printed so the shape can + * be tested and consumed as JSON. */ +export function describeTrainables(targets: readonly TrainableTarget[], cwd: string): readonly DiscoveredRow[] { + return targets.map((target) => ({ + id: target.id, + signature: target.signature, + location: target.artifactRef.startsWith("memory://") + ? target.artifactRef + : relative(cwd, target.artifactRef) || target.artifactRef, + async: target.async, + })); +} + +function table(rows: readonly DiscoveredRow[]): string { + if (rows.length === 0) { + return 'No trainables found. Mark one with the "use training" directive or the @trainable() decorator.'; + } + const width = Math.max(...rows.map((row) => row.id.length)); + const lines = rows.map((row) => `${row.id.padEnd(width)} ${row.location}\n${" ".repeat(width)} ${row.signature}`); + return [ + ...lines, + "", + `${rows.length} trainable${rows.length === 1 ? "" : "s"}.`, + "", + "Bind evals to one with its symbol:", + ` const target = defineTrainable(${JSON.stringify(rows[0]?.id ?? "Class.method")});`, + " await training.train({ trainable: target.symbol, /* ... */ });", + ].join("\n"); +} + +/** Reads the records a run wrote, if any. Absent artifacts are not an error: + * "nothing captured yet" is the normal state before an app has served traffic. */ +async function readRecords(outputDir: string): Promise { + const path = `${outputDir}/records.json`; + try { + const parsed = JSON.parse(await readFile(path, "utf8")) as unknown; + return Array.isArray(parsed) ? parsed as TrainingRecord[] : []; + } catch { + return []; + } +} + +function statusTable(targets: readonly TrainableTarget[], records: readonly TrainingRecord[]): string { + const counts = new Map(); + for (const record of records) { + const entry = counts.get(record.trainableId) ?? { total: 0, succeeded: 0 }; + entry.total += 1; + if (record.succeeded) entry.succeeded += 1; + counts.set(record.trainableId, entry); + } + if (targets.length === 0) return "No trainables found."; + const width = Math.max(...targets.map((target) => target.id.length)); + return targets.map((target) => { + const entry = counts.get(target.id) ?? { total: 0, succeeded: 0 }; + return `${target.id.padEnd(width)} ${entry.succeeded} successful / ${entry.total} captured`; + }).join("\n"); +} + +function sourceSettings(values: Record): SourceSettings { + const files = values["file"] as string[] | undefined; + return { + ...(typeof values["cwd"] === "string" ? { cwd: values["cwd"] } : {}), + ...(typeof values["project"] === "string" ? { tsconfig: values["project"] } : {}), + ...(files && files.length > 0 ? { files } : {}), + }; +} + +/** The CLI as a function, so it is testable without spawning a process. */ +export async function run(argv: readonly string[]): Promise { + let parsed: ReturnType; + try { + parsed = parseArgs({ + args: [...argv], + allowPositionals: true, + options: { + cwd: { type: "string" }, + project: { type: "string" }, + file: { type: "string", multiple: true }, + "output-dir": { type: "string" }, + json: { type: "boolean" }, + help: { type: "boolean", short: "h" }, + }, + }); + } catch (error) { + return { code: 2, stdout: "", stderr: `${(error as Error).message}\n\n${usage}` }; + } + + const command = parsed.positionals[0] ?? (parsed.values["help"] ? "help" : undefined); + if (command === undefined || command === "help") { + return { code: command === undefined ? 2 : 0, stdout: command === "help" ? usage : "", stderr: command === undefined ? usage : "" }; + } + + const settings = sourceSettings(parsed.values); + const cwd = settings.cwd ?? process.cwd(); + + try { + if (command === "discover") { + const rows = describeTrainables(discoverTrainables(settings), cwd); + return { code: 0, stdout: `${parsed.values["json"] ? JSON.stringify(rows, null, 2) : table(rows)}\n`, stderr: "" }; + } + if (command === "status") { + const targets = discoverTrainables(settings); + const records = await readRecords((parsed.values["output-dir"] as string | undefined) ?? ".agentv"); + if (parsed.values["json"]) { + const rows = targets.map((target) => ({ + id: target.id, + captured: records.filter((record) => record.trainableId === target.id).length, + succeeded: records.filter((record) => record.trainableId === target.id && record.succeeded).length, + })); + return { code: 0, stdout: `${JSON.stringify(rows, null, 2)}\n`, stderr: "" }; + } + return { code: 0, stdout: `${statusTable(targets, records)}\n`, stderr: "" }; + } + return { code: 2, stdout: "", stderr: `unknown command: ${command}\n\n${usage}` }; + } catch (error) { + // Library errors already say what to fix; anything else is a real crash. + if (isTsAutocodeError(error)) return { code: 1, stdout: "", stderr: `${error.message}\n` }; + throw error; + } +} + +/** Exposed so `discover`'s suggested snippet stays true to the real API. */ +export { defineTrainable }; diff --git a/test/cli.test.ts b/test/cli.test.ts new file mode 100644 index 0000000..ba456bc --- /dev/null +++ b/test/cli.test.ts @@ -0,0 +1,123 @@ +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { describeTrainables, run, usage } from "../src/cli.js"; +import { discoverInSource } from "ts-autocode-training"; + +// Inspecting what is trainable required writing a script that imports +// discoverTrainables. That matters more here than in most libraries, because +// the identity a user must pass to train() is an exact string with no type +// safety: `defineTrainable("Router.route")`. A typo yields a different symbol +// silently, so `discover` is what makes the marker design usable. + +const directory = "test/output/cli"; +const source = `class Router { + route(input: string): string { + "use training"; + return input; + } + + async enrich(id: string, deep?: boolean): Promise { + "use training"; + return \`\${id}:\${deep}\`; + } +} +`; + +async function project(): Promise { + await mkdir(directory, { recursive: true }); + const file = join(directory, "router.ts"); + await writeFile(file, source, "utf8"); + return file; +} + +describe("ts-autocode discover", () => { + it("lists every marked method with its identity and signature", async () => { + const result = await run(["discover", "--file", await project()]); + expect(result.code).toBe(0); + expect(result.stdout).toContain("Router.route"); + expect(result.stdout).toContain("Router.enrich"); + expect(result.stdout).toContain("route(input: string): string"); + expect(result.stdout).toContain("2 trainables."); + }); + + it("suggests a defineTrainable call using a real discovered id", async () => { + const result = await run(["discover", "--file", await project()]); + // The suggested snippet is the payoff: it removes the guesswork from the + // one stringly-typed seam in an otherwise type-safe design. + expect(result.stdout).toContain('defineTrainable("Router.route")'); + }); + + it("emits machine-readable output", async () => { + const result = await run(["discover", "--file", await project(), "--json"]); + const rows = JSON.parse(result.stdout) as Array>; + expect(rows).toHaveLength(2); + expect(rows[0]).toMatchObject({ id: "Router.route", async: false }); + expect(rows[1]).toMatchObject({ id: "Router.enrich", async: true }); + }); + + it("says so plainly when a project marks nothing", async () => { + await mkdir(directory, { recursive: true }); + const empty = join(directory, "empty.ts"); + await writeFile(empty, "export const nothing = 1;\n", "utf8"); + const result = await run(["discover", "--file", empty]); + expect(result.code).toBe(0); + expect(result.stdout).toContain("No trainables found"); + }); + + it("reports the signature and asyncness the engine will see", () => { + const rows = describeTrainables(discoverInSource(source, "router.ts"), "."); + expect(rows.map((row) => row.id)).toEqual(["Router.route", "Router.enrich"]); + expect(rows[1]?.signature).toBe("enrich(id: string, deep?: boolean): Promise"); + }); +}); + +describe("ts-autocode status", () => { + it("reports zero captures before an app has served traffic", async () => { + const result = await run(["status", "--file", await project(), "--output-dir", `${directory}/absent`]); + expect(result.code).toBe(0); + expect(result.stdout).toContain("0 successful / 0 captured"); + }); + + it("counts captured traces per trainable", async () => { + const artifacts = join(directory, "artifacts"); + await rm(artifacts, { recursive: true, force: true }); + await mkdir(artifacts, { recursive: true }); + await writeFile(join(artifacts, "records.json"), JSON.stringify([ + { trainableId: "Router.route", succeeded: true }, + { trainableId: "Router.route", succeeded: false }, + { trainableId: "Router.enrich", succeeded: true }, + ]), "utf8"); + const result = await run(["status", "--file", await project(), "--output-dir", artifacts, "--json"]); + expect(JSON.parse(result.stdout)).toEqual([ + { id: "Router.route", captured: 2, succeeded: 1 }, + { id: "Router.enrich", captured: 1, succeeded: 1 }, + ]); + }); +}); + +describe("ts-autocode argument handling", () => { + it("prints usage and fails when given no command", async () => { + const result = await run([]); + expect(result.code).toBe(2); + expect(result.stderr).toContain("ts-autocode "); + }); + + it("prints usage successfully when asked", async () => { + expect(await run(["help"])).toMatchObject({ code: 0, stdout: usage }); + expect((await run(["--help"])).code).toBe(0); + }); + + it("rejects an unknown command and an unknown flag", async () => { + expect((await run(["frobnicate"])).stderr).toContain("unknown command: frobnicate"); + expect((await run(["discover", "--nope"])).code).toBe(2); + }); + + it("reports a library failure without a stack trace", async () => { + const result = await run(["discover", "--project", "no-such-tsconfig.json"]); + expect(result.code).toBe(1); + expect(result.stderr).not.toContain(" at "); + }); +}); diff --git a/test/examples.test.ts b/test/examples.test.ts new file mode 100644 index 0000000..3cb6a7e --- /dev/null +++ b/test/examples.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; + +import { optimizeRouter } from "../examples/optimize.js"; +import type { TrainingEngine } from "../src/index.js"; + +// examples/optimize.ts imported "../src/index.js" rather than the package name, +// exported a function rather than running, and was referenced by no test or +// script -- so nothing would have noticed it breaking. CONTRIBUTING asks for a +// runnable example; this executes it on every check. + +const engine: TrainingEngine = { + id: "examples-test", + optimize: async () => ({ implementation: 'return input.includes("invoice") ? "billing" : "fallback";' }), +}; + +describe("examples/optimize.ts", () => { + it("runs end to end against a stub engine", async () => { + const run = await optimizeRouter(engine); + expect(run.outcome).toBe("ready"); + expect(run.canActivate()).toEqual({ ready: true }); + expect(run.final.candidate.trainableId).toBe("Router.route"); + }, 30_000); +}); diff --git a/test/tier1.test.ts b/test/tier1.test.ts index 3f2a497..b922c1b 100644 --- a/test/tier1.test.ts +++ b/test/tier1.test.ts @@ -19,7 +19,7 @@ describe("sideEffects declaration", () => { // leave a consumer with "no training engine is configured" after importing // the package that configures it. it("names the modules whose imports actually wire the runtime", () => { - expect(manifest.sideEffects).toEqual(["./dist/index.js", "./dist/register.js"]); + expect(manifest.sideEffects).toEqual(["./dist/cli-main.js", "./dist/index.js", "./dist/register.js"]); }); }); diff --git a/tsconfig.test.json b/tsconfig.test.json index 005eb88..f1ce650 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -2,6 +2,13 @@ "extends": "./tsconfig.json", "compilerOptions": { "noEmit": true, + "baseUrl": ".", + "paths": { + "ts-autocode": ["src/index.ts"], + "ts-autocode/ax": ["src/providers/ax.ts"], + "ts-autocode/internal": ["src/internal.ts"], + "ts-autocode/grounding": ["src/grounding.ts"] + }, "rootDir": ".", "outDir": null, "declaration": false, From a3fa475d7f463976d835cf83cd10477e276656b2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 17:56:43 +0000 Subject: [PATCH 07/14] refactor: remove the boilerplate the extension points required Five things a consumer had to work around. - Types they had to supply but could not construct. Implementing a custom TrainingLoop means returning a CandidateReview containing a TrainableEvalRun, which only the internals could produce -- so this repo's own tests wrote `{...} as unknown as TrainableEvalRun` and `{} as never`, and a consumer had no better option. Adds createEvalRun, createPromotionDecision and createCandidateReview, and uses them in the two tests that needed the casts, which now have none. - Uninferrable generics. defineTrainingHarness takes three type parameters but its settings mention only TCandidate, so a bare call inferred `unknown` three times and every documented call site wrote them all out. TChallenge was already scoped to `run` and inferred correctly; `inferringHarness()` gives the other two the same treatment. - A lossy argument guess with no way out. Eval inputs were JSON.parsed and spread as arguments, so a trainable taking the literal string "[1,2]" received two numbers. Adds ExecutionSettings.decodeArgs, with the previous behavior exported as `evaluationArgs` and still the default. - `...(x === undefined ? {} : { x })`, written out about twenty-five times because exactOptionalPropertyTypes forbids assigning an explicit undefined, plus a one-off maybeSignal() doing the same for one field. Adds `optional(key, value)` and `defined(values)` and applies them. - Effect as a root runtime dependency to express two try/catch statements. attempt/attemptAsync are now plain try/catch and `effect` is dropped from the root package. It stays where it earns its place: resilience.ts, whose timeout/retry/interruption composition is genuinely hard by hand. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- README.md | 43 ++++++++++- package.json | 1 - packages/harness/src/attempt.ts | 34 +++++---- packages/harness/src/harness.ts | 21 ++++++ packages/harness/src/index.ts | 3 +- packages/harness/test/harness.test.ts | 36 +++++++++ packages/training/src/attempt.ts | 34 +++++---- packages/training/src/builders.ts | 95 ++++++++++++++++++++++++ packages/training/src/index.ts | 6 ++ packages/training/src/optional.ts | 21 ++++++ packages/training/src/training.ts | 55 +++++++++----- packages/training/test/loop.test.ts | 14 ++-- src/attempt.ts | 34 +++++---- src/index.ts | 8 ++ src/providers/ax.ts | 7 +- src/providers/harness.ts | 22 +++--- test/deprecated.test.ts | 16 ++++ test/harness-loop.test.ts | 15 ++-- test/tier5.test.ts | 102 ++++++++++++++++++++++++++ 19 files changed, 473 insertions(+), 94 deletions(-) create mode 100644 packages/training/src/builders.ts create mode 100644 packages/training/src/optional.ts create mode 100644 test/tier5.test.ts diff --git a/README.md b/README.md index fb308b7..82070dc 100644 --- a/README.md +++ b/README.md @@ -304,10 +304,11 @@ Runtime dependencies enter through `TrainingSettings`: }); ``` -- `execution` bounds each candidate run inside the executor: `timeoutMs` caps a - single execution (default 5 seconds). This is distinct from +- `execution` shapes each candidate run inside the executor. `timeoutMs` caps a + single execution (default 5 seconds) — distinct from `resilience.evaluate.timeoutMs`, which bounds the whole attempt and may retry - it. + it. `decodeArgs` turns an eval case's string input into the trainable's + argument list; see below. - `source` overrides TypeScript project discovery when the default `tsconfig.json` is not the desired project. - `outputDir` relocates run artifacts and eval output (default `.agentv`, @@ -460,6 +461,42 @@ const engine: TrainingEngine = { The core validates identity, source digests, and the final candidate regardless of engine. +## Evaluation arguments + +AgentV evaluation is string-in, string-out. By default an eval input is +`JSON.parse`d and a resulting array is spread as the trainable's arguments, +which is a guess: a function taking the single string `"[1,2]"` receives two +numbers instead. Replace it when your arguments are not what the guess +produces: + +```ts +import { configureTraining } from "ts-autocode"; + +configureTraining({ + // Pass the raw eval input through as one string argument. + execution: { decodeArgs: (input) => [input] }, +}); +``` + +## Extending the library + +Implementing a custom `TrainingLoop` means returning a `CandidateReview` +containing a `TrainableEvalRun`. Builders construct both, so extending the +library never requires a cast: + +```ts +import { createCandidateReview, type CandidatePatch, type TrainingLoop } from "ts-autocode"; + +const loop: TrainingLoop = async (input) => { + const candidate: CandidatePatch = await input.propose({ round: 1, slot: 1, feedback: [] }); + const review = createCandidateReview({ candidate, failures: ["not tried yet"] }); + return { outcome: "exhausted", rounds: [{ round: 1, candidate, ...review }] }; +}; +``` + +`createEvalRun` and `createPromotionDecision` build the parts individually when +you have real evidence to carry. + ## Errors Every failure this library raises is a `TsAutocodeError` carrying a `code` you diff --git a/package.json b/package.json index 5ac4e80..7e01a65 100644 --- a/package.json +++ b/package.json @@ -78,7 +78,6 @@ }, "dependencies": { "@ax-llm/ax": "^23.0.0", - "effect": "^3.21.4", "ts-autocode-grounding": "0.1.0", "ts-autocode-harness": "0.1.0", "ts-autocode-rewrite": "0.1.0", diff --git a/packages/harness/src/attempt.ts b/packages/harness/src/attempt.ts index e7d0a83..434ed4d 100644 --- a/packages/harness/src/attempt.ts +++ b/packages/harness/src/attempt.ts @@ -1,7 +1,11 @@ -// Internal Effect-backed fallback helpers. Deliberately duplicated in each -// workspace package (root src/, packages/training, packages/harness) instead -// of adding a shared package for a handful of lines; keep the copies identical. -import { Effect } from "effect"; +// Internal fallback helpers. Deliberately duplicated in each workspace package +// (root src/, packages/training, packages/harness) instead of adding a shared +// package for a handful of lines; keep the copies identical. +// +// These are try/catch, and were written with Effect. That put a large runtime +// dependency in every consumer's tree to express two statements. Effect stays +// where it earns its place: packages/training/src/resilience.ts, whose +// timeout/retry/interruption composition is genuinely hard by hand. export function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); @@ -10,18 +14,18 @@ export function errorMessage(error: unknown): string { /** Runs `fn`, mapping a throw to `fallback(error)` — a sync error-to-value * boundary. The fallback receives the raw thrown value. */ export function attempt(fn: () => T, fallback: (error: unknown) => T): T { - return Effect.runSync( - Effect.try({ try: fn, catch: (error) => error }).pipe( - Effect.catchAll((error) => Effect.sync(() => fallback(error))), - ), - ); + try { + return fn(); + } catch (error) { + return fallback(error); + } } /** Async variant: resolves `fallback(error)` when `fn` throws or rejects. */ -export function attemptAsync(fn: () => Promise, fallback: (error: unknown) => T): Promise { - return Effect.runPromise( - Effect.tryPromise({ try: fn, catch: (error) => error }).pipe( - Effect.catchAll((error) => Effect.sync(() => fallback(error))), - ), - ); +export async function attemptAsync(fn: () => Promise, fallback: (error: unknown) => T): Promise { + try { + return await fn(); + } catch (error) { + return fallback(error); + } } diff --git a/packages/harness/src/harness.ts b/packages/harness/src/harness.ts index dcadbd2..dbb1e6e 100644 --- a/packages/harness/src/harness.ts +++ b/packages/harness/src/harness.ts @@ -155,6 +155,27 @@ export interface TrainingHarness { run(input: HarnessInput): Promise>; } +/** A harness whose roles decide its types. `settings` mentions only + * `TCandidate`, so `defineTrainingHarness()` inferred `unknown` for the other + * two and every documented call site wrote all three out by hand -- + * `defineTrainingHarness()`. `TChallenge` was + * already scoped to `run` and inferred correctly; this gives the other two the + * same treatment, so `inferringHarness().run({ student, teacher })` types + * itself from the callbacks. */ +export interface InferringTrainingHarness { + run( + input: HarnessInput, + ): Promise>; +} + +/** {@link defineTrainingHarness} with the assessment and feedback types + * inferred from the roles rather than written out. */ +export function inferringHarness( + settings: HarnessSettings = {}, +): InferringTrainingHarness { + return defineTrainingHarness(settings) as unknown as InferringTrainingHarness; +} + export function defineTrainingHarness( settings: HarnessSettings = {}, ): TrainingHarness { diff --git a/packages/harness/src/index.ts b/packages/harness/src/index.ts index a0c4ad4..597e23b 100644 --- a/packages/harness/src/index.ts +++ b/packages/harness/src/index.ts @@ -7,7 +7,7 @@ export type { ActionGate, JudgeDecision } from "./dispatch.js"; export { agentBusEntry, agentMessage } from "./schema.js"; export type { AbsolutePath, AgentBusEntry, AgentMessage } from "./schema.js"; -export { defaultHarnessRounds, defaultMaxRounds, defineTrainingHarness } from "./harness.js"; +export { defaultHarnessRounds, defaultMaxRounds, defineTrainingHarness, inferringHarness } from "./harness.js"; export type { AdversaryConfig, AdversaryResult, @@ -17,6 +17,7 @@ export type { HarnessRound, HarnessRun, HarnessSettings, + InferringTrainingHarness, JudgeRequest, RubricRevision, RubricRevisionTurn, diff --git a/packages/harness/test/harness.test.ts b/packages/harness/test/harness.test.ts index 777d05c..ec3bccd 100644 --- a/packages/harness/test/harness.test.ts +++ b/packages/harness/test/harness.test.ts @@ -14,6 +14,7 @@ import { defineTrainingHarness, dispatchAction, HarnessSandbox, + inferringHarness, WriteAheadAgentBus, type AgentBusEntry, } from "../src/index.js"; @@ -317,3 +318,38 @@ async function loopCallbacks(decisions: readonly ("pass" | "fail")[]) { }), }; } + +describe("inferringHarness", () => { + // defineTrainingHarness takes three type parameters but `settings` mentions + // only TCandidate, so a bare call inferred `unknown, unknown, unknown` and + // every documented call site wrote all three out. TChallenge was already + // scoped to `run` and inferred; these two now behave the same way. + it("infers assessment and feedback from the roles", async () => { + const result = await inferringHarness<{ id: string }>().run({ + task: "task", + rubric: "rubric", + student: ({ round }) => ({ id: `candidate-${round}` }), + teacher: (candidate) => ({ + assessment: { score: candidate.id === "candidate-1" ? 0 : 1 }, + feedback: candidate.id === "candidate-1" ? ["needs work"] : [], + }), + }); + + expect(result.outcome).toBe("accepted"); + expect(result.final.candidate.id).toBe("candidate-2"); + // Inferred, not asserted: `assessment` is {score: number} here, so this + // arithmetic typechecks without a cast or an explicit type argument. + expect(result.final.assessment.score + 1).toBe(2); + }); + + it("still honors settings", async () => { + const result = await inferringHarness({ maxRounds: 1 }).run({ + task: "task", + rubric: "rubric", + student: ({ round }) => `candidate-${round}`, + teacher: () => ({ assessment: "no", feedback: ["always rejected"] }), + }); + expect(result.outcome).toBe("exhausted"); + expect(result.rounds).toHaveLength(1); + }); +}); diff --git a/packages/training/src/attempt.ts b/packages/training/src/attempt.ts index e7d0a83..434ed4d 100644 --- a/packages/training/src/attempt.ts +++ b/packages/training/src/attempt.ts @@ -1,7 +1,11 @@ -// Internal Effect-backed fallback helpers. Deliberately duplicated in each -// workspace package (root src/, packages/training, packages/harness) instead -// of adding a shared package for a handful of lines; keep the copies identical. -import { Effect } from "effect"; +// Internal fallback helpers. Deliberately duplicated in each workspace package +// (root src/, packages/training, packages/harness) instead of adding a shared +// package for a handful of lines; keep the copies identical. +// +// These are try/catch, and were written with Effect. That put a large runtime +// dependency in every consumer's tree to express two statements. Effect stays +// where it earns its place: packages/training/src/resilience.ts, whose +// timeout/retry/interruption composition is genuinely hard by hand. export function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); @@ -10,18 +14,18 @@ export function errorMessage(error: unknown): string { /** Runs `fn`, mapping a throw to `fallback(error)` — a sync error-to-value * boundary. The fallback receives the raw thrown value. */ export function attempt(fn: () => T, fallback: (error: unknown) => T): T { - return Effect.runSync( - Effect.try({ try: fn, catch: (error) => error }).pipe( - Effect.catchAll((error) => Effect.sync(() => fallback(error))), - ), - ); + try { + return fn(); + } catch (error) { + return fallback(error); + } } /** Async variant: resolves `fallback(error)` when `fn` throws or rejects. */ -export function attemptAsync(fn: () => Promise, fallback: (error: unknown) => T): Promise { - return Effect.runPromise( - Effect.tryPromise({ try: fn, catch: (error) => error }).pipe( - Effect.catchAll((error) => Effect.sync(() => fallback(error))), - ), - ); +export async function attemptAsync(fn: () => Promise, fallback: (error: unknown) => T): Promise { + try { + return await fn(); + } catch (error) { + return fallback(error); + } } diff --git a/packages/training/src/builders.ts b/packages/training/src/builders.ts new file mode 100644 index 0000000..d18b8e7 --- /dev/null +++ b/packages/training/src/builders.ts @@ -0,0 +1,95 @@ +import type { EvalRunResult } from "@agentv/core"; + +import type { BoundEvaluation, CandidatePatch } from "./engine.js"; +import type { TrainableEvalRun } from "./evaluation.js"; +import type { CandidateReview } from "./loop.js"; +import { defined } from "./optional.js"; +import type { PromotionDecision } from "./promotion.js"; +import { defineTrainable, toTrainableToken, type TrainableIdentity } from "./token.js"; + +// Anyone implementing a custom TrainingLoop must return a CandidateReview +// containing a TrainableEvalRun, which only `evaluateTrainable` could produce. +// So the tests in this repo wrote `{ ... } as unknown as TrainableEvalRun` and +// `{} as never`, and a consumer had no better option. A type a consumer must +// supply but cannot construct is a hole in the contract; these fill it. + +export interface EvalRunInput { + readonly trainable: TrainableIdentity; + readonly evaluations?: readonly BoundEvaluation[]; + /** The AgentV result, when there is a real one to carry. */ + readonly run?: EvalRunResult; +} + +/** Builds a {@link TrainableEvalRun} — the shape a custom `TrainingLoop` must + * return inside its reviews. */ +export function createEvalRun(input: EvalRunInput): TrainableEvalRun { + const token = toTrainableToken(input.trainable); + return Object.freeze({ + token, + run: input.run ?? emptyRun(), + evaluations: Object.freeze([...(input.evaluations ?? [])]), + }); +} + +export interface DecisionInput { + readonly candidateId: string; + readonly promote?: boolean; + readonly failures?: readonly string[]; + readonly meanScore?: number; + readonly passRate?: number; +} + +/** Builds a {@link PromotionDecision}. `promote` defaults to whether there are + * any failures, matching how the real gate decides. */ +export function createPromotionDecision(input: DecisionInput): PromotionDecision { + const failures = Object.freeze([...(input.failures ?? [])]); + const promote = input.promote ?? failures.length === 0; + return Object.freeze({ + candidateId: input.candidateId, + promote, + failures, + meanScore: input.meanScore ?? (promote ? 1 : 0), + passRate: input.passRate ?? (promote ? 1 : 0), + }); +} + +export interface ReviewInput { + readonly candidate: CandidatePatch; + readonly promote?: boolean; + readonly failures?: readonly string[]; + readonly evaluations?: readonly BoundEvaluation[]; + readonly verification?: TrainableEvalRun; + readonly decision?: PromotionDecision; +} + +/** Builds a {@link CandidateReview}, the value a `TrainingLoop` hands back for + * each candidate it reviewed. */ +export function createCandidateReview(input: ReviewInput): CandidateReview { + return Object.freeze({ + verification: input.verification ?? createEvalRun({ + trainable: defineTrainable(input.candidate.trainableId), + ...defined({ evaluations: input.evaluations }), + }), + decision: input.decision ?? createPromotionDecision({ + candidateId: input.candidate.id, + ...defined({ promote: input.promote, failures: input.failures }), + }), + }); +} + +/** An AgentV run result with no cases, for reviews that carry their evidence in + * `evaluations` rather than in a real eval run. Written out in full rather than + * cast: the point of these builders is that nobody has to cast. */ +function emptyRun(): EvalRunResult { + return Object.freeze({ + results: [], + summary: Object.freeze({ + total: 0, + passed: 0, + failed: 0, + executionErrors: 0, + durationMs: 0, + meanScore: 0, + }), + }); +} diff --git a/packages/training/src/index.ts b/packages/training/src/index.ts index 42ee2f9..8233529 100644 --- a/packages/training/src/index.ts +++ b/packages/training/src/index.ts @@ -6,6 +6,7 @@ export { defaultEvolution, defaultObjective, defaultOutputDir, + evaluationArgs, provideTrainingDefaults, training, } from "./training.js"; @@ -92,6 +93,11 @@ export type { export type { TrainableEvalRun } from "./evaluation.js"; +export { createCandidateReview, createEvalRun, createPromotionDecision } from "./builders.js"; +export type { DecisionInput, EvalRunInput, ReviewInput } from "./builders.js"; + +export { defined, optional } from "./optional.js"; + export { defaultMinPassRate, defaultMinScore, defaultPromotionGates, evaluatePromotionGate } from "./promotion.js"; export type { PromotionDecision, PromotionGate, PromotionGateContext, PromotionGateInput } from "./promotion.js"; diff --git a/packages/training/src/optional.ts b/packages/training/src/optional.ts new file mode 100644 index 0000000..8089ecc --- /dev/null +++ b/packages/training/src/optional.ts @@ -0,0 +1,21 @@ +// `exactOptionalPropertyTypes` forbids assigning an explicit `undefined` to an +// optional property, so every optional pass-through in this codebase was +// written as `...(x === undefined ? {} : { x })` -- about twenty-five times, +// plus a one-off `maybeSignal()` helper that did the same thing for one field. + +/** Spreads `{ [key]: value }` when `value` is defined, and nothing when it is + * not. `{ ...optional("signal", signal) }` replaces + * `...(signal === undefined ? {} : { signal })`. */ +export function optional(key: K, value: V | undefined): { [P in K]?: V } { + return (value === undefined ? {} : { [key]: value }) as { [P in K]?: V }; +} + +/** Spreads every defined entry of `values`, dropping the undefined ones. + * `{ ...defined({ signal, timeoutMs }) }` replaces a run of `optional` calls. */ +export function defined(values: T): { [K in keyof T]?: Exclude } { + const result: Record = {}; + for (const [key, value] of Object.entries(values)) { + if (value !== undefined) result[key] = value; + } + return result as { [K in keyof T]?: Exclude }; +} diff --git a/packages/training/src/training.ts b/packages/training/src/training.ts index 3a8a1f0..0952394 100644 --- a/packages/training/src/training.ts +++ b/packages/training/src/training.ts @@ -6,6 +6,7 @@ import { OpenInferenceSpanKind, SemanticConventions } from "@arizeai/openinferen import { SpanStatusCode, trace, type Attributes, type Span, type Tracer } from "@opentelemetry/api"; import { attempt, errorMessage } from "./attempt.js"; +import { optional } from "./optional.js"; import { EngineNotConfiguredError, ExecutorNotConfiguredError, @@ -99,6 +100,15 @@ export const defaultOutputDir = ".agentv"; * is the executor's own per-run limit. */ export interface ExecutionSettings { readonly timeoutMs?: number; + /** How an eval case's string input becomes the trainable's argument list. + * + * AgentV evaluation is string-in, string-out, so by default an input is + * `JSON.parse`d and a resulting array is spread as arguments. That guess is + * lossy: a function legitimately taking the single string `"[1,2]"` + * receives two numbers instead. Set this when your trainable's arguments + * are not what the guess produces — `(input) => [input]` passes the raw + * string through unchanged. */ + readonly decodeArgs?: (input: string) => readonly unknown[]; } export interface TrainingSettings { @@ -334,8 +344,8 @@ class TrainingRuntime implements Training { const run = await this.train({ trainable: token, minTraces, - ...(evolution.objective === undefined ? {} : { objective: evolution.objective }), - ...(evolution.evaluation === undefined ? {} : { evaluation: evolution.evaluation }), + ...optional("objective", evolution.objective), + ...optional("evaluation", evolution.evaluation), }); if (run.outcome !== "ready") { throw TrainingIncompleteError.noPromotableCandidate(run.outcome); @@ -372,6 +382,7 @@ class TrainingRuntime implements Training { const token = defineTrainable(candidate.trainableId); const execute = this.#executorOrThrow(); const timeoutMs = this.#settings.execution?.timeoutMs; + const decodeArgs = this.#settings.execution?.decodeArgs ?? evaluationArgs; const { signal, ...evaluation } = config; signal?.throwIfAborted(); const evaluated = await evaluateTrainable(token, { @@ -383,10 +394,10 @@ class TrainingRuntime implements Training { (attemptSignal) => execute( candidate.target, candidate.implementation, - evaluationArgs(input), + decodeArgs(input), { - ...(timeoutMs === undefined ? {} : { timeoutMs }), - ...(attemptSignal === undefined ? {} : { signal: attemptSignal }), + ...optional("timeoutMs", timeoutMs), + ...optional("signal", attemptSignal), }, ), signal, @@ -416,22 +427,22 @@ class TrainingRuntime implements Training { objective, rubric: promotionRubric(input), outputDir, - ...(options.maxRounds === undefined ? {} : { maxRounds: options.maxRounds }), - ...(options.fanOut === undefined ? {} : { fanOut: options.fanOut }), - ...(input.signal === undefined ? {} : { signal: input.signal }), + ...optional("maxRounds", options.maxRounds), + ...optional("fanOut", options.fanOut), + ...optional("signal", input.signal), propose: ({ feedback, signal }) => this.#propose(token, { objective, constraints: [ ...(input.constraints ?? []), ...feedback.map((failure) => `Previous candidate rejection: ${failure}`), ], - ...(input.engine === undefined ? {} : { engine: input.engine }), - ...(signal === undefined ? {} : { signal }), + ...optional("engine", input.engine), + ...optional("signal", signal), }), review: async (candidate, { label, signal }) => { const verification = await this.#evaluateCandidate(candidate, { ...candidateEvaluation, - ...(signal === undefined ? {} : { signal }), + ...optional("signal", signal), outputDir: `${outputDir}/${label}`, }); const decision = await evaluatePromotionGate({ @@ -439,10 +450,10 @@ class TrainingRuntime implements Training { evaluations: verification.evaluations, // The engine already validated the candidate source. conformance: true, - ...(options.minScore === undefined ? {} : { minScore: options.minScore }), - ...(options.minPassRate === undefined ? {} : { minPassRate: options.minPassRate }), - ...(input.policy === undefined ? {} : { policy: input.policy }), - ...(options.gates === undefined ? {} : { gates: options.gates }), + ...optional("minScore", options.minScore), + ...optional("minPassRate", options.minPassRate), + ...optional("policy", input.policy), + ...optional("gates", options.gates), }); return { verification, decision }; }, @@ -509,9 +520,9 @@ class TrainingRuntime implements Training { }, { variables: this.#variables, - ...(this.#settings.secrets === undefined ? {} : { secrets: this.#settings.secrets }), - ...(this.#settings.model === undefined ? {} : { model: this.#settings.model }), - ...(signal === undefined ? {} : { signal }), + ...optional("secrets", this.#settings.secrets), + ...optional("model", this.#settings.model), + ...optional("signal", signal), }, ), input.signal, @@ -566,7 +577,7 @@ class TrainingRuntime implements Training { ): Result { const startedAt = new Date(); const runId = randomUUID(); - const execution = { args, name, token, runId, startedAt, ...(span === undefined ? {} : { span }) }; + const execution = { args, name, token, runId, startedAt, ...optional("span", span) }; let result: Result; try { result = method.apply(thisValue, args); @@ -784,7 +795,11 @@ function defaultSerialize(value: unknown): string { return attempt(() => JSON.stringify(value) ?? String(value), () => String(value)); } -function evaluationArgs(input: string): readonly unknown[] { +/** The default {@link ExecutionSettings.decodeArgs}: parse the eval input as + * JSON and spread an array as the argument list, falling back to the raw string. + * Ambiguous by nature — a trainable taking the literal string `"[1,2]"` gets + * two numbers — which is why it is replaceable. */ +export function evaluationArgs(input: string): readonly unknown[] { return attempt(() => { const parsed = JSON.parse(input) as unknown; return Array.isArray(parsed) ? parsed : [parsed]; diff --git a/packages/training/test/loop.test.ts b/packages/training/test/loop.test.ts index e972d32..4dedd78 100644 --- a/packages/training/test/loop.test.ts +++ b/packages/training/test/loop.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "vitest"; import { + createCandidateReview, + createPromotionDecision, sequentialLoop, trainingRounds, type CandidatePatch, @@ -22,16 +24,18 @@ function candidate(id: string): CandidatePatch { } function review(candidateId: string, promote: boolean, meanScore = promote ? 1 : 0): CandidateReview { - return { - verification: {} as never, - decision: { + // `createCandidateReview` supplies the verification run, so this no longer + // needs `{} as never` for a type a loop author cannot build. + return createCandidateReview({ + candidate: candidate(candidateId), + decision: createPromotionDecision({ candidateId, promote, failures: promote ? [] : [`rejected ${candidateId}`], meanScore, passRate: promote ? 1 : 0, - }, - }; + }), + }); } function loopInput( diff --git a/src/attempt.ts b/src/attempt.ts index e7d0a83..434ed4d 100644 --- a/src/attempt.ts +++ b/src/attempt.ts @@ -1,7 +1,11 @@ -// Internal Effect-backed fallback helpers. Deliberately duplicated in each -// workspace package (root src/, packages/training, packages/harness) instead -// of adding a shared package for a handful of lines; keep the copies identical. -import { Effect } from "effect"; +// Internal fallback helpers. Deliberately duplicated in each workspace package +// (root src/, packages/training, packages/harness) instead of adding a shared +// package for a handful of lines; keep the copies identical. +// +// These are try/catch, and were written with Effect. That put a large runtime +// dependency in every consumer's tree to express two statements. Effect stays +// where it earns its place: packages/training/src/resilience.ts, whose +// timeout/retry/interruption composition is genuinely hard by hand. export function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); @@ -10,18 +14,18 @@ export function errorMessage(error: unknown): string { /** Runs `fn`, mapping a throw to `fallback(error)` — a sync error-to-value * boundary. The fallback receives the raw thrown value. */ export function attempt(fn: () => T, fallback: (error: unknown) => T): T { - return Effect.runSync( - Effect.try({ try: fn, catch: (error) => error }).pipe( - Effect.catchAll((error) => Effect.sync(() => fallback(error))), - ), - ); + try { + return fn(); + } catch (error) { + return fallback(error); + } } /** Async variant: resolves `fallback(error)` when `fn` throws or rejects. */ -export function attemptAsync(fn: () => Promise, fallback: (error: unknown) => T): Promise { - return Effect.runPromise( - Effect.tryPromise({ try: fn, catch: (error) => error }).pipe( - Effect.catchAll((error) => Effect.sync(() => fallback(error))), - ), - ); +export async function attemptAsync(fn: () => Promise, fallback: (error: unknown) => T): Promise { + try { + return await fn(); + } catch (error) { + return fallback(error); + } } diff --git a/src/index.ts b/src/index.ts index d9296e3..26911db 100644 --- a/src/index.ts +++ b/src/index.ts @@ -34,6 +34,9 @@ export type { TrainableDecorator } from "./instrumentation.js"; export { candidateDeclaration, CandidateSyntaxError, + createCandidateReview, + createEvalRun, + createPromotionDecision, captureTrainable, configureTraining, createTrainingRuntime, @@ -47,12 +50,14 @@ export { defaultPromotionGates, defaultRetry, defaultTsconfig, + defined, defineTrainable, discoverInSource, discoverTrainables, EngineContractError, EngineNotConfiguredError, EngineProposalError, + evaluationArgs, evaluatePromotionGate, ExecutorNotConfiguredError, inMemoryArtifactRef, @@ -64,6 +69,7 @@ export { MemoryTrainingStore, MissingSecretError, OperationInterruptedError, + optional, OperationTimeoutError, parseSetting, PromotionApplierNotConfiguredError, @@ -95,6 +101,7 @@ export type { EngineCandidate, EngineContext, ErrorPhase, + EvalRunInput, EvolutionSettings, ExecutionSettings, ImplementationExecutor, @@ -112,6 +119,7 @@ export type { ResilienceSettings, RetryOptions, ReviewContext, + ReviewInput, RoundObserver, RoundSequence, RoundSettings, diff --git a/src/providers/ax.ts b/src/providers/ax.ts index 82f537b..e7bfc60 100644 --- a/src/providers/ax.ts +++ b/src/providers/ax.ts @@ -11,6 +11,7 @@ import { import { EngineProposalError, MissingSecretError, + optional, type EngineContext, type ModelSelection, type OptimizeRequest, @@ -87,12 +88,12 @@ export function createAxEngine(options: AxEngineOptions = {}): TrainingEngine { scoreImplementation(request, prediction as RewriteOutput, example, options.executionTimeoutMs, context.signal), { ...options.optimize, studentAI, - ...(teacherAI === undefined ? {} : { teacherAI }), + ...optional("teacherAI", teacherAI), }); if (!result.optimizedProgram) throw new EngineProposalError(`Ax did not optimize ${request.trainableId}`); program.applyOptimization(result.optimizedProgram); const output = await program.forward(studentAI, publicInput(examples[0] as Record), { - ...(context.signal === undefined ? {} : { abortSignal: context.signal }), + ...optional("abortSignal", context.signal), }) as RewriteOutput; return { implementation: output[field.output], @@ -203,7 +204,7 @@ async function scoreImplementation( return attemptAsync(async () => { const actual = await executeImplementation(request.target, prediction[field.output], args, { timeoutMs: timeout, - ...(signal === undefined ? {} : { signal }), + ...optional("signal", signal), }); return outputText(actual) === String(exampleValue[field.expected] ?? "") ? 1 : 0; }, () => 0); diff --git a/src/providers/harness.ts b/src/providers/harness.ts index 055e5b1..9e6bfc8 100644 --- a/src/providers/harness.ts +++ b/src/providers/harness.ts @@ -7,7 +7,14 @@ import { type JudgeDecision, type JudgeRequest, } from "ts-autocode-harness"; -import { LoopCapabilityError, type CandidatePatch, type CandidateReview, type TrainingLoop, type TrainingLoopInput } from "ts-autocode-training"; +import { + LoopCapabilityError, + optional, + type CandidatePatch, + type CandidateReview, + type TrainingLoop, + type TrainingLoopInput, +} from "ts-autocode-training"; import { createStorage, type Storage } from "unstorage"; import fsDriver from "unstorage/drivers/fs"; @@ -63,16 +70,16 @@ export function createHarnessLoop(options: HarnessLoopOptions = {}): TrainingLoo ...(options.judge === undefined ? {} : { judge: options.judge }), task: { trainable: input.trainableId, objective: input.objective }, rubric: input.rubric, - ...maybeSignal(input.signal), + ...optional("signal", input.signal), student: ({ round, feedback, signal }) => - input.propose({ round, slot: 1, feedback, ...maybeSignal(signal) }), + input.propose({ round, slot: 1, feedback, ...optional("signal", signal) }), teacher: async (candidate, { round, signal }) => { - const review = await input.review(candidate, { label: `candidate-${round}`, ...maybeSignal(signal) }); + const review = await input.review(candidate, { label: `candidate-${round}`, ...optional("signal", signal) }); return { assessment: review, feedback: review.decision.failures }; }, adversary: { challenge: async (candidate, { signal }) => { - const challenge = await input.review(candidate, { label: `adversary-${candidate.id}`, ...maybeSignal(signal) }); + const challenge = await input.review(candidate, { label: `adversary-${candidate.id}`, ...optional("signal", signal) }); return { challenge, feedback: challenge.decision.failures }; }, }, @@ -84,8 +91,3 @@ export function createHarnessLoop(options: HarnessLoopOptions = {}): TrainingLoo }; } -/** Spreads an abort signal only when one exists, so optional-property types - * never receive an explicit `undefined`. */ -function maybeSignal(signal: AbortSignal | undefined): { signal: AbortSignal } | Record { - return signal === undefined ? {} : { signal }; -} diff --git a/test/deprecated.test.ts b/test/deprecated.test.ts index 961c833..db46226 100644 --- a/test/deprecated.test.ts +++ b/test/deprecated.test.ts @@ -19,6 +19,7 @@ import * as internal from "../src/internal.js"; import { defaultMaxRounds as harnessMaxRounds } from "ts-autocode-harness"; import { digest as groundingDigest, textDigest } from "ts-autocode-grounding"; import { digest as rewriteDigest } from "ts-autocode-rewrite"; +import { defined, optional } from "ts-autocode-training"; // The Tier 3 reshaping is additive: every legacy spelling must keep working. // This file is the enforcement of that promise, not a restatement of it. @@ -193,3 +194,18 @@ describe("ts-autocode/internal", () => { expect(root.commitRewrite).toBe(internal.commitRewrite); }); }); + +describe("optional-spread helper", () => { + it("spreads a defined value and drops an undefined one", () => { + expect({ ...optional("signal", "abc") }).toEqual({ signal: "abc" }); + expect({ ...optional("signal", undefined) }).toEqual({}); + // Crucially it omits the key rather than setting it to undefined, which + // is what exactOptionalPropertyTypes forbids. + expect("signal" in { ...optional("signal", undefined) }).toBe(false); + }); + + it("drops undefined entries from a group", () => { + expect({ ...defined({ a: 1, b: undefined, c: "x" }) }).toEqual({ a: 1, c: "x" }); + expect(Object.keys({ ...defined({ a: undefined }) })).toEqual([]); + }); +}); diff --git a/test/harness-loop.test.ts b/test/harness-loop.test.ts index 5e0dd32..8067394 100644 --- a/test/harness-loop.test.ts +++ b/test/harness-loop.test.ts @@ -4,12 +4,11 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { discoverInSource } from "ts-autocode-training"; +import { createCandidateReview, createEvalRun, discoverInSource } from "ts-autocode-training"; import type { CandidatePatch, CandidateReview, - TrainableEvalRun, } from "../src/index.js"; import { defineTrainable } from "../src/index.js"; import { createHarnessLoop } from "../src/providers/harness.js"; @@ -28,10 +27,14 @@ function candidate(id: string): CandidatePatch { } function review(promote: boolean, failures: readonly string[] = []): CandidateReview { - return { - verification: { token, run: {}, evaluations: [] } as unknown as TrainableEvalRun, - decision: { candidateId: "irrelevant", promote, failures, meanScore: Number(promote), passRate: Number(promote) }, - }; + // No cast: `createCandidateReview` builds the TrainableEvalRun a custom + // loop must return but could not previously construct. + return createCandidateReview({ + candidate: candidate("irrelevant"), + promote, + failures, + verification: createEvalRun({ trainable: token }), + }); } async function outputDir(): Promise { diff --git a/test/tier5.test.ts b/test/tier5.test.ts new file mode 100644 index 0000000..d40354f --- /dev/null +++ b/test/tier5.test.ts @@ -0,0 +1,102 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + createCandidateReview, + createEvalRun, + createPromotionDecision, + createTrainingRuntime, + defineTrainable, + evaluationArgs, + type CandidatePatch, + type ImplementationExecutor, +} from "../src/index.js"; +import { discoverInSource } from "ts-autocode-training"; + +const source = `class Fixture { + route(input: string): string { + "use training"; + return input; + } +} +`; +const target = discoverInSource(source, "fixture.ts")[0]!; +const candidate: CandidatePatch = { + id: "cand-1", trainableId: target.id, engineId: "test", target, implementation: "return input;", +}; + +describe("builders", () => { + // A custom TrainingLoop must return a CandidateReview containing a + // TrainableEvalRun that only the internals could produce, so both this + // repo's tests and any consumer had to write `as unknown as`. + it("builds a review with no cast at all", () => { + const review = createCandidateReview({ candidate, failures: ["nope"] }); + expect(review.decision.promote).toBe(false); + expect(review.decision.failures).toEqual(["nope"]); + expect(review.verification.token.id).toBe(target.id); + expect(review.verification.run.summary.total).toBe(0); + }); + + it("promotes by default when there are no failures", () => { + expect(createCandidateReview({ candidate }).decision).toMatchObject({ + promote: true, meanScore: 1, passRate: 1, + }); + }); + + it("lets the caller supply real evidence", () => { + const verification = createEvalRun({ + trainable: defineTrainable(target.id), + evaluations: [{ trainableId: target.id, result: { score: 1 } as never }], + }); + const review = createCandidateReview({ + candidate, + verification, + decision: createPromotionDecision({ candidateId: candidate.id, promote: true, meanScore: 0.9 }), + }); + expect(review.verification.evaluations).toHaveLength(1); + expect(review.decision.meanScore).toBe(0.9); + }); +}); + +describe("evaluation argument decoding", () => { + it("guesses by default, ambiguously", () => { + expect(evaluationArgs('["a","b"]')).toEqual(["a", "b"]); + expect(evaluationArgs("plain")).toEqual(["plain"]); + // The ambiguity the escape hatch exists for: a trainable taking the + // literal string "[1,2]" receives two numbers instead. + expect(evaluationArgs("[1,2]")).toEqual([1, 2]); + }); + + it("lets a caller decode arguments explicitly", async () => { + const directory = "test/output/tier5"; + await mkdir(directory, { recursive: true }); + const artifact = join(directory, "fixture.ts"); + await writeFile(artifact, source, "utf8"); + + const seen: unknown[][] = []; + const executor: ImplementationExecutor = async (_target, _implementation, args) => { + seen.push([...args]); + return String(args[0]); + }; + const training = createTrainingRuntime({ + engine: { id: "x", optimize: async () => ({ implementation: "return input;" }) }, + executor, + source: { files: [artifact] }, + tracing: { enabled: false }, + // Pass the raw string through rather than letting JSON.parse split it. + execution: { decodeArgs: (input) => [input] }, + }); + await training.train({ + trainable: defineTrainable("Fixture.route").symbol, + evaluation: { + tests: [{ id: "a", input: "[1,2]", assert: [{ type: "equals", value: "[1,2]" }] }], + task: (input) => input, + outputDir: `${directory}/agentv`, + }, + rounds: { max: 1 }, + }); + expect(seen[0]).toEqual(["[1,2]"]); + }, 30_000); +}); From 50957876534d68aeb4497004599949ff94e4491c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 18:01:37 +0000 Subject: [PATCH 08/14] docs: refresh the docs for the new surface Adds the packages/grounding README its own package.json homepage has always linked to and that never existed, and extends the documentation typecheck to cover it and docs/architecture.md, so every fenced TypeScript block in the repo now compiles. Updates prose the preceding commits made stale: onEvent alongside the deprecated onError, rounds.fanOut and promotion.gates, the fail-closed evolve switch, model selection as a neutral descriptor rather than a provider-specific option, and the consumer/author surface split. Records in docs/dx-review.md what shipped, plus two places the remediation deliberately departed from the plan -- configureTraining still replaces by default, and evolution's opt-in polarity was renamed rather than flipped -- and corrects one claim the review got wrong about test/wiring.ts. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- README.md | 12 ++--- docs/architecture.md | 44 +++++++++++++++--- docs/dx-review.md | 38 +++++++++++++-- packages/grounding/README.md | 90 ++++++++++++++++++++++++++++++++++++ packages/harness/README.md | 3 +- packages/training/README.md | 11 +++++ test/docs.test.ts | 2 + 7 files changed, 183 insertions(+), 17 deletions(-) create mode 100644 packages/grounding/README.md diff --git a/README.md b/README.md index 82070dc..18a520a 100644 --- a/README.md +++ b/README.md @@ -207,8 +207,8 @@ The register hook instruments every `"use training"` function at module load. Once a trainable accumulates `evolution.minTraces` successful traces (default 3), it is trained against those traces, verified candidate-bound, gated, and — only when the gate passes — its source body is rewritten. Failures surface -through `TrainingSettings.onError` with the `"evolve"` phase and never block or -alter application calls. Loading the hook is itself the opt-in, so evolution is on unless you turn it +through `TrainingSettings.onEvent` (and the deprecated `onError` with the +`"evolve"` phase) and never block or alter application calls. Loading the hook is itself the opt-in, so evolution is on unless you turn it off: set `TS_AUTOCODE_EVOLVE` to `0`, `false`, `off`, `no`, or `disabled` (or configure `evolution: { enabled: false }`) to capture without rewriting, and use `evolution.onEvolved` to observe applied rewrites. Because the feature rewrites @@ -263,14 +263,14 @@ promotion primitives also remain available. The built-in loop is an observable round sequence (`trainingRounds()` pushes each reviewed round to a subscriber; `sequentialLoop` collects the -subscription into one run). `TrainInput.fanOut` caps how many candidates a +subscription into one run). `TrainInput.rounds.fanOut` caps how many candidates a round proposes and reviews concurrently — the best gated candidate wins the round. Fan-out belongs to `sequentialLoop`: the default governed harness loop reviews exactly one candidate per round, because its judge, adversary and rubric-revision sequence is serial, so it **rejects** a `fanOut` above 1 rather -than accepting one it would ignore. `TrainInput.gates` appends custom promotion -rules to the standard `defaultPromotionGates` set; the configured `policy` runs -as one such rule. +than accepting one it would ignore. `TrainInput.promotion.gates` appends custom +promotion rules to the standard `defaultPromotionGates` set; the deprecated +`policy` runs as one such rule. No Ax program is supplied by the caller. The default engine derives its fields, descriptions, executable examples, and return contract from the TypeScript diff --git a/docs/architecture.md b/docs/architecture.md index 00a3b6e..b354170 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -64,8 +64,11 @@ directive, wiring each discovered class method or function declaration into the same capture path as the decorator. It also enables background evolution by default: after `evolution.minTraces` successful captures, the runtime runs the same train-and-promote pipeline — replay evals, candidate verification, -promotion gate, guarded rewrite — off the hot path, reporting failures through -`onError("evolve")`. Calls made during a module's own top-level evaluation +promotion gate, guarded rewrite — off the hot path, reporting its progress and +failures through `TrainingSettings.onEvent` (and, for the failures, +`onError("evolve")`). Because it rewrites source files, its environment kill +switch fails closed: an unrecognized `TS_AUTOCODE_EVOLVE` value throws rather +than being read as consent. Calls made during a module's own top-level evaluation precede its instrumentation; traffic after startup is captured. The training runtime itself lives in the provider-neutral `ts-autocode-training` package. All cross-package wiring happens in the root `ts-autocode` package: it supplies @@ -95,7 +98,11 @@ signature, creates examples from runtime captures and AgentV results, and scores candidate implementations by running them in Ax's sandbox. Applications can replace it through the provider-neutral `engine` setting without changing capture, evaluation, or promotion. Provider-specific options do not appear in -the root configuration contract. +the root configuration contract — but choosing a provider and model is not a +provider-specific option: `TrainingSettings.model` is a neutral +`ModelSelection` descriptor that the training runtime carries to whatever +engine is configured, exactly as it carries `secrets` and `variables`, so +switching models never means replacing the engine. Candidate bodies are evaluated separately through AgentV before promotion. Baseline results can train the optimizer but cannot satisfy the promotion gate. @@ -110,8 +117,8 @@ its own candidate and promotion types — and ships the default loop as an observable round sequence: `trainingRounds()` pushes each reviewed round to a subscriber as it settles, unsubscribing aborts in-flight work, and `sequentialLoop` is simply the subscription collected into one run. Rounds run -in order, but within a round `fanOut` caps how many candidate propose→review -pipelines run concurrently; duplicate proposals are skipped, a round that +in order, but within a round `rounds.fanOut` caps how many candidate +propose→review pipelines run concurrently; duplicate proposals are skipped, a round that reviews nothing new stalls the run, and when several candidates pass the gate the highest-scoring one is emitted last as the winner. `ts-autocode` (the root package) specifies the connection: its `createHarnessLoop` provider adapts the @@ -153,7 +160,30 @@ each `PromotionGate` is a pure function over one shared `PromotionGateContext` (candidate, candidate-bound results, thresholds, aggregates) returning the failures it sees. The standard rules — conformance, evaluation binding, execution errors, score and pass-rate thresholds — always run; -`PromotionGateInput.gates` (or `TrainInput.gates`) appends extension rules, -and the configured `policy` participates as one more gate. An activation's +`PromotionGateInput.gates` (or `TrainInput.promotion.gates`) appends extension +rules, and the deprecated `policy` participates as one more gate — it always +was one, which is why a single list now expresses both. An activation's rollback stores only the previous and promoted method body and refuses to overwrite subsequent edits. + +## Consumer surface + +The root package exports what an application needs; `ts-autocode/internal` +carries the author-level seams — `captureTrainable`, `provideTrainingDefaults`, +and the rewrite primitives — for building an engine, loop, executor, store, or +instrumentation mechanism. Everything on the subpath is still exported from the +root, so the split is organizational rather than a break. + +Extension points are constructible: a custom `TrainingLoop` returns a +`CandidateReview` containing a `TrainableEvalRun`, and `createCandidateReview`, +`createEvalRun`, and `createPromotionDecision` build those without a cast. + +Every failure the library raises carries a `code` and is recognized by +`isTsAutocodeError`. Errors that have always been `TypeError`s or +`SyntaxError`s still are — family membership is decided by a brand rather than +the prototype chain — and every message string is unchanged. + +`ts-autocode discover` lists the trainables a project marks along with the exact +identity to pass to `defineTrainable`. That identity is the one place this +design falls back to an unchecked string, so printing real ids is what keeps the +marker approach usable. diff --git a/docs/dx-review.md b/docs/dx-review.md index 296f436..4fcde17 100644 --- a/docs/dx-review.md +++ b/docs/dx-review.md @@ -305,13 +305,45 @@ enforced rather than asserted. 23. A documented evaluation-argument contract with an explicit escape hatch. 24. `effect` dropped from the root and from `attempt.ts`. +## What shipped, and two deviations + +All five tiers landed. Two things were done differently from the plan above, +both to avoid trading a stated problem for a worse one: + +**`configureTraining` still replaces by default.** The plan said to make it +merge. Merging silently would carry settings between unrelated calls — one +caller's engine surviving into another's configuration — which is a subtler and +harder-to-debug surprise than the one it fixes. Instead `createTrainingRuntime` +gives genuine isolation (the real gap), `resetTraining()` restores a clean +state, and `{ merge: true }` opts into layering. The replacing default is +documented rather than changed. + +**`evolution.enabled` polarity was not flipped.** The plan called for uniform +`enabled` semantics across `capture`, `tracing`, and `evolution`. But the +asymmetry is justified: the first two are opt-out recording switches, while +evolution rewrites the user's source files and must be opt-in. Defaulting it on +for consistency would be a serious behavioral change. The field is renamed +`auto`, so the name matches the semantics, and `enabled` still works. + +One review claim was also wrong and is corrected here: `packages/training/test/ +wiring.ts` was described as a workaround for the runtime singleton. It is not — +it wires a `PromotionApplier` provider, which is legitimate test setup and +remains. The singleton gap was real; that file was not evidence of it. + ## How these are kept fixed -- A **surface test** asserts every symbol exported by `ts-autocode-training` and - `ts-autocode-rewrite` is reachable from `ts-autocode`, so A5 cannot recur. +- A **surface test** asserts every runtime value exported by + `ts-autocode-training` and `ts-autocode-rewrite` is reachable from + `ts-autocode`, so A5 cannot recur. It has already caught two regressions + during this work. - **Documentation is typechecked**: TypeScript blocks are extracted from the READMEs and compiled in CI. This is what would have caught A1. -- Grounding's generated output is typechecked rather than string-matched, catching A2. +- Grounding's generated output is typechecked rather than string-matched, + catching A2. Verified to fail on the original bug rather than pass vacuously. +- `test/deprecated.test.ts` exercises every legacy spelling, so the + no-breakage promise is enforced rather than asserted. +- `examples/optimize.ts` runs on every check against a stub engine, so the + example cannot rot without CI noticing. - A tree-shaking bundle test asserts the Ax engine survives, catching A3. - Targeted regression tests cover `fanOut`, the rubric thresholds, the evolve kill switch, and the `onError("evolve")` sad path. diff --git a/packages/grounding/README.md b/packages/grounding/README.md new file mode 100644 index 0000000..efdcb74 --- /dev/null +++ b/packages/grounding/README.md @@ -0,0 +1,90 @@ +# ts-autocode-grounding + +Granular grounding decorators, ambient trainable-class scanning, and +deterministic text helpers for trainable TypeScript codegen. + +This package is **host-agnostic**: it never imports a training runtime. It +composes what a class declares into provider-neutral `GroundingOptions`, and a +host — `ts-autocode`, or any other — registers those against its own registry. + +Most applications do not need it. Reach for it when you want an implementation +described one fact at a time, or when trainables are declared ambiently and +their registrations are generated. + +## Granular decorators + +Every decorator is optional. A method with none still grounds: intent is +inferred from the name and the TypeScript signature is the declared shape. + +```ts +import { description, intent, returns } from "ts-autocode/grounding"; + +class Greeter { + @intent("Produce a simple hello-world program") + @returns("Hello World! or Hello, ! when supplied") + greet(name?: string): string { + return name ? `Hello, ${name}!` : "Hello World!"; + } +} +``` + +TC39 stage-3 decorators have no parameter decorators, so parameter descriptions +ride the options object as `params: { name: description("…") }` values. + +## Ambient declarations and codegen + +An `export declare class` is erased at compile time, so no decorator ever runs. +`scanDeclaredTrainables` reads the declaration statically — a real TypeScript +AST walk, never a regex — and `generateDeclaredRegistrations` emits registration +source for it: + +```ts +import { generateDeclaredRegistrations, scanDeclaredTrainables } from "ts-autocode/grounding"; + +const [declared] = scanDeclaredTrainables(` + @trainable + export declare class Program { + @intent("Greet someone") + greet(name?: string): string; + } +`); + +const source = generateDeclaredRegistrations(declared!); +``` + +The emitted source calls `defineGrounding`, exported from this package. It +previously called `training.define`, which does not exist on the `Training` +runtime, so every generated file failed to typecheck; the scan test now +compiles what it emits rather than string-matching it. + +Point the generated import elsewhere with `runtimeModule`: + +```ts +import { generateDeclaredRegistrations, type DeclaredTrainableClass } from "ts-autocode/grounding"; + +declare const declared: DeclaredTrainableClass; + +generateDeclaredRegistrations(declared, { runtimeModule: "@acme/runtime" }); +``` + +## Registering against a host + +`finalizeTrainableClass` registers every granular-declared method (or, when +nothing was annotated, every own prototype method) against a host-provided +`GroundingRegistry`. The registry and metadata symbols are parameters, which is +what keeps this package free of any runtime dependency. + +## Text helpers + +`camelCase`, `pascalCase`, `normalizeText`, `normalizePath`, `stableStringify`, +`toStableValue`, `union`, and `textDigest` back deterministic codegen. + +> `textDigest` hashes line-ending-normalized **text**. It is not the same +> function as `ts-autocode-rewrite`'s `digest`, which canonicalizes an arbitrary +> value as key-sorted JSON. Both emit a `sha256:` prefix, so substituting one +> for the other silently changes every hash. It was called `digest` here too; +> that name remains as a deprecated alias. + +## License + +[MIT](../../LICENSE) diff --git a/packages/harness/README.md b/packages/harness/README.md index 8d40d46..b663f77 100644 --- a/packages/harness/README.md +++ b/packages/harness/README.md @@ -28,7 +28,8 @@ npm install ts-autocode-harness ## Run the loop -The minimal loop is two callbacks: +The minimal loop is two callbacks. `inferringHarness()` types itself from them; +`defineTrainingHarness()` is the explicit form: ```ts import { defineTrainingHarness, type StudentTurn, type TeacherResult } from "ts-autocode-harness"; diff --git a/packages/training/README.md b/packages/training/README.md index 37c058e..d67b5b4 100644 --- a/packages/training/README.md +++ b/packages/training/README.md @@ -30,6 +30,17 @@ provideTrainingDefaults({ }); ``` +`configureTraining` configures one process-wide runtime; `createTrainingRuntime` +builds an isolated one that registers nothing globally, for tests and for hosts +serving several tenants. `resetTraining()` restores a fresh-import state. + +Extension points are constructible: a custom `TrainingLoop` must return a +`CandidateReview` containing a `TrainableEvalRun`, and `createCandidateReview`, +`createEvalRun` and `createPromotionDecision` build those without a cast. + +Every failure carries a `code` and is recognized by `isTsAutocodeError`; the +ones that have always been `TypeError`s still are. + Most applications should depend on [`ts-autocode`](../../README.md), which re-exports this package's API with Ax defaults already registered. diff --git a/test/docs.test.ts b/test/docs.test.ts index 87d4b09..f58486d 100644 --- a/test/docs.test.ts +++ b/test/docs.test.ts @@ -18,6 +18,8 @@ const docs = [ "packages/training/README.md", "packages/harness/README.md", "packages/rewrite/README.md", + "packages/grounding/README.md", + "docs/architecture.md", ]; interface Snippet { From a671a2bf73fabacfae6f4a6a833889efc0e7ff44 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 18:05:29 +0000 Subject: [PATCH 09/14] docs: correct the sideEffects finding to what was actually demonstrated The review claimed a bundler honoring `sideEffects: false` could drop the package's import-time wiring and leave a consumer with "no training engine is configured". Bundling a trivial consumer with esbuild --tree-shaking=true produces byte-identical output with the flag set either way, so that failure was asserted rather than observed. The fix stands and is still correct: the declaration was factually untrue, since importing the root module registers four providers and configures rewrite capture. But the finding now says what it is -- a latent correctness bug in a promise made to bundlers -- and records that esbuild does not collect on it, rather than implying a reproduced breakage. Also drops the planned tree-shaking bundle test, which would pass either way and prove nothing. The manifest assertion in test/tier1.test.ts is what actually guards the declaration. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- docs/dx-review.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/dx-review.md b/docs/dx-review.md index 4fcde17..1d0f53f 100644 --- a/docs/dx-review.md +++ b/docs/dx-review.md @@ -38,13 +38,23 @@ before introducing it. `train`, and `flush` — no `define`. Every generated registration file failed to typecheck. The scan test asserted only the emitted *string*, so nothing caught it. -### A3. `"sideEffects": false` is false, and can silently break the package +### A3. `"sideEffects": false` was a false declaration All five manifests declared it, yet `src/index.ts` performs top-level `provideTrainingDefaults(...)` and `configureRewriteCapture()`, `src/register.ts` is -entirely side effects, and `installInstrumentation` writes to a `globalThis` symbol. A -bundler honoring the hint may legally drop the wiring, leaving the user with -`no training engine is configured; import "ts-autocode"` — after importing `ts-autocode`. +entirely side effects, and `installInstrumentation` writes to a `globalThis` symbol. +The field is a promise to bundlers that dropping an unused module changes nothing, +and that promise was untrue. + +**Scope, honestly:** this is a correctness fix, not a demonstrated failure. The +hazard it licenses — a bundler eliding the module, leaving a consumer with +`no training engine is configured; import "ts-autocode"` after importing +`ts-autocode` — depends on the bundler and on how the package is consumed. +Bundling a trivial consumer with `esbuild --tree-shaking=true` produced +byte-identical output with the flag either way, so esbuild does not act on it in +that configuration. Declaring the truth is still right: the field exists so that +tools *can* rely on it, and asserting something false about your own package is a +latent bug whether or not today's toolchain collects on it. ### A4. `TrainInput.fanOut` was silently ignored under the default configuration @@ -344,6 +354,9 @@ remains. The singleton gap was real; that file was not evidence of it. no-breakage promise is enforced rather than asserted. - `examples/optimize.ts` runs on every check against a stub engine, so the example cannot rot without CI noticing. -- A tree-shaking bundle test asserts the Ax engine survives, catching A3. +- A manifest test pins which modules are declared side-effectful, so A3 cannot + silently revert. (A tree-shaking bundle test was tried and dropped: esbuild + produces identical output either way, so it would pass regardless and prove + nothing.) - Targeted regression tests cover `fanOut`, the rubric thresholds, the evolve kill switch, and the `onError("evolve")` sad path. From 978ffd7ebdaa10106662157f7a0f211d34c0334f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 18:13:11 +0000 Subject: [PATCH 10/14] fix: repair CI, and the Node 20 bug it exposed CI was red on every commit in this branch while `npm run check` passed locally. Two causes, both mine, plus a real pre-existing bug the failure uncovered. **examples/optimize.ts could not resolve `ts-autocode`.** It imports by package name, as a consumer would, but the package entry points at `dist/`, which `npm run check` does not build until *after* the tests run, and Vitest does not read tsconfig `paths`. It passed locally only because a stale `dist/` happened to be lying around -- exactly the works-on-my-machine trap. Mirrors the tsconfig paths as Vitest resolve aliases so the example resolves from source, deterministically. **test/tier1.test.ts imported src/register.ts** to reach a string-parsing function, and importing that module installs a load hook. That surfaced the real bug: **`ts-autocode/register` crashed on Node 20.** `module.registerHooks` is the synchronous in-thread loader API, added in Node 22.15. `engines` declares `node >= 20`, and the README's headline zero-config command is `node --import ts-autocode/register`, so the flagship feature was broken on the minimum supported version -- and failed with an internal `TypeError: registerHooks is not a function` rather than anything actionable. Nothing had ever imported that module in a test: test/register.test.ts exercises only the pure `augmentSource`, so the side-effecting entry was never loaded under test on any Node. The guard now names the requirement, says the rest of the package still works on Node 20, and points at the decorator, which needs no load hook. The README says so too. Also splits the evolve kill switch (src/evolve.ts) and the load-hook guard (src/load-hook.ts) out of the side-effecting entry, so both are testable without installing anything. Verified with a clean dist/ on Node 20.20.2 -- the exact version CI failed on -- and on Node 22: 218 tests pass on both. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- README.md | 5 +++++ docs/dx-review.md | 19 +++++++++++++++++++ src/evolve.ts | 26 ++++++++++++++++++++++++++ src/load-hook.ts | 26 ++++++++++++++++++++++++++ src/register.ts | 27 +++++---------------------- test/tier1.test.ts | 42 +++++++++++++++++++++++++++++++++++++++++- vitest.config.ts | 17 +++++++++++++++++ 7 files changed, 139 insertions(+), 23 deletions(-) create mode 100644 src/evolve.ts create mode 100644 src/load-hook.ts diff --git a/README.md b/README.md index 18a520a..ed85c8d 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,11 @@ guarded source rewrite all apply automatically: node --import ts-autocode/register ./dist/server.js ``` +> This entry point installs a synchronous module load hook via +> `module.registerHooks`, which needs **Node 22.15 or newer**. The rest of +> `ts-autocode` works on Node 20; on an older runtime this entry says so and +> points at the `@trainable()` decorator, which needs no load hook. + The register hook instruments every `"use training"` function at module load. Once a trainable accumulates `evolution.minTraces` successful traces (default 3), it is trained against those traces, verified candidate-bound, gated, and — diff --git a/docs/dx-review.md b/docs/dx-review.md index 1d0f53f..00157b2 100644 --- a/docs/dx-review.md +++ b/docs/dx-review.md @@ -94,6 +94,25 @@ that edits the user's source files, fail-open is the wrong default. *executor* — `executeImplementation`, registered with no options — fell back to a hardcoded 5s with no configuration path through `TrainingSettings` at all. +### A9. `ts-autocode/register` crashed on the minimum supported Node + +Found by CI, not by reading: the documentation-and-example work added the first +test that imports `src/register.ts`, and it threw +`TypeError: registerHooks is not a function` on Node 20.20.2. + +`module.registerHooks` is the synchronous in-thread loader API, added in Node +22.15. `engines` declares `node >= 20`, and the README's headline zero-config +command is `node --import ts-autocode/register ./dist/server.js` — so the +flagship feature was broken on the minimum version the package claims to +support, and failed with an internal `TypeError` rather than anything a user +could act on. + +Nothing had imported that module in a test. `test/register.test.ts` exercises +only `src/register/hook.js`, the pure `augmentSource` function, so the +side-effecting entry was never loaded under test on any Node version. This is +the clearest single argument for the documentation and example tests added +here: the bug was not subtle, it was simply never executed. + ## B. Missing capabilities ### B1. No supported way to choose a model or provider diff --git a/src/evolve.ts b/src/evolve.ts new file mode 100644 index 0000000..dcfd119 --- /dev/null +++ b/src/evolve.ts @@ -0,0 +1,26 @@ +// The evolve kill switch, kept apart from `ts-autocode/register` so it can be +// read and tested without installing a module load hook. Importing +// `src/register.ts` runs that installation, which is not something a unit test +// of a string-parsing rule should require -- and on a Node without +// `module.registerHooks` it throws outright. + +/** Environment switch for zero-config evolution. Loading `ts-autocode/register` + * is itself the opt-in, so an unset variable leaves evolution on; the variable + * exists to turn it back off without changing the command line. */ +export const evolveVariable = "TS_AUTOCODE_EVOLVE"; + +const evolveOff = ["0", "false", "off", "no", "disabled"]; +const evolveOn = ["1", "true", "on", "yes", "enabled"]; + +/** Reads the kill switch, failing closed: an unrecognized value throws rather + * than being guessed at. Evolution rewrites the user's source files, so a + * misspelled `TS_AUTOCODE_EVOLVE=nope` must never be read as consent. */ +export function evolutionEnabled(value: string | undefined): boolean { + const flag = (value ?? "").trim().toLowerCase(); + if (flag === "") return true; + if (evolveOff.includes(flag)) return false; + if (evolveOn.includes(flag)) return true; + throw new Error( + `${evolveVariable} must be one of ${[...evolveOn, ...evolveOff].join(", ")}; received ${JSON.stringify(value)}`, + ); +} diff --git a/src/load-hook.ts b/src/load-hook.ts new file mode 100644 index 0000000..667f4f8 --- /dev/null +++ b/src/load-hook.ts @@ -0,0 +1,26 @@ +import module from "node:module"; + +// `module.registerHooks` is the synchronous in-thread loader API. Node 20 does +// not provide it, so `ts-autocode/register` -- the documented zero-config entry +// point, `node --import ts-autocode/register` -- threw +// `TypeError: registerHooks is not a function` there, despite `engines` +// declaring Node 20 support. Nothing imported that module in a test, so it went +// unnoticed until CI ran the suite on 20.20.2. + +/** True when this runtime can install a synchronous module load hook. */ +export function canRegisterLoadHook(): boolean { + return typeof module.registerHooks === "function"; +} + +/** Installs the load hook, or explains what is missing and what to do instead + * of surfacing an internal TypeError. */ +export function registerLoadHook(hooks: Parameters[0]): void { + if (!canRegisterLoadHook()) { + throw new Error( + `ts-autocode/register needs module.registerHooks, which ${process.version} does not provide (Node 22.15 or newer does). ` + + "The rest of ts-autocode works on Node 20; only the load-time instrumentation this entry installs does not. " + + "Mark methods with the @trainable() decorator and call training.train(...) directly, or upgrade Node.", + ); + } + module.registerHooks(hooks); +} diff --git a/src/register.ts b/src/register.ts index c884d7d..c94b26b 100644 --- a/src/register.ts +++ b/src/register.ts @@ -1,42 +1,25 @@ -import { registerHooks } from "node:module"; import { fileURLToPath } from "node:url"; import { installInstrumentation } from "ts-autocode-rewrite"; import { provideTrainingDefaults } from "ts-autocode-training"; +import { evolutionEnabled, evolveVariable } from "./evolve.js"; +import { registerLoadHook } from "./load-hook.js"; import { instrumentTrainable, wrapTrainable } from "./instrumentation.js"; import { augmentSource } from "./register/hook.js"; // Importing the package entry wires the Ax engine and executor defaults, the // harness loop, the promotion applier, and rewrite capture interception. import "./index.js"; -installInstrumentation({ method: instrumentTrainable, wrap: wrapTrainable }); - -/** Environment switch for zero-config evolution. Loading this module is itself - * the opt-in, so an unset variable leaves evolution on; the variable exists to - * turn it back off without changing the command line. */ -export const evolveVariable = "TS_AUTOCODE_EVOLVE"; -const evolveOff = ["0", "false", "off", "no", "disabled"]; -const evolveOn = ["1", "true", "on", "yes", "enabled"]; +export { evolutionEnabled, evolveVariable } from "./evolve.js"; -/** Reads the kill switch, failing closed: an unrecognized value throws rather - * than being guessed at. Evolution rewrites the user's source files, so a - * misspelled `TS_AUTOCODE_EVOLVE=nope` must never be read as consent. */ -export function evolutionEnabled(value: string | undefined): boolean { - const flag = (value ?? "").trim().toLowerCase(); - if (flag === "") return true; - if (evolveOff.includes(flag)) return false; - if (evolveOn.includes(flag)) return true; - throw new Error( - `${evolveVariable} must be one of ${[...evolveOn, ...evolveOff].join(", ")}; received ${JSON.stringify(value)}`, - ); -} +installInstrumentation({ method: instrumentTrainable, wrap: wrapTrainable }); if (evolutionEnabled(process.env[evolveVariable])) { provideTrainingDefaults({ evolution: { enabled: true } }); } -registerHooks({ +registerLoadHook({ load(url, context, nextLoad) { const result = nextLoad(url, context); if (!url.startsWith("file:") || url.includes("/node_modules/")) return result; diff --git a/test/tier1.test.ts b/test/tier1.test.ts index b922c1b..9543178 100644 --- a/test/tier1.test.ts +++ b/test/tier1.test.ts @@ -4,7 +4,10 @@ import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; import { createHarnessLoop } from "../src/providers/harness.js"; -import { evolutionEnabled, evolveVariable } from "../src/register.js"; +import nodeModule from "node:module"; + +import { evolutionEnabled, evolveVariable } from "../src/evolve.js"; +import { canRegisterLoadHook, registerLoadHook } from "../src/load-hook.js"; import { defaultMinPassRate, defaultMinScore } from "ts-autocode-training"; const repoRoot = fileURLToPath(new URL("..", import.meta.url)); @@ -109,3 +112,40 @@ describe("harness loop fan-out", () => { await expect(run).rejects.toThrow("proposed"); }); }); + +describe("ts-autocode/register on older Node", () => { + // CI on Node 20.20.2 surfaced `TypeError: registerHooks is not a function` + // from src/register.ts. `module.registerHooks` is the synchronous in-thread + // loader API and Node 20 does not have it, so the documented zero-config + // entry point -- `node --import ts-autocode/register` -- crashed on the + // minimum version `engines` declares. Nothing imported that module in a + // test before, which is why it went unnoticed. + it("installs the hook when the runtime provides it", () => { + expect(canRegisterLoadHook()).toBe(typeof nodeModule.registerHooks === "function"); + if (!canRegisterLoadHook()) return; + const installed: unknown[] = []; + const original = nodeModule.registerHooks; + try { + (nodeModule as { registerHooks?: unknown }).registerHooks = (hooks: unknown) => installed.push(hooks); + registerLoadHook({ load: (url, context, nextLoad) => nextLoad(url, context) }); + expect(installed).toHaveLength(1); + } finally { + (nodeModule as { registerHooks?: unknown }).registerHooks = original; + } + }); + + it("explains the requirement instead of surfacing a TypeError", () => { + const original = nodeModule.registerHooks; + try { + delete (nodeModule as { registerHooks?: unknown }).registerHooks; + expect(canRegisterLoadHook()).toBe(false); + expect(() => registerLoadHook({ load: (url, context, nextLoad) => nextLoad(url, context) })) + .toThrow(/needs module\.registerHooks/); + // The message must say what still works and what to do, not just fail. + expect(() => registerLoadHook({ load: (url, context, nextLoad) => nextLoad(url, context) })) + .toThrow(/@trainable\(\) decorator/); + } finally { + (nodeModule as { registerHooks?: unknown }).registerHooks = original; + } + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index b48a9f9..4c75c18 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,6 +1,23 @@ +import { fileURLToPath } from "node:url"; + import { defineConfig } from "vitest/config"; +const src = (path: string) => fileURLToPath(new URL(path, import.meta.url)); + export default defineConfig({ + resolve: { + // `examples/` imports by package name, as a consumer would, but the + // package's own entry points at `dist/`, which `npm run check` does not + // build until after the tests run. Vitest also does not read + // tsconfig `paths`, so mirror them here: without this the example + // resolves only when a stale `dist/` happens to be lying around. + alias: [ + { find: /^ts-autocode$/, replacement: src("./src/index.ts") }, + { find: /^ts-autocode\/ax$/, replacement: src("./src/providers/ax.ts") }, + { find: /^ts-autocode\/internal$/, replacement: src("./src/internal.ts") }, + { find: /^ts-autocode\/grounding$/, replacement: src("./src/grounding.ts") }, + ], + }, test: { include: ["test/**/*.test.ts", "packages/grounding/test/**/*.test.ts", "packages/harness/test/**/*.test.ts", "packages/rewrite/test/**/*.test.ts", "packages/training/test/**/*.test.ts"], }, From 5b6ae674145197e158505ffd30d72279ff98bb3d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 18:47:35 +0000 Subject: [PATCH 11/14] test: add coverage enforcement and close the atomic unit gaps The suite had 218 tests and no coverage measurement at all, which made "we have tests" unfalsifiable. Measuring it first showed 85.35% statements and 73.14% branches -- and that the three files added in the previous PR were among the worst covered in the workspace: errors.ts at 16.66% branch, builders.ts at 55%, optional.ts at 75%. Adds @vitest/coverage-v8, wires coverage into `npm run check`, and sets thresholds as a ratchet (90/78/93/93 -- what is actually achieved now, to be raised as later suites land, never lowered). New atomic unit suites, 184 tests: - errors.ts: every constructor, static factory, payload accessor and message string; both directions of the brand-based `hasInstance`, including that a hand-rolled look-alike is not admitted and that subclass `instanceof` stays exact; and the Zod boundary. - optional.ts / defined(): asserts key *presence*, not deep equality -- `{a: undefined}` and `{}` compare equal under toEqual, so the distinction the helpers exist for would otherwise go unchecked. - builders.ts: every defaulting rule a consumer will rely on without reading the source, plus that supplied evaluations are copied rather than aliased. - token.ts: the normalization and rejection rules standing between a typo and a silently different identity. - canonical.ts, component.ts: pre-existing gaps at 46% and 50% branch. The class-instance case matters -- if `isRecord` wrongly accepted a Date, every Date would hash identically. - attempt.ts (both copies), and the CLI's status and option paths. Also adds test/digest-protocol.test.ts. Training and rewrite each implement the body digest and never import each other, so guarded application depends on two independent implementations agreeing; that agreement was assumed, and is now asserted. Boy-scout fix: test/docs.test.ts built one ts.Program per snippet, taking ~36s and relying on the default 5s per-test timeout. Under coverage instrumentation it blew that timeout and failed 15 of 26. It now builds one program for all snippets: 3s, and robust under instrumentation. Verified it still fails when a README snippet breaks rather than having gone vacuous. 218 -> 402 tests; 85.35 -> 90.15% statements, 73.14 -> 79% branches. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- .gitignore | 1 + docs/testing.md | 41 ++ package-lock.json | 740 ++++++++++++++-------- package.json | 5 +- packages/grounding/test/component.test.ts | 222 +++++++ packages/harness/test/attempt.test.ts | 85 +++ packages/rewrite/test/canonical.test.ts | 66 ++ packages/training/test/builders.test.ts | 123 ++++ packages/training/test/errors.test.ts | 182 ++++++ packages/training/test/optional.test.ts | 72 +++ packages/training/test/token.test.ts | 105 +++ test/attempt.test.ts | 85 +++ test/cli.test.ts | 73 +++ test/digest-protocol.test.ts | 49 ++ test/docs.test.ts | 98 +-- vitest.config.ts | 18 + 16 files changed, 1661 insertions(+), 304 deletions(-) create mode 100644 docs/testing.md create mode 100644 packages/grounding/test/component.test.ts create mode 100644 packages/harness/test/attempt.test.ts create mode 100644 packages/rewrite/test/canonical.test.ts create mode 100644 packages/training/test/builders.test.ts create mode 100644 packages/training/test/errors.test.ts create mode 100644 packages/training/test/optional.test.ts create mode 100644 packages/training/test/token.test.ts create mode 100644 test/attempt.test.ts create mode 100644 test/digest-protocol.test.ts diff --git a/.gitignore b/.gitignore index ebb0b6c..3965396 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ node_modules/ dist/ coverage/ +test/output/ *.tsbuildinfo .env .agentv/ diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..8c9cbd6 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,41 @@ +# Testing strategy + +The suite is organized by what each layer can actually catch. Coverage is +enforced as a ratchet in `vitest.config.ts` — raise the thresholds as suites +land, never lower them to get a build green. + +| Layer | Where | Catches | +|---|---|---| +| Atomic unit | `packages/*/test/*.test.ts` | Behavior of one function or class, including every branch of its defaulting and rejection rules | +| Functional | `test/*.test.ts` | A whole path through the runtime: mark, capture, train, gate, activate, roll back | +| Documentation | `test/docs.test.ts` | README and architecture snippets that no longer compile | +| Surface | `test/surface.test.ts` | Re-export drift between the root package and its siblings | +| Protocol | `test/digest-protocol.test.ts` | Two packages that must agree without importing each other | +| Compatibility | `test/deprecated.test.ts` | A deprecated spelling that silently stopped working | + +## Running + +```bash +npm test # suite only +npm run test:coverage # suite with coverage and thresholds +npm run check # typecheck + coverage + build +``` + +Coverage reports land in `test/output/coverage` (git-ignored). `lcov.info` is +there for editor and CI integrations. + +## Conventions + +- **A test names the defect it prevents.** Where a test exists because + something was once wrong, the comment says what was wrong. That is what makes + it safe to change later: a reader can tell whether the constraint still + matters. +- **Assert on structure, not incidental formatting.** Column padding and + message wording change; `Router.route 0 successful` broke on a one-space + alignment shift, and the fix was to match the meaning instead. +- **A test that cannot fail is worse than no test.** Where a check guards a + specific past bug, verify it fails when that bug is reintroduced before + trusting it. The grounding codegen test and the documentation test were both + confirmed this way. +- **Fixtures and artifacts belong under `test/output/`**, which is git-ignored + and excluded from TypeScript compilation. diff --git a/package-lock.json b/package-lock.json index cd2da54..6f25942 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,6 @@ ], "dependencies": { "@ax-llm/ax": "^23.0.0", - "effect": "^3.21.4", "ts-autocode-grounding": "0.1.0", "ts-autocode-harness": "0.1.0", "ts-autocode-rewrite": "0.1.0", @@ -23,8 +22,13 @@ "unstorage": "^1.17.5", "zod": "^4.4.3" }, + "bin": { + "ts-autocode": "dist/cli-main.js" + }, "devDependencies": { "@types/node": "^22.15.3", + "@vitest/coverage-v8": "^4.1.11", + "fast-check": "^4.9.0", "vitest": "^4.1.9" }, "engines": { @@ -491,6 +495,42 @@ } } }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.8.tgz", + "integrity": "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.8" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/@babel/runtime": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", @@ -500,6 +540,30 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/types": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", + "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@cfworker/json-schema": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", @@ -530,40 +594,6 @@ "node": ">=20.0.0" } }, - "node_modules/@emnapi/core": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", - "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.2", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", - "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", - "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@github/copilot": { "version": "1.0.70", "resolved": "https://registry.npmjs.org/@github/copilot/-/copilot-1.0.70.tgz", @@ -752,13 +782,34 @@ } } }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", "dev": true, "license": "MIT" }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "node_modules/@langchain/core": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.2.tgz", @@ -938,25 +989,6 @@ } } }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.6.tgz", - "integrity": "sha512-ZLv/JdUfkvOy9eCnnBaGfiO+XimbjebAeO+MRQqD/B+FR1tnRN0tpKSJHRbE8sFfS6aqsXZ67TQjfwfsxULVbg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@tybys/wasm-util": "^0.10.3" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - }, - "peerDependencies": { - "@emnapi/core": "^1.7.1", - "@emnapi/runtime": "^1.7.1" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1353,9 +1385,9 @@ } }, "node_modules/@oxc-project/types": { - "version": "0.139.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.139.0.tgz", - "integrity": "sha512-r9gHphtCs+1M7J0pw6Sn/hh/Wpa/iQrOOkrNAlVLF/gHq+/CJmHIWKKUUhdWjcD6CIa8idarspCsASiXCXvFUw==", + "version": "0.147.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.147.0.tgz", + "integrity": "sha512-IJ3s6ltHLp45S0bh7phkX+gJO7A1Wuz2EaqpAhb8WjqDwbzMiWKHhyyT42tskaWjEYXtHtVCPpnBJVT9+dcRLg==", "dev": true, "license": "MIT", "funding": { @@ -1419,10 +1451,27 @@ "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", "license": "BSD-3-Clause" }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.6.tgz", + "integrity": "sha512-b+jTcARdTiFLI6jB4a5XjTm0RWd6KcRfQj/I2356fxUZemiho9zQLxo0RtCuMDAyKcLo6cEltkgbQp6d1+sjjQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.1.5.tgz", - "integrity": "sha512-lZg8fqIv2v7FF237bwMgzGZEJvGL79/s5knJ/i6FmsGF4XXlzccZ4jb+TrFIxtSSxFtIpdsgrPZeMk1I9AFcyQ==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.6.tgz", + "integrity": "sha512-lkWU8ZJaRk9q3CIEY1Tc7vIFALp3Xw5NfGJo2hQg5oIqNgxWi1zI+IiDEK3r70BF5Dzol1tcXsnzsRc8NLhG+Q==", "cpu": [ "arm64" ], @@ -1437,9 +1486,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.1.5.tgz", - "integrity": "sha512-51Bnx9pNiMRKSUNtBfySkNJ9vMU9Hh3I1ozDd6gyPPYzaXCfnptUcEZxXGYFn+ul2dtcMUiqGR1Yai2K10uoTw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.6.tgz", + "integrity": "sha512-dgR56NYnvAszm7Ob1B2/Vn0e8bUQYZH2UjVaMMtMVOCKFSfjhfLmuA/9+O+F+ajUdG6B/bSssrKW6JJYASa8jA==", "cpu": [ "arm64" ], @@ -1454,9 +1503,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.1.5.tgz", - "integrity": "sha512-Tm+gbfC0aHu1tBA/JvKQh32S0K6YgCHkiAF4/W6xX0K0RmNuc94VeK419dJoE65R5aRxmo+noZQSWrAMF6yb6g==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.6.tgz", + "integrity": "sha512-vpVxFvUCFioJqug7OTvqptkc4yb8UX0AwfDmJpaR/0sWz+BUmqSVAf7c8JkUgnN8YLspb4a/N6NhTyMAmdyQ7Q==", "cpu": [ "x64" ], @@ -1471,9 +1520,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.1.5.tgz", - "integrity": "sha512-JMzDKCCXq93YccG5gz3hvOs1oXRKAf0XYpfOS88e+wZrC8Iugj6j68867vrYZkvpDDpKn/KoKORThmchMpF6TA==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.6.tgz", + "integrity": "sha512-h1wG6Y6K3JlRswxsI64qQJqBAy4vrLuHgRbc8CZMGSWTOFRY6ghMApM1NKzB2I0n5xV1fjkE18SuVl2QpLeNpA==", "cpu": [ "x64" ], @@ -1488,9 +1537,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.1.5.tgz", - "integrity": "sha512-uML21j2K5TfPGutKxub+M+nLjZIrWjXQ5Grx4lCe/nimTj9B4L63zHpjXLl4y0L3mcm2htEQIb06oCG/szerNw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.6.tgz", + "integrity": "sha512-tbCiqub0q2MVWJKgF5PoAlNWCtQydiOYSLIkd8sByqK/6MMYLJRcSXSYodqYtd0O+Fw7QaVmKKlS4oL94YRZ0w==", "cpu": [ "arm" ], @@ -1505,9 +1554,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.1.5.tgz", - "integrity": "sha512-navSiuTMogvnQoZoM/v+l3ZWo50/NTwSHSzheABx/RCnmUPaKwq9qSo4Br2OYRs21+Fz8uFqITZM3H4opOB0/Q==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.6.tgz", + "integrity": "sha512-oxK9+baEBPhZG5HB4URY+uU04zJWeZlH6Tb9rB5DK4DF9XR1uXNLXt5Q5ZsugTKayNCNLhkcwz/ye74hRI98dg==", "cpu": [ "arm64" ], @@ -1522,9 +1571,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.1.5.tgz", - "integrity": "sha512-lAryqH7IteztmCXQXk0etKj4wBQ7Gx5S6LjKhsgp9zb8I5bsuvU/2llH1hDQcjsFeqIsovMVN339/8pUDDBXxA==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.6.tgz", + "integrity": "sha512-muWCk27FVBEZtv0MsK8gnfSmgczA8KQ0uRVJbTABKhkRfQc38aUrcb7fhi3BNiyseFmgcRsoMfQsSNJ+DbZdSw==", "cpu": [ "arm64" ], @@ -1539,9 +1588,9 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.1.5.tgz", - "integrity": "sha512-fsK/sNBnxzBlL4O1JNrZakVQxPspqpED5dLtNsZS9oOKmtSpdNIzxH2kkol5HYTWJN47sE20ztMJPxfZ89qGOg==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.6.tgz", + "integrity": "sha512-eWDoSfU7Co2qj3vgB3Dt4lj1mG6CoWbcJQkRMP3XJplyCMtuaq3LHvPFjS9QIPvMGWVadJC04Xiy0IdcVPtnwQ==", "cpu": [ "ppc64" ], @@ -1556,9 +1605,9 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.1.5.tgz", - "integrity": "sha512-gLYb4BIadlfTOYT5gO503n8zQjXflgzpD0FcyKh0Mzx3rqCZKnHoJWV9xe1KXUJ5lx2JfcSHr/mhzS0PC/McAA==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.6.tgz", + "integrity": "sha512-2bWNjRSIayvupRKxXUY2tWG9fYdoUlTqWywHRvE8Eq3GvuQ+f2HeIkve697fIt+IQs/PV8yFsdWuhp1aJ1PdnA==", "cpu": [ "s390x" ], @@ -1573,9 +1622,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.1.5.tgz", - "integrity": "sha512-FjcpEKUyJygHgs1o50VYNvkt5+7Le/VEdYt0AkRpkL33MnyQfwr8l5mXwMmfmTbyMPr5vJLC+8/Gd9gXnwU1QQ==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.6.tgz", + "integrity": "sha512-KekI0gS0wLxe1UBSQSjenBVwou/JkcQPDzBPICGZjxUv9k3RteHDPBQaiOicZUFKRIH2wKEimGwVpnJsbPzu7w==", "cpu": [ "x64" ], @@ -1590,9 +1639,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.1.5.tgz", - "integrity": "sha512-Me+PfPI2TMeOQk0gYWfLQZtTktrmzbr8cDboqX83XKc7UrgAi55gF+2dUkWdxd19n55Essp2yeca+O9N5rBxHg==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.6.tgz", + "integrity": "sha512-TvtPnfVr+HtyGiDmPK4VWmlNm7QhNNAcK5Q9A7aOXsI8545yCyaoMaicXrFZ72JzeYjaUVk7yT243zT0jzjFKQ==", "cpu": [ "x64" ], @@ -1607,9 +1656,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.1.5.tgz", - "integrity": "sha512-yc5WrLzXks6zCQfn9Oxr8pORKyl/pF+QjHmW/Qx3qu0oyrrNC+y2JLTU1E2rcWYAmzlnqngWXHQjy51VzW70Vw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.6.tgz", + "integrity": "sha512-iOo0VEay2XFhaCcH0sps5XIimkSuOnNaZrf6+ZkoSOQBJPKNU48RkmJv0/lSpipexu5P+ouFgafe5IGr/DiQfg==", "cpu": [ "arm64" ], @@ -1623,29 +1672,10 @@ "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.1.5.tgz", - "integrity": "sha512-VbQGPX2b4r48TAMIM2cjgluIM1HYutm4pcTEJsle7iEP7sB1dFqtPLBVbdLAZCxy1txCcPxf4QFf4v8uvltPqA==", - "cpu": [ - "wasm32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "1.11.1", - "@emnapi/runtime": "1.11.1", - "@napi-rs/wasm-runtime": "^1.1.6" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.1.5.tgz", - "integrity": "sha512-gHv82k63z4qpV5+Q1y/12KrK0ltWBukVDI8nZcbT7Tt/ZlOIVwppazneq0F93oDxTo3IgAMEDIoQh3E2n6mVsw==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.6.tgz", + "integrity": "sha512-y5NTmmasMS455JlOCO4ZM9krIchv3Mvm1crL1iUPGOPgEzSkves9n0SdC5Sjz6+qWDFhd8/JpfWMH8NSWNHe+A==", "cpu": [ "arm64" ], @@ -1660,9 +1690,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.1.5.tgz", - "integrity": "sha512-tTZuDBPw85tEN5PQi1pnEBzDy0Z49HtScLAbD5t6hyeU92A95pRWaSMw1GZZi/RwgSgUIl0xrSlXIT/9QzvYSA==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.6.tgz", + "integrity": "sha512-np8iZSLfXlAD4kWhiyq/u0Yt8oZDtRQ8lGhQaCXo2rl37KNjeU0GjJuwr4P3oeZ++ROfofsKNBqR5LTO8aXyWQ==", "cpu": [ "x64" ], @@ -1770,17 +1800,6 @@ "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", "license": "MIT" }, - "node_modules/@tybys/wasm-util": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", - "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", - "dev": true, - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -1828,17 +1847,48 @@ "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", "license": "MIT" }, + "node_modules/@vitest/coverage-v8": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.11.tgz", + "integrity": "sha512-8MVGEFnJIcdGjcbfKmeq8z0pZHH0JlVtoVZH9Q/qwUp6wyFnEJUBMrw9DCaj+ra3vShGmhavjalMIhPNxZAUcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^1.0.2", + "@vitest/utils": "4.1.11", + "ast-v8-to-istanbul": "^1.0.0", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.2.0", + "magicast": "^0.5.2", + "obug": "^2.1.1", + "std-env": "^4.0.0-rc.1", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "4.1.11", + "vitest": "4.1.11" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } + } + }, "node_modules/@vitest/expect": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.10.tgz", - "integrity": "sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.10", - "@vitest/utils": "4.1.10", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -1847,13 +1897,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.10.tgz", - "integrity": "sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.10", + "@vitest/spy": "4.1.11", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -1874,9 +1924,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.10.tgz", - "integrity": "sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", "dev": true, "license": "MIT", "dependencies": { @@ -1887,13 +1937,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.10.tgz", - "integrity": "sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.10", + "@vitest/utils": "4.1.11", "pathe": "^2.0.3" }, "funding": { @@ -1901,14 +1951,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.10.tgz", - "integrity": "sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.10", - "@vitest/utils": "4.1.10", + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -1917,9 +1967,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.10.tgz", - "integrity": "sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", "dev": true, "license": "MIT", "funding": { @@ -1927,13 +1977,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.10.tgz", - "integrity": "sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.10", + "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -1985,6 +2035,18 @@ "node": ">=12" } }, + "node_modules/ast-v8-to-istanbul": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-1.0.5.tgz", + "integrity": "sha512-UPAgKJFSEGMWSDr3LX4tqnAb4f7KGT8O40Tyx8wbYmmZ/yn58lNCm8h3svs3eXgiGd5AXxz8NDOvXWvicq+rJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "estree-walker": "^3.0.3", + "js-tokens": "^10.0.0" + } + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -2171,6 +2233,44 @@ "fast-check": "^3.23.1" } }, + "node_modules/effect/node_modules/fast-check": { + "version": "3.23.2", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-3.23.2.tgz", + "integrity": "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^6.1.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/effect/node_modules/pure-rand": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", + "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, "node_modules/es-module-lexer": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.0.tgz", @@ -2212,9 +2312,10 @@ "license": "MIT" }, "node_modules/fast-check": { - "version": "3.23.2", - "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-3.23.2.tgz", - "integrity": "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A==", + "version": "4.9.0", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-4.9.0.tgz", + "integrity": "sha512-7ms6T7SybUev/PQITciI0yLM2pOSFy5zpG8Ty7tQofcVaQUvrMXp6CBwqF6fThLCLOrfBtuHAtwq6Yu4XPCllg==", + "dev": true, "funding": [ { "type": "individual", @@ -2227,10 +2328,10 @@ ], "license": "MIT", "dependencies": { - "pure-rand": "^6.1.0" + "pure-rand": "^8.0.0" }, "engines": { - "node": ">=8.0.0" + "node": ">=12.17.0" } }, "node_modules/fast-glob": { @@ -2421,6 +2522,23 @@ "uncrypto": "^0.1.3" } }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, "node_modules/http-proxy-agent": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", @@ -2499,6 +2617,45 @@ "node": ">=0.12.0" } }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/js-tiktoken": { "version": "1.0.21", "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", @@ -2509,6 +2666,13 @@ "base64-js": "^1.5.1" } }, + "node_modules/js-tokens": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", + "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==", + "dev": true, + "license": "MIT" + }, "node_modules/json-bigint": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", @@ -2618,9 +2782,9 @@ } }, "node_modules/lightningcss": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", - "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", "dev": true, "license": "MPL-2.0", "dependencies": { @@ -2634,23 +2798,23 @@ "url": "https://opencollective.com/parcel" }, "optionalDependencies": { - "lightningcss-android-arm64": "1.32.0", - "lightningcss-darwin-arm64": "1.32.0", - "lightningcss-darwin-x64": "1.32.0", - "lightningcss-freebsd-x64": "1.32.0", - "lightningcss-linux-arm-gnueabihf": "1.32.0", - "lightningcss-linux-arm64-gnu": "1.32.0", - "lightningcss-linux-arm64-musl": "1.32.0", - "lightningcss-linux-x64-gnu": "1.32.0", - "lightningcss-linux-x64-musl": "1.32.0", - "lightningcss-win32-arm64-msvc": "1.32.0", - "lightningcss-win32-x64-msvc": "1.32.0" + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" } }, "node_modules/lightningcss-android-arm64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", - "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", "cpu": [ "arm64" ], @@ -2669,9 +2833,9 @@ } }, "node_modules/lightningcss-darwin-arm64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", - "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", "cpu": [ "arm64" ], @@ -2690,9 +2854,9 @@ } }, "node_modules/lightningcss-darwin-x64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", - "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", "cpu": [ "x64" ], @@ -2711,9 +2875,9 @@ } }, "node_modules/lightningcss-freebsd-x64": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", - "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", "cpu": [ "x64" ], @@ -2732,9 +2896,9 @@ } }, "node_modules/lightningcss-linux-arm-gnueabihf": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", - "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", "cpu": [ "arm" ], @@ -2753,9 +2917,9 @@ } }, "node_modules/lightningcss-linux-arm64-gnu": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", - "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", "cpu": [ "arm64" ], @@ -2774,9 +2938,9 @@ } }, "node_modules/lightningcss-linux-arm64-musl": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", - "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", "cpu": [ "arm64" ], @@ -2795,9 +2959,9 @@ } }, "node_modules/lightningcss-linux-x64-gnu": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", - "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", "cpu": [ "x64" ], @@ -2816,9 +2980,9 @@ } }, "node_modules/lightningcss-linux-x64-musl": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", - "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", "cpu": [ "x64" ], @@ -2837,9 +3001,9 @@ } }, "node_modules/lightningcss-win32-arm64-msvc": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", - "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", "cpu": [ "arm64" ], @@ -2858,9 +3022,9 @@ } }, "node_modules/lightningcss-win32-x64-msvc": { - "version": "1.32.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", - "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", "cpu": [ "x64" ], @@ -2903,6 +3067,34 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/magicast": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.4.tgz", + "integrity": "sha512-llBEhWm1SacoRwgHUoQJYtwp4PBLF4faQi5TCpIGyGs9n4y5+juI0tDgyKIfpqxckRHaHzouUEph3THklWh03w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", + "source-map-js": "^1.2.1" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -2954,9 +3146,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.15", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.15.tgz", - "integrity": "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { @@ -3167,9 +3359,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", - "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", "dev": true, "license": "MIT", "engines": { @@ -3180,9 +3372,9 @@ } }, "node_modules/postcss": { - "version": "8.5.16", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.16.tgz", - "integrity": "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg==", + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", "dev": true, "funding": [ { @@ -3200,7 +3392,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.12", + "nanoid": "^3.3.17", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -3232,9 +3424,10 @@ } }, "node_modules/pure-rand": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", - "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-8.4.2.tgz", + "integrity": "sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==", + "dev": true, "funding": [ { "type": "individual", @@ -3313,13 +3506,13 @@ } }, "node_modules/rolldown": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.1.5.tgz", - "integrity": "sha512-t9z29cJjXf/vxQ8dyhCSpt6H6aSwHTk8cT5I3iy6SMXuFpk5mB6PL6XfC8PCwrPTx93udwKUm9HRteAlTGBLiA==", + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.6.tgz", + "integrity": "sha512-vMM4q3aixf46GiF1Kok8jDPFsEpXgFWGjUHXNkNHNm+Y2adXAG2dbX91jkti3i0ZRsOlcmbuzAz1poObSHCmUA==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.139.0", + "@oxc-project/types": "=0.147.0", "@rolldown/pluginutils": "^1.0.0" }, "bin": { @@ -3329,21 +3522,21 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.1.5", - "@rolldown/binding-darwin-arm64": "1.1.5", - "@rolldown/binding-darwin-x64": "1.1.5", - "@rolldown/binding-freebsd-x64": "1.1.5", - "@rolldown/binding-linux-arm-gnueabihf": "1.1.5", - "@rolldown/binding-linux-arm64-gnu": "1.1.5", - "@rolldown/binding-linux-arm64-musl": "1.1.5", - "@rolldown/binding-linux-ppc64-gnu": "1.1.5", - "@rolldown/binding-linux-s390x-gnu": "1.1.5", - "@rolldown/binding-linux-x64-gnu": "1.1.5", - "@rolldown/binding-linux-x64-musl": "1.1.5", - "@rolldown/binding-openharmony-arm64": "1.1.5", - "@rolldown/binding-wasm32-wasi": "1.1.5", - "@rolldown/binding-win32-arm64-msvc": "1.1.5", - "@rolldown/binding-win32-x64-msvc": "1.1.5" + "@rolldown/binding-android-arm-eabi": "1.2.6", + "@rolldown/binding-android-arm64": "1.2.6", + "@rolldown/binding-darwin-arm64": "1.2.6", + "@rolldown/binding-darwin-x64": "1.2.6", + "@rolldown/binding-freebsd-x64": "1.2.6", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.6", + "@rolldown/binding-linux-arm64-gnu": "1.2.6", + "@rolldown/binding-linux-arm64-musl": "1.2.6", + "@rolldown/binding-linux-ppc64-gnu": "1.2.6", + "@rolldown/binding-linux-s390x-gnu": "1.2.6", + "@rolldown/binding-linux-x64-gnu": "1.2.6", + "@rolldown/binding-linux-x64-musl": "1.2.6", + "@rolldown/binding-openharmony-arm64": "1.2.6", + "@rolldown/binding-win32-arm64-msvc": "1.2.6", + "@rolldown/binding-win32-x64-msvc": "1.2.6" } }, "node_modules/run-parallel": { @@ -3432,6 +3625,19 @@ "dev": true, "license": "MIT" }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -3650,16 +3856,16 @@ } }, "node_modules/vite": { - "version": "8.1.4", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.4.tgz", - "integrity": "sha512-bTT9PsdWO+MQMNG9ZXIP/qM9wGh37DFxTV/sPq9cFpHr3w4jkgef032PkAL9jAqhk3Nz8NQw3O8n6/xFkqO4QQ==", + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.2.tgz", + "integrity": "sha512-cFKLV/PRgAUlIRm5WjMjJ86jrftzpqcgH+Us+DS8mI3CDNiH30Whrz8uHL3+MOLPAgqbMBAqWdAHAphOAM+z/Q==", "dev": true, "license": "MIT", "dependencies": { - "lightningcss": "^1.32.0", + "lightningcss": "^1.33.0", "picomatch": "^4.0.5", - "postcss": "^8.5.16", - "rolldown": "~1.1.4", + "postcss": "^8.5.26", + "rolldown": "~1.2.4", "tinyglobby": "^0.2.17" }, "bin": { @@ -3676,7 +3882,7 @@ }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.3.0", + "@vitejs/devtools": "^0.4.0 || ^0.5.0", "esbuild": "^0.27.0 || ^0.28.0", "jiti": ">=1.21.0", "less": "^4.0.0", @@ -3728,19 +3934,19 @@ } }, "node_modules/vitest": { - "version": "4.1.10", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.10.tgz", - "integrity": "sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.10", - "@vitest/mocker": "4.1.10", - "@vitest/pretty-format": "4.1.10", - "@vitest/runner": "4.1.10", - "@vitest/snapshot": "4.1.10", - "@vitest/spy": "4.1.10", - "@vitest/utils": "4.1.10", + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -3768,12 +3974,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.10", - "@vitest/browser-preview": "4.1.10", - "@vitest/browser-webdriverio": "4.1.10", - "@vitest/coverage-istanbul": "4.1.10", - "@vitest/coverage-v8": "4.1.10", - "@vitest/ui": "4.1.10", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" diff --git a/package.json b/package.json index 7e01a65..b9a7d90 100644 --- a/package.json +++ b/package.json @@ -60,7 +60,8 @@ "build:harness": "node -e \"require('node:fs').rmSync('packages/harness/dist', { recursive: true, force: true })\" && tsc -p packages/harness/tsconfig.json", "typecheck": "npm run build:grounding && npm run build:harness && npm run build:rewrite && npm run build:training && tsc --noEmit -p tsconfig.test.json && tsc --noEmit -p packages/grounding/tsconfig.test.json && tsc --noEmit -p packages/harness/tsconfig.test.json && tsc --noEmit -p packages/rewrite/tsconfig.test.json && tsc --noEmit -p packages/training/tsconfig.test.json", "test": "node test/run.mjs", - "check": "npm run typecheck && npm test && npm run build:core", + "test:coverage": "node test/run.mjs --coverage", + "check": "npm run typecheck && npm run test:coverage && npm run build:core", "prepublishOnly": "npm run check", "build:training": "node -e \"require('node:fs').rmSync('packages/training/dist', { recursive: true, force: true })\" && tsc -p packages/training/tsconfig.json", "build:rewrite": "node -e \"require('node:fs').rmSync('packages/rewrite/dist', { recursive: true, force: true })\" && tsc -p packages/rewrite/tsconfig.json" @@ -74,6 +75,8 @@ ], "devDependencies": { "@types/node": "^22.15.3", + "@vitest/coverage-v8": "^4.1.11", + "fast-check": "^4.9.0", "vitest": "^4.1.9" }, "dependencies": { diff --git a/packages/grounding/test/component.test.ts b/packages/grounding/test/component.test.ts new file mode 100644 index 0000000..18dcbc2 --- /dev/null +++ b/packages/grounding/test/component.test.ts @@ -0,0 +1,222 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + COMPONENT_METADATA, + componentMetadataOf, + createComponentDecorator, + finalizeTrainableClass, + REGISTERED_METHODS, + type GroundingOptions, + type GroundingRegistry, +} from "../src/index.js"; +import { intent, PENDING_GROUNDINGS } from "../src/decorators.js"; + +// component.ts is the host seam: it registers a class's methods against a +// registry the host owns, and records component metadata. It was the least +// covered file in the workspace (58% statements, 50% branches), and the +// uncovered half is exactly the fallback behavior a host depends on. + +function registry() { + const registered = new Map(); + const spy: GroundingRegistry = { + has: (methodRef) => registered.has(methodRef), + register: (_baseline, options) => { + registered.set(options.methodRef, options); + return { methodRef: options.methodRef }; + }, + }; + return { registry: spy, registered }; +} + +describe("finalizeTrainableClass", () => { + it("registers every own prototype method when nothing was annotated", () => { + class Bare { + one(): string { return "one"; } + two(): string { return "two"; } + } + const { registry: host, registered } = registry(); + finalizeTrainableClass(Bare, undefined, host); + expect([...registered.keys()].sort()).toEqual(["Bare.one", "Bare.two"]); + }); + + it("registers only the annotated methods when some were", () => { + const metadata: Record = {}; + const pending = new Map([["only", { intent: "declared" }]]); + metadata[PENDING_GROUNDINGS] = pending; + class Partial { + only(): string { return "only"; } + other(): string { return "other"; } + } + const { registry: host, registered } = registry(); + finalizeTrainableClass(Partial, metadata, host); + expect([...registered.keys()]).toEqual(["Partial.only"]); + expect(registered.get("Partial.only")?.intent).toBe("declared"); + }); + + it("skips the constructor and non-function members", () => { + class WithField { + value = 1; + method(): number { return this.value; } + } + const { registry: host, registered } = registry(); + finalizeTrainableClass(WithField, undefined, host); + expect([...registered.keys()]).toEqual(["WithField.method"]); + }); + + it("first registration wins, so a re-run does not duplicate", () => { + class Twice { method(): void { /* no-op */ } } + const { registry: host, registered } = registry(); + finalizeTrainableClass(Twice, undefined, host); + finalizeTrainableClass(Twice, undefined, host); + expect(registered.size).toBe(1); + }); + + it("binds the baseline to a lazily constructed instance", () => { + let constructed = 0; + class Counted { + constructor() { constructed += 1; } + echo(input: unknown): unknown { return input; } + } + const baselines: Array<(input: unknown) => unknown> = []; + finalizeTrainableClass(Counted, undefined, { + has: () => false, + register: (baseline, options) => { baselines.push(baseline); return { methodRef: options.methodRef }; }, + }); + // Not constructed just by registering. + expect(constructed).toBe(0); + expect(baselines[0]?.("hi")).toBe("hi"); + expect(constructed).toBe(1); + // Reused, not rebuilt, on a second call. + baselines[0]?.("again"); + expect(constructed).toBe(1); + }); + + it("falls back to the prototype when the class cannot be constructed", () => { + class Unconstructible { + constructor() { throw new Error("no"); } + echo(input: unknown): unknown { return input; } + } + const baselines: Array<(input: unknown) => unknown> = []; + finalizeTrainableClass(Unconstructible as unknown as new () => object, undefined, { + has: () => false, + register: (baseline, options) => { baselines.push(baseline); return { methodRef: options.methodRef }; }, + }); + expect(baselines[0]?.("hi")).toBe("hi"); + }); + + it("accumulates registered refs on the metadata record", () => { + class Recorded { a(): void { /* no-op */ } b(): void { /* no-op */ } } + const metadata: Record = {}; + const { registry: host } = registry(); + finalizeTrainableClass(Recorded, metadata, host); + expect(metadata[REGISTERED_METHODS]).toEqual(["Recorded.a", "Recorded.b"]); + }); + + it("honors a custom registered-methods symbol", () => { + const slot = Symbol("custom"); + class Custom { a(): void { /* no-op */ } } + const metadata: Record = {}; + finalizeTrainableClass(Custom, metadata, registry().registry, slot); + expect(metadata[slot]).toEqual(["Custom.a"]); + expect(metadata[REGISTERED_METHODS]).toBeUndefined(); + }); + + it("registers nothing for a class with no own methods", () => { + class Empty {} + const { registry: host, registered } = registry(); + finalizeTrainableClass(Empty, undefined, host); + expect(registered.size).toBe(0); + }); +}); + +describe("createComponentDecorator", () => { + const symbols = { component: COMPONENT_METADATA, operations: REGISTERED_METHODS }; + + it("records intent, name and operation refs on the class metadata", () => { + const component = createComponentDecorator(symbols); + const metadata: Record = { [REGISTERED_METHODS]: ["Widget.render"] }; + class Widget {} + component({ intent: "Render a widget" })(Widget, { + kind: "class", name: "Widget", metadata, + } as unknown as ClassDecoratorContext); + expect(metadata[COMPONENT_METADATA]).toEqual({ + intent: "Render a widget", name: "Widget", operations: ["Widget.render"], + }); + expect(Object.isFrozen(metadata[COMPONENT_METADATA])).toBe(true); + }); + + it("defaults operations to empty when no method decorator ran", () => { + const component = createComponentDecorator(symbols); + const metadata: Record = {}; + class Widget {} + component({ intent: "i" })(Widget, { kind: "class", name: "Widget", metadata } as never); + expect((metadata[COMPONENT_METADATA] as { operations: string[] }).operations).toEqual([]); + }); + + it("falls back to the class's own name when the context has none", () => { + const component = createComponentDecorator(symbols); + const metadata: Record = {}; + class Fallback {} + component({ intent: "i" })(Fallback, { kind: "class", metadata } as never); + expect((metadata[COMPONENT_METADATA] as { name: string }).name).toBe("Fallback"); + }); + + it("does nothing when the runtime supplies no metadata record", () => { + const component = createComponentDecorator(symbols); + class Widget {} + expect(() => component({ intent: "i" })(Widget, { kind: "class", name: "Widget" } as never)).not.toThrow(); + }); +}); + +describe("componentMetadataOf", () => { + it("returns undefined for a class that was never decorated", () => { + class Plain {} + expect(componentMetadataOf(Plain)).toBeUndefined(); + }); + + it("returns undefined for non-classes and nullish input", () => { + expect(componentMetadataOf(undefined)).toBeUndefined(); + expect(componentMetadataOf(null)).toBeUndefined(); + expect(componentMetadataOf(42)).toBeUndefined(); + }); + + it("reads metadata back off a decorated class", () => { + const metadataSymbol = (Symbol as { metadata?: symbol }).metadata; + if (metadataSymbol === undefined) return; + const value = { intent: "i", name: "N", operations: [] }; + const holder = { [metadataSymbol]: { [COMPONENT_METADATA]: value } }; + expect(componentMetadataOf(holder)).toBe(value); + }); + + it("honors a custom component symbol", () => { + const metadataSymbol = (Symbol as { metadata?: symbol }).metadata; + if (metadataSymbol === undefined) return; + const slot = Symbol.for("custom.component"); + const value = { intent: "i", name: "N", operations: [] }; + const holder = { [metadataSymbol]: { [slot]: value } }; + expect(componentMetadataOf(holder, slot)).toBe(value); + expect(componentMetadataOf(holder)).toBeUndefined(); + }); + + it("returns undefined when Symbol.metadata is unavailable", () => { + const original = (Symbol as { metadata?: symbol }).metadata; + try { + delete (Symbol as { metadata?: symbol }).metadata; + expect(componentMetadataOf(class {})).toBeUndefined(); + } finally { + if (original !== undefined) (Symbol as { metadata?: symbol }).metadata = original; + } + }); +}); + +describe("granular decorators feeding the finalizer", () => { + it("carries a declared intent through to registration", () => { + const metadata: Record = {}; + const context = { name: "route", metadata } as never; + intent("Route the request")(vi.fn(), context); + class Router { route(): void { /* no-op */ } } + const { registry: host, registered } = registry(); + finalizeTrainableClass(Router, metadata, host); + expect(registered.get("Router.route")?.intent).toBe("Route the request"); + }); +}); diff --git a/packages/harness/test/attempt.test.ts b/packages/harness/test/attempt.test.ts new file mode 100644 index 0000000..e0ef3eb --- /dev/null +++ b/packages/harness/test/attempt.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; + +import { attempt, attemptAsync, errorMessage } from "../src/attempt.js"; + +// These were Effect-backed and are now plain try/catch. The fallback branch had +// zero coverage in the root copy, which is the branch that matters: it is the +// boundary that keeps a capture or serialization failure from breaking a +// traced application call. + +describe("errorMessage", () => { + it("uses an Error's message", () => { + expect(errorMessage(new Error("boom"))).toBe("boom"); + expect(errorMessage(new TypeError("typed"))).toBe("typed"); + }); + + it("stringifies anything else", () => { + expect(errorMessage("plain")).toBe("plain"); + expect(errorMessage(42)).toBe("42"); + expect(errorMessage(null)).toBe("null"); + expect(errorMessage(undefined)).toBe("undefined"); + expect(errorMessage({ toString: () => "custom" })).toBe("custom"); + }); + + it("keeps a subclass message", () => { + class Custom extends Error {} + expect(errorMessage(new Custom("sub"))).toBe("sub"); + }); +}); + +describe("attempt", () => { + it("returns the value when nothing throws", () => { + expect(attempt(() => 1, () => 2)).toBe(1); + }); + + it("returns the fallback when the body throws", () => { + expect(attempt(() => { throw new Error("x"); }, () => 2)).toBe(2); + }); + + it("hands the raw thrown value to the fallback, not a wrapper", () => { + const thrown = { code: "E" }; + expect(attempt(() => { throw thrown; }, (error) => error)).toBe(thrown); + }); + + it("propagates a throw from the fallback itself", () => { + expect(() => attempt(() => { throw new Error("first"); }, () => { throw new Error("second"); })) + .toThrow("second"); + }); + + it("preserves falsy return values rather than treating them as failure", () => { + expect(attempt(() => 0, () => 99)).toBe(0); + expect(attempt(() => undefined, () => 99)).toBeUndefined(); + }); + + it("runs synchronously", () => { + const order: string[] = []; + order.push("before"); + attempt(() => order.push("body"), () => order.push("fallback")); + order.push("after"); + expect(order).toEqual(["before", "body", "after"]); + }); +}); + +describe("attemptAsync", () => { + it("resolves the value when nothing rejects", async () => { + await expect(attemptAsync(async () => 1, () => 2)).resolves.toBe(1); + }); + + it("resolves the fallback when the promise rejects", async () => { + await expect(attemptAsync(async () => { throw new Error("x"); }, () => 2)).resolves.toBe(2); + }); + + it("resolves the fallback when the body throws synchronously", async () => { + await expect(attemptAsync(() => { throw new Error("sync"); }, () => 3)).resolves.toBe(3); + }); + + it("hands the raw rejection value to the fallback", async () => { + const thrown = { code: "E" }; + await expect(attemptAsync(async () => { throw thrown; }, (error) => error)).resolves.toBe(thrown); + }); + + it("rejects when the fallback throws", async () => { + await expect(attemptAsync(async () => { throw new Error("first"); }, () => { throw new Error("second"); })) + .rejects.toThrow("second"); + }); +}); diff --git a/packages/rewrite/test/canonical.test.ts b/packages/rewrite/test/canonical.test.ts new file mode 100644 index 0000000..5a5566d --- /dev/null +++ b/packages/rewrite/test/canonical.test.ts @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; + +import { check, digest } from "../src/canonical.js"; + +// The digest is a cross-package protocol: guarded rewriting refuses a candidate +// whose target body digest no longer matches, so training and rewrite must hash +// identical content identically. `isRecord`'s prototype check is the subtle +// part -- class instances must NOT be key-sorted into `{}`. + +describe("digest", () => { + it("is stable for the same value", () => { + expect(digest({ a: 1 })).toBe(digest({ a: 1 })); + }); + + it("ignores key order, which is the point of canonicalization", () => { + expect(digest({ a: 1, b: 2 })).toBe(digest({ b: 2, a: 1 })); + expect(digest({ outer: { x: 1, y: 2 } })).toBe(digest({ outer: { y: 2, x: 1 } })); + }); + + it("respects array order, which is meaningful", () => { + expect(digest([1, 2])).not.toBe(digest([2, 1])); + }); + + it("sorts keys inside arrays too", () => { + expect(digest([{ a: 1, b: 2 }])).toBe(digest([{ b: 2, a: 1 }])); + }); + + it("distinguishes different values", () => { + expect(digest({ a: 1 })).not.toBe(digest({ a: 2 })); + expect(digest("a")).not.toBe(digest("b")); + }); + + it("handles primitives and null-prototype objects", () => { + for (const value of ["s", 1, true, null, [], {}]) { + expect(digest(value)).toMatch(/^sha256:[0-9a-f]{64}$/); + } + const bare = Object.create(null) as Record; + bare["b"] = 1; + bare["a"] = 2; + expect(digest(bare)).toBe(digest({ a: 2, b: 1 })); + }); + + it("does not canonicalize class instances into empty objects", () => { + // A Date serializes through JSON.stringify; if isRecord wrongly accepted + // it, every Date would hash identically. + expect(digest(new Date("2020-01-01"))).not.toBe(digest(new Date("2021-01-01"))); + expect(digest(new Map([["a", 1]]))).toBe(digest({})); + }); + + it("returns the documented prefix and hex length", () => { + expect(digest("x")).toMatch(/^sha256:[0-9a-f]{64}$/); + }); +}); + +describe("check", () => { + it("passes a truthy condition through", () => { + expect(() => check(1, "unused")).not.toThrow(); + expect(() => check("non-empty", "unused")).not.toThrow(); + }); + + it("throws the given message for every falsy condition", () => { + for (const value of [false, 0, "", null, undefined, Number.NaN]) { + expect(() => check(value, "boom")).toThrow("boom"); + } + }); +}); diff --git a/packages/training/test/builders.test.ts b/packages/training/test/builders.test.ts new file mode 100644 index 0000000..bd295ac --- /dev/null +++ b/packages/training/test/builders.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, it } from "vitest"; + +import { createCandidateReview, createEvalRun, createPromotionDecision } from "../src/builders.js"; +import { discoverInSource } from "../src/source.js"; +import { defineTrainable } from "../src/token.js"; +import type { CandidatePatch } from "../src/engine.js"; + +// These builders exist so that implementing a TrainingLoop never needs a cast. +// Their defaulting rules are the part a consumer will rely on without reading +// the source, so each default is pinned here. + +const target = discoverInSource(`class Fixture { + route(input: string): string { + "use training"; + return input; + } +}`, "fixture.ts")[0]!; + +const candidate: CandidatePatch = { + id: "cand-1", trainableId: target.id, engineId: "test", target, implementation: "return input;", +}; + +describe("createEvalRun", () => { + it("accepts a token or a symbol as the identity", () => { + const token = defineTrainable(target.id); + expect(createEvalRun({ trainable: token }).token.id).toBe(target.id); + expect(createEvalRun({ trainable: token.symbol }).token.id).toBe(target.id); + }); + + it("defaults to an empty but well-formed AgentV run", () => { + const run = createEvalRun({ trainable: defineTrainable(target.id) }); + expect(run.evaluations).toEqual([]); + expect(run.run.results).toEqual([]); + expect(run.run.summary).toEqual({ + total: 0, passed: 0, failed: 0, executionErrors: 0, durationMs: 0, meanScore: 0, + }); + }); + + it("carries supplied evaluations and freezes the result", () => { + const evaluations = [{ trainableId: target.id, result: { score: 1 } as never }]; + const run = createEvalRun({ trainable: defineTrainable(target.id), evaluations }); + expect(run.evaluations).toHaveLength(1); + expect(Object.isFrozen(run)).toBe(true); + expect(Object.isFrozen(run.evaluations)).toBe(true); + }); + + it("copies the evaluations rather than aliasing the caller's array", () => { + const evaluations = [{ trainableId: target.id, result: { score: 1 } as never }]; + const run = createEvalRun({ trainable: defineTrainable(target.id), evaluations }); + evaluations.push({ trainableId: target.id, result: { score: 0 } as never }); + expect(run.evaluations).toHaveLength(1); + }); + + it("uses a supplied run verbatim", () => { + const supplied = { results: [], summary: { total: 7, passed: 7, failed: 0, executionErrors: 0, durationMs: 1, meanScore: 1 } }; + expect(createEvalRun({ trainable: defineTrainable(target.id), run: supplied }).run).toBe(supplied); + }); +}); + +describe("createPromotionDecision", () => { + it("promotes by default when no failures are given", () => { + expect(createPromotionDecision({ candidateId: "c" })) + .toEqual({ candidateId: "c", promote: true, failures: [], meanScore: 1, passRate: 1 }); + }); + + it("refuses by default when failures are given", () => { + expect(createPromotionDecision({ candidateId: "c", failures: ["bad"] })) + .toEqual({ candidateId: "c", promote: false, failures: ["bad"], meanScore: 0, passRate: 0 }); + }); + + it("lets an explicit promote override the inference in both directions", () => { + expect(createPromotionDecision({ candidateId: "c", failures: ["bad"], promote: true }).promote).toBe(true); + expect(createPromotionDecision({ candidateId: "c", promote: false }).promote).toBe(false); + // And the derived scores follow the explicit flag, not the failures. + expect(createPromotionDecision({ candidateId: "c", failures: ["bad"], promote: true }).meanScore).toBe(1); + }); + + it("honors explicit scores, including zero", () => { + expect(createPromotionDecision({ candidateId: "c", meanScore: 0, passRate: 0.5 })) + .toMatchObject({ meanScore: 0, passRate: 0.5 }); + }); + + it("freezes the decision and its failures", () => { + const decision = createPromotionDecision({ candidateId: "c", failures: ["x"] }); + expect(Object.isFrozen(decision)).toBe(true); + expect(Object.isFrozen(decision.failures)).toBe(true); + }); +}); + +describe("createCandidateReview", () => { + it("derives both halves from the candidate alone", () => { + const review = createCandidateReview({ candidate }); + expect(review.decision.candidateId).toBe(candidate.id); + expect(review.decision.promote).toBe(true); + expect(review.verification.token.id).toBe(target.id); + }); + + it("threads failures into the derived decision", () => { + expect(createCandidateReview({ candidate, failures: ["nope"] }).decision) + .toMatchObject({ promote: false, failures: ["nope"] }); + }); + + it("threads evaluations into the derived verification", () => { + const review = createCandidateReview({ + candidate, + evaluations: [{ trainableId: target.id, result: { score: 1 } as never }], + }); + expect(review.verification.evaluations).toHaveLength(1); + }); + + it("prefers an explicitly supplied verification or decision", () => { + const verification = createEvalRun({ trainable: defineTrainable(target.id) }); + const decision = createPromotionDecision({ candidateId: "other", promote: false }); + const review = createCandidateReview({ candidate, verification, decision, failures: ["ignored"] }); + expect(review.verification).toBe(verification); + expect(review.decision).toBe(decision); + expect(review.decision.failures).toEqual([]); + }); + + it("freezes the review", () => { + expect(Object.isFrozen(createCandidateReview({ candidate }))).toBe(true); + }); +}); diff --git a/packages/training/test/errors.test.ts b/packages/training/test/errors.test.ts new file mode 100644 index 0000000..faedfc6 --- /dev/null +++ b/packages/training/test/errors.test.ts @@ -0,0 +1,182 @@ +import { z } from "zod"; +import { describe, expect, it } from "vitest"; + +import { + CandidateSyntaxError, + EngineContractError, + EngineNotConfiguredError, + EngineProposalError, + ExecutorNotConfiguredError, + InsufficientTracesError, + InvalidSettingsError, + InvalidTrainableIdentityError, + isTsAutocodeError, + LoopCapabilityError, + MissingSecretError, + OperationInterruptedError, + parseSetting, + PromotionApplierNotConfiguredError, + PromotionRejectedError, + SourceDiscoveryError, + TraceNotFoundError, + TrainingIncompleteError, + TsAutocodeError, + TsAutocodeSyntaxError, + TsAutocodeTypeError, + type TsAutocodeErrorCode, +} from "../src/errors.js"; + +// Atomic coverage of the error family: every constructor, every static factory, +// every branch of the brand-based `hasInstance`, and the Zod boundary. The +// family was added with only a handful of these exercised (16% branch), which +// is exactly the sort of gap a "we have tests" claim hides. + +/** Every concrete error, with the code and message each must produce. */ +const cases: ReadonlyArray Error, TsAutocodeErrorCode, string]> = [ + ["EngineNotConfiguredError", () => new EngineNotConfiguredError(), "engine_not_configured", + 'no training engine is configured; import "ts-autocode" for the Ax default or set TrainingSettings.engine'], + ["ExecutorNotConfiguredError", () => new ExecutorNotConfiguredError(), "executor_not_configured", + 'candidate execution requires an executor; import "ts-autocode" or set TrainingSettings.executor'], + ["PromotionApplierNotConfiguredError", () => new PromotionApplierNotConfiguredError(), "applier_not_configured", + 'activation requires a promotion applier; import "ts-autocode" for the default or set TrainingProviders.promote'], + ["PromotionRejectedError", () => new PromotionRejectedError("cand-9"), "promotion_rejected", + "candidate has not passed the promotion gate: cand-9"], + ["InsufficientTracesError", () => new InsufficientTracesError(4, 2), "insufficient_traces", + "training from captured traffic requires 4 distinct successful runtime traces; found 2"], + ["TraceNotFoundError", () => new TraceNotFoundError("abc"), "trace_not_found", + "live trace was not found for eval input: abc"], + ["CandidateSyntaxError", () => new CandidateSyntaxError("Router.route"), "candidate_syntax", + "engine returned invalid TypeScript for Router.route"], + ["EngineContractError", () => new EngineContractError("bad contract"), "engine_contract", "bad contract"], + ["EngineProposalError", () => new EngineProposalError("no candidate"), "engine_proposal", "no candidate"], + ["MissingSecretError", () => new MissingSecretError("K", "needs K"), "missing_secret", "needs K"], + ["LoopCapabilityError", () => new LoopCapabilityError("cannot fan out"), "loop_capability", "cannot fan out"], + ["InvalidTrainableIdentityError", () => new InvalidTrainableIdentityError("bad id"), "invalid_identity", "bad id"], + ["SourceDiscoveryError", () => new SourceDiscoveryError("not found"), "source_discovery", "not found"], + ["OperationInterruptedError", () => new OperationInterruptedError("propose"), "operation_interrupted", + "propose was interrupted"], + ["InvalidSettingsError", () => new InvalidSettingsError("bad setting"), "invalid_settings", "bad setting"], + ["TrainingIncompleteError.noRounds", () => TrainingIncompleteError.noRounds("stalled"), "training_incomplete", + "training loop returned no rounds: stalled"], + ["TrainingIncompleteError.noPromotableCandidate", () => TrainingIncompleteError.noPromotableCandidate("exhausted"), + "training_incomplete", "background training did not produce a promotable candidate: exhausted"], +]; + +describe.each(cases)("%s", (name, build, code, message) => { + it("carries its code, message and name", () => { + const error = build(); + expect((error as Error & { code: string }).code).toBe(code); + expect(error.message).toBe(message); + expect(error.name).toBe(name.split(".")[0]); + }); + + it("is recognized as part of the family", () => { + expect(isTsAutocodeError(build())).toBe(true); + expect(build()).toBeInstanceOf(TsAutocodeError); + expect(build()).toBeInstanceOf(Error); + }); + + it("has a usable stack", () => { + expect(typeof build().stack).toBe("string"); + }); +}); + +describe("brand-based family membership", () => { + it("rejects non-errors and foreign errors", () => { + for (const value of [undefined, null, 0, "", "err", {}, [], new Error("plain"), new TypeError("plain")]) { + expect(isTsAutocodeError(value)).toBe(false); + expect(value).not.toBeInstanceOf(TsAutocodeError); + } + }); + + it("keeps subclass instanceof exact rather than brand-wide", () => { + const engine = new EngineNotConfiguredError(); + expect(engine).toBeInstanceOf(EngineNotConfiguredError); + expect(engine).not.toBeInstanceOf(ExecutorNotConfiguredError); + expect(engine).not.toBeInstanceOf(LoopCapabilityError); + // And a TypeError-rooted member is not an instance of an Error-rooted one. + expect(new InvalidSettingsError("x")).not.toBeInstanceOf(EngineNotConfiguredError); + }); + + it("keeps the builtin prototypes the family grafts onto", () => { + expect(new InvalidTrainableIdentityError("x")).toBeInstanceOf(TypeError); + expect(new InvalidTrainableIdentityError("x")).toBeInstanceOf(TsAutocodeTypeError); + expect(new CandidateSyntaxError("x")).toBeInstanceOf(SyntaxError); + expect(new CandidateSyntaxError("x")).toBeInstanceOf(TsAutocodeSyntaxError); + // All three roots still answer to the family check. + for (const error of [new EngineNotConfiguredError(), new InvalidSettingsError("x"), new CandidateSyntaxError("x")]) { + expect(error).toBeInstanceOf(TsAutocodeError); + } + }); + + it("does not treat a hand-rolled look-alike as a member", () => { + class Impostor extends Error { readonly code = "engine_not_configured"; } + expect(isTsAutocodeError(new Impostor("nope"))).toBe(false); + }); +}); + +describe("error payloads", () => { + it("PromotionRejectedError exposes failures only when it has a decision", () => { + expect(new PromotionRejectedError("c1").failures).toEqual([]); + expect(new PromotionRejectedError("c1").decision).toBeUndefined(); + const decision = { candidateId: "c1", promote: false, failures: ["a", "b"], meanScore: 0.2, passRate: 0.5 }; + const rejected = new PromotionRejectedError("c1", decision); + expect(rejected.failures).toEqual(["a", "b"]); + expect(rejected.decision).toBe(decision); + expect(rejected.candidateId).toBe("c1"); + }); + + it("InsufficientTracesError pluralizes only when it should", () => { + expect(new InsufficientTracesError(1, 0).message).toContain("1 distinct successful runtime trace;"); + expect(new InsufficientTracesError(2, 0).message).toContain("2 distinct successful runtime traces;"); + expect(new InsufficientTracesError(3, 1)).toMatchObject({ required: 3, found: 1 }); + }); + + it("TrainingIncompleteError keeps the outcome it was built from", () => { + expect(TrainingIncompleteError.noRounds("stalled").outcome).toBe("stalled"); + expect(TrainingIncompleteError.noPromotableCandidate("exhausted").outcome).toBe("exhausted"); + }); + + it("carries the identifying detail each error was given", () => { + expect(new TraceNotFoundError("in").input).toBe("in"); + expect(new CandidateSyntaxError("T.m").trainableId).toBe("T.m"); + expect(new MissingSecretError("K", "m").secret).toBe("K"); + expect(new OperationInterruptedError("op").operation).toBe("op"); + }); + + it("supports a cause, so a wrapped failure is not lost", () => { + const cause = new Error("underlying"); + expect(new InvalidSettingsError("outer", { cause }).cause).toBe(cause); + }); +}); + +describe("parseSetting", () => { + const schema = z.number().int().positive("must be a positive integer"); + + it("returns the parsed value unchanged when valid", () => { + expect(parseSetting(schema, 3)).toBe(3); + }); + + it("raises the schema's own message as a library error, not a ZodError", () => { + let thrown: unknown; + try { + parseSetting(schema, -1); + } catch (error) { + thrown = error; + } + expect(isTsAutocodeError(thrown)).toBe(true); + expect(thrown).toBeInstanceOf(InvalidSettingsError); + expect((thrown as Error).message).toBe("must be a positive integer"); + // The underlying ZodError is preserved rather than discarded. + expect((thrown as Error).cause).toBeDefined(); + }); + + it("falls back to a generic message when the schema supplies no issue text", () => { + const empty = z.custom(() => false, { message: "" }); + expect(() => parseSetting(empty, 1)).toThrow(InvalidSettingsError); + }); + + it("transforms as the schema directs", () => { + expect(parseSetting(z.string().transform((value) => value.length), "abcd")).toBe(4); + }); +}); diff --git a/packages/training/test/optional.test.ts b/packages/training/test/optional.test.ts new file mode 100644 index 0000000..08cca4d --- /dev/null +++ b/packages/training/test/optional.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; + +import { defined, optional } from "../src/optional.js"; + +// `exactOptionalPropertyTypes` forbids assigning an explicit `undefined` to an +// optional property, so the distinction these helpers exist for is "key absent" +// versus "key present with value undefined". Asserting only deep equality would +// miss that entirely, since `{a: undefined}` and `{}` compare equal under +// toEqual — so these check key presence directly. + +describe("optional", () => { + it("includes the key when the value is defined", () => { + expect({ ...optional("signal", "abc") }).toEqual({ signal: "abc" }); + expect(Object.keys({ ...optional("signal", "abc") })).toEqual(["signal"]); + }); + + it("omits the key entirely when the value is undefined", () => { + const spread = { ...optional("signal", undefined) }; + expect(Object.keys(spread)).toEqual([]); + expect("signal" in spread).toBe(false); + }); + + it("keeps falsy-but-defined values, which is the whole point", () => { + for (const value of [0, "", false, Number.NaN, null] as const) { + const spread = { ...optional("v", value) }; + expect("v" in spread).toBe(true); + expect((spread as { v: unknown }).v).toBe(value === value ? value : (spread as { v: number }).v); + } + expect(Number.isNaN(({ ...optional("v", Number.NaN) } as { v: number }).v)).toBe(true); + }); + + it("preserves object identity rather than cloning", () => { + const signal = new AbortController().signal; + expect(({ ...optional("signal", signal) } as { signal: AbortSignal }).signal).toBe(signal); + }); + + it("composes so later spreads win, as object spread does", () => { + expect({ a: 1, ...optional("a", 2) }).toEqual({ a: 2 }); + expect({ a: 1, ...optional("a", undefined) }).toEqual({ a: 1 }); + }); +}); + +describe("defined", () => { + it("drops only the undefined entries", () => { + expect({ ...defined({ a: 1, b: undefined, c: "x" }) }).toEqual({ a: 1, c: "x" }); + }); + + it("keeps null and other falsy values", () => { + const spread = { ...defined({ a: null, b: 0, c: false, d: "", e: undefined }) }; + expect(Object.keys(spread).sort()).toEqual(["a", "b", "c", "d"]); + }); + + it("returns an empty object when everything is undefined", () => { + expect(Object.keys({ ...defined({ a: undefined, b: undefined }) })).toEqual([]); + }); + + it("handles an empty input", () => { + expect({ ...defined({}) }).toEqual({}); + }); + + it("does not mutate its input", () => { + const input = { a: 1, b: undefined }; + defined(input); + expect("b" in input).toBe(true); + }); + + it("copies own enumerable keys only", () => { + const base = Object.create({ inherited: "no" }) as Record; + base["own"] = "yes"; + expect({ ...defined(base) }).toEqual({ own: "yes" }); + }); +}); diff --git a/packages/training/test/token.test.ts b/packages/training/test/token.test.ts new file mode 100644 index 0000000..0e5a436 --- /dev/null +++ b/packages/training/test/token.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "vitest"; + +import { + defineTrainable, + toTrainableToken, + trainableIdFromKey, + trainableTokenFromSymbol, +} from "../src/token.js"; +import { InvalidTrainableIdentityError } from "../src/errors.js"; + +// The token is the durable join key binding a method to its captures, evals, +// candidate and promotion decision. Its normalization and rejection rules are +// the only thing standing between a typo and a silently different identity. + +describe("defineTrainable", () => { + it("produces a stable id and registry symbol", () => { + const first = defineTrainable("Router.route"); + const second = defineTrainable("Router.route"); + expect(first.id).toBe("Router.route"); + expect(first.symbol).toBe(second.symbol); + expect(Symbol.keyFor(first.symbol)).toBe("ts-autocode.trainable:Router.route"); + }); + + it("trims surrounding whitespace so equivalent spellings agree", () => { + expect(defineTrainable(" Router.route ").id).toBe("Router.route"); + expect(defineTrainable(" Router.route ").symbol).toBe(defineTrainable("Router.route").symbol); + }); + + it("freezes the token", () => { + expect(Object.isFrozen(defineTrainable("A.b"))).toBe(true); + }); + + it("rejects an empty or whitespace-only id", () => { + for (const value of ["", " ", "\t", "\n"]) { + expect(() => defineTrainable(value)).toThrow(InvalidTrainableIdentityError); + expect(() => defineTrainable(value)).toThrow("trainable id must be a non-empty string"); + } + }); + + it("distinguishes ids that differ only in case", () => { + expect(defineTrainable("Router.route").symbol).not.toBe(defineTrainable("router.route").symbol); + }); +}); + +describe("toTrainableToken", () => { + it("passes a token through unchanged", () => { + const token = defineTrainable("A.b"); + expect(toTrainableToken(token)).toBe(token); + }); + + it("resolves a registry symbol back to its token", () => { + const token = defineTrainable("A.b"); + expect(toTrainableToken(token.symbol).id).toBe("A.b"); + }); + + it("accepts a bare Symbol.for key without the library prefix", () => { + expect(toTrainableToken(Symbol.for("Custom.method")).id).toBe("Custom.method"); + }); + + it("rejects anything that is not a symbol or token", () => { + for (const value of ["Router.route", 42, null, undefined, {}, { id: 1 }, []]) { + expect(() => toTrainableToken(value as never)).toThrow(InvalidTrainableIdentityError); + expect(() => toTrainableToken(value as never)).toThrow("must be a symbol or TrainableToken"); + } + }); +}); + +describe("trainableTokenFromSymbol", () => { + it("uses the description when a symbol is not in the registry", () => { + expect(trainableTokenFromSymbol(Symbol("Described.method")).id).toBe("Described.method"); + }); + + it("rejects a symbol with neither key nor description", () => { + expect(() => trainableTokenFromSymbol(Symbol())).toThrow("must carry a registry key or description"); + }); + + it("rejects a symbol whose description is only whitespace", () => { + expect(() => trainableTokenFromSymbol(Symbol(" "))).toThrow("must carry a registry key or description"); + }); + + it("round-trips every token symbol", () => { + for (const id of ["A.b", "acme.route", "deeply.nested.name", "with-dash", "with_underscore"]) { + expect(trainableTokenFromSymbol(defineTrainable(id).symbol).id).toBe(id); + } + }); +}); + +describe("trainableIdFromKey", () => { + it("strips the library prefix", () => { + expect(trainableIdFromKey("ts-autocode.trainable:Router.route")).toBe("Router.route"); + }); + + it("leaves an unprefixed key alone", () => { + expect(trainableIdFromKey("Router.route")).toBe("Router.route"); + }); + + it("strips only the leading prefix, not a later occurrence", () => { + expect(trainableIdFromKey("ts-autocode.trainable:a:ts-autocode.trainable:b")) + .toBe("a:ts-autocode.trainable:b"); + }); + + it("handles an empty key", () => { + expect(trainableIdFromKey("")).toBe(""); + }); +}); diff --git a/test/attempt.test.ts b/test/attempt.test.ts new file mode 100644 index 0000000..e0ef3eb --- /dev/null +++ b/test/attempt.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; + +import { attempt, attemptAsync, errorMessage } from "../src/attempt.js"; + +// These were Effect-backed and are now plain try/catch. The fallback branch had +// zero coverage in the root copy, which is the branch that matters: it is the +// boundary that keeps a capture or serialization failure from breaking a +// traced application call. + +describe("errorMessage", () => { + it("uses an Error's message", () => { + expect(errorMessage(new Error("boom"))).toBe("boom"); + expect(errorMessage(new TypeError("typed"))).toBe("typed"); + }); + + it("stringifies anything else", () => { + expect(errorMessage("plain")).toBe("plain"); + expect(errorMessage(42)).toBe("42"); + expect(errorMessage(null)).toBe("null"); + expect(errorMessage(undefined)).toBe("undefined"); + expect(errorMessage({ toString: () => "custom" })).toBe("custom"); + }); + + it("keeps a subclass message", () => { + class Custom extends Error {} + expect(errorMessage(new Custom("sub"))).toBe("sub"); + }); +}); + +describe("attempt", () => { + it("returns the value when nothing throws", () => { + expect(attempt(() => 1, () => 2)).toBe(1); + }); + + it("returns the fallback when the body throws", () => { + expect(attempt(() => { throw new Error("x"); }, () => 2)).toBe(2); + }); + + it("hands the raw thrown value to the fallback, not a wrapper", () => { + const thrown = { code: "E" }; + expect(attempt(() => { throw thrown; }, (error) => error)).toBe(thrown); + }); + + it("propagates a throw from the fallback itself", () => { + expect(() => attempt(() => { throw new Error("first"); }, () => { throw new Error("second"); })) + .toThrow("second"); + }); + + it("preserves falsy return values rather than treating them as failure", () => { + expect(attempt(() => 0, () => 99)).toBe(0); + expect(attempt(() => undefined, () => 99)).toBeUndefined(); + }); + + it("runs synchronously", () => { + const order: string[] = []; + order.push("before"); + attempt(() => order.push("body"), () => order.push("fallback")); + order.push("after"); + expect(order).toEqual(["before", "body", "after"]); + }); +}); + +describe("attemptAsync", () => { + it("resolves the value when nothing rejects", async () => { + await expect(attemptAsync(async () => 1, () => 2)).resolves.toBe(1); + }); + + it("resolves the fallback when the promise rejects", async () => { + await expect(attemptAsync(async () => { throw new Error("x"); }, () => 2)).resolves.toBe(2); + }); + + it("resolves the fallback when the body throws synchronously", async () => { + await expect(attemptAsync(() => { throw new Error("sync"); }, () => 3)).resolves.toBe(3); + }); + + it("hands the raw rejection value to the fallback", async () => { + const thrown = { code: "E" }; + await expect(attemptAsync(async () => { throw thrown; }, (error) => error)).resolves.toBe(thrown); + }); + + it("rejects when the fallback throws", async () => { + await expect(attemptAsync(async () => { throw new Error("first"); }, () => { throw new Error("second"); })) + .rejects.toThrow("second"); + }); +}); diff --git a/test/cli.test.ts b/test/cli.test.ts index ba456bc..cc6d677 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -121,3 +121,76 @@ describe("ts-autocode argument handling", () => { expect(result.stderr).not.toContain(" at "); }); }); + +describe("ts-autocode status output paths", () => { + it("reports no trainables in table form when the project marks none", async () => { + await mkdir(directory, { recursive: true }); + const empty = join(directory, "nothing.ts"); + await writeFile(empty, "export const x = 1;\n", "utf8"); + const result = await run(["status", "--file", empty]); + expect(result.code).toBe(0); + expect(result.stdout).toContain("No trainables found."); + }); + + it("reports zero for a trainable that has no records at all", async () => { + const artifacts = join(directory, "empty-artifacts"); + await mkdir(artifacts, { recursive: true }); + await writeFile(join(artifacts, "records.json"), "[]", "utf8"); + const result = await run(["status", "--file", await project(), "--output-dir", artifacts]); + expect(result.stdout).toMatch(/Router\.route\s+0 successful \/ 0 captured/); + }); + + it("ignores a malformed records file rather than crashing", async () => { + const artifacts = join(directory, "bad-artifacts"); + await mkdir(artifacts, { recursive: true }); + await writeFile(join(artifacts, "records.json"), "{ not json", "utf8"); + const result = await run(["status", "--file", await project(), "--output-dir", artifacts]); + expect(result.code).toBe(0); + expect(result.stdout).toContain("0 successful / 0 captured"); + }); + + it("ignores a records file that is valid JSON but not an array", async () => { + const artifacts = join(directory, "object-artifacts"); + await mkdir(artifacts, { recursive: true }); + await writeFile(join(artifacts, "records.json"), '{"trainableId":"Router.route"}', "utf8"); + expect((await run(["status", "--file", await project(), "--output-dir", artifacts])).code).toBe(0); + }); + + it("counts records for trainables the project no longer declares", async () => { + const artifacts = join(directory, "stale-artifacts"); + await mkdir(artifacts, { recursive: true }); + await writeFile(join(artifacts, "records.json"), JSON.stringify([ + { trainableId: "Gone.method", succeeded: true }, + { trainableId: "Router.route", succeeded: true }, + ]), "utf8"); + const result = await run(["status", "--file", await project(), "--output-dir", artifacts, "--json"]); + // Only declared trainables are reported; a stale record is simply not shown. + expect(JSON.parse(result.stdout)).toEqual([ + { id: "Router.route", captured: 1, succeeded: 1 }, + { id: "Router.enrich", captured: 0, succeeded: 0 }, + ]); + }); +}); + +describe("ts-autocode option handling", () => { + it("accepts repeated --file", async () => { + await mkdir(directory, { recursive: true }); + const second = join(directory, "second.ts"); + await writeFile(second, 'class Other {\n\tgo(): string {\n\t\t"use training";\n\t\treturn "x";\n\t}\n}\n', "utf8"); + const result = await run(["discover", "--file", await project(), "--file", second, "--json"]); + const rows = JSON.parse(result.stdout) as Array<{ id: string }>; + expect(rows.map((row) => row.id)).toContain("Other.go"); + }); + + it("resolves --file relative to --cwd", async () => { + await project(); + const result = await run(["discover", "--cwd", directory, "--file", "router.ts", "--json"]); + expect((JSON.parse(result.stdout) as unknown[]).length).toBe(2); + }); + + it("reports paths relative to the working directory", async () => { + const result = await run(["discover", "--cwd", directory, "--file", "router.ts", "--json"]); + const rows = JSON.parse(result.stdout) as Array<{ location: string }>; + expect(rows[0]?.location).toBe("router.ts"); + }); +}); diff --git a/test/digest-protocol.test.ts b/test/digest-protocol.test.ts new file mode 100644 index 0000000..7fd2832 --- /dev/null +++ b/test/digest-protocol.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; + +import { digest as rewriteDigest } from "ts-autocode-rewrite"; +import { textDigest } from "ts-autocode-grounding"; + +import { digest as trainingDigest } from "../packages/training/src/digest.js"; + +// Body digests are the shared protocol between training and rewrite: guarded +// application refuses a candidate whose target body digest no longer matches, +// so the two packages must hash identical content to identical values. They are +// separate implementations by design (neither package imports the other), which +// is exactly why the agreement needs a test rather than an assumption. This +// lives at the root because it is the only place both packages are importable. + + +const values: readonly unknown[] = [ + "return input;", + "", + { a: 1, b: [2, { c: 3 }] }, + { b: [2, { c: 3 }], a: 1 }, + [1, 2, 3], + null, + 0, + true, + { nested: { deeply: { keys: "sorted" } } }, +]; + +describe("cross-package digest protocol", () => { + it.each(values.map((value, index) => [index, value] as const))( + "training and rewrite agree on value %i", + (_index, value) => { + expect(rewriteDigest(value)).toBe(trainingDigest(value)); + }, + ); + + it("both are insensitive to key order", () => { + expect(rewriteDigest({ a: 1, b: 2 })).toBe(rewriteDigest({ b: 2, a: 1 })); + expect(trainingDigest({ a: 1, b: 2 })).toBe(trainingDigest({ b: 2, a: 1 })); + }); + + it("grounding's text digest is deliberately a different function", () => { + // Same `sha256:` prefix, different algorithm -- swapping them silently + // changes every hash, which is why it was renamed `textDigest`. + expect(textDigest("a\nb")).not.toBe(rewriteDigest("a\nb")); + // And it normalizes line endings, which the value digest does not. + expect(textDigest("a\r\nb")).toBe(textDigest("a\nb")); + expect(rewriteDigest("a\r\nb")).not.toBe(rewriteDigest("a\nb")); + }); +}); diff --git a/test/docs.test.ts b/test/docs.test.ts index f58486d..931f842 100644 --- a/test/docs.test.ts +++ b/test/docs.test.ts @@ -59,10 +59,65 @@ const snippets = docs.flatMap((doc) => // consumer. const directory = join(repoRoot, "test", "output", "docs"); +/** Compiler options a consumer effectively gets, plus path mappings because the + * package cannot import itself by name from inside its own repo. Snippets stay + * written exactly as a consumer would write them. */ +const compilerOptions: ts.CompilerOptions = { + target: ts.ScriptTarget.ES2023, + module: ts.ModuleKind.NodeNext, + moduleResolution: ts.ModuleResolutionKind.NodeNext, + lib: ["lib.es2023.d.ts", "lib.esnext.decorators.d.ts"], + strict: true, + noUncheckedIndexedAccess: true, + exactOptionalPropertyTypes: true, + noEmit: true, + skipLibCheck: true, + types: ["node"], + baseUrl: repoRoot, + paths: { + "ts-autocode": ["src/index.ts"], + "ts-autocode/ax": ["src/providers/ax.ts"], + "ts-autocode/internal": ["src/internal.ts"], + "ts-autocode/grounding": ["src/grounding.ts"], + }, +}; + +function snippetPath(snippet: Snippet): string { + return join(directory, `snippet-${snippet.doc.replace(/\W/g, "_")}-${snippet.line}.ts`); +} + +function describeDiagnostic(diagnostic: ts.Diagnostic): string { + const line = diagnostic.file && diagnostic.start !== undefined + ? diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start).line + 1 + : 0; + return `line ${line}: ${ts.flattenDiagnosticMessageText(diagnostic.messageText, " ")}`; +} + +/** Errors per snippet file. One `ts.Program` covers every snippet: building 26 + * of them re-parsed the whole dependency graph each time, which took ~36s + * normally and blew the default 5s per-test timeout under coverage + * instrumentation. Each snippet is still its own module, so a stray + * declaration in one cannot satisfy another. */ +const diagnosticsBySnippet = new Map(); + beforeAll(async () => { await rm(directory, { recursive: true, force: true }); await mkdir(directory, { recursive: true }); -}); + const files = await Promise.all(snippets.map(async (snippet) => { + const file = snippetPath(snippet); + await writeFile(file, snippet.code, "utf8"); + return file; + })); + const program = ts.createProgram(files, compilerOptions); + for (const file of files) diagnosticsBySnippet.set(file, []); + for (const diagnostic of ts.getPreEmitDiagnostics(program)) { + const name = diagnostic.file?.fileName; + if (name === undefined) continue; + const key = files.find((file) => file.replace(/\\/g, "/") === name); + if (key === undefined) continue; + diagnosticsBySnippet.set(key, [...(diagnosticsBySnippet.get(key) ?? []), describeDiagnostic(diagnostic)]); + } +}, 180_000); afterAll(async () => { await rm(directory, { recursive: true, force: true }); @@ -73,43 +128,14 @@ describe("documentation snippets", () => { expect(snippets.length).toBeGreaterThan(5); }); + it("compiles every snippet it found", () => { + expect(diagnosticsBySnippet.size).toBe(snippets.length); + }); + it.each(snippets.map((snippet) => [`${snippet.doc}:${snippet.line}`, snippet] as const))( "%s compiles", - async (_label, snippet) => { - // Snippets are top-level-await narratives, so compile each as its own - // module resolving `ts-autocode` through the repo's real node_modules. - const file = join(directory, `snippet-${snippet.doc.replace(/\W/g, "_")}-${snippet.line}.ts`); - await writeFile(file, snippet.code, "utf8"); - const program = ts.createProgram([file], { - target: ts.ScriptTarget.ES2023, - module: ts.ModuleKind.NodeNext, - moduleResolution: ts.ModuleResolutionKind.NodeNext, - lib: ["lib.es2023.d.ts", "lib.esnext.decorators.d.ts"], - strict: true, - noUncheckedIndexedAccess: true, - exactOptionalPropertyTypes: true, - noEmit: true, - skipLibCheck: true, - types: ["node"], - baseUrl: repoRoot, - // The package cannot import itself by name from inside its own - // repo, so resolve its public entries to their sources — the - // snippets stay written exactly as a consumer would write them. - paths: { - "ts-autocode": ["src/index.ts"], - "ts-autocode/ax": ["src/providers/ax.ts"], - "ts-autocode/grounding": ["src/grounding.ts"], - }, - }); - const errors = ts.getPreEmitDiagnostics(program) - .filter((diagnostic) => diagnostic.file?.fileName === file.replace(/\\/g, "/")) - .map((diagnostic) => { - const position = diagnostic.file && diagnostic.start !== undefined - ? diagnostic.file.getLineAndCharacterOfPosition(diagnostic.start).line + 1 - : 0; - return `line ${position}: ${ts.flattenDiagnosticMessageText(diagnostic.messageText, " ")}`; - }); - expect(errors).toEqual([]); + (_label, snippet) => { + expect(diagnosticsBySnippet.get(snippetPath(snippet))).toEqual([]); }, ); }); diff --git a/vitest.config.ts b/vitest.config.ts index 4c75c18..fea8656 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -20,5 +20,23 @@ export default defineConfig({ }, test: { include: ["test/**/*.test.ts", "packages/grounding/test/**/*.test.ts", "packages/harness/test/**/*.test.ts", "packages/rewrite/test/**/*.test.ts", "packages/training/test/**/*.test.ts"], + coverage: { + provider: "v8", + include: ["src/**/*.ts", "packages/*/src/**/*.ts"], + // `cli-main.ts` is a four-line bin wrapper around `run()`, which is + // tested directly; `register.ts` installs a module load hook at + // import time, so importing it to measure it would install one. + // Both are covered by behavior elsewhere, not by line counting. + exclude: ["src/cli-main.ts", "src/register.ts"], + reporter: ["text", "json-summary", "lcov"], + reportsDirectory: "test/output/coverage", + // Ratchets: raise these as suites land, never lower them to get green. + thresholds: { + statements: 90, + branches: 78, + functions: 93, + lines: 93, + }, + }, }, }); From 039e937340138a4ba0ad84832ee140821b4f3ea2 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 18:53:22 +0000 Subject: [PATCH 12/14] test: add Verify-style characterization snapshots The workspace had no snapshot tests at all. For a library whose product is rewritten source, the generated text *is* the product, and a diff of it is the only review that shows what actually changed -- `toContain("return input")` says almost nothing about an emitted module. test/support/verify.ts follows the Verify model rather than Vitest's inline snapshots: one named file per subject under test/snapshots/, committed and reviewed like any other artifact. An inline .snap blob keyed by test name is hard to read in a diff and a rename silently orphans it. `scrub()` removes digests, UUIDs, timestamps and absolute paths, because a snapshot that churns is one everyone learns to re-approve without reading. Approved: discovered source targets (the whole contract handed to an optimizer), emitted instrumentation and the augmented module, the synthetic candidate declaration for sync and async targets, applied rewrites, grounding codegen, promotion decisions, CLI usage/discover/status, the export surface of all seven entry points with each export's kind, the error message catalogue, the promotion rubric read by the judge, and the Ax program signature. The last two are the ones nothing else could pin: both are read by a model rather than by code, and neither has a natural assertion. The rubric is where the literal string "evaluation default" once shipped in place of a threshold. Verified the snapshots actually fail on a real change rather than passing vacuously -- which also surfaced that the root suite runs against siblings' built dist/, so a sibling source edit needs a rebuild before it is visible. Boy-scout fix, found by the Ax snapshot: a parameter with a literal default and no annotation (`retries = 2`) was reported as type `unknown`, which the Ax field mapper turned into `json` -- so the optimizer was told a plainly numeric argument had an opaque shape. Source discovery now infers string, number, boolean, bigint and uniform array types from a literal initializer, exactly where TypeScript would. Anything non-literal stays `unknown`. The snapshot now shows `{"name": "number"}` where it showed `{"name": "json"}`. 402 -> 443 tests; branches 79 -> 80.29%. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- docs/testing.md | 28 +++ packages/training/src/source.ts | 33 +++- packages/training/test/source.test.ts | 43 ++++ test/characterization-messages.test.ts | 60 ++++++ test/characterization-prompts.test.ts | 124 ++++++++++++ test/characterization-surface.test.ts | 48 +++++ test/characterization.test.ts | 186 ++++++++++++++++++ test/snapshots/cli/discover.verified.txt | 12 ++ test/snapshots/cli/status.verified.txt | 3 + test/snapshots/cli/usage.verified.txt | 13 ++ .../candidate-declaration-async.verified.ts | 3 + .../engine/candidate-declaration.verified.ts | 3 + test/snapshots/errors/catalogue.verified.txt | 19 ++ .../declared-registrations.verified.ts | 48 +++++ .../promotion/accepted-decision.verified.json | 7 + .../promotion/rejected-decision.verified.json | 12 ++ .../ax-program-signature.verified.json | 50 +++++ .../prompts/rubric-configured.verified.txt | 1 + .../prompts/rubric-defaults.verified.txt | 2 + .../rewrite/applied-candidate.verified.ts | 16 ++ .../rewrite/augmented-module.verified.ts | 22 +++ .../emitted-instrumentation.verified.ts | 4 + .../rewrite/register-hook-output.verified.ts | 22 +++ .../source/discovered-decorated.verified.json | 31 +++ .../source/discovered-targets.verified.json | 76 +++++++ .../surface/root-export-kinds.verified.json | 85 ++++++++ .../surface/ts-autocode-ax.verified.txt | 3 + .../ts-autocode-grounding.verified.txt | 28 +++ .../surface/ts-autocode-harness.verified.txt | 11 ++ .../surface/ts-autocode-internal.verified.txt | 24 +++ .../surface/ts-autocode-rewrite.verified.txt | 17 ++ .../surface/ts-autocode-training.verified.txt | 56 ++++++ .../surface/ts-autocode.verified.txt | 83 ++++++++ test/support/verify.ts | 73 +++++++ tsconfig.test.json | 5 +- 35 files changed, 1249 insertions(+), 2 deletions(-) create mode 100644 test/characterization-messages.test.ts create mode 100644 test/characterization-prompts.test.ts create mode 100644 test/characterization-surface.test.ts create mode 100644 test/characterization.test.ts create mode 100644 test/snapshots/cli/discover.verified.txt create mode 100644 test/snapshots/cli/status.verified.txt create mode 100644 test/snapshots/cli/usage.verified.txt create mode 100644 test/snapshots/engine/candidate-declaration-async.verified.ts create mode 100644 test/snapshots/engine/candidate-declaration.verified.ts create mode 100644 test/snapshots/errors/catalogue.verified.txt create mode 100644 test/snapshots/grounding/declared-registrations.verified.ts create mode 100644 test/snapshots/promotion/accepted-decision.verified.json create mode 100644 test/snapshots/promotion/rejected-decision.verified.json create mode 100644 test/snapshots/prompts/ax-program-signature.verified.json create mode 100644 test/snapshots/prompts/rubric-configured.verified.txt create mode 100644 test/snapshots/prompts/rubric-defaults.verified.txt create mode 100644 test/snapshots/rewrite/applied-candidate.verified.ts create mode 100644 test/snapshots/rewrite/augmented-module.verified.ts create mode 100644 test/snapshots/rewrite/emitted-instrumentation.verified.ts create mode 100644 test/snapshots/rewrite/register-hook-output.verified.ts create mode 100644 test/snapshots/source/discovered-decorated.verified.json create mode 100644 test/snapshots/source/discovered-targets.verified.json create mode 100644 test/snapshots/surface/root-export-kinds.verified.json create mode 100644 test/snapshots/surface/ts-autocode-ax.verified.txt create mode 100644 test/snapshots/surface/ts-autocode-grounding.verified.txt create mode 100644 test/snapshots/surface/ts-autocode-harness.verified.txt create mode 100644 test/snapshots/surface/ts-autocode-internal.verified.txt create mode 100644 test/snapshots/surface/ts-autocode-rewrite.verified.txt create mode 100644 test/snapshots/surface/ts-autocode-training.verified.txt create mode 100644 test/snapshots/surface/ts-autocode.verified.txt create mode 100644 test/support/verify.ts diff --git a/docs/testing.md b/docs/testing.md index 8c9cbd6..5e1d32b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -12,6 +12,7 @@ land, never lower them to get a build green. | Surface | `test/surface.test.ts` | Re-export drift between the root package and its siblings | | Protocol | `test/digest-protocol.test.ts` | Two packages that must agree without importing each other | | Compatibility | `test/deprecated.test.ts` | A deprecated spelling that silently stopped working | +| Characterization | `test/characterization*.test.ts` | A change to anything this library *generates* — rewritten source, emitted instrumentation, prompts, CLI output, the export surface | ## Running @@ -24,6 +25,33 @@ npm run check # typecheck + coverage + build Coverage reports land in `test/output/coverage` (git-ignored). `lcov.info` is there for editor and CI integrations. +## Characterization snapshots + +For a library whose product is rewritten source, the generated text *is* the +product, and a diff of it is the only review that shows what actually changed. +`toContain("return input")` says almost nothing about an emitted module. + +`test/support/verify.ts` follows the [Verify](https://github.com/VerifyTests/Verify) +model rather than Vitest's inline snapshots: one named file per subject under +`test/snapshots/`, committed and reviewed like any other artifact. An inline +`.snap` blob keyed by test name is hard to read in a diff, and a test rename +silently orphans it. + +``` +test/snapshots/rewrite/emitted-instrumentation.verified.ts +test/snapshots/prompts/ax-program-signature.verified.json +test/snapshots/surface/ts-autocode.txt +``` + +Approve a deliberate change with `npm test -- -u`, and **read the diff** — that +is the entire value. `scrub()` removes digests, UUIDs, timestamps and absolute +paths first, because a snapshot that churns is one everyone learns to +re-approve without reading. + +Snapshots are excluded from `tsconfig.test.json`: they are generated artifacts, +and the emitted instrumentation deliberately references names from the module it +is appended to, so it does not typecheck standalone. + ## Conventions - **A test names the defect it prevents.** Where a test exists because diff --git a/packages/training/src/source.ts b/packages/training/src/source.ts index 15e7d48..89ab984 100644 --- a/packages/training/src/source.ts +++ b/packages/training/src/source.ts @@ -86,6 +86,37 @@ export function discoverInSource(source: string, artifactRef = inMemoryArtifactR return discoverSourceFile(sourceFile, artifactRef); } +/** A parameter's declared type, or one inferred from a literal default. + * + * `retries = 2` has no type annotation, and reporting it as `unknown` reached + * the default engine's field mapper as `json` -- so the optimizer was told a + * plainly numeric argument had an opaque shape. TypeScript infers these from + * the initializer and so can we, for the literal forms that cover almost every + * real default. Anything else stays `unknown`, as before. */ +function parameterType(parameter: ts.ParameterDeclaration, sourceFile: ts.SourceFile): string { + const declared = parameter.type?.getText(sourceFile); + if (declared !== undefined) return declared; + return inferredLiteralType(parameter.initializer) ?? "unknown"; +} + +function inferredLiteralType(initializer: ts.Expression | undefined): string | undefined { + if (initializer === undefined) return undefined; + if (ts.isStringLiteralLike(initializer)) return "string"; + if (ts.isNumericLiteral(initializer)) return "number"; + if (initializer.kind === ts.SyntaxKind.TrueKeyword || initializer.kind === ts.SyntaxKind.FalseKeyword) return "boolean"; + if (ts.isBigIntLiteral(initializer)) return "bigint"; + // `-1` and friends: a negated numeric literal is still a number. + if (ts.isPrefixUnaryExpression(initializer) && ts.isNumericLiteral(initializer.operand)) { + return inferredLiteralType(initializer.operand); + } + if (ts.isArrayLiteralExpression(initializer)) { + const elements = initializer.elements.map((element) => inferredLiteralType(element)); + const first = elements[0]; + return first !== undefined && elements.every((element) => element === first) ? `${first}[]` : undefined; + } + return undefined; +} + function discoverSourceFile( sourceFile: ts.SourceFile, artifactRef: string, @@ -138,7 +169,7 @@ function targetFor( const parameters = node.parameters.map((parameter, index): TrainableParameter => ({ name: ts.isIdentifier(parameter.name) ? parameter.name.text : `arg${index}`, declaration: parameter.getText(sourceFile), - type: parameter.type?.getText(sourceFile) ?? "unknown", + type: parameterType(parameter, sourceFile), optional: parameter.questionToken !== undefined || parameter.initializer !== undefined, })); const returnType = node.type?.getText(sourceFile) ?? "unknown"; diff --git a/packages/training/test/source.test.ts b/packages/training/test/source.test.ts index 07fd7e9..bb96d2e 100644 --- a/packages/training/test/source.test.ts +++ b/packages/training/test/source.test.ts @@ -71,3 +71,46 @@ class Router { expect(discoverTrainables({ files: [`${output}/router.ts`] })[0]?.id).toBe("custom.route"); }); }); + +describe("parameter types inferred from literal defaults", () => { + // A defaulted parameter has no type annotation, and reporting it as + // `unknown` reached the Ax field mapper as `json` -- so the optimizer was + // told a plainly numeric argument had an opaque shape. Found by the + // characterization snapshot of the generated program signature. + const declare = (parameters: string) => discoverInSource(`class Fixture { + method(${parameters}): void { + "use training"; + } +}`, "fixture.ts")[0]?.parameters ?? []; + + it.each([ + ["retries = 2", "number"], + ["name = \"x\"", "string"], + ["flag = true", "boolean"], + ["flag = false", "boolean"], + ["offset = -1", "number"], + ["big = 1n", "bigint"], + ["tags = [\"a\", \"b\"]", "string[]"], + ["counts = [1, 2]", "number[]"], + ])("infers %s as %s", (declaration, expected) => { + expect(declare(declaration)[0]?.type).toBe(expected); + }); + + it("prefers an explicit annotation over the initializer", () => { + expect(declare("retries: 1 | 2 = 2")[0]?.type).toBe("1 | 2"); + }); + + it("leaves a non-literal default unknown rather than guessing", () => { + for (const declaration of ["value = compute()", "value = {}", "value = []", "mixed = [1, \"a\"]"]) { + expect(declare(declaration)[0]?.type).toBe("unknown"); + } + }); + + it("still marks a defaulted parameter optional", () => { + expect(declare("retries = 2")[0]?.optional).toBe(true); + }); + + it("leaves an undefaulted, unannotated parameter unknown", () => { + expect(declare("value")[0]?.type).toBe("unknown"); + }); +}); diff --git a/test/characterization-messages.test.ts b/test/characterization-messages.test.ts new file mode 100644 index 0000000..7812f5a --- /dev/null +++ b/test/characterization-messages.test.ts @@ -0,0 +1,60 @@ +import { describe, it } from "vitest"; + +import { + CandidateSyntaxError, + EngineContractError, + EngineNotConfiguredError, + EngineProposalError, + ExecutorNotConfiguredError, + InsufficientTracesError, + InvalidSettingsError, + InvalidTrainableIdentityError, + LoopCapabilityError, + MissingSecretError, + OperationInterruptedError, + OperationTimeoutError, + PromotionApplierNotConfiguredError, + PromotionRejectedError, + SourceDiscoveryError, + TraceNotFoundError, + TrainingIncompleteError, +} from "../src/index.js"; +import { verify } from "./support/verify.js"; + +// Error text is the library's most-read documentation: it is what a stuck user +// sees. It is also load-bearing here, because the typed hierarchy promised to +// preserve every message byte for byte so existing catch blocks and substring +// assertions keep working. An approved catalogue makes any drift reviewable, +// and reads as a table of contents for what can go wrong. + +const catalogue: ReadonlyArray = [ + new EngineNotConfiguredError(), + new ExecutorNotConfiguredError(), + new PromotionApplierNotConfiguredError(), + new PromotionRejectedError("cand-1"), + new InsufficientTracesError(1, 0), + new InsufficientTracesError(3, 2), + new TraceNotFoundError("Where is my invoice?"), + new CandidateSyntaxError("Router.route"), + new EngineContractError("engine returned an empty implementation"), + new EngineProposalError("Ax did not optimize Router.route"), + new MissingSecretError("OPENAI_API_KEY", "default optimizer requires OPENAI_API_KEY or a custom TrainingSettings.engine"), + new LoopCapabilityError("the governed harness loop reviews one candidate per round"), + new InvalidTrainableIdentityError("trainable id must be a non-empty string"), + new SourceDiscoveryError("trainable source was not found: Router.route"), + new OperationInterruptedError("engine.propose"), + new InvalidSettingsError("minScore must be between 0 and 1"), + new OperationTimeoutError({ operation: "engine.propose", timeoutMs: 30_000 }), + TrainingIncompleteError.noRounds("stalled"), + TrainingIncompleteError.noPromotableCandidate("exhausted"), +]; + +describe("error message catalogue", () => { + it("reads as the user sees it", async () => { + const lines = catalogue.map((error) => { + const code = (error as Error & { code?: string }).code ?? "-"; + return `${error.name.padEnd(36)} ${String(code).padEnd(26)} ${error.message}`; + }); + await verify("errors/catalogue.txt", `${lines.join("\n")}\n`); + }); +}); diff --git a/test/characterization-prompts.test.ts b/test/characterization-prompts.test.ts new file mode 100644 index 0000000..934c121 --- /dev/null +++ b/test/characterization-prompts.test.ts @@ -0,0 +1,124 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { createTrainingRuntime, defineTrainable, discoverInSource } from "../src/index.js"; +import type { TrainingLoopInput } from "../src/index.js"; +import { verify, verifyJson } from "./support/verify.js"; + +const mocks = vi.hoisted(() => ({ ax: vi.fn(), optimize: vi.fn(), ai: vi.fn(), forward: vi.fn(), apply: vi.fn() })); + +vi.mock("@ax-llm/ax", async (importOriginal) => ({ + ...await importOriginal(), + ai: mocks.ai, + ax: mocks.ax, + optimize: mocks.optimize, +})); + +// Two artifacts nothing else pins, both of which are read by a model rather +// than by code: +// +// - the promotion rubric handed to the harness judge, which once shipped the +// literal string "evaluation default" in place of the real threshold; +// - the Ax program signature derived from the TypeScript method, which is +// the prompt an optimizer actually receives. +// +// Neither has a natural assertion -- they are prose and field descriptors -- +// so an approved file is the only review that shows a change to them. + +const directory = "test/output/prompts"; +const source = `class Router { + route(input: string, retries = 2, tags?: string[]): string { + "use training"; + return input; + } +} +`; + +async function fixture(): Promise { + await mkdir(directory, { recursive: true }); + const file = join(directory, "router.ts"); + await writeFile(file, source, "utf8"); + return file; +} + +/** Runs a train() far enough to capture what the loop is handed, then stops. */ +async function captureLoopInput(input: Record = {}): Promise { + const artifact = await fixture(); + let captured: TrainingLoopInput | undefined; + const training = createTrainingRuntime({ + engine: { id: "characterization", optimize: async () => ({ implementation: "return input;" }) }, + executor: async () => "x", + source: { files: [artifact] }, + tracing: { enabled: false }, + loop: async (loopInput) => { + captured = loopInput; + return { outcome: "exhausted", rounds: [] }; + }, + }); + await training.train({ + trainable: defineTrainable("Router.route").symbol, + evaluation: { + tests: [{ id: "a", input: "a", assert: [{ type: "equals", value: "a" }] }], + task: (value) => value, + outputDir: `${directory}/agentv`, + }, + ...input, + }).catch(() => undefined); + if (captured === undefined) throw new Error("loop was never invoked"); + return captured; +} + +describe("promotion rubric handed to the judge", () => { + it("names resolved thresholds at the defaults", async () => { + const { rubric, objective } = await captureLoopInput(); + await verify("prompts/rubric-defaults.txt", `objective: ${objective}\nrubric: ${rubric}\n`); + }); + + it("reflects configured thresholds and a policy", async () => { + const { rubric } = await captureLoopInput({ + promotion: { minScore: 0.95, minPassRate: 0.5 }, + policy: () => true, + }); + await verify("prompts/rubric-configured.txt", `${rubric}\n`); + }); + + it("never contains a placeholder where a number belongs", async () => { + const { rubric } = await captureLoopInput(); + expect(rubric).not.toContain("evaluation default"); + expect(rubric).toMatch(/Minimum evaluation score: [\d.]+\./); + }); +}); + +describe("Ax program derived from the method signature", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.stubEnv("OPENAI_API_KEY", "test-key"); + mocks.ax.mockReturnValue({ applyOptimization: mocks.apply, forward: mocks.forward }); + mocks.optimize.mockResolvedValue({ optimizedProgram: { optimizerType: "t", converged: true, totalRounds: 1 }, bestScore: 1 }); + mocks.forward.mockResolvedValue({ optimizedMethodImplementation: "return input;" }); + }); + + afterEach(() => vi.unstubAllEnvs()); + + it("turns parameters into named, typed, described input fields", async () => { + const { createAxEngine } = await import("../src/providers/ax.js"); + const target = discoverInSource(source, "router.ts")[0]!; + await createAxEngine().optimize({ + trainableId: target.id, + objective: "Preserve routing", + target, + records: [], + evaluations: [{ + trainableId: target.id, + test: { id: "a", input: '["hello",1,["x"]]', assert: [{ type: "equals", value: "hello" }] }, + result: { input: [{ role: "user", content: '["hello",1,["x"]]' }], output: "hello", executionStatus: "ok" } as never, + }], + constraints: ["Do not call the network."], + }, { variables: {} }); + // This object is the prompt. Its description and field types decide what + // the model is asked for. + await verifyJson("prompts/ax-program-signature", mocks.ax.mock.calls[0]?.[0]); + }); +}); diff --git a/test/characterization-surface.test.ts b/test/characterization-surface.test.ts new file mode 100644 index 0000000..d8f1115 --- /dev/null +++ b/test/characterization-surface.test.ts @@ -0,0 +1,48 @@ +import { describe, it } from "vitest"; + +import * as root from "../src/index.js"; +import * as internal from "../src/internal.js"; +import * as ax from "../src/providers/ax.js"; +import * as grounding from "../src/grounding.js"; +import * as harness from "ts-autocode-harness"; +import * as rewrite from "ts-autocode-rewrite"; +import * as training from "ts-autocode-training"; +import { verify, verifyJson } from "./support/verify.js"; + +// The public surface, captured as an approved file. An export added or removed +// shows up as a diff in the pull request that does it, which is the only place +// anyone will notice. `test/surface.test.ts` proves the root re-exports its +// siblings exhaustively; this says what that set actually *is*. + +const entries: ReadonlyArray = [ + ["ts-autocode", root], + ["ts-autocode-internal", internal], + ["ts-autocode-ax", ax], + ["ts-autocode-grounding", grounding], + ["ts-autocode-harness", harness], + ["ts-autocode-rewrite", rewrite], + ["ts-autocode-training", training], +]; + +describe("public export surface", () => { + it.each(entries)("%s exports", async (name, module) => { + await verify(`surface/${name}.txt`, + `${Object.keys(module).filter((key) => key !== "default").sort().join("\n")}\n`); + }); + + it("records what each export is, so a value silently becoming a type is visible", async () => { + await verifyJson("surface/root-export-kinds", Object.fromEntries( + Object.keys(root).filter((key) => key !== "default").sort() + .map((key) => [key, kindOf((root as Record)[key])]), + )); + }); +}); + +function kindOf(value: unknown): string { + if (typeof value === "function") { + return /^class\s/.test(Function.prototype.toString.call(value)) ? "class" : "function"; + } + if (Array.isArray(value)) return "array"; + if (value === null) return "null"; + return typeof value; +} diff --git a/test/characterization.test.ts b/test/characterization.test.ts new file mode 100644 index 0000000..dd724eb --- /dev/null +++ b/test/characterization.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it } from "vitest"; + +import { + applyCandidate, + candidateDeclaration, + commitRewrite, + createRewriter, + discoverInSource, + emitInstrumentation, + evaluatePromotionGate, + revertRewrite, + type CandidatePatch, +} from "../src/index.js"; +import { generateDeclaredRegistrations, scanDeclaredTrainables } from "ts-autocode-grounding"; +import { augmentSource } from "../src/register/hook.js"; +import { run, usage } from "../src/cli.js"; +import { verify, verifyJson } from "./support/verify.js"; + +// Characterization tests over everything this library *generates*. For a +// library whose product is rewritten source, the generated text is the product +// -- and a diff of it is the only review that shows what actually changed. +// Assertions like `toContain("return input")` say almost nothing about the +// emitted module; an approved file says all of it. +// +// These are deliberately not correctness tests. They pin current behavior so +// that an unintended change to it shows up in a pull request diff. + +const routerSource = `class Router { + route(input: string): string { + "use training"; + return input.includes("invoice") ? "billing" : "fallback"; + } + + async enrich(id: string, deep?: boolean): Promise { + "use training"; + return \`\${id}:\${deep}\`; + } +} + +export function normalize(input: string): string { + "use training"; + return input.trim(); +} +`; + +const target = discoverInSource(routerSource, "router.ts")[0]!; + +function candidate(implementation: string): CandidatePatch { + return { id: "cand-1", trainableId: target.id, engineId: "characterization", target, implementation }; +} + +describe("discovered source targets", () => { + it("records the signature, parameters and offsets an engine will see", async () => { + // This is the entire contract handed to an optimizer. A change to any + // field silently changes every generated prompt. + await verifyJson("source/discovered-targets", discoverInSource(routerSource, "router.ts")); + }); + + it("describes a decorated class the same way", async () => { + const decorated = `import { defineTrainable, trainable } from "ts-autocode"; + +const route = defineTrainable("acme.route"); + +class Service { + @trainable(route.symbol) + handle(input: string, retries = 2): string { + return input; + } +} +`; + await verifyJson("source/discovered-decorated", discoverInSource(decorated, "service.ts")); + }); +}); + +describe("emitted instrumentation", () => { + it("appends a registration statement that binds no names", async () => { + await verify("rewrite/emitted-instrumentation.ts", emitInstrumentation([ + { id: "Router.route", methodName: "route", className: "Router" }, + { id: "normalize", methodName: "normalize" }, + ])); + }); + + it("rewrites a whole module append-only, leaving original lines untouched", async () => { + const rewriter = createRewriter((source, path) => + discoverInSource(source, path).map((found) => ({ + id: found.id, + methodName: found.methodName, + ...(found.className === undefined ? {} : { className: found.className }), + })), "use training"); + await verify("rewrite/augmented-module.ts", rewriter(routerSource, "router.ts")); + }); + + it("produces the same module through the register hook", async () => { + await verify("rewrite/register-hook-output.ts", augmentSource(routerSource, "router.ts")); + }); + + it("leaves an unmarked module byte-identical", () => { + const plain = "export const value = 1;\n"; + expect(augmentSource(plain, "plain.ts")).toBe(plain); + }); +}); + +describe("candidate application", () => { + it("wraps a proposed body in the synthetic declaration an executor runs", async () => { + await verify("engine/candidate-declaration.ts", + candidateDeclaration(target, "\treturn input.toUpperCase();")); + }); + + it("preserves the async signature for an async target", async () => { + const asyncTarget = discoverInSource(routerSource, "router.ts")[1]!; + await verify("engine/candidate-declaration-async.ts", + candidateDeclaration(asyncTarget, "\treturn `${id}`;")); + }); + + it("rewrites only the marked body, keeping the directive and indentation", async () => { + await verify("rewrite/applied-candidate.ts", + applyCandidate(routerSource, candidate("\t\treturn input.toUpperCase();"))); + }); + + it("round-trips exactly through revert", () => { + const committed = commitRewrite(routerSource, candidate("\t\treturn input.toUpperCase();")); + expect(revertRewrite(committed.source, committed.snapshot)).toBe(routerSource); + }); +}); + +describe("grounding codegen", () => { + it("emits registration source for an ambient declaration", async () => { + const [declared] = scanDeclaredTrainables(`@trainable +export declare class Program { + @intent("Produce a greeting") + @returns("Hello World! or Hello, !") + greet( + @description("Optional person to greet") + name?: string, + ): string; + + other(count: number): number; +} +`); + await verify("grounding/declared-registrations.ts", + generateDeclaredRegistrations(declared!)); + }); +}); + +describe("promotion decisions", () => { + it("reports the standard gate failures a rejected candidate collects", async () => { + const decision = await evaluatePromotionGate({ + candidate: candidate("return input;"), + evaluations: [], + conformance: false, + }); + await verifyJson("promotion/rejected-decision", decision); + }); + + it("reports a passing decision", async () => { + const decision = await evaluatePromotionGate({ + candidate: candidate("return input;"), + evaluations: [{ + trainableId: target.id, + candidateId: "cand-1", + result: { testId: "a", score: 1, executionStatus: "ok", output: "x" } as never, + }], + conformance: true, + }); + await verifyJson("promotion/accepted-decision", decision); + }); +}); + +describe("command line output", () => { + it("prints usage", async () => { + await verify("cli/usage.txt", usage); + }); + + it("prints the discover table a user copies identities from", async () => { + const { mkdir, writeFile } = await import("node:fs/promises"); + await mkdir("test/output/characterization", { recursive: true }); + await writeFile("test/output/characterization/router.ts", routerSource, "utf8"); + const result = await run(["discover", "--cwd", "test/output/characterization", "--file", "router.ts"]); + await verify("cli/discover.txt", result.stdout); + }); + + it("prints the status table", async () => { + const result = await run(["status", "--cwd", "test/output/characterization", "--file", "router.ts"]); + await verify("cli/status.txt", result.stdout); + }); +}); diff --git a/test/snapshots/cli/discover.verified.txt b/test/snapshots/cli/discover.verified.txt new file mode 100644 index 0000000..498949e --- /dev/null +++ b/test/snapshots/cli/discover.verified.txt @@ -0,0 +1,12 @@ +Router.route router.ts + route(input: string): string +Router.enrich router.ts + enrich(id: string, deep?: boolean): Promise +normalize router.ts + normalize(input: string): string + +3 trainables. + +Bind evals to one with its symbol: + const target = defineTrainable("Router.route"); + await training.train({ trainable: target.symbol, /* ... */ }); diff --git a/test/snapshots/cli/status.verified.txt b/test/snapshots/cli/status.verified.txt new file mode 100644 index 0000000..9e75717 --- /dev/null +++ b/test/snapshots/cli/status.verified.txt @@ -0,0 +1,3 @@ +Router.route 0 successful / 0 captured +Router.enrich 0 successful / 0 captured +normalize 0 successful / 0 captured diff --git a/test/snapshots/cli/usage.verified.txt b/test/snapshots/cli/usage.verified.txt new file mode 100644 index 0000000..56c82c6 --- /dev/null +++ b/test/snapshots/cli/usage.verified.txt @@ -0,0 +1,13 @@ +ts-autocode [options] + +Commands: + discover List every trainable the TypeScript project marks. + status Show captured traces per trainable from a run's artifacts. + help Show this message. + +Options: + --cwd Project root (default: the working directory). + --project tsconfig to read (default: tsconfig.json). + --file Scan only these files; repeatable. + --output-dir Where run artifacts live (default: .agentv). + --json Emit JSON instead of a table. diff --git a/test/snapshots/engine/candidate-declaration-async.verified.ts b/test/snapshots/engine/candidate-declaration-async.verified.ts new file mode 100644 index 0000000..50276a1 --- /dev/null +++ b/test/snapshots/engine/candidate-declaration-async.verified.ts @@ -0,0 +1,3 @@ +async function candidate(id: string, deep?: boolean): Promise { + return `${id}`; +} diff --git a/test/snapshots/engine/candidate-declaration.verified.ts b/test/snapshots/engine/candidate-declaration.verified.ts new file mode 100644 index 0000000..da680e2 --- /dev/null +++ b/test/snapshots/engine/candidate-declaration.verified.ts @@ -0,0 +1,3 @@ +function candidate(input: string): string { + return input.toUpperCase(); +} diff --git a/test/snapshots/errors/catalogue.verified.txt b/test/snapshots/errors/catalogue.verified.txt new file mode 100644 index 0000000..24d41a7 --- /dev/null +++ b/test/snapshots/errors/catalogue.verified.txt @@ -0,0 +1,19 @@ +EngineNotConfiguredError engine_not_configured no training engine is configured; import "ts-autocode" for the Ax default or set TrainingSettings.engine +ExecutorNotConfiguredError executor_not_configured candidate execution requires an executor; import "ts-autocode" or set TrainingSettings.executor +PromotionApplierNotConfiguredError applier_not_configured activation requires a promotion applier; import "ts-autocode" for the default or set TrainingProviders.promote +PromotionRejectedError promotion_rejected candidate has not passed the promotion gate: cand-1 +InsufficientTracesError insufficient_traces training from captured traffic requires 1 distinct successful runtime trace; found 0 +InsufficientTracesError insufficient_traces training from captured traffic requires 3 distinct successful runtime traces; found 2 +TraceNotFoundError trace_not_found live trace was not found for eval input: Where is my invoice? +CandidateSyntaxError candidate_syntax engine returned invalid TypeScript for Router.route +EngineContractError engine_contract engine returned an empty implementation +EngineProposalError engine_proposal Ax did not optimize Router.route +MissingSecretError missing_secret default optimizer requires OPENAI_API_KEY or a custom TrainingSettings.engine +LoopCapabilityError loop_capability the governed harness loop reviews one candidate per round +InvalidTrainableIdentityError invalid_identity trainable id must be a non-empty string +SourceDiscoveryError source_discovery trainable source was not found: Router.route +OperationInterruptedError operation_interrupted engine.propose was interrupted +InvalidSettingsError invalid_settings minScore must be between 0 and 1 +OperationTimeout - engine.propose timed out after 30000ms +TrainingIncompleteError training_incomplete training loop returned no rounds: stalled +TrainingIncompleteError training_incomplete background training did not produce a promotable candidate: exhausted diff --git a/test/snapshots/grounding/declared-registrations.verified.ts b/test/snapshots/grounding/declared-registrations.verified.ts new file mode 100644 index 0000000..3636dea --- /dev/null +++ b/test/snapshots/grounding/declared-registrations.verified.ts @@ -0,0 +1,48 @@ +// Generated from an ambient trainable declaration — do not edit by hand. +import { defineGrounding } from "ts-autocode/grounding"; + +export const greet = defineGrounding({ + "methodRef": "Program.greet", + "intent": "Produce a greeting", + "contract": { + "ref": "decl://Program.greet", + "input": { + "name": { + "type": "string", + "optional": true, + "description": "Optional person to greet" + } + }, + "output": { + "type": "string", + "description": "Hello World! or Hello, !" + } + }, + "params": { + "name": { + "description": "Optional person to greet" + } + }, + "output": { + "returns": { + "description": "Hello World! or Hello, !" + } + } +}); + +export const other = defineGrounding({ + "methodRef": "Program.other", + "intent": "Inferred: implement Program.other to satisfy its declared signature and descriptions.", + "contract": { + "ref": "decl://Program.other", + "input": { + "count": { + "type": "number", + "optional": false + } + }, + "output": { + "type": "number" + } + } +}); diff --git a/test/snapshots/promotion/accepted-decision.verified.json b/test/snapshots/promotion/accepted-decision.verified.json new file mode 100644 index 0000000..338f4b3 --- /dev/null +++ b/test/snapshots/promotion/accepted-decision.verified.json @@ -0,0 +1,7 @@ +{ + "candidateId": "cand-1", + "failures": [], + "meanScore": 1, + "passRate": 1, + "promote": true +} diff --git a/test/snapshots/promotion/rejected-decision.verified.json b/test/snapshots/promotion/rejected-decision.verified.json new file mode 100644 index 0000000..b86e8cc --- /dev/null +++ b/test/snapshots/promotion/rejected-decision.verified.json @@ -0,0 +1,12 @@ +{ + "candidateId": "cand-1", + "failures": [ + "conformance failed", + "candidate-specific AgentV evaluations are required", + "mean AgentV score 0 is below 0.8", + "AgentV pass rate 0 is below 1" + ], + "meanScore": 0, + "passRate": 0, + "promote": false +} diff --git a/test/snapshots/prompts/ax-program-signature.verified.json b/test/snapshots/prompts/ax-program-signature.verified.json new file mode 100644 index 0000000..d3b8337 --- /dev/null +++ b/test/snapshots/prompts/ax-program-signature.verified.json @@ -0,0 +1,50 @@ +{ + "description": "Rewrite only the TypeScript body of route(input: string, retries = 2, tags?: string[]): string.\nPreserve routing\nDo not call the network.\nReturn the method body without braces or markdown fences.", + "inputs": [ + { + "description": "input: string", + "name": "methodArgumentInput", + "type": { + "name": "string" + } + }, + { + "description": "retries = 2", + "isOptional": true, + "name": "methodArgumentRetries", + "type": { + "name": "number" + } + }, + { + "description": "tags?: string[]", + "isOptional": true, + "name": "methodArgumentTags", + "type": { + "isArray": true, + "name": "string" + } + }, + { + "name": "trainingObjective", + "type": { + "name": "string" + } + }, + { + "name": "currentMethodImplementation", + "type": { + "name": "code" + } + } + ], + "outputs": [ + { + "description": "A complete replacement body for route(input: string, retries = 2, tags?: string[]): string", + "name": "optimizedMethodImplementation", + "type": { + "name": "code" + } + } + ] +} diff --git a/test/snapshots/prompts/rubric-configured.verified.txt b/test/snapshots/prompts/rubric-configured.verified.txt new file mode 100644 index 0000000..da96264 --- /dev/null +++ b/test/snapshots/prompts/rubric-configured.verified.txt @@ -0,0 +1 @@ +Candidate must pass source conformance checks. Minimum evaluation score: 0.95. Minimum evaluation pass rate: 0.5. Candidate must pass the configured promotion policy. diff --git a/test/snapshots/prompts/rubric-defaults.verified.txt b/test/snapshots/prompts/rubric-defaults.verified.txt new file mode 100644 index 0000000..b11ad4d --- /dev/null +++ b/test/snapshots/prompts/rubric-defaults.verified.txt @@ -0,0 +1,2 @@ +objective: Preserve behavior demonstrated by the evaluation cases +rubric: Candidate must pass source conformance checks. Minimum evaluation score: 0.8. Minimum evaluation pass rate: 1. No additional promotion policy. diff --git a/test/snapshots/rewrite/applied-candidate.verified.ts b/test/snapshots/rewrite/applied-candidate.verified.ts new file mode 100644 index 0000000..6e597c0 --- /dev/null +++ b/test/snapshots/rewrite/applied-candidate.verified.ts @@ -0,0 +1,16 @@ +class Router { + route(input: string): string { + "use training"; + return input.toUpperCase(); + } + + async enrich(id: string, deep?: boolean): Promise { + "use training"; + return `${id}:${deep}`; + } +} + +export function normalize(input: string): string { + "use training"; + return input.trim(); +} diff --git a/test/snapshots/rewrite/augmented-module.verified.ts b/test/snapshots/rewrite/augmented-module.verified.ts new file mode 100644 index 0000000..20ac6af --- /dev/null +++ b/test/snapshots/rewrite/augmented-module.verified.ts @@ -0,0 +1,22 @@ +class Router { + route(input: string): string { + "use training"; + return input.includes("invoice") ? "billing" : "fallback"; + } + + async enrich(id: string, deep?: boolean): Promise { + "use training"; + return `${id}:${deep}`; + } +} + +export function normalize(input: string): string { + "use training"; + return input.trim(); +} + +;globalThis[Symbol.for("ts-autocode.instrument")]?.([ + { id: "Router.route", name: "route", owner: () => Router }, + { id: "Router.enrich", name: "enrich", owner: () => Router }, + { id: "normalize", get: () => normalize, set: __fn => { normalize = __fn; } } +]); diff --git a/test/snapshots/rewrite/emitted-instrumentation.verified.ts b/test/snapshots/rewrite/emitted-instrumentation.verified.ts new file mode 100644 index 0000000..08891bf --- /dev/null +++ b/test/snapshots/rewrite/emitted-instrumentation.verified.ts @@ -0,0 +1,4 @@ +globalThis[Symbol.for("ts-autocode.instrument")]?.([ + { id: "Router.route", name: "route", owner: () => Router }, + { id: "normalize", get: () => normalize, set: __fn => { normalize = __fn; } } +]); diff --git a/test/snapshots/rewrite/register-hook-output.verified.ts b/test/snapshots/rewrite/register-hook-output.verified.ts new file mode 100644 index 0000000..20ac6af --- /dev/null +++ b/test/snapshots/rewrite/register-hook-output.verified.ts @@ -0,0 +1,22 @@ +class Router { + route(input: string): string { + "use training"; + return input.includes("invoice") ? "billing" : "fallback"; + } + + async enrich(id: string, deep?: boolean): Promise { + "use training"; + return `${id}:${deep}`; + } +} + +export function normalize(input: string): string { + "use training"; + return input.trim(); +} + +;globalThis[Symbol.for("ts-autocode.instrument")]?.([ + { id: "Router.route", name: "route", owner: () => Router }, + { id: "Router.enrich", name: "enrich", owner: () => Router }, + { id: "normalize", get: () => normalize, set: __fn => { normalize = __fn; } } +]); diff --git a/test/snapshots/source/discovered-decorated.verified.json b/test/snapshots/source/discovered-decorated.verified.json new file mode 100644 index 0000000..d2a0ac5 --- /dev/null +++ b/test/snapshots/source/discovered-decorated.verified.json @@ -0,0 +1,31 @@ +[ + { + "artifactRef": "service.ts", + "async": false, + "bodyDigest": "sha256:", + "bodyEnd": 210, + "bodyStart": 192, + "className": "Service", + "exported": false, + "id": "acme.route", + "implementation": "return input;", + "indentation": "\t", + "methodName": "handle", + "parameters": [ + { + "declaration": "input: string", + "name": "input", + "optional": false, + "type": "string" + }, + { + "declaration": "retries = 2", + "name": "retries", + "optional": true, + "type": "number" + } + ], + "returnType": "string", + "signature": "handle(input: string, retries = 2): string" + } +] diff --git a/test/snapshots/source/discovered-targets.verified.json b/test/snapshots/source/discovered-targets.verified.json new file mode 100644 index 0000000..0ddf802 --- /dev/null +++ b/test/snapshots/source/discovered-targets.verified.json @@ -0,0 +1,76 @@ +[ + { + "artifactRef": "router.ts", + "async": false, + "bodyDigest": "sha256:", + "bodyEnd": 127, + "bodyStart": 64, + "className": "Router", + "exported": false, + "id": "Router.route", + "implementation": "return input.includes(\"invoice\") ? \"billing\" : \"fallback\";", + "indentation": "\t", + "methodName": "route", + "parameters": [ + { + "declaration": "input: string", + "name": "input", + "optional": false, + "type": "string" + } + ], + "returnType": "string", + "signature": "route(input: string): string" + }, + { + "artifactRef": "router.ts", + "async": true, + "bodyDigest": "sha256:", + "bodyEnd": 236, + "bodyStart": 208, + "className": "Router", + "exported": false, + "id": "Router.enrich", + "implementation": "return `${id}:${deep}`;", + "indentation": "\t", + "methodName": "enrich", + "parameters": [ + { + "declaration": "id: string", + "name": "id", + "optional": false, + "type": "string" + }, + { + "declaration": "deep?: boolean", + "name": "deep", + "optional": true, + "type": "boolean" + } + ], + "returnType": "Promise", + "signature": "enrich(id: string, deep?: boolean): Promise" + }, + { + "artifactRef": "router.ts", + "async": false, + "bodyDigest": "sha256:", + "bodyEnd": 331, + "bodyStart": 308, + "exported": true, + "id": "normalize", + "implementation": "return input.trim();", + "indentation": "", + "methodName": "normalize", + "parameters": [ + { + "declaration": "input: string", + "name": "input", + "optional": false, + "type": "string" + } + ], + "returnType": "string", + "signature": "normalize(input: string): string" + } +] diff --git a/test/snapshots/surface/root-export-kinds.verified.json b/test/snapshots/surface/root-export-kinds.verified.json new file mode 100644 index 0000000..c21a05c --- /dev/null +++ b/test/snapshots/surface/root-export-kinds.verified.json @@ -0,0 +1,85 @@ +{ + "CandidateSyntaxError": "class", + "EngineContractError": "class", + "EngineNotConfiguredError": "class", + "EngineProposalError": "class", + "ExecutorNotConfiguredError": "class", + "InsufficientTracesError": "class", + "InvalidSettingsError": "class", + "InvalidTrainableIdentityError": "class", + "LoopCapabilityError": "class", + "MemoryTrainingStore": "class", + "MissingSecretError": "class", + "OperationInterruptedError": "class", + "OperationTimeoutError": "class", + "PromotionApplierNotConfiguredError": "class", + "PromotionRejectedError": "class", + "SourceDiscoveryError": "class", + "TraceNotFoundError": "class", + "TrainingIncompleteError": "class", + "TsAutocodeError": "class", + "TsAutocodeSyntaxError": "class", + "TsAutocodeTypeError": "class", + "annotateRewrite": "function", + "applyCandidate": "function", + "candidateDeclaration": "function", + "captureTrainable": "function", + "check": "function", + "commitRewrite": "function", + "configureRewrite": "function", + "configureRewriteCapture": "function", + "configureTraining": "function", + "createCandidateReview": "function", + "createEvalRun": "function", + "createHarnessLoop": "function", + "createPromotionDecision": "function", + "createRewriter": "function", + "createTrainingRuntime": "function", + "declaringContainer": "function", + "defaultActionLogDir": "string", + "defaultContextWindow": "number", + "defaultEvolution": "object", + "defaultFanOut": "number", + "defaultHarnessRounds": "number", + "defaultMaxRounds": "number", + "defaultMinPassRate": "number", + "defaultMinScore": "number", + "defaultObjective": "string", + "defaultOutputDir": "string", + "defaultPromotionGates": "array", + "defaultRetry": "object", + "defaultTsconfig": "string", + "defineTrainable": "function", + "defined": "function", + "digest": "function", + "discoverInSource": "function", + "discoverTrainables": "function", + "dispatchRewrite": "function", + "emitInstrumentation": "function", + "evaluatePromotionGate": "function", + "evaluationArgs": "function", + "inMemoryArtifactRef": "string", + "installInstrumentation": "function", + "installedInstrumentation": "function", + "instrumentKey": "string", + "instrumentTrainable": "function", + "isTsAutocodeError": "function", + "optional": "function", + "parseSetting": "function", + "provideTrainingDefaults": "function", + "resetTraining": "function", + "restoreImplementation": "function", + "revertRewrite": "function", + "rewritePromotion": "function", + "sequentialLoop": "function", + "swapImplementation": "function", + "swappedImplementation": "function", + "trainable": "function", + "trainableTokenFromSymbol": "function", + "training": "object", + "trainingMarker": "string", + "trainingRounds": "function", + "windowedContext": "function", + "withPolicy": "function", + "wrapTrainable": "function" +} diff --git a/test/snapshots/surface/ts-autocode-ax.verified.txt b/test/snapshots/surface/ts-autocode-ax.verified.txt new file mode 100644 index 0000000..963b637 --- /dev/null +++ b/test/snapshots/surface/ts-autocode-ax.verified.txt @@ -0,0 +1,3 @@ +apiKeyNamesFor +createAxEngine +defaultExecutionTimeoutMs diff --git a/test/snapshots/surface/ts-autocode-grounding.verified.txt b/test/snapshots/surface/ts-autocode-grounding.verified.txt new file mode 100644 index 0000000..4981e5a --- /dev/null +++ b/test/snapshots/surface/ts-autocode-grounding.verified.txt @@ -0,0 +1,28 @@ +COMPONENT_METADATA +PENDING_GROUNDINGS +REGISTERED_METHODS +camelCase +componentMetadata +componentMetadataOf +composeOptions +createComponentDecorator +defineGrounding +description +digest +finalizeTrainableClass +generateDeclaredRegistrations +granularOptionsFor +inferredIntent +intent +normalizePath +normalizeText +param +pascalCase +pendingGroundings +registeredMethods +returns +scanDeclaredTrainables +stableStringify +textDigest +toStableValue +union diff --git a/test/snapshots/surface/ts-autocode-harness.verified.txt b/test/snapshots/surface/ts-autocode-harness.verified.txt new file mode 100644 index 0000000..6a9bb24 --- /dev/null +++ b/test/snapshots/surface/ts-autocode-harness.verified.txt @@ -0,0 +1,11 @@ +AgentActionDeniedError +HarnessSandbox +WriteAheadAgentBus +agentBusEntry +agentMessage +createSandboxPolicy +defaultHarnessRounds +defaultMaxRounds +defineTrainingHarness +dispatchAction +inferringHarness diff --git a/test/snapshots/surface/ts-autocode-internal.verified.txt b/test/snapshots/surface/ts-autocode-internal.verified.txt new file mode 100644 index 0000000..2e73e85 --- /dev/null +++ b/test/snapshots/surface/ts-autocode-internal.verified.txt @@ -0,0 +1,24 @@ +annotateRewrite +applyCandidate +candidateDeclaration +captureTrainable +commitRewrite +configureRewrite +configureRewriteCapture +createRewriter +declaringContainer +digest +dispatchRewrite +emitInstrumentation +installInstrumentation +installedInstrumentation +instrumentKey +instrumentTrainable +provideTrainingDefaults +restoreImplementation +revertRewrite +rewritePromotion +swapImplementation +swappedImplementation +withPolicy +wrapTrainable diff --git a/test/snapshots/surface/ts-autocode-rewrite.verified.txt b/test/snapshots/surface/ts-autocode-rewrite.verified.txt new file mode 100644 index 0000000..3d7c0b2 --- /dev/null +++ b/test/snapshots/surface/ts-autocode-rewrite.verified.txt @@ -0,0 +1,17 @@ +annotateRewrite +applyCandidate +check +commitRewrite +configureRewrite +createRewriter +declaringContainer +digest +dispatchRewrite +emitInstrumentation +installInstrumentation +installedInstrumentation +instrumentKey +restoreImplementation +revertRewrite +swapImplementation +swappedImplementation diff --git a/test/snapshots/surface/ts-autocode-training.verified.txt b/test/snapshots/surface/ts-autocode-training.verified.txt new file mode 100644 index 0000000..78b96be --- /dev/null +++ b/test/snapshots/surface/ts-autocode-training.verified.txt @@ -0,0 +1,56 @@ +CandidateSyntaxError +EngineContractError +EngineNotConfiguredError +EngineProposalError +ExecutorNotConfiguredError +InsufficientTracesError +InvalidSettingsError +InvalidTrainableIdentityError +LoopCapabilityError +MemoryTrainingStore +MissingSecretError +OperationInterruptedError +OperationTimeoutError +PromotionApplierNotConfiguredError +PromotionRejectedError +SourceDiscoveryError +TraceNotFoundError +TrainingIncompleteError +TsAutocodeError +TsAutocodeSyntaxError +TsAutocodeTypeError +candidateDeclaration +captureTrainable +configureTraining +createCandidateReview +createEvalRun +createPromotionDecision +createTrainingRuntime +defaultEvolution +defaultFanOut +defaultMaxRounds +defaultMinPassRate +defaultMinScore +defaultObjective +defaultOutputDir +defaultPromotionGates +defaultRetry +defaultTsconfig +defineTrainable +defined +discoverInSource +discoverTrainables +evaluatePromotionGate +evaluationArgs +inMemoryArtifactRef +isTsAutocodeError +optional +parseSetting +provideTrainingDefaults +resetTraining +sequentialLoop +trainableTokenFromSymbol +training +trainingMarker +trainingRounds +withPolicy diff --git a/test/snapshots/surface/ts-autocode.verified.txt b/test/snapshots/surface/ts-autocode.verified.txt new file mode 100644 index 0000000..8730f78 --- /dev/null +++ b/test/snapshots/surface/ts-autocode.verified.txt @@ -0,0 +1,83 @@ +CandidateSyntaxError +EngineContractError +EngineNotConfiguredError +EngineProposalError +ExecutorNotConfiguredError +InsufficientTracesError +InvalidSettingsError +InvalidTrainableIdentityError +LoopCapabilityError +MemoryTrainingStore +MissingSecretError +OperationInterruptedError +OperationTimeoutError +PromotionApplierNotConfiguredError +PromotionRejectedError +SourceDiscoveryError +TraceNotFoundError +TrainingIncompleteError +TsAutocodeError +TsAutocodeSyntaxError +TsAutocodeTypeError +annotateRewrite +applyCandidate +candidateDeclaration +captureTrainable +check +commitRewrite +configureRewrite +configureRewriteCapture +configureTraining +createCandidateReview +createEvalRun +createHarnessLoop +createPromotionDecision +createRewriter +createTrainingRuntime +declaringContainer +defaultActionLogDir +defaultContextWindow +defaultEvolution +defaultFanOut +defaultHarnessRounds +defaultMaxRounds +defaultMinPassRate +defaultMinScore +defaultObjective +defaultOutputDir +defaultPromotionGates +defaultRetry +defaultTsconfig +defineTrainable +defined +digest +discoverInSource +discoverTrainables +dispatchRewrite +emitInstrumentation +evaluatePromotionGate +evaluationArgs +inMemoryArtifactRef +installInstrumentation +installedInstrumentation +instrumentKey +instrumentTrainable +isTsAutocodeError +optional +parseSetting +provideTrainingDefaults +resetTraining +restoreImplementation +revertRewrite +rewritePromotion +sequentialLoop +swapImplementation +swappedImplementation +trainable +trainableTokenFromSymbol +training +trainingMarker +trainingRounds +windowedContext +withPolicy +wrapTrainable diff --git a/test/support/verify.ts b/test/support/verify.ts new file mode 100644 index 0000000..730aa6f --- /dev/null +++ b/test/support/verify.ts @@ -0,0 +1,73 @@ +import { fileURLToPath } from "node:url"; + +import { expect } from "vitest"; + +// A Verify-style characterization helper (https://github.com/VerifyTests/Verify). +// +// Vitest's inline `toMatchSnapshot` hides the approved value inside a `.snap` +// blob keyed by test name, which makes a diff hard to read and a rename silently +// orphan the snapshot. Verify's model is better for characterizing generated +// text: one named file per subject, committed and reviewed like any other +// artifact, so a diff in a pull request shows exactly what the generator's +// output became. +// +// `test/snapshots/.verified.` is the approved value. A mismatch fails +// and Vitest writes the received value beside it; `npm test -- -u` approves. + +const snapshotDirectory = new URL("../snapshots/", import.meta.url); + +/** Values that would make a snapshot churn between machines or runs. */ +export interface Scrubbers { + /** Absolute paths to replace with a stable placeholder. */ + readonly paths?: Readonly>; + /** Extra replacements applied in order. */ + readonly replace?: ReadonlyArray; +} + +/** Removes machine- and run-specific detail so a snapshot means the same thing + * everywhere. A snapshot that churns is a snapshot everyone learns to re-approve + * without reading, which defeats the point. */ +export function scrub(value: string, scrubbers: Scrubbers = {}): string { + let text = value; + for (const [absolute, placeholder] of Object.entries(scrubbers.paths ?? {})) { + text = text.split(absolute).join(placeholder); + } + text = text + .replace(/sha256:[0-9a-f]{64}/g, "sha256:") + .replace(/\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/g, "") + .replace(/\d{4}-\d{2}-\d{2}T[\d:.]+Z/g, "") + .replace(/\r\n/g, "\n"); + for (const [pattern, replacement] of scrubbers.replace ?? []) { + text = text.replace(pattern, replacement); + } + return text.trimEnd() + "\n"; +} + +/** Compare `value` against the approved snapshot named `name`. */ +export async function verify(name: string, value: string, scrubbers?: Scrubbers): Promise { + await expect(scrub(value, scrubbers)).toMatchFileSnapshot(fileFor(name)); +} + +/** Approved-snapshot path for a subject name. `a/b.ts` keeps its extension so + * editors syntax-highlight the snapshot. */ +export function fileFor(name: string): string { + const dot = name.lastIndexOf("."); + const stem = dot > 0 ? name.slice(0, dot) : name; + const extension = dot > 0 ? name.slice(dot) : ".txt"; + return fileURLToPath(new URL(`${stem}.verified${extension}`, snapshotDirectory)); +} + +/** Characterize a JSON-serializable value with stable key ordering. */ +export async function verifyJson(name: string, value: unknown, scrubbers?: Scrubbers): Promise { + await verify(`${name}.json`, `${JSON.stringify(sortKeys(value), null, 2)}\n`, scrubbers); +} + +function sortKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortKeys); + if (typeof value !== "object" || value === null) return value; + if (Object.getPrototypeOf(value) !== Object.prototype) return value; + return Object.fromEntries( + Object.keys(value as Record).sort() + .map((key) => [key, sortKeys((value as Record)[key])]), + ); +} diff --git a/tsconfig.test.json b/tsconfig.test.json index f1ce650..e7658b0 100644 --- a/tsconfig.test.json +++ b/tsconfig.test.json @@ -16,5 +16,8 @@ "sourceMap": false }, "include": ["src", "test", "examples", "vitest.config.ts"], - "exclude": ["test/output", "test/.agentv", "examples/output"] + // Approved snapshots are generated artifacts, not sources. The emitted + // instrumentation deliberately references names from the module it is + // appended to, so it does not typecheck standalone -- and must not be asked to. + "exclude": ["test/output", "test/snapshots", "test/.agentv", "examples/output"] } From d89997cadb26dce6ab35aa76ff063f94e0468296 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 19:03:50 +0000 Subject: [PATCH 13/14] test: add property-based and fuzz suites The workspace had neither. Properties state the law and let fast-check hunt for a counterexample; fuzzing feeds the parsers input they were not written for. Both found real defects on the first run. Properties cover identity round-trips, digest canonicalization (load-bearing: guarded rewriting refuses a candidate whose body digest changed), the spread helpers, evaluation-argument decoding, and promotion-gate aggregation. Fuzzing covers source discovery, the register load hook, ambient declaration scanning, the evolve kill switch, and the CLI. Four findings, all fixed here: 1. `toTrainableToken` was not exported from either barrel. It takes the public `TrainableIdentity` type and is the canonical validator, so anyone implementing a loop, engine or store needs it. Now exported; the surface snapshot shows the single added line. 2. `minScore: Infinity` reported "expected number, received number" -- Zod's base schema rejected it before the .finite() message could apply, so a user who passed a bad threshold was told nothing useful. Every rejection now reports the range. 3. Discovery could report `bodyEnd` past the end of the source. TypeScript's error recovery synthesizes a body for an unterminated block whose `end` sits past EOF, so a truncated file produced a target claiming offsets outside its own source. Slicing clamps, so nothing was corrupted, but publishing an out-of-range range is malformed data crossing a public boundary. Now clamped -- a no-op for source that parses. 4. `TrainableTarget`'s body fields had undocumented and subtly different relationships to the source: `implementation` is trimmed, `bodyDigest` hashes the raw slice, and guarded application depends on the digest side. Nothing said so until a property asked. Now documented and pinned. The fuzz corpus is itself tested. An early version used random punctuation; instrumenting it showed 1 input in 3000 produced a discovered target, so every property about offsets and rewriting was passing vacuously. test/support/sources.ts generates structurally plausible marked modules and then damages them, and the suite asserts the corpus still reaches real work. One property documents a limitation rather than a bug: `-0` cannot round-trip through an eval input, because JSON.stringify(-0) is "0". 443 -> 500 tests. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- docs/testing.md | 22 +- packages/training/src/index.ts | 2 +- packages/training/src/promotion.ts | 11 +- packages/training/src/source.ts | 20 +- packages/training/test/promotion.test.ts | 52 ++++ packages/training/test/source.test.ts | 28 ++ src/index.ts | 1 + test/fuzz.test.ts | 234 +++++++++++++++ test/property.test.ts | 281 ++++++++++++++++++ .../surface/root-export-kinds.verified.json | 1 + .../surface/ts-autocode-training.verified.txt | 1 + .../surface/ts-autocode.verified.txt | 1 + test/support/sources.ts | Bin 0 -> 4765 bytes 13 files changed, 648 insertions(+), 6 deletions(-) create mode 100644 test/fuzz.test.ts create mode 100644 test/property.test.ts create mode 100644 test/support/sources.ts diff --git a/docs/testing.md b/docs/testing.md index 5e1d32b..4ab4b7c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -12,6 +12,8 @@ land, never lower them to get a build green. | Surface | `test/surface.test.ts` | Re-export drift between the root package and its siblings | | Protocol | `test/digest-protocol.test.ts` | Two packages that must agree without importing each other | | Compatibility | `test/deprecated.test.ts` | A deprecated spelling that silently stopped working | +| Property | `test/property.test.ts` | A law that holds for chosen examples but not in general | +| Fuzz | `test/fuzz.test.ts` | A parser crashing, hanging, or corrupting source it did not write | | Characterization | `test/characterization*.test.ts` | A change to anything this library *generates* — rewritten source, emitted instrumentation, prompts, CLI output, the export surface | ## Running @@ -52,7 +54,25 @@ Snapshots are excluded from `tsconfig.test.json`: they are generated artifacts, and the emitted instrumentation deliberately references names from the module it is appended to, so it does not typecheck standalone. -## Conventions +## Property and fuzz tests + +`fast-check` states the law and hunts for a counterexample, rather than sampling +a few inputs by hand. Failures print a shrunk counterexample and a seed, so a +regression is reproducible rather than "it failed once on CI". + +Properties target the pure, total functions where examples can only sample: +identity round-trips, digest canonicalization, gate aggregation, and the spread +helpers. Fuzzing targets the parsers, because every one of them runs against +code this library did not write — `augmentSource` sees every module a user +loads. + +**A fuzz corpus must reach the code.** An early version used random punctuation; +instrumenting it showed **1 input in 3000** produced a discovered target, so +every property about offsets and rewriting was passing vacuously. +`test/support/sources.ts` now generates structurally plausible marked modules +and then damages them, and `test/fuzz.test.ts` asserts the corpus still reaches +real work — so the suite cannot quietly decay back into theatre. + - **A test names the defect it prevents.** Where a test exists because something was once wrong, the comment says what was wrong. That is what makes diff --git a/packages/training/src/index.ts b/packages/training/src/index.ts index 8233529..a966c84 100644 --- a/packages/training/src/index.ts +++ b/packages/training/src/index.ts @@ -72,7 +72,7 @@ export type { TrainingRound, } from "./loop.js"; -export { defineTrainable, trainableTokenFromSymbol } from "./token.js"; +export { defineTrainable, toTrainableToken, trainableTokenFromSymbol } from "./token.js"; export type { TrainableId, TrainableIdentity, TrainableToken } from "./token.js"; export { defaultTsconfig, discoverInSource, discoverTrainables, inMemoryArtifactRef, trainingMarker } from "./source.js"; diff --git a/packages/training/src/promotion.ts b/packages/training/src/promotion.ts index 42e637b..d2100ec 100644 --- a/packages/training/src/promotion.ts +++ b/packages/training/src/promotion.ts @@ -4,8 +4,15 @@ import { z } from "zod"; import type { BoundEvaluation, CandidatePatch } from "./engine.js"; import { parseSetting } from "./errors.js"; -const unitInterval = (name: string) => - z.number().finite(`${name} must be between 0 and 1`).min(0, `${name} must be between 0 and 1`).max(1, `${name} must be between 0 and 1`); +/** A threshold in [0, 1]. Every rejection -- wrong type, NaN, Infinity, or + * merely out of range -- reports the same message, because a user who passed a + * bad threshold wants to know the range, not Zod's type vocabulary. Without the + * base-schema message, `minScore: Infinity` reported "expected number, + * received number", which says nothing useful. */ +const unitInterval = (name: string) => { + const message = `${name} must be between 0 and 1`; + return z.number({ error: message }).finite(message).min(0, message).max(1, message); +}; const minScoreThreshold = unitInterval("minScore"); const minPassRateThreshold = unitInterval("minPassRate"); diff --git a/packages/training/src/source.ts b/packages/training/src/source.ts index 89ab984..344dc0a 100644 --- a/packages/training/src/source.ts +++ b/packages/training/src/source.ts @@ -44,9 +44,19 @@ export interface TrainableTarget { readonly parameters: readonly TrainableParameter[]; readonly returnType: string; readonly async: boolean; + /** Offset of the first character after the directive (or after the body's + * opening brace when the marker is a decorator). Guarded rewriting replaces + * exactly `[bodyStart, bodyEnd)`. */ readonly bodyStart: number; + /** Offset of the body's closing brace. */ readonly bodyEnd: number; + /** Digest of the **raw** `[bodyStart, bodyEnd)` slice, whitespace included. + * This is what guarded application re-computes to decide whether the file + * changed since discovery, so it must not be derived from + * {@link TrainableTarget.implementation}, which is trimmed. */ readonly bodyDigest: string; + /** The body as an engine should read it: the same slice, **trimmed**. The + * untrimmed slice is recoverable as `source.slice(bodyStart, bodyEnd)`. */ readonly implementation: string; readonly indentation: string; } @@ -162,8 +172,14 @@ function targetFor( if (node.asteriskToken) throw new InvalidTrainableIdentityError(`generator methods cannot be trainable: ${artifactRef}`); const body = node.body as ts.Block; const directive = firstDirective(body); - const bodyStart = directive?.end ?? body.getStart(sourceFile) + 1; - const bodyEnd = body.end - 1; + // TypeScript's error recovery synthesizes a body for an unterminated block, + // whose `end` can sit past EOF -- so a truncated file yielded a target + // claiming offsets outside its own source. Slicing clamps, so nothing was + // corrupted, but publishing an out-of-range range is malformed data crossing + // a public boundary. Clamp it: a no-op for source that parses. + const limit = sourceFile.text.length; + const bodyStart = Math.min(directive?.end ?? body.getStart(sourceFile) + 1, limit); + const bodyEnd = Math.max(bodyStart, Math.min(body.end - 1, limit)); const implementation = sourceFile.text.slice(bodyStart, bodyEnd); const methodName = node.name?.getText(sourceFile) ?? "anonymous"; const parameters = node.parameters.map((parameter, index): TrainableParameter => ({ diff --git a/packages/training/test/promotion.test.ts b/packages/training/test/promotion.test.ts index e2c61fb..56ef314 100644 --- a/packages/training/test/promotion.test.ts +++ b/packages/training/test/promotion.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { commitRewrite, revertRewrite } from "ts-autocode-rewrite"; import { + defaultMinScore, defineTrainable, evaluatePromotionGate, type CandidatePatch, @@ -105,3 +106,54 @@ describe("promotion", () => { expect(decision.failures).toContain("AgentV evaluations must be run against the candidate"); }); }); + +describe("threshold validation messages", () => { + // A user who passes a bad threshold wants to know the range, not Zod's type + // vocabulary. `minScore: Infinity` used to report "expected number, received + // number", because the base schema rejected it before the .finite() message + // could apply. Found by a property test over non-unit-interval doubles. + const bad: ReadonlyArray = [ + ["negative", -0.5], + ["above one", 1.5], + ["positive infinity", Number.POSITIVE_INFINITY], + ["negative infinity", Number.NEGATIVE_INFINITY], + ["NaN", Number.NaN], + ["a string", "0.5"], + ["a boolean", true], + ["an object", {}], + ]; + + it.each(bad)("reports the range for %s", async (_label, value) => { + const promise = evaluatePromotionGate({ + candidate: candidate(), evaluations: [], conformance: true, minScore: value as number, + }); + await expect(promise).rejects.toThrow("minScore must be between 0 and 1"); + }); + + it("names the offending setting, not a generic one", async () => { + await expect(evaluatePromotionGate({ + candidate: candidate(), evaluations: [], conformance: true, minPassRate: 2, + })).rejects.toThrow("minPassRate must be between 0 and 1"); + }); + + it("treats null and undefined as unset, applying the default", async () => { + // `minScore ?? defaultMinScore` coalesces both. Neither is type-valid, so + // this pins the behavior rather than endorsing it as an input. + for (const value of [undefined, null]) { + // `exactOptionalPropertyTypes` forbids passing an explicit undefined, + // which is the point: neither spelling is type-valid, so this pins + // runtime behavior rather than endorsing either as an input. + const input = { candidate: candidate(), evaluations: [], conformance: true, minScore: value }; + const decision = await evaluatePromotionGate(input as unknown as Parameters[0]); + expect(decision.failures.some((failure) => failure.includes(`below ${defaultMinScore}`))).toBe(true); + } + }); + + it("accepts the closed interval's endpoints", async () => { + for (const value of [0, 1]) { + await expect(evaluatePromotionGate({ + candidate: candidate(), evaluations: [], conformance: true, minScore: value, minPassRate: value, + })).resolves.toBeDefined(); + } + }); +}); diff --git a/packages/training/test/source.test.ts b/packages/training/test/source.test.ts index bb96d2e..7699603 100644 --- a/packages/training/test/source.test.ts +++ b/packages/training/test/source.test.ts @@ -114,3 +114,31 @@ describe("parameter types inferred from literal defaults", () => { expect(declare("value")[0]?.type).toBe("unknown"); }); }); + +describe("offsets on source TypeScript could not fully parse", () => { + // Error recovery synthesizes a body for an unterminated block whose `end` + // can sit past EOF, so a truncated file produced a target claiming offsets + // outside its own source. Found by fuzzing generated-then-damaged modules. + const truncated = 'class Router {\n\troute(input: string): string {\n\t\t"use training";\n\t\t// } a brace in a comm'; + + it("never reports offsets outside the source", () => { + for (const target of discoverInSource(truncated, "truncated.ts")) { + expect(target.bodyStart).toBeGreaterThanOrEqual(0); + expect(target.bodyEnd).toBeLessThanOrEqual(truncated.length); + expect(target.bodyStart).toBeLessThanOrEqual(target.bodyEnd); + } + }); + + it("keeps the digest consistent with the clamped slice", () => { + for (const target of discoverInSource(truncated, "truncated.ts")) { + const raw = truncated.slice(target.bodyStart, target.bodyEnd); + expect(raw.trim()).toBe(target.implementation); + } + }); + + it("leaves offsets untouched for source that parses", () => { + const valid = 'class Router {\n\troute(input: string): string {\n\t\t"use training";\n\t\treturn input;\n\t}\n}\n'; + const target = discoverInSource(valid, "valid.ts")[0]!; + expect(valid.slice(target.bodyStart, target.bodyEnd).trim()).toBe("return input;"); + }); +}); diff --git a/src/index.ts b/src/index.ts index 26911db..9ef0469 100644 --- a/src/index.ts +++ b/src/index.ts @@ -78,6 +78,7 @@ export { resetTraining, sequentialLoop, SourceDiscoveryError, + toTrainableToken, trainableTokenFromSymbol, training, TraceNotFoundError, diff --git a/test/fuzz.test.ts b/test/fuzz.test.ts new file mode 100644 index 0000000..f19271f --- /dev/null +++ b/test/fuzz.test.ts @@ -0,0 +1,234 @@ +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; + +import { discoverInSource, isTsAutocodeError } from "../src/index.js"; +import { augmentSource } from "../src/register/hook.js"; +import { run } from "../src/cli.js"; +import { scanDeclaredTrainables } from "ts-autocode-grounding"; +import { commitRewrite, digest as rewriteDigest, revertRewrite } from "ts-autocode-rewrite"; +import { evolutionEnabled } from "../src/evolve.js"; +import { anyModule, damagedModule, markedModule } from "./support/sources.js"; + +// Fuzzing: feed the parsers arbitrary and deliberately hostile input and assert +// they fail predictably rather than crashing, hanging, or -- worst for this +// library -- silently corrupting a user's source file. +// +// Every one of these functions runs against code the library did not write: +// `augmentSource` sees every module a user loads, and `scanDeclaredTrainables` +// and `discoverInSource` see whatever is in their project. + +const runs = 200; + +// The corpus is generated marked modules and damaged variants of them (see +// test/support/sources.ts), not random punctuation: instrumenting a +// punctuation corpus showed 1 in 3000 inputs produced a discovered target, so +// every property about offsets and rewriting was passing vacuously. +const hostileSource = anyModule; + +describe("source discovery", () => { + it("never crashes on arbitrary text", () => { + fc.assert(fc.property(fc.string({ maxLength: 400 }), (source) => { + expect(() => discoverInSource(source, "fuzz.ts")).not.toThrow(); + }), { numRuns: runs }); + }); + + it("never crashes on hostile fragments", () => { + fc.assert(fc.property(hostileSource, (source) => { + // A throw is acceptable only as a typed library error, never as a + // TypeError from an unchecked property access. + try { + discoverInSource(source, "fuzz.ts"); + } catch (error) { + expect(isTsAutocodeError(error)).toBe(true); + } + }), { numRuns: runs }); + }); + + it("only ever reports targets whose offsets lie inside the source", () => { + fc.assert(fc.property(hostileSource, (source) => { + for (const target of safeDiscover(source)) { + expect(target.bodyStart).toBeGreaterThanOrEqual(0); + expect(target.bodyEnd).toBeLessThanOrEqual(source.length); + expect(target.bodyStart).toBeLessThanOrEqual(target.bodyEnd); + } + }), { numRuns: runs }); + }); + + it("relates the body slice, implementation and digest exactly as documented", () => { + // These three fields have subtly different relationships to the source + // -- `implementation` is trimmed, `bodyDigest` hashes the raw slice -- + // and nothing said so until a property test asked. Guarded application + // depends on the digest side, so the distinction is load-bearing. + fc.assert(fc.property(hostileSource, (source) => { + for (const target of safeDiscover(source)) { + const raw = source.slice(target.bodyStart, target.bodyEnd); + expect(raw.trim()).toBe(target.implementation); + expect(rewriteDigest(raw)).toBe(target.bodyDigest); + } + }), { numRuns: runs }); + }); + + it("produces targets a guarded rewrite accepts and can revert", () => { + // The end-to-end invariant that matters: whatever discovery reports must + // survive commit and revert byte for byte. + fc.assert(fc.property(markedModule, (source) => { + for (const target of safeDiscover(source)) { + const candidate = { + id: "fuzz", trainableId: target.id, engineId: "fuzz", target, + implementation: "return \"fuzzed\";", + }; + const committed = commitRewrite(source, candidate); + expect(committed.source).not.toBe(source); + expect(revertRewrite(committed.source, committed.snapshot)).toBe(source); + } + }), { numRuns: 100 }); + }); + + it("never reports an empty identity", () => { + fc.assert(fc.property(hostileSource, (source) => { + for (const target of safeDiscover(source)) { + expect(target.id.trim().length).toBeGreaterThan(0); + } + }), { numRuns: runs }); + }); +}); + +describe("the register load hook", () => { + // This rewrites every module a user loads. Corrupting one would be the worst + // failure this library could have. + it("never throws, whatever the module contains", () => { + fc.assert(fc.property(fc.string({ maxLength: 400 }), (source) => { + expect(() => augmentSource(source, "fuzz.ts")).not.toThrow(); + }), { numRuns: runs }); + }); + + it("is append-only: the original source is always a prefix of the result", () => { + fc.assert(fc.property(hostileSource, (source) => { + // Line numbers and sourcemaps of the original module depend on this. + expect(augmentSource(source, "fuzz.ts").startsWith(source)).toBe(true); + }), { numRuns: runs }); + }); + + it("leaves a module without the marker byte-identical", () => { + fc.assert(fc.property(fc.string({ maxLength: 400 }), (source) => { + fc.pre(!source.includes("use training")); + expect(augmentSource(source, "fuzz.ts")).toBe(source); + }), { numRuns: runs }); + }); + + it("is idempotent for sources it leaves alone", () => { + fc.assert(fc.property(hostileSource, (source) => { + const once = augmentSource(source, "fuzz.ts"); + fc.pre(once === source); + expect(augmentSource(once, "fuzz.ts")).toBe(source); + }), { numRuns: runs }); + }); +}); + +describe("ambient declaration scanning", () => { + it("never crashes on arbitrary text", () => { + fc.assert(fc.property(fc.string({ maxLength: 400 }), (source) => { + expect(() => scanDeclaredTrainables(source)).not.toThrow(); + }), { numRuns: runs }); + }); + + it("rejects non-string input with a TypeError rather than coercing", () => { + for (const value of [undefined, null, 1, {}, [], true]) { + expect(() => scanDeclaredTrainables(value as never)).toThrow(TypeError); + } + }); + + it("only reports operations with a non-empty method and contract ref", () => { + fc.assert(fc.property(hostileSource, (source) => { + for (const declared of scanDeclaredTrainables(source)) { + for (const operation of declared.operations) { + expect(operation.method.length).toBeGreaterThan(0); + expect(operation.contractRef.length).toBeGreaterThan(0); + expect(operation.intent.length).toBeGreaterThan(0); + } + } + }), { numRuns: runs }); + }); +}); + +describe("the evolve kill switch", () => { + // This decides whether the library rewrites the user's source. It must never + // read an unrecognized value as consent. + it("only ever enables on a recognized affirmative or an unset value", () => { + fc.assert(fc.property(fc.string({ maxLength: 20 }), (value) => { + let enabled: boolean; + try { + enabled = evolutionEnabled(value); + } catch { + return; // Refusing to guess is the correct outcome. + } + const flag = value.trim().toLowerCase(); + expect(enabled).toBe(flag === "" || ["1", "true", "on", "yes", "enabled"].includes(flag)); + }), { numRuns: 500 }); + }); +}); + +describe("the command line", () => { + it("never throws for arbitrary argv, and always reports a usable exit code", async () => { + await fc.assert(fc.asyncProperty( + fc.array(fc.string({ maxLength: 20 }), { maxLength: 8 }), + async (argv) => { + const result = await run(argv); + expect([0, 1, 2]).toContain(result.code); + // A non-zero exit must say something; a zero exit must not be silent. + expect((result.code === 0 ? result.stdout : result.stderr).length).toBeGreaterThan(0); + }, + ), { numRuns: 150 }); + }); + + it("never leaks a stack trace to stderr", async () => { + await fc.assert(fc.asyncProperty( + fc.array(fc.oneof(fc.constantFrom("discover", "status", "help", "--json", "--cwd", "--file", "--nope"), + fc.string({ maxLength: 12 })), { maxLength: 6 }), + async (argv) => { + const result = await run(argv); + expect(result.stderr).not.toMatch(/\n\s+at /); + }, + ), { numRuns: 150 }); + }); +}); + +/** Discovery result, or nothing when the source is rejected outright. */ +function safeDiscover(source: string) { + try { + return discoverInSource(source, "fuzz.ts"); + } catch { + return []; + } +} + +describe("the fuzz corpus itself", () => { + // A corpus that never reaches the code under test makes every property + // above vacuously true. This asserts the corpus does its job, so the suite + // cannot quietly decay into theatre. + it("produces modules that discovery actually finds targets in", () => { + let withTargets = 0; + const samples = fc.sample(markedModule, 100); + for (const source of samples) { + if (discoverInSource(source, "fuzz.ts").length > 0) withTargets += 1; + } + expect(withTargets).toBeGreaterThan(80); + }); + + it("produces damaged modules that still often parse", () => { + let withTargets = 0; + for (const source of fc.sample(damagedModule, 200)) { + if (safeDiscover(source).length > 0) withTargets += 1; + } + // Damaged input should be a genuine mix, not all-or-nothing. + expect(withTargets).toBeGreaterThan(10); + }); + + it("produces modules the load hook actually rewrites", () => { + let rewritten = 0; + for (const source of fc.sample(markedModule, 100)) { + if (augmentSource(source, "fuzz.ts") !== source) rewritten += 1; + } + expect(rewritten).toBeGreaterThan(80); + }); +}); diff --git a/test/property.test.ts b/test/property.test.ts new file mode 100644 index 0000000..3bc714f --- /dev/null +++ b/test/property.test.ts @@ -0,0 +1,281 @@ +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; + +import { digest as rewriteDigest } from "ts-autocode-rewrite"; +import { textDigest } from "ts-autocode-grounding"; +import { + defined, + defineTrainable, + evaluationArgs, + evaluatePromotionGate, + optional, + toTrainableToken, + trainableTokenFromSymbol, + type BoundEvaluation, + type CandidatePatch, +} from "../src/index.js"; +import { discoverInSource } from "../src/index.js"; +import { trainableIdFromKey } from "../packages/training/src/token.js"; + +// Property-based tests: instead of a handful of chosen inputs, state the law +// that must hold and let fast-check hunt for a counterexample. These target the +// pure, total functions where an example-based test can only ever sample -- +// identity round-trips, canonicalization, aggregation, and the spread helpers. +// +// Every failure prints a shrunk counterexample and a seed, so a regression is +// reproducible rather than "it failed once on CI". + +const runs = 300; + +/** Ids the library accepts: non-empty once trimmed. */ +const trainableId = fc.string({ minLength: 1, maxLength: 64 }) + .filter((value) => value.trim().length > 0); + +describe("trainable identity", () => { + it("round-trips through its symbol for any acceptable id", () => { + fc.assert(fc.property(trainableId, (id) => { + const token = defineTrainable(id); + expect(trainableTokenFromSymbol(token.symbol).id).toBe(id.trim()); + expect(toTrainableToken(token.symbol)).toEqual(token); + }), { numRuns: runs }); + }); + + it("is idempotent: defining twice yields the same symbol", () => { + fc.assert(fc.property(trainableId, (id) => { + expect(defineTrainable(id).symbol).toBe(defineTrainable(id).symbol); + }), { numRuns: runs }); + }); + + it("ignores surrounding whitespace but nothing else", () => { + fc.assert(fc.property(trainableId, fc.stringMatching(/^[ \t\n]*$/), (id, pad) => { + expect(defineTrainable(`${pad}${id}${pad}`).id).toBe(id.trim()); + }), { numRuns: runs }); + }); + + it("distinct trimmed ids never collide", () => { + fc.assert(fc.property(trainableId, trainableId, (left, right) => { + fc.pre(left.trim() !== right.trim()); + expect(defineTrainable(left).symbol).not.toBe(defineTrainable(right).symbol); + }), { numRuns: runs }); + }); + + it("stripping the key prefix is idempotent for unprefixed keys", () => { + fc.assert(fc.property(fc.string(), (key) => { + fc.pre(!key.startsWith("ts-autocode.trainable:")); + expect(trainableIdFromKey(key)).toBe(key); + }), { numRuns: runs }); + }); + + it("rejects every blank id rather than producing an empty identity", () => { + fc.assert(fc.property(fc.stringMatching(/^[ \t\n\r]*$/), (blank) => { + expect(() => defineTrainable(blank)).toThrow(); + }), { numRuns: runs }); + }); +}); + +describe("digest canonicalization", () => { + // Guarded rewriting refuses a candidate whose body digest changed, so these + // laws are load-bearing rather than cosmetic. + const json = fc.jsonValue(); + + it("is deterministic", () => { + fc.assert(fc.property(json, (value) => { + expect(rewriteDigest(value)).toBe(rewriteDigest(value)); + }), { numRuns: runs }); + }); + + it("is insensitive to object key order at any depth", () => { + fc.assert(fc.property(json, (value) => { + expect(rewriteDigest(value)).toBe(rewriteDigest(shuffleKeys(value))); + }), { numRuns: runs }); + }); + + it("is sensitive to array order whenever order is observable", () => { + fc.assert(fc.property(fc.uniqueArray(fc.integer(), { minLength: 2, maxLength: 8 }), (values) => { + expect(rewriteDigest(values)).not.toBe(rewriteDigest([...values].reverse())); + }), { numRuns: runs }); + }); + + it("always produces the documented shape", () => { + fc.assert(fc.property(json, (value) => { + expect(rewriteDigest(value)).toMatch(/^sha256:[0-9a-f]{64}$/); + }), { numRuns: runs }); + }); + + it("text digest normalizes line endings and nothing else", () => { + fc.assert(fc.property(fc.array(fc.string({ maxLength: 12 }), { maxLength: 8 }), (lines) => { + expect(textDigest(lines.join("\r\n"))).toBe(textDigest(lines.join("\n"))); + }), { numRuns: runs }); + }); +}); + +describe("optional and defined", () => { + it("optional includes a key exactly when the value is defined", () => { + fc.assert(fc.property(fc.string({ minLength: 1 }), fc.option(fc.jsonValue(), { nil: undefined }), (key, value) => { + const spread = { ...optional(key, value) } as Record; + expect(key in spread).toBe(value !== undefined); + }), { numRuns: runs }); + }); + + it("defined keeps exactly the defined entries", () => { + fc.assert(fc.property( + fc.dictionary(fc.string({ minLength: 1 }), fc.option(fc.integer(), { nil: undefined })), + (record) => { + const spread = { ...defined(record) } as Record; + const expected = Object.entries(record).filter(([, value]) => value !== undefined).map(([key]) => key); + expect(Object.keys(spread).sort()).toEqual(expected.sort()); + }, + ), { numRuns: runs }); + }); + + it("defined never introduces an explicit undefined", () => { + fc.assert(fc.property( + fc.dictionary(fc.string({ minLength: 1 }), fc.option(fc.integer(), { nil: undefined })), + (record) => { + for (const value of Object.values({ ...defined(record) })) expect(value).toBeDefined(); + }, + ), { numRuns: runs }); + }); +}); + +describe("evaluation argument decoding", () => { + it("never throws, whatever the eval input is", () => { + fc.assert(fc.property(fc.string(), (input) => { + expect(Array.isArray(evaluationArgs(input))).toBe(true); + }), { numRuns: runs }); + }); + + it("round-trips a JSON array of arguments", () => { + // `-0` is excluded because JSON cannot represent it: JSON.stringify(-0) + // is "0", so no decoder could return it. That is a property of the wire + // format, not something this library can or should fix -- but it is + // worth having stated, since eval inputs are JSON strings. + fc.assert(fc.property(fc.array(fc.jsonValue(), { maxLength: 6 }), (args) => { + fc.pre(!JSON.stringify(args).includes("-0") && !hasNegativeZero(args)); + expect(evaluationArgs(JSON.stringify(args))).toEqual(args); + }), { numRuns: runs }); + }); + + it("cannot recover negative zero, because JSON does not carry it", () => { + expect(JSON.stringify([-0])).toBe("[0]"); + expect(Object.is(evaluationArgs("[-0]")[0], -0)).toBe(true); + expect(Object.is(evaluationArgs(JSON.stringify([-0]))[0], -0)).toBe(false); + }); + + it("wraps a non-array JSON value as a single argument", () => { + fc.assert(fc.property(fc.oneof(fc.integer(), fc.boolean(), fc.constant(null)), (value) => { + expect(evaluationArgs(JSON.stringify(value))).toEqual([value]); + }), { numRuns: runs }); + }); + + it("falls back to the raw string when the input is not JSON", () => { + fc.assert(fc.property(fc.string(), (input) => { + fc.pre(!isJson(input)); + expect(evaluationArgs(input)).toEqual([input]); + }), { numRuns: runs }); + }); +}); + +describe("promotion gate aggregation", () => { + const target = discoverInSource(`class F { + m(input: string): string { + "use training"; + return input; + } +}`, "f.ts")[0]!; + const candidate: CandidatePatch = { + id: "c", trainableId: target.id, engineId: "prop", target, implementation: "return input;", + }; + const score = fc.float({ min: 0, max: 1, noNaN: true }); + + const evaluationsOf = (scores: readonly number[]): BoundEvaluation[] => scores.map((value, index) => ({ + trainableId: target.id, + candidateId: candidate.id, + result: { testId: `t${index}`, score: value, executionStatus: "ok", output: "" } as never, + })); + + it("mean score is the arithmetic mean of the bound results", async () => { + await fc.assert(fc.asyncProperty(fc.array(score, { minLength: 1, maxLength: 8 }), async (scores) => { + const decision = await evaluatePromotionGate({ + candidate, evaluations: evaluationsOf(scores), conformance: true, + }); + const mean = scores.reduce((sum, value) => sum + value, 0) / scores.length; + expect(decision.meanScore).toBeCloseTo(mean, 10); + }), { numRuns: 100 }); + }); + + it("mean score and pass rate stay within the unit interval", async () => { + await fc.assert(fc.asyncProperty(fc.array(score, { minLength: 1, maxLength: 8 }), async (scores) => { + const decision = await evaluatePromotionGate({ + candidate, evaluations: evaluationsOf(scores), conformance: true, + }); + for (const value of [decision.meanScore, decision.passRate]) { + expect(value).toBeGreaterThanOrEqual(0); + expect(value).toBeLessThanOrEqual(1); + } + }), { numRuns: 100 }); + }); + + it("promotes only when it reports no failures, always", async () => { + await fc.assert(fc.asyncProperty( + fc.array(score, { maxLength: 6 }), fc.boolean(), score, score, + async (scores, conformance, minScore, minPassRate) => { + const decision = await evaluatePromotionGate({ + candidate, evaluations: evaluationsOf(scores), conformance, minScore, minPassRate, + }); + expect(decision.promote).toBe(decision.failures.length === 0); + }, + ), { numRuns: 200 }); + }); + + it("an extra gate that always refuses always blocks promotion", async () => { + await fc.assert(fc.asyncProperty(fc.array(score, { minLength: 1, maxLength: 6 }), async (scores) => { + const decision = await evaluatePromotionGate({ + candidate, + evaluations: evaluationsOf(scores), + conformance: true, + minScore: 0, + minPassRate: 0, + gates: [() => "always refuses"], + }); + expect(decision.promote).toBe(false); + expect(decision.failures).toContain("always refuses"); + }), { numRuns: 100 }); + }); + + it("rejects a threshold outside the unit interval rather than clamping it", async () => { + await fc.assert(fc.asyncProperty( + fc.double({ noNaN: true }).filter((value) => value < 0 || value > 1), + async (minScore) => { + await expect(evaluatePromotionGate({ + candidate, evaluations: [], conformance: true, minScore, + })).rejects.toThrow(/between 0 and 1/); + }, + ), { numRuns: 100 }); + }); +}); + +/** Recursively reorders object keys, leaving arrays and scalars alone. */ +function shuffleKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(shuffleKeys); + if (typeof value !== "object" || value === null) return value; + const entries = Object.entries(value as Record).reverse(); + return Object.fromEntries(entries.map(([key, nested]) => [key, shuffleKeys(nested)])); +} + +/** True when any nested value is `-0`, which JSON flattens to `0`. */ +function hasNegativeZero(value: unknown): boolean { + if (Object.is(value, -0)) return true; + if (Array.isArray(value)) return value.some(hasNegativeZero); + if (typeof value === "object" && value !== null) return Object.values(value).some(hasNegativeZero); + return false; +} + +function isJson(value: string): boolean { + try { + JSON.parse(value); + return true; + } catch { + return false; + } +} diff --git a/test/snapshots/surface/root-export-kinds.verified.json b/test/snapshots/surface/root-export-kinds.verified.json index c21a05c..15392ff 100644 --- a/test/snapshots/surface/root-export-kinds.verified.json +++ b/test/snapshots/surface/root-export-kinds.verified.json @@ -74,6 +74,7 @@ "sequentialLoop": "function", "swapImplementation": "function", "swappedImplementation": "function", + "toTrainableToken": "function", "trainable": "function", "trainableTokenFromSymbol": "function", "training": "object", diff --git a/test/snapshots/surface/ts-autocode-training.verified.txt b/test/snapshots/surface/ts-autocode-training.verified.txt index 78b96be..bbb43f2 100644 --- a/test/snapshots/surface/ts-autocode-training.verified.txt +++ b/test/snapshots/surface/ts-autocode-training.verified.txt @@ -49,6 +49,7 @@ parseSetting provideTrainingDefaults resetTraining sequentialLoop +toTrainableToken trainableTokenFromSymbol training trainingMarker diff --git a/test/snapshots/surface/ts-autocode.verified.txt b/test/snapshots/surface/ts-autocode.verified.txt index 8730f78..3ef8878 100644 --- a/test/snapshots/surface/ts-autocode.verified.txt +++ b/test/snapshots/surface/ts-autocode.verified.txt @@ -73,6 +73,7 @@ rewritePromotion sequentialLoop swapImplementation swappedImplementation +toTrainableToken trainable trainableTokenFromSymbol training diff --git a/test/support/sources.ts b/test/support/sources.ts new file mode 100644 index 0000000000000000000000000000000000000000..a264d6dba504ab135cf172dceb2514f7170fdee6 GIT binary patch literal 4765 zcmb_g!EPHl65Xry6@`WXwLI-sCb`Lz1m0a_=P+3elG()qmNBBbDU0J4TOg^(iqLxv7_w%!PFq#X=T$$z`uM z8qwdRmPWYRSen;{+(MGltF^ev3);>cxu}0SrqkgD4AF5mYK$)YuL`oG`c2(ZvOv^Uz`1 zN*bpE9AZ`j$a+4v5>D{eVzrXGyr?zyFmhv*cVPo#K-OCJXC?~)e;I}a|K11>t;CL< zt#kTmAu;RraD354;^2U%3FlS8n3`3xdrWg>tjpM8O{G|hTglJMtc^lx2al}xZ6(|S zzQaKy1rmpG#EF?zZDro;;l!;YRWg`!oP(v;be$N)Et8BAU7Mw-)Mx)=0R$@lB(+h+ zf+x88?_%l4`ej{J!tmhFYh6nJ^zdK(@LTclpYq|?qUMx(n?u?myE}VOGtm<_adB{xOp>r~ zNcR~2Yb+CE{F#kG7!5U-N2@;iXwxt`7x^r>*hVvLAuFlB^KLY>QEq`>lO!yfBu&J4 zdK6v4{NPMr+87L|umL%FsF}P34q(X~cjVIn!WOBdi3XTt(-2?5EV@UAVJbQh6_*_A zk=YiYeTm}uwx`oC94zNw*b5B>m5iMEL4inoMF?ZWHbrT;^{SHTp&&q6Nb8KNCKhKp z9A-FxrMUk=>RY!M(<^F*y|a8NR%yDoZHM&w4fn3FmF=Cy8P^&k8Qm#eW>l}Z1&Qh= zL-3`%_lu_e9nWd_j4+1(?KDoPw?3hgW{gu~Bh3_>pXI;QN~g>W!?w__+J@+Bl@Rfw zE<5=i+U0vc98f?_fK<{Q@%G~m#b%vA#)oIdFNZCn8{vgm$GtzfpSxkbPHcd-j$V0(4h%C*}6GR(8=j z1yGpMzw;2Hs{!c~aK`rmFAgK3@+0q&dNjJxcIbw?-&|=H8&E^s_&u?7f^3s%Zlv5x ztWF#OfOk=Q{onIR-m|Eb&;U22_QYPoWoq^}ctb zBZY)T@q*y?B#3?LdD`U%lX207PzkI;~aU!qAzhEShtV@Ld^T) z<7s6|!rX?A10+h;sQXrf@qr3g4zECV^e;4vaP`PX_xUe2K`swqyBP+KZrLZs4svgA z3>k&(+uQ&Xhk+(=XiNg8ivBJxkIV~#(8&cngy_M$Edd4P(CXwkTMTtEog&?uvZnn5Jru^!R_dWOl_LK8D}qPwtU_NL>#N7roH zYcF5rVLHS%Bv(~+$~kCs9Nq`u&3A_@?vUT5;YKu$y2`h!s7jG*5KeJIc;RnPxfgKh zi;Uhw*XQ%9u1%bkSH6SNh+d+peqr$!R=#>}rIAS4LZ(m~$eiS{eH?9wrD>+8w{8s# z9@X&x33oii$#p97FIY$hT|qUteBxhlF8_jw!RQ>~m5yDx{F*O`&t!(Alz4;y4+CEb zA!+lOSGLdRU-LTI@$}v9xp@|;?@}BN;^f=3Ooy>to?Fr4F`DSnEo@H=m)#@y4`=(Q zXU^8u|G>4_{6+AVS&y$ud^fo|s=6%RRQdYuxpyPBPx&@fO8LG+xIsjAiP#HBzT@6X z1AVnrdQ7MYh{$o=#J0t7!NPvH7nIi_5u!f4@r9Mf>b&;)@P^*Lr8Gs~SVOdhYV_0m z$Z*(}WUE*H=`URTX&}1Z-qRsc@M!`3lhI~@%QAl%Vcv;fPc~@43;&S>{QDFKp|@QO zLJ#Sk?Pw#_?ZS=eB{<2<6fO}4`mT!(H8{ZMCLZH^JQJO#V-hVaM9&Bx Date: Fri, 28 Aug 2026 19:18:52 +0000 Subject: [PATCH 14/14] fix: repair a flaky property, and the prototype bug behind it CI on this branch failed with `Counterexample: ["toString", undefined]`, seed 1960220664. Two separate defects, both here: 1. The assertion was wrong. `key in spread` walks the prototype chain, so any key named after an Object.prototype member -- "toString", "constructor", "valueOf" -- read as present whether or not it had been added. It takes fast-check about 87 runs to find one, which is why it passed locally and failed in CI: the seed is random per run. Now `Object.hasOwn`. 2. `defined()` was genuinely broken for the same class of key. It built its result with `result[key] = value`, an assignment that goes through the `__proto__` setter on Object.prototype: a `__proto__` key was silently dropped, and with an object value it replaced the result's prototype instead of adding a key. Now built with Object.fromEntries, which defines own properties. `optional()` was already safe, because a computed key in an object literal defines rather than assigns. The second is the reason this belongs in this PR rather than a later one: the dictionary property that finds it was introduced here, so leaving the fix downstream would leave this branch intermittently red. Reproduced the original failure with CI's exact seed before fixing, then confirmed 2000 runs of that seed and 5000 of the dictionary property pass, plus eight full suite runs on random seeds. Adds regression tests for every key that shadows Object.prototype, including that Object.prototype itself is never polluted. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QchTdrzVJeexx1sjWjkJVb --- packages/training/src/optional.ts | 13 +++++--- packages/training/test/optional.test.ts | 41 +++++++++++++++++++++++-- test/property.test.ts | 18 ++++++++++- 3 files changed, 64 insertions(+), 8 deletions(-) diff --git a/packages/training/src/optional.ts b/packages/training/src/optional.ts index 8089ecc..81be7d0 100644 --- a/packages/training/src/optional.ts +++ b/packages/training/src/optional.ts @@ -13,9 +13,12 @@ export function optional(key: K, value: V | undefined): { [ /** Spreads every defined entry of `values`, dropping the undefined ones. * `{ ...defined({ signal, timeoutMs }) }` replaces a run of `optional` calls. */ export function defined(values: T): { [K in keyof T]?: Exclude } { - const result: Record = {}; - for (const [key, value] of Object.entries(values)) { - if (value !== undefined) result[key] = value; - } - return result as { [K in keyof T]?: Exclude }; + // `Object.fromEntries`, not `result[key] = value`. Assignment goes through + // the `__proto__` setter on Object.prototype, so a `__proto__` key was + // silently dropped -- and with an object value it replaced the result's + // prototype instead of adding a key. `fromEntries` defines own properties, + // which is what a key/value copy should do. Found by a property test. + return Object.fromEntries( + Object.entries(values).filter(([, value]) => value !== undefined), + ) as { [K in keyof T]?: Exclude }; } diff --git a/packages/training/test/optional.test.ts b/packages/training/test/optional.test.ts index 08cca4d..5ff3668 100644 --- a/packages/training/test/optional.test.ts +++ b/packages/training/test/optional.test.ts @@ -17,13 +17,14 @@ describe("optional", () => { it("omits the key entirely when the value is undefined", () => { const spread = { ...optional("signal", undefined) }; expect(Object.keys(spread)).toEqual([]); - expect("signal" in spread).toBe(false); + // `Object.hasOwn`, not `in`: `in` walks the prototype chain. + expect(Object.hasOwn(spread, "signal")).toBe(false); }); it("keeps falsy-but-defined values, which is the whole point", () => { for (const value of [0, "", false, Number.NaN, null] as const) { const spread = { ...optional("v", value) }; - expect("v" in spread).toBe(true); + expect(Object.hasOwn(spread, "v")).toBe(true); expect((spread as { v: unknown }).v).toBe(value === value ? value : (spread as { v: number }).v); } expect(Number.isNaN(({ ...optional("v", Number.NaN) } as { v: number }).v)).toBe(true); @@ -70,3 +71,39 @@ describe("defined", () => { expect({ ...defined(base) }).toEqual({ own: "yes" }); }); }); + +describe("keys that shadow Object.prototype", () => { + // `defined` built its result with `result[key] = value`, which goes through + // the `__proto__` setter: a `__proto__` key was silently dropped, and with + // an object value it replaced the result's prototype instead of adding a + // key. Found by a property test over arbitrary dictionaries. + const dangerous = ["__proto__", "constructor", "prototype", "toString", "hasOwnProperty", "valueOf"]; + + it.each(dangerous)("defined keeps a %s key as an own property", (key) => { + const result = defined(JSON.parse(`{"${key}": 1}`) as Record); + expect(Object.hasOwn(result, key)).toBe(true); + expect((result as Record)[key]).toBe(1); + }); + + it.each(dangerous)("optional keeps a %s key as an own property", (key) => { + const result = { ...optional(key, 1) }; + expect(Object.hasOwn(result, key)).toBe(true); + }); + + it("never lets a __proto__ value replace the result's prototype", () => { + const result = defined(JSON.parse('{"__proto__": {"isAdmin": true}}') as Record); + expect(Object.getPrototypeOf(result)).toBe(Object.prototype); + expect((result as Record)["isAdmin"]).toBeUndefined(); + expect(Object.getPrototypeOf({ ...optional("__proto__", { isAdmin: true }) })).toBe(Object.prototype); + }); + + it("does not pollute Object.prototype itself", () => { + defined(JSON.parse('{"__proto__": {"polluted": true}}') as Record); + expect(({} as Record)["polluted"]).toBeUndefined(); + }); + + it("still drops an undefined value under a dangerous key", () => { + const result = defined({ ["__proto__"]: undefined } as Record); + expect(Object.getOwnPropertyNames(result)).toEqual([]); + }); +}); diff --git a/test/property.test.ts b/test/property.test.ts index 3bc714f..3ccd0e0 100644 --- a/test/property.test.ts +++ b/test/property.test.ts @@ -111,12 +111,28 @@ describe("digest canonicalization", () => { describe("optional and defined", () => { it("optional includes a key exactly when the value is defined", () => { + // `Object.hasOwn`, not `in`: `in` walks the prototype chain, so any key + // named after an Object.prototype member ("toString", "constructor") + // reads as present whether or not it was added. fast-check found that + // mistake in this assertion within a few hundred runs. fc.assert(fc.property(fc.string({ minLength: 1 }), fc.option(fc.jsonValue(), { nil: undefined }), (key, value) => { const spread = { ...optional(key, value) } as Record; - expect(key in spread).toBe(value !== undefined); + expect(Object.hasOwn(spread, key)).toBe(value !== undefined); }), { numRuns: runs }); }); + it("optional is safe for keys that shadow Object.prototype", () => { + for (const key of ["__proto__", "constructor", "toString", "hasOwnProperty", "valueOf"]) { + const present = { ...optional(key, 1) } as Record; + expect(Object.hasOwn(present, key)).toBe(true); + expect(present[key]).toBe(1); + // A computed `__proto__` defines an own property rather than setting + // the prototype, so the object stays a plain object. + expect(Object.getPrototypeOf({ ...optional("__proto__", { polluted: true }) })).toBe(Object.prototype); + expect(Object.hasOwn({ ...optional(key, undefined) } as object, key)).toBe(false); + } + }); + it("defined keeps exactly the defined entries", () => { fc.assert(fc.property( fc.dictionary(fc.string({ minLength: 1 }), fc.option(fc.integer(), { nil: undefined })),