diff --git a/docs/testing.md b/docs/testing.md index 5e1d32b..4ab4b7c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -12,6 +12,8 @@ land, never lower them to get a build green. | Surface | `test/surface.test.ts` | Re-export drift between the root package and its siblings | | Protocol | `test/digest-protocol.test.ts` | Two packages that must agree without importing each other | | Compatibility | `test/deprecated.test.ts` | A deprecated spelling that silently stopped working | +| Property | `test/property.test.ts` | A law that holds for chosen examples but not in general | +| Fuzz | `test/fuzz.test.ts` | A parser crashing, hanging, or corrupting source it did not write | | Characterization | `test/characterization*.test.ts` | A change to anything this library *generates* — rewritten source, emitted instrumentation, prompts, CLI output, the export surface | ## Running @@ -52,7 +54,25 @@ Snapshots are excluded from `tsconfig.test.json`: they are generated artifacts, and the emitted instrumentation deliberately references names from the module it is appended to, so it does not typecheck standalone. -## Conventions +## Property and fuzz tests + +`fast-check` states the law and hunts for a counterexample, rather than sampling +a few inputs by hand. Failures print a shrunk counterexample and a seed, so a +regression is reproducible rather than "it failed once on CI". + +Properties target the pure, total functions where examples can only sample: +identity round-trips, digest canonicalization, gate aggregation, and the spread +helpers. Fuzzing targets the parsers, because every one of them runs against +code this library did not write — `augmentSource` sees every module a user +loads. + +**A fuzz corpus must reach the code.** An early version used random punctuation; +instrumenting it showed **1 input in 3000** produced a discovered target, so +every property about offsets and rewriting was passing vacuously. +`test/support/sources.ts` now generates structurally plausible marked modules +and then damages them, and `test/fuzz.test.ts` asserts the corpus still reaches +real work — so the suite cannot quietly decay back into theatre. + - **A test names the defect it prevents.** Where a test exists because something was once wrong, the comment says what was wrong. That is what makes diff --git a/packages/training/src/index.ts b/packages/training/src/index.ts index 8233529..a966c84 100644 --- a/packages/training/src/index.ts +++ b/packages/training/src/index.ts @@ -72,7 +72,7 @@ export type { TrainingRound, } from "./loop.js"; -export { defineTrainable, trainableTokenFromSymbol } from "./token.js"; +export { defineTrainable, toTrainableToken, trainableTokenFromSymbol } from "./token.js"; export type { TrainableId, TrainableIdentity, TrainableToken } from "./token.js"; export { defaultTsconfig, discoverInSource, discoverTrainables, inMemoryArtifactRef, trainingMarker } from "./source.js"; diff --git a/packages/training/src/optional.ts b/packages/training/src/optional.ts index 8089ecc..81be7d0 100644 --- a/packages/training/src/optional.ts +++ b/packages/training/src/optional.ts @@ -13,9 +13,12 @@ export function optional(key: K, value: V | undefined): { [ /** Spreads every defined entry of `values`, dropping the undefined ones. * `{ ...defined({ signal, timeoutMs }) }` replaces a run of `optional` calls. */ export function defined(values: T): { [K in keyof T]?: Exclude } { - const result: Record = {}; - for (const [key, value] of Object.entries(values)) { - if (value !== undefined) result[key] = value; - } - return result as { [K in keyof T]?: Exclude }; + // `Object.fromEntries`, not `result[key] = value`. Assignment goes through + // the `__proto__` setter on Object.prototype, so a `__proto__` key was + // silently dropped -- and with an object value it replaced the result's + // prototype instead of adding a key. `fromEntries` defines own properties, + // which is what a key/value copy should do. Found by a property test. + return Object.fromEntries( + Object.entries(values).filter(([, value]) => value !== undefined), + ) as { [K in keyof T]?: Exclude }; } diff --git a/packages/training/src/promotion.ts b/packages/training/src/promotion.ts index 42e637b..d2100ec 100644 --- a/packages/training/src/promotion.ts +++ b/packages/training/src/promotion.ts @@ -4,8 +4,15 @@ import { z } from "zod"; import type { BoundEvaluation, CandidatePatch } from "./engine.js"; import { parseSetting } from "./errors.js"; -const unitInterval = (name: string) => - z.number().finite(`${name} must be between 0 and 1`).min(0, `${name} must be between 0 and 1`).max(1, `${name} must be between 0 and 1`); +/** A threshold in [0, 1]. Every rejection -- wrong type, NaN, Infinity, or + * merely out of range -- reports the same message, because a user who passed a + * bad threshold wants to know the range, not Zod's type vocabulary. Without the + * base-schema message, `minScore: Infinity` reported "expected number, + * received number", which says nothing useful. */ +const unitInterval = (name: string) => { + const message = `${name} must be between 0 and 1`; + return z.number({ error: message }).finite(message).min(0, message).max(1, message); +}; const minScoreThreshold = unitInterval("minScore"); const minPassRateThreshold = unitInterval("minPassRate"); diff --git a/packages/training/src/source.ts b/packages/training/src/source.ts index 89ab984..344dc0a 100644 --- a/packages/training/src/source.ts +++ b/packages/training/src/source.ts @@ -44,9 +44,19 @@ export interface TrainableTarget { readonly parameters: readonly TrainableParameter[]; readonly returnType: string; readonly async: boolean; + /** Offset of the first character after the directive (or after the body's + * opening brace when the marker is a decorator). Guarded rewriting replaces + * exactly `[bodyStart, bodyEnd)`. */ readonly bodyStart: number; + /** Offset of the body's closing brace. */ readonly bodyEnd: number; + /** Digest of the **raw** `[bodyStart, bodyEnd)` slice, whitespace included. + * This is what guarded application re-computes to decide whether the file + * changed since discovery, so it must not be derived from + * {@link TrainableTarget.implementation}, which is trimmed. */ readonly bodyDigest: string; + /** The body as an engine should read it: the same slice, **trimmed**. The + * untrimmed slice is recoverable as `source.slice(bodyStart, bodyEnd)`. */ readonly implementation: string; readonly indentation: string; } @@ -162,8 +172,14 @@ function targetFor( if (node.asteriskToken) throw new InvalidTrainableIdentityError(`generator methods cannot be trainable: ${artifactRef}`); const body = node.body as ts.Block; const directive = firstDirective(body); - const bodyStart = directive?.end ?? body.getStart(sourceFile) + 1; - const bodyEnd = body.end - 1; + // TypeScript's error recovery synthesizes a body for an unterminated block, + // whose `end` can sit past EOF -- so a truncated file yielded a target + // claiming offsets outside its own source. Slicing clamps, so nothing was + // corrupted, but publishing an out-of-range range is malformed data crossing + // a public boundary. Clamp it: a no-op for source that parses. + const limit = sourceFile.text.length; + const bodyStart = Math.min(directive?.end ?? body.getStart(sourceFile) + 1, limit); + const bodyEnd = Math.max(bodyStart, Math.min(body.end - 1, limit)); const implementation = sourceFile.text.slice(bodyStart, bodyEnd); const methodName = node.name?.getText(sourceFile) ?? "anonymous"; const parameters = node.parameters.map((parameter, index): TrainableParameter => ({ diff --git a/packages/training/test/optional.test.ts b/packages/training/test/optional.test.ts index 08cca4d..5ff3668 100644 --- a/packages/training/test/optional.test.ts +++ b/packages/training/test/optional.test.ts @@ -17,13 +17,14 @@ describe("optional", () => { it("omits the key entirely when the value is undefined", () => { const spread = { ...optional("signal", undefined) }; expect(Object.keys(spread)).toEqual([]); - expect("signal" in spread).toBe(false); + // `Object.hasOwn`, not `in`: `in` walks the prototype chain. + expect(Object.hasOwn(spread, "signal")).toBe(false); }); it("keeps falsy-but-defined values, which is the whole point", () => { for (const value of [0, "", false, Number.NaN, null] as const) { const spread = { ...optional("v", value) }; - expect("v" in spread).toBe(true); + expect(Object.hasOwn(spread, "v")).toBe(true); expect((spread as { v: unknown }).v).toBe(value === value ? value : (spread as { v: number }).v); } expect(Number.isNaN(({ ...optional("v", Number.NaN) } as { v: number }).v)).toBe(true); @@ -70,3 +71,39 @@ describe("defined", () => { expect({ ...defined(base) }).toEqual({ own: "yes" }); }); }); + +describe("keys that shadow Object.prototype", () => { + // `defined` built its result with `result[key] = value`, which goes through + // the `__proto__` setter: a `__proto__` key was silently dropped, and with + // an object value it replaced the result's prototype instead of adding a + // key. Found by a property test over arbitrary dictionaries. + const dangerous = ["__proto__", "constructor", "prototype", "toString", "hasOwnProperty", "valueOf"]; + + it.each(dangerous)("defined keeps a %s key as an own property", (key) => { + const result = defined(JSON.parse(`{"${key}": 1}`) as Record); + expect(Object.hasOwn(result, key)).toBe(true); + expect((result as Record)[key]).toBe(1); + }); + + it.each(dangerous)("optional keeps a %s key as an own property", (key) => { + const result = { ...optional(key, 1) }; + expect(Object.hasOwn(result, key)).toBe(true); + }); + + it("never lets a __proto__ value replace the result's prototype", () => { + const result = defined(JSON.parse('{"__proto__": {"isAdmin": true}}') as Record); + expect(Object.getPrototypeOf(result)).toBe(Object.prototype); + expect((result as Record)["isAdmin"]).toBeUndefined(); + expect(Object.getPrototypeOf({ ...optional("__proto__", { isAdmin: true }) })).toBe(Object.prototype); + }); + + it("does not pollute Object.prototype itself", () => { + defined(JSON.parse('{"__proto__": {"polluted": true}}') as Record); + expect(({} as Record)["polluted"]).toBeUndefined(); + }); + + it("still drops an undefined value under a dangerous key", () => { + const result = defined({ ["__proto__"]: undefined } as Record); + expect(Object.getOwnPropertyNames(result)).toEqual([]); + }); +}); diff --git a/packages/training/test/promotion.test.ts b/packages/training/test/promotion.test.ts index e2c61fb..56ef314 100644 --- a/packages/training/test/promotion.test.ts +++ b/packages/training/test/promotion.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { commitRewrite, revertRewrite } from "ts-autocode-rewrite"; import { + defaultMinScore, defineTrainable, evaluatePromotionGate, type CandidatePatch, @@ -105,3 +106,54 @@ describe("promotion", () => { expect(decision.failures).toContain("AgentV evaluations must be run against the candidate"); }); }); + +describe("threshold validation messages", () => { + // A user who passes a bad threshold wants to know the range, not Zod's type + // vocabulary. `minScore: Infinity` used to report "expected number, received + // number", because the base schema rejected it before the .finite() message + // could apply. Found by a property test over non-unit-interval doubles. + const bad: ReadonlyArray = [ + ["negative", -0.5], + ["above one", 1.5], + ["positive infinity", Number.POSITIVE_INFINITY], + ["negative infinity", Number.NEGATIVE_INFINITY], + ["NaN", Number.NaN], + ["a string", "0.5"], + ["a boolean", true], + ["an object", {}], + ]; + + it.each(bad)("reports the range for %s", async (_label, value) => { + const promise = evaluatePromotionGate({ + candidate: candidate(), evaluations: [], conformance: true, minScore: value as number, + }); + await expect(promise).rejects.toThrow("minScore must be between 0 and 1"); + }); + + it("names the offending setting, not a generic one", async () => { + await expect(evaluatePromotionGate({ + candidate: candidate(), evaluations: [], conformance: true, minPassRate: 2, + })).rejects.toThrow("minPassRate must be between 0 and 1"); + }); + + it("treats null and undefined as unset, applying the default", async () => { + // `minScore ?? defaultMinScore` coalesces both. Neither is type-valid, so + // this pins the behavior rather than endorsing it as an input. + for (const value of [undefined, null]) { + // `exactOptionalPropertyTypes` forbids passing an explicit undefined, + // which is the point: neither spelling is type-valid, so this pins + // runtime behavior rather than endorsing either as an input. + const input = { candidate: candidate(), evaluations: [], conformance: true, minScore: value }; + const decision = await evaluatePromotionGate(input as unknown as Parameters[0]); + expect(decision.failures.some((failure) => failure.includes(`below ${defaultMinScore}`))).toBe(true); + } + }); + + it("accepts the closed interval's endpoints", async () => { + for (const value of [0, 1]) { + await expect(evaluatePromotionGate({ + candidate: candidate(), evaluations: [], conformance: true, minScore: value, minPassRate: value, + })).resolves.toBeDefined(); + } + }); +}); diff --git a/packages/training/test/source.test.ts b/packages/training/test/source.test.ts index bb96d2e..7699603 100644 --- a/packages/training/test/source.test.ts +++ b/packages/training/test/source.test.ts @@ -114,3 +114,31 @@ describe("parameter types inferred from literal defaults", () => { expect(declare("value")[0]?.type).toBe("unknown"); }); }); + +describe("offsets on source TypeScript could not fully parse", () => { + // Error recovery synthesizes a body for an unterminated block whose `end` + // can sit past EOF, so a truncated file produced a target claiming offsets + // outside its own source. Found by fuzzing generated-then-damaged modules. + const truncated = 'class Router {\n\troute(input: string): string {\n\t\t"use training";\n\t\t// } a brace in a comm'; + + it("never reports offsets outside the source", () => { + for (const target of discoverInSource(truncated, "truncated.ts")) { + expect(target.bodyStart).toBeGreaterThanOrEqual(0); + expect(target.bodyEnd).toBeLessThanOrEqual(truncated.length); + expect(target.bodyStart).toBeLessThanOrEqual(target.bodyEnd); + } + }); + + it("keeps the digest consistent with the clamped slice", () => { + for (const target of discoverInSource(truncated, "truncated.ts")) { + const raw = truncated.slice(target.bodyStart, target.bodyEnd); + expect(raw.trim()).toBe(target.implementation); + } + }); + + it("leaves offsets untouched for source that parses", () => { + const valid = 'class Router {\n\troute(input: string): string {\n\t\t"use training";\n\t\treturn input;\n\t}\n}\n'; + const target = discoverInSource(valid, "valid.ts")[0]!; + expect(valid.slice(target.bodyStart, target.bodyEnd).trim()).toBe("return input;"); + }); +}); diff --git a/src/index.ts b/src/index.ts index 26911db..9ef0469 100644 --- a/src/index.ts +++ b/src/index.ts @@ -78,6 +78,7 @@ export { resetTraining, sequentialLoop, SourceDiscoveryError, + toTrainableToken, trainableTokenFromSymbol, training, TraceNotFoundError, diff --git a/test/fuzz.test.ts b/test/fuzz.test.ts new file mode 100644 index 0000000..f19271f --- /dev/null +++ b/test/fuzz.test.ts @@ -0,0 +1,234 @@ +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; + +import { discoverInSource, isTsAutocodeError } from "../src/index.js"; +import { augmentSource } from "../src/register/hook.js"; +import { run } from "../src/cli.js"; +import { scanDeclaredTrainables } from "ts-autocode-grounding"; +import { commitRewrite, digest as rewriteDigest, revertRewrite } from "ts-autocode-rewrite"; +import { evolutionEnabled } from "../src/evolve.js"; +import { anyModule, damagedModule, markedModule } from "./support/sources.js"; + +// Fuzzing: feed the parsers arbitrary and deliberately hostile input and assert +// they fail predictably rather than crashing, hanging, or -- worst for this +// library -- silently corrupting a user's source file. +// +// Every one of these functions runs against code the library did not write: +// `augmentSource` sees every module a user loads, and `scanDeclaredTrainables` +// and `discoverInSource` see whatever is in their project. + +const runs = 200; + +// The corpus is generated marked modules and damaged variants of them (see +// test/support/sources.ts), not random punctuation: instrumenting a +// punctuation corpus showed 1 in 3000 inputs produced a discovered target, so +// every property about offsets and rewriting was passing vacuously. +const hostileSource = anyModule; + +describe("source discovery", () => { + it("never crashes on arbitrary text", () => { + fc.assert(fc.property(fc.string({ maxLength: 400 }), (source) => { + expect(() => discoverInSource(source, "fuzz.ts")).not.toThrow(); + }), { numRuns: runs }); + }); + + it("never crashes on hostile fragments", () => { + fc.assert(fc.property(hostileSource, (source) => { + // A throw is acceptable only as a typed library error, never as a + // TypeError from an unchecked property access. + try { + discoverInSource(source, "fuzz.ts"); + } catch (error) { + expect(isTsAutocodeError(error)).toBe(true); + } + }), { numRuns: runs }); + }); + + it("only ever reports targets whose offsets lie inside the source", () => { + fc.assert(fc.property(hostileSource, (source) => { + for (const target of safeDiscover(source)) { + expect(target.bodyStart).toBeGreaterThanOrEqual(0); + expect(target.bodyEnd).toBeLessThanOrEqual(source.length); + expect(target.bodyStart).toBeLessThanOrEqual(target.bodyEnd); + } + }), { numRuns: runs }); + }); + + it("relates the body slice, implementation and digest exactly as documented", () => { + // These three fields have subtly different relationships to the source + // -- `implementation` is trimmed, `bodyDigest` hashes the raw slice -- + // and nothing said so until a property test asked. Guarded application + // depends on the digest side, so the distinction is load-bearing. + fc.assert(fc.property(hostileSource, (source) => { + for (const target of safeDiscover(source)) { + const raw = source.slice(target.bodyStart, target.bodyEnd); + expect(raw.trim()).toBe(target.implementation); + expect(rewriteDigest(raw)).toBe(target.bodyDigest); + } + }), { numRuns: runs }); + }); + + it("produces targets a guarded rewrite accepts and can revert", () => { + // The end-to-end invariant that matters: whatever discovery reports must + // survive commit and revert byte for byte. + fc.assert(fc.property(markedModule, (source) => { + for (const target of safeDiscover(source)) { + const candidate = { + id: "fuzz", trainableId: target.id, engineId: "fuzz", target, + implementation: "return \"fuzzed\";", + }; + const committed = commitRewrite(source, candidate); + expect(committed.source).not.toBe(source); + expect(revertRewrite(committed.source, committed.snapshot)).toBe(source); + } + }), { numRuns: 100 }); + }); + + it("never reports an empty identity", () => { + fc.assert(fc.property(hostileSource, (source) => { + for (const target of safeDiscover(source)) { + expect(target.id.trim().length).toBeGreaterThan(0); + } + }), { numRuns: runs }); + }); +}); + +describe("the register load hook", () => { + // This rewrites every module a user loads. Corrupting one would be the worst + // failure this library could have. + it("never throws, whatever the module contains", () => { + fc.assert(fc.property(fc.string({ maxLength: 400 }), (source) => { + expect(() => augmentSource(source, "fuzz.ts")).not.toThrow(); + }), { numRuns: runs }); + }); + + it("is append-only: the original source is always a prefix of the result", () => { + fc.assert(fc.property(hostileSource, (source) => { + // Line numbers and sourcemaps of the original module depend on this. + expect(augmentSource(source, "fuzz.ts").startsWith(source)).toBe(true); + }), { numRuns: runs }); + }); + + it("leaves a module without the marker byte-identical", () => { + fc.assert(fc.property(fc.string({ maxLength: 400 }), (source) => { + fc.pre(!source.includes("use training")); + expect(augmentSource(source, "fuzz.ts")).toBe(source); + }), { numRuns: runs }); + }); + + it("is idempotent for sources it leaves alone", () => { + fc.assert(fc.property(hostileSource, (source) => { + const once = augmentSource(source, "fuzz.ts"); + fc.pre(once === source); + expect(augmentSource(once, "fuzz.ts")).toBe(source); + }), { numRuns: runs }); + }); +}); + +describe("ambient declaration scanning", () => { + it("never crashes on arbitrary text", () => { + fc.assert(fc.property(fc.string({ maxLength: 400 }), (source) => { + expect(() => scanDeclaredTrainables(source)).not.toThrow(); + }), { numRuns: runs }); + }); + + it("rejects non-string input with a TypeError rather than coercing", () => { + for (const value of [undefined, null, 1, {}, [], true]) { + expect(() => scanDeclaredTrainables(value as never)).toThrow(TypeError); + } + }); + + it("only reports operations with a non-empty method and contract ref", () => { + fc.assert(fc.property(hostileSource, (source) => { + for (const declared of scanDeclaredTrainables(source)) { + for (const operation of declared.operations) { + expect(operation.method.length).toBeGreaterThan(0); + expect(operation.contractRef.length).toBeGreaterThan(0); + expect(operation.intent.length).toBeGreaterThan(0); + } + } + }), { numRuns: runs }); + }); +}); + +describe("the evolve kill switch", () => { + // This decides whether the library rewrites the user's source. It must never + // read an unrecognized value as consent. + it("only ever enables on a recognized affirmative or an unset value", () => { + fc.assert(fc.property(fc.string({ maxLength: 20 }), (value) => { + let enabled: boolean; + try { + enabled = evolutionEnabled(value); + } catch { + return; // Refusing to guess is the correct outcome. + } + const flag = value.trim().toLowerCase(); + expect(enabled).toBe(flag === "" || ["1", "true", "on", "yes", "enabled"].includes(flag)); + }), { numRuns: 500 }); + }); +}); + +describe("the command line", () => { + it("never throws for arbitrary argv, and always reports a usable exit code", async () => { + await fc.assert(fc.asyncProperty( + fc.array(fc.string({ maxLength: 20 }), { maxLength: 8 }), + async (argv) => { + const result = await run(argv); + expect([0, 1, 2]).toContain(result.code); + // A non-zero exit must say something; a zero exit must not be silent. + expect((result.code === 0 ? result.stdout : result.stderr).length).toBeGreaterThan(0); + }, + ), { numRuns: 150 }); + }); + + it("never leaks a stack trace to stderr", async () => { + await fc.assert(fc.asyncProperty( + fc.array(fc.oneof(fc.constantFrom("discover", "status", "help", "--json", "--cwd", "--file", "--nope"), + fc.string({ maxLength: 12 })), { maxLength: 6 }), + async (argv) => { + const result = await run(argv); + expect(result.stderr).not.toMatch(/\n\s+at /); + }, + ), { numRuns: 150 }); + }); +}); + +/** Discovery result, or nothing when the source is rejected outright. */ +function safeDiscover(source: string) { + try { + return discoverInSource(source, "fuzz.ts"); + } catch { + return []; + } +} + +describe("the fuzz corpus itself", () => { + // A corpus that never reaches the code under test makes every property + // above vacuously true. This asserts the corpus does its job, so the suite + // cannot quietly decay into theatre. + it("produces modules that discovery actually finds targets in", () => { + let withTargets = 0; + const samples = fc.sample(markedModule, 100); + for (const source of samples) { + if (discoverInSource(source, "fuzz.ts").length > 0) withTargets += 1; + } + expect(withTargets).toBeGreaterThan(80); + }); + + it("produces damaged modules that still often parse", () => { + let withTargets = 0; + for (const source of fc.sample(damagedModule, 200)) { + if (safeDiscover(source).length > 0) withTargets += 1; + } + // Damaged input should be a genuine mix, not all-or-nothing. + expect(withTargets).toBeGreaterThan(10); + }); + + it("produces modules the load hook actually rewrites", () => { + let rewritten = 0; + for (const source of fc.sample(markedModule, 100)) { + if (augmentSource(source, "fuzz.ts") !== source) rewritten += 1; + } + expect(rewritten).toBeGreaterThan(80); + }); +}); diff --git a/test/property.test.ts b/test/property.test.ts new file mode 100644 index 0000000..3ccd0e0 --- /dev/null +++ b/test/property.test.ts @@ -0,0 +1,297 @@ +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; + +import { digest as rewriteDigest } from "ts-autocode-rewrite"; +import { textDigest } from "ts-autocode-grounding"; +import { + defined, + defineTrainable, + evaluationArgs, + evaluatePromotionGate, + optional, + toTrainableToken, + trainableTokenFromSymbol, + type BoundEvaluation, + type CandidatePatch, +} from "../src/index.js"; +import { discoverInSource } from "../src/index.js"; +import { trainableIdFromKey } from "../packages/training/src/token.js"; + +// Property-based tests: instead of a handful of chosen inputs, state the law +// that must hold and let fast-check hunt for a counterexample. These target the +// pure, total functions where an example-based test can only ever sample -- +// identity round-trips, canonicalization, aggregation, and the spread helpers. +// +// Every failure prints a shrunk counterexample and a seed, so a regression is +// reproducible rather than "it failed once on CI". + +const runs = 300; + +/** Ids the library accepts: non-empty once trimmed. */ +const trainableId = fc.string({ minLength: 1, maxLength: 64 }) + .filter((value) => value.trim().length > 0); + +describe("trainable identity", () => { + it("round-trips through its symbol for any acceptable id", () => { + fc.assert(fc.property(trainableId, (id) => { + const token = defineTrainable(id); + expect(trainableTokenFromSymbol(token.symbol).id).toBe(id.trim()); + expect(toTrainableToken(token.symbol)).toEqual(token); + }), { numRuns: runs }); + }); + + it("is idempotent: defining twice yields the same symbol", () => { + fc.assert(fc.property(trainableId, (id) => { + expect(defineTrainable(id).symbol).toBe(defineTrainable(id).symbol); + }), { numRuns: runs }); + }); + + it("ignores surrounding whitespace but nothing else", () => { + fc.assert(fc.property(trainableId, fc.stringMatching(/^[ \t\n]*$/), (id, pad) => { + expect(defineTrainable(`${pad}${id}${pad}`).id).toBe(id.trim()); + }), { numRuns: runs }); + }); + + it("distinct trimmed ids never collide", () => { + fc.assert(fc.property(trainableId, trainableId, (left, right) => { + fc.pre(left.trim() !== right.trim()); + expect(defineTrainable(left).symbol).not.toBe(defineTrainable(right).symbol); + }), { numRuns: runs }); + }); + + it("stripping the key prefix is idempotent for unprefixed keys", () => { + fc.assert(fc.property(fc.string(), (key) => { + fc.pre(!key.startsWith("ts-autocode.trainable:")); + expect(trainableIdFromKey(key)).toBe(key); + }), { numRuns: runs }); + }); + + it("rejects every blank id rather than producing an empty identity", () => { + fc.assert(fc.property(fc.stringMatching(/^[ \t\n\r]*$/), (blank) => { + expect(() => defineTrainable(blank)).toThrow(); + }), { numRuns: runs }); + }); +}); + +describe("digest canonicalization", () => { + // Guarded rewriting refuses a candidate whose body digest changed, so these + // laws are load-bearing rather than cosmetic. + const json = fc.jsonValue(); + + it("is deterministic", () => { + fc.assert(fc.property(json, (value) => { + expect(rewriteDigest(value)).toBe(rewriteDigest(value)); + }), { numRuns: runs }); + }); + + it("is insensitive to object key order at any depth", () => { + fc.assert(fc.property(json, (value) => { + expect(rewriteDigest(value)).toBe(rewriteDigest(shuffleKeys(value))); + }), { numRuns: runs }); + }); + + it("is sensitive to array order whenever order is observable", () => { + fc.assert(fc.property(fc.uniqueArray(fc.integer(), { minLength: 2, maxLength: 8 }), (values) => { + expect(rewriteDigest(values)).not.toBe(rewriteDigest([...values].reverse())); + }), { numRuns: runs }); + }); + + it("always produces the documented shape", () => { + fc.assert(fc.property(json, (value) => { + expect(rewriteDigest(value)).toMatch(/^sha256:[0-9a-f]{64}$/); + }), { numRuns: runs }); + }); + + it("text digest normalizes line endings and nothing else", () => { + fc.assert(fc.property(fc.array(fc.string({ maxLength: 12 }), { maxLength: 8 }), (lines) => { + expect(textDigest(lines.join("\r\n"))).toBe(textDigest(lines.join("\n"))); + }), { numRuns: runs }); + }); +}); + +describe("optional and defined", () => { + it("optional includes a key exactly when the value is defined", () => { + // `Object.hasOwn`, not `in`: `in` walks the prototype chain, so any key + // named after an Object.prototype member ("toString", "constructor") + // reads as present whether or not it was added. fast-check found that + // mistake in this assertion within a few hundred runs. + fc.assert(fc.property(fc.string({ minLength: 1 }), fc.option(fc.jsonValue(), { nil: undefined }), (key, value) => { + const spread = { ...optional(key, value) } as Record; + expect(Object.hasOwn(spread, key)).toBe(value !== undefined); + }), { numRuns: runs }); + }); + + it("optional is safe for keys that shadow Object.prototype", () => { + for (const key of ["__proto__", "constructor", "toString", "hasOwnProperty", "valueOf"]) { + const present = { ...optional(key, 1) } as Record; + expect(Object.hasOwn(present, key)).toBe(true); + expect(present[key]).toBe(1); + // A computed `__proto__` defines an own property rather than setting + // the prototype, so the object stays a plain object. + expect(Object.getPrototypeOf({ ...optional("__proto__", { polluted: true }) })).toBe(Object.prototype); + expect(Object.hasOwn({ ...optional(key, undefined) } as object, key)).toBe(false); + } + }); + + it("defined keeps exactly the defined entries", () => { + fc.assert(fc.property( + fc.dictionary(fc.string({ minLength: 1 }), fc.option(fc.integer(), { nil: undefined })), + (record) => { + const spread = { ...defined(record) } as Record; + const expected = Object.entries(record).filter(([, value]) => value !== undefined).map(([key]) => key); + expect(Object.keys(spread).sort()).toEqual(expected.sort()); + }, + ), { numRuns: runs }); + }); + + it("defined never introduces an explicit undefined", () => { + fc.assert(fc.property( + fc.dictionary(fc.string({ minLength: 1 }), fc.option(fc.integer(), { nil: undefined })), + (record) => { + for (const value of Object.values({ ...defined(record) })) expect(value).toBeDefined(); + }, + ), { numRuns: runs }); + }); +}); + +describe("evaluation argument decoding", () => { + it("never throws, whatever the eval input is", () => { + fc.assert(fc.property(fc.string(), (input) => { + expect(Array.isArray(evaluationArgs(input))).toBe(true); + }), { numRuns: runs }); + }); + + it("round-trips a JSON array of arguments", () => { + // `-0` is excluded because JSON cannot represent it: JSON.stringify(-0) + // is "0", so no decoder could return it. That is a property of the wire + // format, not something this library can or should fix -- but it is + // worth having stated, since eval inputs are JSON strings. + fc.assert(fc.property(fc.array(fc.jsonValue(), { maxLength: 6 }), (args) => { + fc.pre(!JSON.stringify(args).includes("-0") && !hasNegativeZero(args)); + expect(evaluationArgs(JSON.stringify(args))).toEqual(args); + }), { numRuns: runs }); + }); + + it("cannot recover negative zero, because JSON does not carry it", () => { + expect(JSON.stringify([-0])).toBe("[0]"); + expect(Object.is(evaluationArgs("[-0]")[0], -0)).toBe(true); + expect(Object.is(evaluationArgs(JSON.stringify([-0]))[0], -0)).toBe(false); + }); + + it("wraps a non-array JSON value as a single argument", () => { + fc.assert(fc.property(fc.oneof(fc.integer(), fc.boolean(), fc.constant(null)), (value) => { + expect(evaluationArgs(JSON.stringify(value))).toEqual([value]); + }), { numRuns: runs }); + }); + + it("falls back to the raw string when the input is not JSON", () => { + fc.assert(fc.property(fc.string(), (input) => { + fc.pre(!isJson(input)); + expect(evaluationArgs(input)).toEqual([input]); + }), { numRuns: runs }); + }); +}); + +describe("promotion gate aggregation", () => { + const target = discoverInSource(`class F { + m(input: string): string { + "use training"; + return input; + } +}`, "f.ts")[0]!; + const candidate: CandidatePatch = { + id: "c", trainableId: target.id, engineId: "prop", target, implementation: "return input;", + }; + const score = fc.float({ min: 0, max: 1, noNaN: true }); + + const evaluationsOf = (scores: readonly number[]): BoundEvaluation[] => scores.map((value, index) => ({ + trainableId: target.id, + candidateId: candidate.id, + result: { testId: `t${index}`, score: value, executionStatus: "ok", output: "" } as never, + })); + + it("mean score is the arithmetic mean of the bound results", async () => { + await fc.assert(fc.asyncProperty(fc.array(score, { minLength: 1, maxLength: 8 }), async (scores) => { + const decision = await evaluatePromotionGate({ + candidate, evaluations: evaluationsOf(scores), conformance: true, + }); + const mean = scores.reduce((sum, value) => sum + value, 0) / scores.length; + expect(decision.meanScore).toBeCloseTo(mean, 10); + }), { numRuns: 100 }); + }); + + it("mean score and pass rate stay within the unit interval", async () => { + await fc.assert(fc.asyncProperty(fc.array(score, { minLength: 1, maxLength: 8 }), async (scores) => { + const decision = await evaluatePromotionGate({ + candidate, evaluations: evaluationsOf(scores), conformance: true, + }); + for (const value of [decision.meanScore, decision.passRate]) { + expect(value).toBeGreaterThanOrEqual(0); + expect(value).toBeLessThanOrEqual(1); + } + }), { numRuns: 100 }); + }); + + it("promotes only when it reports no failures, always", async () => { + await fc.assert(fc.asyncProperty( + fc.array(score, { maxLength: 6 }), fc.boolean(), score, score, + async (scores, conformance, minScore, minPassRate) => { + const decision = await evaluatePromotionGate({ + candidate, evaluations: evaluationsOf(scores), conformance, minScore, minPassRate, + }); + expect(decision.promote).toBe(decision.failures.length === 0); + }, + ), { numRuns: 200 }); + }); + + it("an extra gate that always refuses always blocks promotion", async () => { + await fc.assert(fc.asyncProperty(fc.array(score, { minLength: 1, maxLength: 6 }), async (scores) => { + const decision = await evaluatePromotionGate({ + candidate, + evaluations: evaluationsOf(scores), + conformance: true, + minScore: 0, + minPassRate: 0, + gates: [() => "always refuses"], + }); + expect(decision.promote).toBe(false); + expect(decision.failures).toContain("always refuses"); + }), { numRuns: 100 }); + }); + + it("rejects a threshold outside the unit interval rather than clamping it", async () => { + await fc.assert(fc.asyncProperty( + fc.double({ noNaN: true }).filter((value) => value < 0 || value > 1), + async (minScore) => { + await expect(evaluatePromotionGate({ + candidate, evaluations: [], conformance: true, minScore, + })).rejects.toThrow(/between 0 and 1/); + }, + ), { numRuns: 100 }); + }); +}); + +/** Recursively reorders object keys, leaving arrays and scalars alone. */ +function shuffleKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(shuffleKeys); + if (typeof value !== "object" || value === null) return value; + const entries = Object.entries(value as Record).reverse(); + return Object.fromEntries(entries.map(([key, nested]) => [key, shuffleKeys(nested)])); +} + +/** True when any nested value is `-0`, which JSON flattens to `0`. */ +function hasNegativeZero(value: unknown): boolean { + if (Object.is(value, -0)) return true; + if (Array.isArray(value)) return value.some(hasNegativeZero); + if (typeof value === "object" && value !== null) return Object.values(value).some(hasNegativeZero); + return false; +} + +function isJson(value: string): boolean { + try { + JSON.parse(value); + return true; + } catch { + return false; + } +} diff --git a/test/snapshots/surface/root-export-kinds.verified.json b/test/snapshots/surface/root-export-kinds.verified.json index c21a05c..15392ff 100644 --- a/test/snapshots/surface/root-export-kinds.verified.json +++ b/test/snapshots/surface/root-export-kinds.verified.json @@ -74,6 +74,7 @@ "sequentialLoop": "function", "swapImplementation": "function", "swappedImplementation": "function", + "toTrainableToken": "function", "trainable": "function", "trainableTokenFromSymbol": "function", "training": "object", diff --git a/test/snapshots/surface/ts-autocode-training.verified.txt b/test/snapshots/surface/ts-autocode-training.verified.txt index 78b96be..bbb43f2 100644 --- a/test/snapshots/surface/ts-autocode-training.verified.txt +++ b/test/snapshots/surface/ts-autocode-training.verified.txt @@ -49,6 +49,7 @@ parseSetting provideTrainingDefaults resetTraining sequentialLoop +toTrainableToken trainableTokenFromSymbol training trainingMarker diff --git a/test/snapshots/surface/ts-autocode.verified.txt b/test/snapshots/surface/ts-autocode.verified.txt index 8730f78..3ef8878 100644 --- a/test/snapshots/surface/ts-autocode.verified.txt +++ b/test/snapshots/surface/ts-autocode.verified.txt @@ -73,6 +73,7 @@ rewritePromotion sequentialLoop swapImplementation swappedImplementation +toTrainableToken trainable trainableTokenFromSymbol training diff --git a/test/support/sources.ts b/test/support/sources.ts new file mode 100644 index 0000000..a264d6d Binary files /dev/null and b/test/support/sources.ts differ