test(5/7): chaos / fault injection, and completing runtime isolation

docs/testing.md

@@ -15,6 +15,7 @@ land, never lower them to get a build green.
 | Property | `test/property.test.ts` | A law that holds for chosen examples but not in general |
 | Fuzz | `test/fuzz.test.ts` | A parser crashing, hanging, or corrupting source it did not write |
 | Contract | `test/contract.test.ts` | A provider implementation that satisfies the types but not the contract |
+| Chaos | `test/chaos.test.ts` | A dependency failing, hanging, or racing — and the damage that leaves behind |
 | Characterization | `test/characterization*.test.ts` | A change to anything this library *generates* — rewritten source, emitted instrumentation, prompts, CLI output, the export surface |
 
 ## Running

packages/training/src/training.ts

@@ -279,6 +279,20 @@ export interface Training {
 	records(trainable?: TrainableIdentity): Promise<readonly TrainingRecord[]>;
 	evaluate(trainable: TrainableIdentity, config: EvalConfig): Promise<TrainableEvalRun>;
 	train(input: TrainInput): Promise<TrainingRun>;
+	/** Route one call of a marked trainable through *this* runtime's capture.
+	 *
+	 * {@link captureTrainable} does the same thing for the process-wide runtime,
+	 * and is what installed instrumentation calls. A runtime built with
+	 * {@link createTrainingRuntime} is not reachable that way -- it registers
+	 * nothing globally, by design -- so without this an isolated runtime could
+	 * train and evaluate but never capture, which is half a runtime. */
+	capture<This, Args extends unknown[], Result>(
+		trainable: TrainableIdentity,
+		methodName: string,
+		thisValue: This,
+		method: (this: This, ...args: Args) => Result,
+		args: Args,
+	): Result;
 	flush(): Promise<void>;
 }
 
@@ -547,6 +561,16 @@ class TrainingRuntime implements Training {
 		await Promise.all([...this.#pending]);
 	}
 
+	capture<This, Args extends unknown[], Result>(
+		trainable: TrainableIdentity,
+		methodName: string,
+		thisValue: This,
+		method: (this: This, ...args: Args) => Result,
+		args: Args,
+	): Result {
+		return this.invoke(thisValue, method, args, toTrainableToken(trainable), methodName);
+	}
+
 	invoke<This, Args extends unknown[], Result>(
 		thisValue: This,
 		method: (this: This, ...args: Args) => Result,
@@ -756,6 +780,8 @@ export const training: Training = Object.freeze<Training>({
 	records: (identity) => runtime().records(identity),
 	evaluate: (identity, config) => runtime().evaluate(identity, config),
 	train: (input) => runtime().train(input),
+	capture: (trainable, methodName, thisValue, method, args) =>
+		runtime().capture(trainable, methodName, thisValue, method, args),
 	flush: () => runtime().flush(),
 });
 

packages/training/test/training.test.ts

@@ -8,6 +8,7 @@ import "./wiring.js";
 
 import * as publicApi from "../src/index.js";
 import {
+	createTrainingRuntime,
 	captureTrainable,
 	configureTraining,
 	defineTrainable,
@@ -325,3 +326,56 @@ describe("training resilience policies", () => {
 
 const functionExecutor: ImplementationExecutor = async (target, implementation, args) =>
 	new Function(...target.parameters.map((parameter) => parameter.name), implementation)(...args);
+
+describe("capture on an isolated runtime", () => {
+	// `captureTrainable` routes to the process-wide runtime, so a runtime built
+	// with `createTrainingRuntime` could train and evaluate but never capture --
+	// half a runtime, and exactly the case isolation exists for. Found while
+	// writing fault-injection tests against an isolated runtime.
+	it("records through the runtime it was called on, not the global one", async () => {
+		const isolated = createTrainingRuntime({ tracing: { enabled: false } });
+		const other = createTrainingRuntime({ tracing: { enabled: false } });
+		const token = defineTrainable("Isolated.route");
+
+		expect(isolated.capture(token, "route", undefined, (input: string) => input.toUpperCase(), ["abc"]))
+			.toBe("ABC");
+		await isolated.flush();
+		await other.flush();
+
+		expect((await isolated.records(token)).length).toBe(1);
+		expect((await other.records(token)).length).toBe(0);
+	});
+
+	it("accepts a symbol identity as well as a token", async () => {
+		const isolated = createTrainingRuntime({ tracing: { enabled: false } });
+		const token = defineTrainable("Isolated.symbol");
+		isolated.capture(token.symbol, "route", undefined, (input: string) => input, ["x"]);
+		await isolated.flush();
+		expect((await isolated.records(token)).length).toBe(1);
+	});
+
+	it("preserves this, arguments, return values and thrown errors", async () => {
+		const isolated = createTrainingRuntime({ tracing: { enabled: false } });
+		const token = defineTrainable("Isolated.receiver");
+		const receiver = { suffix: "!" };
+		function method(this: typeof receiver, input: string): string {
+			return `${input}${this.suffix}`;
+		}
+		expect(isolated.capture(token, "method", receiver, method, ["hi"])).toBe("hi!");
+
+		const boom = () => { throw new Error("thrown"); };
+		expect(() => isolated.capture(token, "boom", undefined, boom, [])).toThrow("thrown");
+		await isolated.flush();
+		const records = await isolated.records(token);
+		expect(records.map((record) => record.succeeded)).toEqual([true, false]);
+	});
+
+	it("awaits an async method and records its settled outcome", async () => {
+		const isolated = createTrainingRuntime({ tracing: { enabled: false } });
+		const token = defineTrainable("Isolated.async");
+		await expect(isolated.capture(token, "slow", undefined, async (input: string) => input, ["x"]))
+			.resolves.toBe("x");
+		await isolated.flush();
+		expect((await isolated.records(token))[0]?.succeeded).toBe(true);
+	});
+});