diff --git a/docs/chapter_6.md b/docs/chapter_6.md index 2686a5c..5ee4cb4 100644 --- a/docs/chapter_6.md +++ b/docs/chapter_6.md @@ -312,6 +312,7 @@ The regression file stores failures grouped by test type (property-based tests v "seed": 901717247, "dial": "example/sip010.cjs", "numRuns": 15, + "counterexamplePath": "2:1:0", "timestamp": 1767886534833 }, { @@ -341,6 +342,8 @@ Each failure record includes: - `numRuns` – Number of test iterations needed for the failure to occur - `timestamp` – When the failure was discovered (Unix timestamp in milliseconds) - `dial` (optional) – Path to the dialer file used during the test +- `counterexamplePath` (optional) – fast-check shrink metadata for diagnostics; + it is not an exact replay trace Failures are sorted by timestamp in descending order, with the most recent failures first. diff --git a/heatstroke.tests.ts b/heatstroke.tests.ts index 731d4ef..0c1b4a9 100644 --- a/heatstroke.tests.ts +++ b/heatstroke.tests.ts @@ -290,6 +290,7 @@ describe("Custom reporter logging", () => { }); const runDetails = { + counterexamplePath: `${r.path}:preferred`, path: r.path, failed: r.failed, numRuns: r.numRuns, @@ -320,7 +321,7 @@ describe("Custom reporter logging", () => { const expectedMessages = [ `\nError: Property failed after ${r.numRuns} tests.`, `Seed : ${r.seed}`, - `Path : ${r.path}`, + `Path : ${r.path}:preferred`, `\nCounterexample:`, `- Contract : ${getContractNameFromContractId( rendezvousContractId, diff --git a/heatstroke.ts b/heatstroke.ts index f137940..e5859bc 100644 --- a/heatstroke.ts +++ b/heatstroke.ts @@ -26,7 +26,9 @@ import { getContractNameFromContractId } from "./shared"; * @property `runDetails.counterexample`: The input that caused the failure. * @property `runDetails.numRuns`: The number of test cases that were run. * @property `runDetails.seed`: The replay seed for reproducing the failure. - * @property `runDetails.path`: The replay path for reproducing the failure. + * @property `runDetails.counterexamplePath`: Fast-check shrink metadata for + * diagnostics; it is not an exact replay trace. + * @property `runDetails.path`: The legacy path fallback for older callers. * @property `runDetails.error`: The error thrown during the test. * @param radio The event emitter to log messages. * @param type The type of test that failed: invariant or property. @@ -42,6 +44,7 @@ export const reporter = ( if (failed) { const error = runDetails.errorInstance || runDetails.error; + const counterexamplePathValue = runDetails.counterexamplePath ?? path; // Extract the actual Clarity error once for both error types. const clarityError = @@ -56,8 +59,8 @@ export const reporter = ( `\nError: Property failed after ${numRuns} tests.`, ); radio.emit("logFailure", `Seed : ${seed}`); - if (path) { - radio.emit("logFailure", `Path : ${path}`); + if (counterexamplePathValue) { + radio.emit("logFailure", `Path : ${counterexamplePathValue}`); } switch (type) { case "invariant": { diff --git a/heatstroke.types.ts b/heatstroke.types.ts index 6cd73c0..0acdbd3 100644 --- a/heatstroke.types.ts +++ b/heatstroke.types.ts @@ -5,6 +5,7 @@ export interface RunDetails { counterexample: CounterExample[]; numRuns: number; seed: number; + counterexamplePath?: string | null; path?: string; error?: Error; errorInstance?: Error; diff --git a/persistence.tests.ts b/persistence.tests.ts index 508508f..26b329a 100644 --- a/persistence.tests.ts +++ b/persistence.tests.ts @@ -1,4 +1,4 @@ -import { mkdirSync, rmSync, statSync } from "node:fs"; +import { mkdirSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; @@ -409,6 +409,51 @@ describe("Failure Persistence", () => { ), ); }); + + it("persists paths and reads records from older files", () => { + const customBaseDir = createTemporaryCustomTestBaseDir("paths"); + const options = { baseDir: customBaseDir }; + + try { + persistFailure( + { + ...createMockRunDetails(1), + counterexamplePath: "0:1", + path: "legacy", + }, + "test", + TEST_CONTRACT_ID, + undefined, + options, + ); + persistFailure( + { ...createMockRunDetails(2), path: "legacy" }, + "test", + TEST_CONTRACT_ID, + undefined, + options, + ); + + const failures = loadFailures(TEST_CONTRACT_ID, "test", options); + expect( + failures.find(({ seed }) => seed === 1)?.counterexamplePath, + ).toBe("0:1"); + expect( + failures.find(({ seed }) => seed === 2)?.counterexamplePath, + ).toBe("legacy"); + + const record = { seed: 17, numRuns: 20, timestamp: 123 }; + writeFileSync( + getFailureFilePath(TEST_CONTRACT_ID, customBaseDir), + JSON.stringify({ invariant: [record], test: [] }), + ); + expect(loadFailures(TEST_CONTRACT_ID, "invariant", options)).toEqual([ + record, + ]); + } finally { + rmSync(customBaseDir, { recursive: true, force: true }); + } + }); }); describe("Loading Failures", () => { diff --git a/persistence.ts b/persistence.ts index a7d0956..5631992 100644 --- a/persistence.ts +++ b/persistence.ts @@ -11,6 +11,8 @@ interface FailureRecord { seed: number; /** The number of runs needed to reproduce the failure*/ numRuns: number; + /** The fast-check path to the counterexample */ + counterexamplePath?: string | null; /** The path to the dialer file used for this test run */ dial: string | undefined; /** Timestamp when the failure was recorded */ @@ -115,6 +117,7 @@ export const persistFailure = ( seed: runDetails.seed, dial: dial, numRuns: runDetails.numRuns, + counterexamplePath: runDetails.counterexamplePath ?? runDetails.path, timestamp: Date.now(), };