From de135d2243cc7c6415f9cb84cf9121d3d6bd1564 Mon Sep 17 00:00:00 2001 From: Luis Schaab Date: Sat, 13 Jun 2026 19:33:10 +0000 Subject: [PATCH 1/7] fix: make solidity --snapshot-check read-only for function gas snapshots --- .../gas-analytics/function-gas-snapshots.ts | 51 +++-- .../tasks/solidity-test/task-action.ts | 2 +- .../tasks/solidity-test/task-action.ts | 186 +++++++++++------- 3 files changed, 136 insertions(+), 103 deletions(-) diff --git a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/function-gas-snapshots.ts b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/function-gas-snapshots.ts index 2a7f0a16fe0..ade917b744a 100644 --- a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/function-gas-snapshots.ts +++ b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/function-gas-snapshots.ts @@ -64,7 +64,7 @@ export interface FunctionGasSnapshotChange { export interface FunctionGasSnapshotCheckResult { passed: boolean; comparison: FunctionGasSnapshotComparison; - written: boolean; + noBaseline: boolean; } export function getFunctionGasSnapshotsPath(basePath: string): string { @@ -323,8 +323,6 @@ export async function checkFunctionGasSnapshots( previousFunctionGasSnapshots = await readFunctionGasSnapshots(basePath); } catch (error) { if (error instanceof FileNotFoundError) { - await writeFunctionGasSnapshots(basePath, functionGasSnapshots); - return { passed: true, comparison: { @@ -332,7 +330,7 @@ export async function checkFunctionGasSnapshots( removed: [], changed: [], }, - written: true, + noBaseline: functionGasSnapshots.length > 0, }; } @@ -344,17 +342,10 @@ export async function checkFunctionGasSnapshots( functionGasSnapshots, ); - // Update snapshots when functions are added or removed (but not changed) - const hasAddedOrRemoved = - comparison.added.length > 0 || comparison.removed.length > 0; - if (comparison.changed.length === 0 && hasAddedOrRemoved) { - await writeFunctionGasSnapshots(basePath, functionGasSnapshots); - } - return { passed: comparison.changed.length === 0, comparison, - written: hasAddedOrRemoved, + noBaseline: false, }; } @@ -362,7 +353,7 @@ export function logFunctionGasSnapshotsSection( result: FunctionGasSnapshotCheckResult, logger: typeof console.log = console.log, ): void { - const { comparison, written } = result; + const { comparison, noBaseline } = result; const changedLength = comparison.changed.length; const addedLength = comparison.added.length; const removedLength = comparison.removed.length; @@ -370,33 +361,31 @@ export function logFunctionGasSnapshotsSection( const hasAdded = addedLength > 0; const hasRemoved = removedLength > 0; const hasAnyDifferences = hasChanges || hasAdded || hasRemoved; - const isFirstTimeWrite = written && !hasAnyDifferences; // Nothing to report - if (!isFirstTimeWrite && !hasAnyDifferences) { + if (!noBaseline && !hasAnyDifferences) { return; } - logger( - formatSectionHeader("Function gas snapshots", { - changedLength, - addedLength, - removedLength, - }), - ); - - if (isFirstTimeWrite) { - logger(); + if (noBaseline) { logger( styleText( - "green", - " No existing snapshots found. Function gas snapshots written successfully", + "yellow", + "Function gas snapshots: no baseline found. Run your tests with --snapshot to create one.", ), ); logger(); return; } + logger( + formatSectionHeader("Function gas snapshots", { + changedLength, + addedLength, + removedLength, + }), + ); + if (hasChanges) { logger(); printFunctionGasSnapshotChanges(comparison.changed, logger); @@ -404,7 +393,9 @@ export function logFunctionGasSnapshotsSection( if (hasAdded) { logger(); - logger(` Added ${comparison.added.length} function(s):`); + logger( + ` ${comparison.added.length} function(s) produced by this run are not in the baseline:`, + ); const addedLines = stringifyFunctionGasSnapshots(comparison.added).split( "\n", ); @@ -415,7 +406,9 @@ export function logFunctionGasSnapshotsSection( if (hasRemoved) { logger(); - logger(` Removed ${comparison.removed.length} function(s):`); + logger( + ` ${comparison.removed.length} stored function(s) were not produced by this run:`, + ); const removedLines = stringifyFunctionGasSnapshots( comparison.removed, ).split("\n"); diff --git a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts index 7e6baf3460f..7703ddaa159 100644 --- a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts +++ b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts @@ -149,7 +149,7 @@ export function logSnapshotCheckResult( ); const functionGasHasOutput = - functionGasSnapshotsCheck.written || + functionGasSnapshotsCheck.noBaseline || functionGasSnapshotsCheck.comparison.changed.length > 0 || functionGasSnapshotsCheck.comparison.added.length > 0 || functionGasSnapshotsCheck.comparison.removed.length > 0; diff --git a/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts b/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts index 04bdc011ae4..3a245e67266 100644 --- a/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts +++ b/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts @@ -163,7 +163,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { const tmp = createTmpDir("snapshots-check-test", "test"); describe("function gas snapshots", () => { - it("should write function gas snapshots on first run (no existing file)", async () => { + it("should not write function gas snapshots on first run (no existing file)", async () => { const suiteResults = [ createSuiteResult("MyContract", [ createStandardTestResult("testA", 10000n), @@ -176,13 +176,14 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { ); assert.equal(functionGasSnapshotsCheck.passed, true); - assert.equal(functionGasSnapshotsCheck.written, true); + assert.equal(functionGasSnapshotsCheck.noBaseline, true); assert.equal(functionGasSnapshotsCheck.comparison.added.length, 0); assert.equal(functionGasSnapshotsCheck.comparison.removed.length, 0); assert.equal(functionGasSnapshotsCheck.comparison.changed.length, 0); + + // --snapshot-check is read-only: it must not bootstrap the baseline. const snapshotPath = getFunctionGasSnapshotsPath(tmp.path); - const savedContent = await readUtf8File(snapshotPath); - assert.equal(savedContent, "MyContract#testA (gas: 10000)"); + assert.equal(await exists(snapshotPath), false); }); it("should pass when function gas snapshots are unchanged", async () => { @@ -200,7 +201,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { ); assert.equal(functionGasSnapshotsCheck.passed, true); - assert.equal(functionGasSnapshotsCheck.written, false); + assert.equal(functionGasSnapshotsCheck.noBaseline, false); assert.equal(functionGasSnapshotsCheck.comparison.added.length, 0); assert.equal(functionGasSnapshotsCheck.comparison.removed.length, 0); assert.equal(functionGasSnapshotsCheck.comparison.changed.length, 0); @@ -226,13 +227,13 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { ); assert.equal(functionGasSnapshotsCheck.passed, false); - assert.equal(functionGasSnapshotsCheck.written, false); + assert.equal(functionGasSnapshotsCheck.noBaseline, false); assert.equal(functionGasSnapshotsCheck.comparison.added.length, 0); assert.equal(functionGasSnapshotsCheck.comparison.removed.length, 0); assert.equal(functionGasSnapshotsCheck.comparison.changed.length, 1); }); - it("should pass and update file when functions are added", async () => { + it("should pass and not modify the baseline when functions are added", async () => { const initialResults = [ createSuiteResult("MyContract", [ createStandardTestResult("testA", 10000n), @@ -247,23 +248,27 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { await handleSnapshot(tmp.path, initialResults, true); + const snapshotPath = getFunctionGasSnapshotsPath(tmp.path); + const before = await readUtf8File(snapshotPath); + const { functionGasSnapshotsCheck } = await handleSnapshotCheck( tmp.path, withAddedResults, ); assert.equal(functionGasSnapshotsCheck.passed, true); - assert.equal(functionGasSnapshotsCheck.written, true); + assert.equal(functionGasSnapshotsCheck.noBaseline, false); assert.equal(functionGasSnapshotsCheck.comparison.added.length, 1); assert.equal(functionGasSnapshotsCheck.comparison.removed.length, 0); assert.equal(functionGasSnapshotsCheck.comparison.changed.length, 0); - const snapshotPath = getFunctionGasSnapshotsPath(tmp.path); - const savedContent = await readUtf8File(snapshotPath); - assert.match(savedContent, /MyContract#testB \(gas: 20000\)/); + // Read-only: the added function must not be written to the baseline. + const after = await readUtf8File(snapshotPath); + assert.equal(after, before); + assert.doesNotMatch(after, /testB/); }); - it("should pass and update file when functions are removed", async () => { + it("should pass and not modify the baseline when functions are removed", async () => { const initialResults = [ createSuiteResult("MyContract", [ createStandardTestResult("testA", 10000n), @@ -278,20 +283,57 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { await handleSnapshot(tmp.path, initialResults, true); + const snapshotPath = getFunctionGasSnapshotsPath(tmp.path); + const before = await readUtf8File(snapshotPath); + const { functionGasSnapshotsCheck } = await handleSnapshotCheck( tmp.path, withRemovedResults, ); assert.equal(functionGasSnapshotsCheck.passed, true); - assert.equal(functionGasSnapshotsCheck.written, true); + assert.equal(functionGasSnapshotsCheck.noBaseline, false); assert.equal(functionGasSnapshotsCheck.comparison.added.length, 0); assert.equal(functionGasSnapshotsCheck.comparison.removed.length, 1); assert.equal(functionGasSnapshotsCheck.comparison.changed.length, 0); + // Read-only: the missing function must remain in the baseline. + const after = await readUtf8File(snapshotPath); + assert.equal(after, before); + assert.match(after, /testB/); + }); + + it("should not delete the baseline when a scoped run produces no function gas snapshots", async () => { + const initialResults = [ + createSuiteResult("MyContract", [ + createStandardTestResult("testA", 10000n), + createStandardTestResult("testB", 20000n), + ]), + ]; + + await handleSnapshot(tmp.path, initialResults, true); + const snapshotPath = getFunctionGasSnapshotsPath(tmp.path); - const savedContent = await readUtf8File(snapshotPath); - assert.doesNotMatch(savedContent, /testB/); + const before = await readUtf8File(snapshotPath); + + // Simulates `--grep`/positional filtering down to a single test. + const scopedResults = [ + createSuiteResult("MyContract", [ + createStandardTestResult("testA", 10000n), + ]), + ]; + + const { functionGasSnapshotsCheck } = await handleSnapshotCheck( + tmp.path, + scopedResults, + ); + + assert.equal(functionGasSnapshotsCheck.passed, true); + assert.equal(functionGasSnapshotsCheck.noBaseline, false); + assert.equal(functionGasSnapshotsCheck.comparison.removed.length, 1); + + const after = await readUtf8File(snapshotPath); + assert.equal(after, before); }); }); @@ -534,7 +576,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { }, ], }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -557,7 +599,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { functionGasSnapshotsCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: false, @@ -587,7 +629,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { assert.match(text, /To update snapshots, run your tests with --snapshot/); }); - it("should log first-time write message when function gas snapshots written with no changes", () => { + it("should log no-baseline message when there's no function gas baseline", () => { const result = { functionGasSnapshotsCheck: { passed: true, @@ -596,7 +638,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { removed: [], changed: [], }, - written: true, + noBaseline: true, }, snapshotCheatcodesCheck: { passed: true, @@ -610,10 +652,9 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { const text = getLoggerOutput(); assert.match(text, /Snapshot check passed/); - assert.match(text, /Function gas snapshots:/); assert.match( text, - /No existing snapshots found\. Function gas snapshots written successfully/, + /Function gas snapshots: no baseline found\. Run your tests with --snapshot to create one\./, ); }); @@ -622,7 +663,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { functionGasSnapshotsCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -656,7 +697,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { removed: [], changed: [], }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -690,7 +731,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { removed: [], changed: [], }, - written: true, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -705,7 +746,10 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { const text = getLoggerOutput(); assert.match(text, /Snapshot check passed/); assert.match(text, /Function gas snapshots: 1 added/); - assert.match(text, /Added 1 function\(s\):/); + assert.match( + text, + /1 function\(s\) produced by this run are not in the baseline:/, + ); assert.match(text, /\+ MyContract#testB \(gas: 20000\)/); }); @@ -714,7 +758,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { functionGasSnapshotsCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -761,7 +805,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { ], changed: [], }, - written: true, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -776,7 +820,10 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { const text = getLoggerOutput(); assert.match(text, /Snapshot check passed/); assert.match(text, /Function gas snapshots: 1 removed/); - assert.match(text, /Removed 1 function\(s\):/); + assert.match( + text, + /1 stored function\(s\) were not produced by this run:/, + ); assert.match(text, /- MyContract#testB \(gas: 20000\)/); }); @@ -785,7 +832,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { functionGasSnapshotsCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -821,7 +868,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { functionGasSnapshotsCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -854,7 +901,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { removed: [], changed: [], }, - written: true, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -871,7 +918,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { Function gas snapshots: 1 added - Added 1 function(s): + 1 function(s) produced by this run are not in the baseline: + NewContract#testA() (gas: 7500) `; @@ -893,7 +940,7 @@ Function gas snapshots: 1 added ], changed: [], }, - written: true, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -910,7 +957,7 @@ Function gas snapshots: 1 added Function gas snapshots: 1 removed - Removed 1 function(s): + 1 stored function(s) were not produced by this run: - OldContract#testDeprecated() (gas: 3000) `; @@ -948,7 +995,7 @@ Function gas snapshots: 1 removed ], changed: [], }, - written: true, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -965,23 +1012,23 @@ Function gas snapshots: 1 removed Function gas snapshots: 2 added, 1 removed - Added 2 function(s): + 2 function(s) produced by this run are not in the baseline: + ContractA#testNew() (gas: 10000) + ContractB#testFuzz(uint256) (runs: 256, μ: 15000, ~: 14500) - Removed 1 function(s): + 1 stored function(s) were not produced by this run: - ContractA#testOld() (gas: 8000) `; assert.equal(text, expected); }); - it("function gas first-time write", () => { + it("function gas no baseline", () => { const result = { functionGasSnapshotsCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: true, + noBaseline: true, }, snapshotCheatcodesCheck: { passed: true, @@ -994,12 +1041,9 @@ Function gas snapshots: 2 added, 1 removed logSnapshotCheckResult(result, logger); const text = getLoggerOutput(); - // Note: "Function gas snapshots: " has a trailing space when there are no counts const expected = `Snapshot check passed -Function gas snapshots: - - No existing snapshots found. Function gas snapshots written successfully +Function gas snapshots: no baseline found. Run your tests with --snapshot to create one. `; assert.equal(text, expected); @@ -1014,7 +1058,7 @@ Function gas snapshots: removed: [], changed: [], }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -1057,7 +1101,7 @@ Snapshot cheatcodes: 1 added removed: [], changed: [], }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -1100,7 +1144,7 @@ Snapshot cheatcodes: 1 removed removed: [], changed: [], }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -1154,7 +1198,7 @@ Snapshot cheatcodes: 1 added, 2 removed functionGasSnapshotsCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -1178,12 +1222,12 @@ Snapshot cheatcodes: assert.equal(text, expected); }); - it("both function gas and cheatcodes first-time write", () => { + it("both function gas and cheatcodes no baseline", () => { const result = { functionGasSnapshotsCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: true, + noBaseline: true, }, snapshotCheatcodesCheck: { passed: true, @@ -1196,12 +1240,10 @@ Snapshot cheatcodes: logSnapshotCheckResult(result, logger); const text = getLoggerOutput(); - // Note: "Function gas snapshots: " has a trailing space when there are no counts + // Note: "Snapshot cheatcodes: " has a trailing space when there are no counts const expected = `Snapshot check passed -Function gas snapshots: - - No existing snapshots found. Function gas snapshots written successfully +Function gas snapshots: no baseline found. Run your tests with --snapshot to create one. Snapshot cheatcodes: @@ -1231,7 +1273,7 @@ Snapshot cheatcodes: added: [], removed: [], }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -1264,7 +1306,7 @@ To update snapshots, run your tests with --snapshot functionGasSnapshotsCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: false, @@ -1322,7 +1364,7 @@ To update snapshots, run your tests with --snapshot }, ], }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: false, @@ -1408,7 +1450,7 @@ To update snapshots, run your tests with --snapshot }, ], }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: true, @@ -1436,10 +1478,10 @@ Function gas snapshots: 2 changed, 1 added, 1 removed Expected (~): 20000 Actual (~): 18000 (-10.00%, Δ-2000) - Added 1 function(s): + 1 function(s) produced by this run are not in the baseline: + ContractA#testNew() (gas: 12000) - Removed 1 function(s): + 1 stored function(s) were not produced by this run: - ContractA#testOld() (gas: 9000) To update snapshots, run your tests with --snapshot @@ -1457,7 +1499,7 @@ To update snapshots, run your tests with --snapshot removed: [], changed: [], }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: false, @@ -1545,7 +1587,7 @@ To update snapshots, run your tests with --snapshot }, ], }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: false, @@ -1591,10 +1633,10 @@ Function gas snapshots: 1 changed, 1 added, 1 removed Expected (gas): 5000 Actual (gas): 6000 (+20.00%, Δ+1000) - Added 1 function(s): + 1 function(s) produced by this run are not in the baseline: + NewContract#testA() (gas: 7500) - Removed 1 function(s): + 1 stored function(s) were not produced by this run: - OldContract#testDeprecated() (gas: 3000) Snapshot cheatcodes: 1 changed, 1 added, 1 removed @@ -1637,7 +1679,7 @@ To update snapshots, run your tests with --snapshot ], changed: [], }, - written: true, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: false, @@ -1666,10 +1708,10 @@ To update snapshots, run your tests with --snapshot Function gas snapshots: 1 added, 1 removed - Added 1 function(s): + 1 function(s) produced by this run are not in the baseline: + NewContract#testA() (gas: 7500) - Removed 1 function(s): + 1 stored function(s) were not produced by this run: - OldContract#testDeprecated() (gas: 3000) Snapshot cheatcodes: 1 changed @@ -1703,7 +1745,7 @@ To update snapshots, run your tests with --snapshot }, ], }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: false, @@ -1758,13 +1800,13 @@ To update snapshots, run your tests with --snapshot it("function gas first-time write, cheatcodes changed", () => { const result = { functionGasSnapshotsCheck: { - passed: false, + passed: true, comparison: { added: [], removed: [], changed: [], }, - written: true, + noBaseline: true, }, snapshotCheatcodesCheck: { passed: false, @@ -1791,9 +1833,7 @@ To update snapshots, run your tests with --snapshot const text = getLoggerOutput(); const expected = `Snapshot check failed -Function gas snapshots: - - No existing snapshots found. Function gas snapshots written successfully +Function gas snapshots: no baseline found. Run your tests with --snapshot to create one. Snapshot cheatcodes: 1 changed @@ -1826,7 +1866,7 @@ To update snapshots, run your tests with --snapshot }, ], }, - written: false, + noBaseline: false, }, snapshotCheatcodesCheck: { passed: false, From eb12514c9e03eb83fe64decc39d2e688af1c3181 Mon Sep 17 00:00:00 2001 From: Luis Schaab Date: Sat, 13 Jun 2026 19:51:24 +0000 Subject: [PATCH 2/7] fix: make solidity --snapshot-check read-only --- .../gas-analytics/snapshot-cheatcodes.ts | 55 ++-- .../tasks/solidity-test/task-action.ts | 2 +- .../tasks/solidity-test/task-action.ts | 235 +++++++++++------- 3 files changed, 174 insertions(+), 118 deletions(-) diff --git a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/snapshot-cheatcodes.ts b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/snapshot-cheatcodes.ts index 032072200ac..6a40d4f7194 100644 --- a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/snapshot-cheatcodes.ts +++ b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/snapshot-cheatcodes.ts @@ -78,7 +78,7 @@ export interface SnapshotCheatcodesComparison { export interface SnapshotCheatcodesCheckResult { passed: boolean; comparison: SnapshotCheatcodesComparison; - written: boolean; + noBaseline: boolean; renamedGroups: RenamedSnapshotGroup[]; } @@ -363,12 +363,6 @@ export async function checkSnapshotCheatcodes( previousSnapshotCheatcodes = await readSnapshotCheatcodes(basePath); } catch (error) { if (error instanceof FileNotFoundError) { - // Only write if there are cheatcodes to save - const written = snapshotCheatcodes.size > 0; - if (written) { - await writeSnapshotCheatcodes(basePath, snapshotCheatcodes); - } - return { passed: true, comparison: { @@ -376,7 +370,7 @@ export async function checkSnapshotCheatcodes( removed: [], changed: [], }, - written, + noBaseline: snapshotCheatcodes.size > 0, renamedGroups, }; } @@ -389,17 +383,10 @@ export async function checkSnapshotCheatcodes( snapshotCheatcodes, ); - // Update snapshots when functions are added or removed (but not changed) - const hasAddedOrRemoved = - comparison.added.length > 0 || comparison.removed.length > 0; - if (comparison.changed.length === 0 && hasAddedOrRemoved) { - await writeSnapshotCheatcodes(basePath, snapshotCheatcodes); - } - return { passed: comparison.changed.length === 0, comparison, - written: hasAddedOrRemoved, + noBaseline: false, renamedGroups, }; } @@ -408,7 +395,7 @@ export function logSnapshotCheatcodesSection( result: SnapshotCheatcodesCheckResult, logger: typeof console.log = console.log, ): void { - const { comparison, written } = result; + const { comparison, noBaseline } = result; const changedLength = comparison.changed.length; const addedLength = comparison.added.length; const removedLength = comparison.removed.length; @@ -416,33 +403,31 @@ export function logSnapshotCheatcodesSection( const hasAdded = addedLength > 0; const hasRemoved = removedLength > 0; const hasAnyDifferences = hasChanges || hasAdded || hasRemoved; - const isFirstTimeWrite = written && !hasAnyDifferences; // Nothing to report - if (!isFirstTimeWrite && !hasAnyDifferences) { + if (!noBaseline && !hasAnyDifferences) { return; } - logger( - formatSectionHeader("Snapshot cheatcodes", { - changedLength, - addedLength, - removedLength, - }), - ); - - if (isFirstTimeWrite) { - logger(); + if (noBaseline) { logger( styleText( - "green", - " No existing snapshots found. Snapshot cheatcodes written successfully", + "yellow", + "Snapshot cheatcodes: no baseline found. Run your tests with --snapshot to create one.", ), ); logger(); return; } + logger( + formatSectionHeader("Snapshot cheatcodes", { + changedLength, + addedLength, + removedLength, + }), + ); + if (hasChanges) { logger(); printSnapshotCheatcodeChanges(comparison.changed, logger); @@ -450,7 +435,9 @@ export function logSnapshotCheatcodesSection( if (hasAdded) { logger(); - logger(` Added ${comparison.added.length} snapshot(s):`); + logger( + ` ${comparison.added.length} snapshot(s) produced by this run are not in the baseline:`, + ); const addedLines = stringifySnapshotCheatcodes(comparison.added).split( "\n", ); @@ -461,7 +448,9 @@ export function logSnapshotCheatcodesSection( if (hasRemoved) { logger(); - logger(` Removed ${comparison.removed.length} snapshot(s):`); + logger( + ` ${comparison.removed.length} stored snapshot(s) were not produced by this run:`, + ); const removedLines = stringifySnapshotCheatcodes(comparison.removed).split( "\n", ); diff --git a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts index 7703ddaa159..051ca4af85b 100644 --- a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts +++ b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts @@ -154,7 +154,7 @@ export function logSnapshotCheckResult( functionGasSnapshotsCheck.comparison.added.length > 0 || functionGasSnapshotsCheck.comparison.removed.length > 0; const snapshotCheatcodesHasOutput = - snapshotCheatcodesCheck.written || + snapshotCheatcodesCheck.noBaseline || snapshotCheatcodesCheck.comparison.changed.length > 0 || snapshotCheatcodesCheck.comparison.added.length > 0 || snapshotCheatcodesCheck.comparison.removed.length > 0; diff --git a/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts b/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts index 3a245e67266..fb416818d11 100644 --- a/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts +++ b/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts @@ -338,7 +338,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { }); describe("snapshot cheatcodes", () => { - it("should write snapshot cheatcodes on first run (no existing file)", async () => { + it("should not write snapshot cheatcodes on first run (no existing file)", async () => { const suiteResults = [ createSuiteResult("MyContract", [ createTestResultWithSnapshots([ @@ -356,20 +356,20 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { ); assert.equal(snapshotCheatcodesCheck.passed, true); - assert.equal(snapshotCheatcodesCheck.written, true); + assert.equal(snapshotCheatcodesCheck.noBaseline, true); assert.equal(snapshotCheatcodesCheck.comparison.added.length, 0); assert.equal(snapshotCheatcodesCheck.comparison.removed.length, 0); assert.equal(snapshotCheatcodesCheck.comparison.changed.length, 0); + // --snapshot-check is read-only: it must not bootstrap the baseline. const cheatcodePath = getSnapshotCheatcodesPath( tmp.path, "TestGroup.json", ); - const cheatcodeContent = await readJsonFile(cheatcodePath); - assert.deepEqual(cheatcodeContent, { "test-entry": "42" }); + assert.equal(await exists(cheatcodePath), false); }); - it("should not write when no existing file and no cheatcodes in current run", async () => { + it("should report no baseline when no existing file and no cheatcodes in current run", async () => { const suiteResults = [ createSuiteResult("MyContract", [ createTestResultWithSnapshots(undefined), @@ -382,7 +382,8 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { ); assert.equal(snapshotCheatcodesCheck.passed, true); - assert.equal(snapshotCheatcodesCheck.written, false); + // Nothing was produced, so there's nothing to flag as missing a baseline. + assert.equal(snapshotCheatcodesCheck.noBaseline, false); assert.equal(snapshotCheatcodesCheck.comparison.added.length, 0); assert.equal(snapshotCheatcodesCheck.comparison.removed.length, 0); assert.equal(snapshotCheatcodesCheck.comparison.changed.length, 0); @@ -408,7 +409,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { ); assert.equal(snapshotCheatcodesCheck.passed, true); - assert.equal(snapshotCheatcodesCheck.written, false); + assert.equal(snapshotCheatcodesCheck.noBaseline, false); assert.equal(snapshotCheatcodesCheck.comparison.added.length, 0); assert.equal(snapshotCheatcodesCheck.comparison.removed.length, 0); assert.equal(snapshotCheatcodesCheck.comparison.changed.length, 0); @@ -438,19 +439,29 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { await handleSnapshot(tmp.path, initialResults, true); + const cheatcodePath = getSnapshotCheatcodesPath( + tmp.path, + "TestGroup.json", + ); + const before = await readJsonFile(cheatcodePath); + const { snapshotCheatcodesCheck } = await handleSnapshotCheck( tmp.path, changedResults, ); assert.equal(snapshotCheatcodesCheck.passed, false); - assert.equal(snapshotCheatcodesCheck.written, false); + assert.equal(snapshotCheatcodesCheck.noBaseline, false); assert.equal(snapshotCheatcodesCheck.comparison.added.length, 0); assert.equal(snapshotCheatcodesCheck.comparison.removed.length, 0); assert.equal(snapshotCheatcodesCheck.comparison.changed.length, 1); + + // Read-only: a changed value must not be written to the baseline. + const after = await readJsonFile(cheatcodePath); + assert.deepEqual(after, before); }); - it("should pass and update file when cheatcodes are added", async () => { + it("should pass and not modify the baseline when cheatcodes are added", async () => { const initialResults = [ createSuiteResult("MyContract", [ createTestResultWithSnapshots([ @@ -477,29 +488,29 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { await handleSnapshot(tmp.path, initialResults, true); + const cheatcodePath = getSnapshotCheatcodesPath( + tmp.path, + "TestGroup.json", + ); + const before = await readJsonFile(cheatcodePath); + const { snapshotCheatcodesCheck } = await handleSnapshotCheck( tmp.path, withAddedResults, ); assert.equal(snapshotCheatcodesCheck.passed, true); - assert.equal(snapshotCheatcodesCheck.written, true); + assert.equal(snapshotCheatcodesCheck.noBaseline, false); assert.equal(snapshotCheatcodesCheck.comparison.added.length, 1); assert.equal(snapshotCheatcodesCheck.comparison.removed.length, 0); assert.equal(snapshotCheatcodesCheck.comparison.changed.length, 0); - const cheatcodePath = getSnapshotCheatcodesPath( - tmp.path, - "TestGroup.json", - ); - const cheatcodeContent = await readJsonFile(cheatcodePath); - assert.deepEqual(cheatcodeContent, { - "entry-a": "42", - "entry-b": "100", - }); + // Read-only: the added entry must not be written to the baseline. + const after = await readJsonFile(cheatcodePath); + assert.deepEqual(after, before); }); - it("should pass and update file when cheatcodes are removed", async () => { + it("should pass and not modify the baseline when cheatcodes are removed", async () => { const initialResults = [ createSuiteResult("MyContract", [ createTestResultWithSnapshots([ @@ -526,23 +537,82 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { await handleSnapshot(tmp.path, initialResults, true); + const cheatcodePath = getSnapshotCheatcodesPath( + tmp.path, + "TestGroup.json", + ); + const before = await readJsonFile(cheatcodePath); + const { snapshotCheatcodesCheck } = await handleSnapshotCheck( tmp.path, withRemovedResults, ); assert.equal(snapshotCheatcodesCheck.passed, true); - assert.equal(snapshotCheatcodesCheck.written, true); + assert.equal(snapshotCheatcodesCheck.noBaseline, false); assert.equal(snapshotCheatcodesCheck.comparison.added.length, 0); assert.equal(snapshotCheatcodesCheck.comparison.removed.length, 1); assert.equal(snapshotCheatcodesCheck.comparison.changed.length, 0); - const cheatcodePath = getSnapshotCheatcodesPath( + // Read-only: the removed entry must remain in the baseline. + const after = await readJsonFile(cheatcodePath); + assert.deepEqual(after, before); + }); + + it("should not delete group files when a scoped run omits a group", async () => { + // Reproduces the issue: a baseline with two groups, then a scoped run + // (e.g. via --grep) that only exercises one of them. + const initialResults = [ + createSuiteResult("GroupAContract", [ + createTestResultWithSnapshots([ + { + name: "A", + entries: [{ name: "a1", value: "42" }], + }, + ]), + ]), + createSuiteResult("GroupBContract", [ + createTestResultWithSnapshots([ + { + name: "B", + entries: [{ name: "b1", value: "100" }], + }, + ]), + ]), + ]; + + await handleSnapshot(tmp.path, initialResults, true); + + const pathA = getSnapshotCheatcodesPath(tmp.path, "A.json"); + const pathB = getSnapshotCheatcodesPath(tmp.path, "B.json"); + const beforeA = await readJsonFile(pathA); + const beforeB = await readJsonFile(pathB); + + // Only group A is produced this run; group B's tests weren't run. + const scopedResults = [ + createSuiteResult("GroupAContract", [ + createTestResultWithSnapshots([ + { + name: "A", + entries: [{ name: "a1", value: "42" }], + }, + ]), + ]), + ]; + + const { snapshotCheatcodesCheck } = await handleSnapshotCheck( tmp.path, - "TestGroup.json", + scopedResults, ); - const cheatcodeContent = await readJsonFile(cheatcodePath); - assert.deepEqual(cheatcodeContent, { "entry-a": "42" }); + + assert.equal(snapshotCheatcodesCheck.passed, true); + assert.equal(snapshotCheatcodesCheck.noBaseline, false); + assert.equal(snapshotCheatcodesCheck.comparison.removed.length, 1); + + // The orphaned group file must NOT be deleted, and neither file changed. + assert.equal(await exists(pathB), true); + assert.deepEqual(await readJsonFile(pathA), beforeA); + assert.deepEqual(await readJsonFile(pathB), beforeB); }); }); }); @@ -581,7 +651,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -616,7 +686,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { }, ], }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -643,7 +713,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -658,7 +728,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { ); }); - it("should log first-time write message when snapshot cheatcodes written with no changes", () => { + it("should log no-baseline message when there's no snapshot cheatcodes baseline", () => { const result = { functionGasSnapshotsCheck: { passed: true, @@ -672,7 +742,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { removed: [], changed: [], }, - written: true, + noBaseline: true, renamedGroups: [], }, }; @@ -681,10 +751,9 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { const text = getLoggerOutput(); assert.match(text, /Snapshot check passed/); - assert.match(text, /Snapshot cheatcodes:/); assert.match( text, - /No existing snapshots found\. Snapshot cheatcodes written successfully/, + /Snapshot cheatcodes: no baseline found\. Run your tests with --snapshot to create one\./, ); }); @@ -702,7 +771,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -736,7 +805,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -773,7 +842,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { removed: [], changed: [], }, - written: true, + noBaseline: false, renamedGroups: [], }, }; @@ -783,7 +852,10 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { const text = getLoggerOutput(); assert.match(text, /Snapshot check passed/); assert.match(text, /Snapshot cheatcodes: 1 added/); - assert.match(text, /Added 1 snapshot\(s\):/); + assert.match( + text, + /1 snapshot\(s\) produced by this run are not in the baseline:/, + ); assert.match(text, /\+ TestGroup#test-entry: 42/); }); @@ -810,7 +882,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -847,7 +919,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { ], changed: [], }, - written: true, + noBaseline: false, renamedGroups: [], }, }; @@ -857,7 +929,10 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { const text = getLoggerOutput(); assert.match(text, /Snapshot check passed/); assert.match(text, /Snapshot cheatcodes: 1 removed/); - assert.match(text, /Removed 1 snapshot\(s\):/); + assert.match( + text, + /1 stored snapshot\(s\) were not produced by this run:/, + ); assert.match(text, /- TestGroup#test-entry: 42/); }); @@ -873,7 +948,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -906,7 +981,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -945,7 +1020,7 @@ Function gas snapshots: 1 added snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -1000,7 +1075,7 @@ Function gas snapshots: 1 removed snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -1033,7 +1108,7 @@ Function gas snapshots: 2 added, 1 removed snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -1073,7 +1148,7 @@ Function gas snapshots: no baseline found. Run your tests with --snapshot to cre removed: [], changed: [], }, - written: true, + noBaseline: false, renamedGroups: [], }, }; @@ -1085,7 +1160,7 @@ Function gas snapshots: no baseline found. Run your tests with --snapshot to cre Snapshot cheatcodes: 1 added - Added 1 snapshot(s): + 1 snapshot(s) produced by this run are not in the baseline: + TestGroup#test-entry: 42 `; @@ -1116,7 +1191,7 @@ Snapshot cheatcodes: 1 added ], changed: [], }, - written: true, + noBaseline: false, renamedGroups: [], }, }; @@ -1128,7 +1203,7 @@ Snapshot cheatcodes: 1 added Snapshot cheatcodes: 1 removed - Removed 1 snapshot(s): + 1 stored snapshot(s) were not produced by this run: - TestGroup#test-entry: 42 `; @@ -1170,7 +1245,7 @@ Snapshot cheatcodes: 1 removed ], changed: [], }, - written: true, + noBaseline: false, renamedGroups: [], }, }; @@ -1182,10 +1257,10 @@ Snapshot cheatcodes: 1 removed Snapshot cheatcodes: 1 added, 2 removed - Added 1 snapshot(s): + 1 snapshot(s) produced by this run are not in the baseline: + TestGroup#test-entry: 42 - Removed 2 snapshot(s): + 2 stored snapshot(s) were not produced by this run: - AnotherGroup#old-entry: 150 - TestGroup#removed-entry: 100 `; @@ -1193,7 +1268,7 @@ Snapshot cheatcodes: 1 added, 2 removed assert.equal(text, expected); }); - it("cheatcodes first-time write", () => { + it("cheatcodes no baseline", () => { const result = { functionGasSnapshotsCheck: { passed: true, @@ -1203,7 +1278,7 @@ Snapshot cheatcodes: 1 added, 2 removed snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: true, + noBaseline: true, renamedGroups: [], }, }; @@ -1211,12 +1286,9 @@ Snapshot cheatcodes: 1 added, 2 removed logSnapshotCheckResult(result, logger); const text = getLoggerOutput(); - // Note: "Snapshot cheatcodes: " has a trailing space when there are no counts const expected = `Snapshot check passed -Snapshot cheatcodes: - - No existing snapshots found. Snapshot cheatcodes written successfully +Snapshot cheatcodes: no baseline found. Run your tests with --snapshot to create one. `; assert.equal(text, expected); @@ -1232,7 +1304,7 @@ Snapshot cheatcodes: snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: true, + noBaseline: true, renamedGroups: [], }, }; @@ -1240,14 +1312,11 @@ Snapshot cheatcodes: logSnapshotCheckResult(result, logger); const text = getLoggerOutput(); - // Note: "Snapshot cheatcodes: " has a trailing space when there are no counts const expected = `Snapshot check passed Function gas snapshots: no baseline found. Run your tests with --snapshot to create one. -Snapshot cheatcodes: - - No existing snapshots found. Snapshot cheatcodes written successfully +Snapshot cheatcodes: no baseline found. Run your tests with --snapshot to create one. `; assert.equal(text, expected); @@ -1278,7 +1347,7 @@ Snapshot cheatcodes: snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -1323,7 +1392,7 @@ To update snapshots, run your tests with --snapshot }, ], }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -1381,7 +1450,7 @@ To update snapshots, run your tests with --snapshot }, ], }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -1455,7 +1524,7 @@ To update snapshots, run your tests with --snapshot snapshotCheatcodesCheck: { passed: true, comparison: { added: [], removed: [], changed: [] }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -1528,7 +1597,7 @@ To update snapshots, run your tests with --snapshot }, ], }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -1545,10 +1614,10 @@ Snapshot cheatcodes: 1 changed, 1 added, 1 removed Expected: 42 Actual: 100 (+138.10%, Δ+58) - Added 1 snapshot(s): + 1 snapshot(s) produced by this run are not in the baseline: + GroupA#new-entry: 256 - Removed 1 snapshot(s): + 1 stored snapshot(s) were not produced by this run: - GroupB#old-entry: 128 To update snapshots, run your tests with --snapshot @@ -1616,7 +1685,7 @@ To update snapshots, run your tests with --snapshot }, ], }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -1646,10 +1715,10 @@ Snapshot cheatcodes: 1 changed, 1 added, 1 removed Expected: 42 Actual: 100 (+138.10%, Δ+58) - Added 1 snapshot(s): + 1 snapshot(s) produced by this run are not in the baseline: + GroupA#new-entry: 256 - Removed 1 snapshot(s): + 1 stored snapshot(s) were not produced by this run: - GroupB#old-entry: 128 To update snapshots, run your tests with --snapshot @@ -1696,7 +1765,7 @@ To update snapshots, run your tests with --snapshot }, ], }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -1766,7 +1835,7 @@ To update snapshots, run your tests with --snapshot ], changed: [], }, - written: true, + noBaseline: false, renamedGroups: [], }, }; @@ -1785,10 +1854,10 @@ Function gas snapshots: 1 changed Snapshot cheatcodes: 1 added, 1 removed - Added 1 snapshot(s): + 1 snapshot(s) produced by this run are not in the baseline: + GroupA#new-entry: 256 - Removed 1 snapshot(s): + 1 stored snapshot(s) were not produced by this run: - GroupB#old-entry: 128 To update snapshots, run your tests with --snapshot @@ -1797,7 +1866,7 @@ To update snapshots, run your tests with --snapshot assert.equal(text, expected); }); - it("function gas first-time write, cheatcodes changed", () => { + it("function gas no baseline, cheatcodes changed", () => { const result = { functionGasSnapshotsCheck: { passed: true, @@ -1823,7 +1892,7 @@ To update snapshots, run your tests with --snapshot }, ], }, - written: false, + noBaseline: false, renamedGroups: [], }, }; @@ -1848,7 +1917,7 @@ To update snapshots, run your tests with --snapshot assert.equal(text, expected); }); - it("cheatcodes first-time write, function gas changed", () => { + it("cheatcodes no baseline, function gas changed", () => { const result = { functionGasSnapshotsCheck: { passed: false, @@ -1869,13 +1938,13 @@ To update snapshots, run your tests with --snapshot noBaseline: false, }, snapshotCheatcodesCheck: { - passed: false, + passed: true, comparison: { added: [], removed: [], changed: [], }, - written: true, + noBaseline: true, renamedGroups: [], }, }; @@ -1892,9 +1961,7 @@ Function gas snapshots: 1 changed Expected (gas): 5000 Actual (gas): 6000 (+20.00%, Δ+1000) -Snapshot cheatcodes: - - No existing snapshots found. Snapshot cheatcodes written successfully +Snapshot cheatcodes: no baseline found. Run your tests with --snapshot to create one. To update snapshots, run your tests with --snapshot `; From f3c5727d9df507135a37e782ea1c032fdc31c588 Mon Sep 17 00:00:00 2001 From: Luis Schaab Date: Sat, 13 Jun 2026 19:55:33 +0000 Subject: [PATCH 3/7] chore: add changeset for snapshot-check fix --- .changeset/quiet-melons-wave.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/quiet-melons-wave.md diff --git a/.changeset/quiet-melons-wave.md b/.changeset/quiet-melons-wave.md new file mode 100644 index 00000000000..5e98514fea2 --- /dev/null +++ b/.changeset/quiet-melons-wave.md @@ -0,0 +1,5 @@ +--- +"hardhat": patch +--- + +Make `hardhat test solidity --snapshot-check` read-only so it no longer rewrites or deletes `.gas-snapshot` or `snapshots/*.json` files; it now reports differences, fails when a stored gas value changed, and suggests `--snapshot` when there's no baseline. From 1757effe47f007faec73b35e42d424462ff164e0 Mon Sep 17 00:00:00 2001 From: Luis Schaab Date: Sat, 13 Jun 2026 20:05:27 +0000 Subject: [PATCH 4/7] test: clean up snapshot-check tests --- .../tasks/solidity-test/task-action.ts | 40 ++++++++----------- 1 file changed, 16 insertions(+), 24 deletions(-) diff --git a/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts b/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts index fb416818d11..eca190873a4 100644 --- a/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts +++ b/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts @@ -221,6 +221,9 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { await handleSnapshot(tmp.path, initialResults, true); + const snapshotPath = getFunctionGasSnapshotsPath(tmp.path); + const before = await readUtf8File(snapshotPath); + const { functionGasSnapshotsCheck } = await handleSnapshotCheck( tmp.path, changedResults, @@ -231,6 +234,10 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { assert.equal(functionGasSnapshotsCheck.comparison.added.length, 0); assert.equal(functionGasSnapshotsCheck.comparison.removed.length, 0); assert.equal(functionGasSnapshotsCheck.comparison.changed.length, 1); + + // Read-only: a changed value must not be written to the baseline. + const after = await readUtf8File(snapshotPath); + assert.equal(after, before); }); it("should pass and not modify the baseline when functions are added", async () => { @@ -303,37 +310,22 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { assert.match(after, /testB/); }); - it("should not delete the baseline when a scoped run produces no function gas snapshots", async () => { - const initialResults = [ - createSuiteResult("MyContract", [ - createStandardTestResult("testA", 10000n), - createStandardTestResult("testB", 20000n), - ]), - ]; - - await handleSnapshot(tmp.path, initialResults, true); - - const snapshotPath = getFunctionGasSnapshotsPath(tmp.path); - const before = await readUtf8File(snapshotPath); - - // Simulates `--grep`/positional filtering down to a single test. - const scopedResults = [ - createSuiteResult("MyContract", [ - createStandardTestResult("testA", 10000n), - ]), - ]; + it("should not flag no baseline when no existing file and no function gas snapshots produced", async () => { + // e.g. a project that only uses snapshot cheatcodes never produces a + // `.gas-snapshot`; the check should stay quiet rather than nag about it. + const suiteResults = [createSuiteResult("MyContract", [])]; const { functionGasSnapshotsCheck } = await handleSnapshotCheck( tmp.path, - scopedResults, + suiteResults, ); assert.equal(functionGasSnapshotsCheck.passed, true); assert.equal(functionGasSnapshotsCheck.noBaseline, false); - assert.equal(functionGasSnapshotsCheck.comparison.removed.length, 1); + assert.equal(functionGasSnapshotsCheck.comparison.changed.length, 0); - const after = await readUtf8File(snapshotPath); - assert.equal(after, before); + const snapshotPath = getFunctionGasSnapshotsPath(tmp.path); + assert.equal(await exists(snapshotPath), false); }); }); @@ -369,7 +361,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { assert.equal(await exists(cheatcodePath), false); }); - it("should report no baseline when no existing file and no cheatcodes in current run", async () => { + it("should not flag no baseline when no existing file and no cheatcodes produced", async () => { const suiteResults = [ createSuiteResult("MyContract", [ createTestResultWithSnapshots(undefined), From 4afd6e82b9bd3592f10099dbc20002c734a23ab3 Mon Sep 17 00:00:00 2001 From: Luis Schaab Date: Sat, 13 Jun 2026 20:41:47 +0000 Subject: [PATCH 5/7] chore: link docs PR in changeset --- .changeset/quiet-melons-wave.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.changeset/quiet-melons-wave.md b/.changeset/quiet-melons-wave.md index 5e98514fea2..d752589fcdc 100644 --- a/.changeset/quiet-melons-wave.md +++ b/.changeset/quiet-melons-wave.md @@ -1,4 +1,5 @@ --- +# docs: https://github.com/NomicFoundation/hardhat-website/pull/278 "hardhat": patch --- From 9c799e6a0cf06db51f97716932f80971ba000fae Mon Sep 17 00:00:00 2001 From: Luis Schaab Date: Mon, 22 Jun 2026 14:02:04 +0000 Subject: [PATCH 6/7] fix: fail solidity --snapshot-check on missing snapshots and skip added/removed on filtered runs --- .../gas-analytics/function-gas-snapshots.ts | 19 +- .../gas-analytics/snapshot-cheatcodes.ts | 19 +- .../tasks/solidity-test/task-action.ts | 27 +- .../tasks/solidity-test/task-action.ts | 263 +++++++++++++----- 4 files changed, 243 insertions(+), 85 deletions(-) diff --git a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/function-gas-snapshots.ts b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/function-gas-snapshots.ts index ade917b744a..801358dd0cd 100644 --- a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/function-gas-snapshots.ts +++ b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/function-gas-snapshots.ts @@ -323,14 +323,17 @@ export async function checkFunctionGasSnapshots( previousFunctionGasSnapshots = await readFunctionGasSnapshots(basePath); } catch (error) { if (error instanceof FileNotFoundError) { + // Running a check without a stored snapshot is a mistake: fail so it's + // caught, but only when this run actually produced something to check. + const noBaseline = functionGasSnapshots.length > 0; return { - passed: true, + passed: !noBaseline, comparison: { added: [], removed: [], changed: [], }, - noBaseline: functionGasSnapshots.length > 0, + noBaseline, }; } @@ -352,12 +355,16 @@ export async function checkFunctionGasSnapshots( export function logFunctionGasSnapshotsSection( result: FunctionGasSnapshotCheckResult, logger: typeof console.log = console.log, + isFiltered = false, ): void { const { comparison, noBaseline } = result; const changedLength = comparison.changed.length; - const addedLength = comparison.added.length; - const removedLength = comparison.removed.length; const hasChanges = changedLength > 0; + // On a filtered run (--grep or specific files), added and missing snapshots + // are mostly artifacts of the filter rather than real differences, so we + // don't report them. + const addedLength = isFiltered ? 0 : comparison.added.length; + const removedLength = isFiltered ? 0 : comparison.removed.length; const hasAdded = addedLength > 0; const hasRemoved = removedLength > 0; const hasAnyDifferences = hasChanges || hasAdded || hasRemoved; @@ -371,7 +378,7 @@ export function logFunctionGasSnapshotsSection( logger( styleText( "yellow", - "Function gas snapshots: no baseline found. Run your tests with --snapshot to create one.", + "Function gas snapshots: no snapshot found. Run your tests with --snapshot to create one.", ), ); logger(); @@ -394,7 +401,7 @@ export function logFunctionGasSnapshotsSection( if (hasAdded) { logger(); logger( - ` ${comparison.added.length} function(s) produced by this run are not in the baseline:`, + ` ${comparison.added.length} function(s) produced by this run are not in the snapshot:`, ); const addedLines = stringifyFunctionGasSnapshots(comparison.added).split( "\n", diff --git a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/snapshot-cheatcodes.ts b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/snapshot-cheatcodes.ts index 6a40d4f7194..f04545d528b 100644 --- a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/snapshot-cheatcodes.ts +++ b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/snapshot-cheatcodes.ts @@ -363,14 +363,17 @@ export async function checkSnapshotCheatcodes( previousSnapshotCheatcodes = await readSnapshotCheatcodes(basePath); } catch (error) { if (error instanceof FileNotFoundError) { + // Running a check without stored snapshots is a mistake: fail so it's + // caught, but only when this run actually produced something to check. + const noBaseline = snapshotCheatcodes.size > 0; return { - passed: true, + passed: !noBaseline, comparison: { added: [], removed: [], changed: [], }, - noBaseline: snapshotCheatcodes.size > 0, + noBaseline, renamedGroups, }; } @@ -394,12 +397,16 @@ export async function checkSnapshotCheatcodes( export function logSnapshotCheatcodesSection( result: SnapshotCheatcodesCheckResult, logger: typeof console.log = console.log, + isFiltered = false, ): void { const { comparison, noBaseline } = result; const changedLength = comparison.changed.length; - const addedLength = comparison.added.length; - const removedLength = comparison.removed.length; const hasChanges = changedLength > 0; + // On a filtered run (--grep or specific files), added and missing snapshots + // are mostly artifacts of the filter rather than real differences, so we + // don't report them. + const addedLength = isFiltered ? 0 : comparison.added.length; + const removedLength = isFiltered ? 0 : comparison.removed.length; const hasAdded = addedLength > 0; const hasRemoved = removedLength > 0; const hasAnyDifferences = hasChanges || hasAdded || hasRemoved; @@ -413,7 +420,7 @@ export function logSnapshotCheatcodesSection( logger( styleText( "yellow", - "Snapshot cheatcodes: no baseline found. Run your tests with --snapshot to create one.", + "Snapshot cheatcodes: no snapshots found. Run your tests with --snapshot to create one.", ), ); logger(); @@ -436,7 +443,7 @@ export function logSnapshotCheatcodesSection( if (hasAdded) { logger(); logger( - ` ${comparison.added.length} snapshot(s) produced by this run are not in the baseline:`, + ` ${comparison.added.length} snapshot(s) produced by this run are not in the snapshot:`, ); const addedLines = stringifySnapshotCheatcodes(comparison.added).split( "\n", diff --git a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts index 051ca4af85b..6d0fd3f9e7e 100644 --- a/packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts +++ b/packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts @@ -31,6 +31,9 @@ import { interface GasAnalyticsTestActionArguments { snapshot: boolean; snapshotCheck: boolean; + // Forwarded from the base `test solidity` task; used to detect scoped runs. + grep?: string; + testFiles?: string[]; } export interface SnapshotResult { @@ -74,7 +77,11 @@ const runSolidityTests: TaskOverrideActionFunction< rootPath, suiteResults, ); - logSnapshotCheckResult(snapshotCheckResult); + // On a scoped run, added/missing snapshots are expected, so don't report + // them (they'd be noise from the filter, not real differences). + const isFiltered = + args.grep !== undefined || (args.testFiles?.length ?? 0) > 0; + logSnapshotCheckResult(snapshotCheckResult, console.log, isFiltered); snapshotCheckPassed = snapshotCheckResult.functionGasSnapshotsCheck.passed && snapshotCheckResult.snapshotCheatcodesCheck.passed; @@ -141,6 +148,7 @@ export async function handleSnapshotCheck( export function logSnapshotCheckResult( { functionGasSnapshotsCheck, snapshotCheatcodesCheck }: SnapshotCheckResult, logger: typeof console.log = console.log, + isFiltered = false, ): void { logger( functionGasSnapshotsCheck.passed && snapshotCheatcodesCheck.passed @@ -148,23 +156,28 @@ export function logSnapshotCheckResult( : styleText("red", "Snapshot check failed"), ); + // On a filtered run, added/removed aren't reported, so they don't count + // towards a section having output (which drives the spacing below). + const showAddedRemoved = !isFiltered; const functionGasHasOutput = functionGasSnapshotsCheck.noBaseline || functionGasSnapshotsCheck.comparison.changed.length > 0 || - functionGasSnapshotsCheck.comparison.added.length > 0 || - functionGasSnapshotsCheck.comparison.removed.length > 0; + (showAddedRemoved && + (functionGasSnapshotsCheck.comparison.added.length > 0 || + functionGasSnapshotsCheck.comparison.removed.length > 0)); const snapshotCheatcodesHasOutput = snapshotCheatcodesCheck.noBaseline || snapshotCheatcodesCheck.comparison.changed.length > 0 || - snapshotCheatcodesCheck.comparison.added.length > 0 || - snapshotCheatcodesCheck.comparison.removed.length > 0; + (showAddedRemoved && + (snapshotCheatcodesCheck.comparison.added.length > 0 || + snapshotCheatcodesCheck.comparison.removed.length > 0)); // Add an extra newline if function gas snapshots have output if (functionGasHasOutput) { logger(); } - logFunctionGasSnapshotsSection(functionGasSnapshotsCheck, logger); + logFunctionGasSnapshotsSection(functionGasSnapshotsCheck, logger, isFiltered); // Add an extra newline if only snapshot cheatcodes have output // (logFunctionGasSnapshotsSection adds one if it has output) @@ -172,7 +185,7 @@ export function logSnapshotCheckResult( logger(); } - logSnapshotCheatcodesSection(snapshotCheatcodesCheck, logger); + logSnapshotCheatcodesSection(snapshotCheatcodesCheck, logger, isFiltered); // Add an extra newline if only the rename warnings have output // (logSnapshotCheatcodesSection adds one if it has output) diff --git a/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts b/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts index eca190873a4..4a48771abaf 100644 --- a/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts +++ b/packages/hardhat/test/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts @@ -163,7 +163,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { const tmp = createTmpDir("snapshots-check-test", "test"); describe("function gas snapshots", () => { - it("should not write function gas snapshots on first run (no existing file)", async () => { + it("should fail on first run when there's no function gas snapshot", async () => { const suiteResults = [ createSuiteResult("MyContract", [ createStandardTestResult("testA", 10000n), @@ -175,13 +175,14 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { suiteResults, ); - assert.equal(functionGasSnapshotsCheck.passed, true); + // Checking with no stored snapshot is a mistake: it must fail. + assert.equal(functionGasSnapshotsCheck.passed, false); assert.equal(functionGasSnapshotsCheck.noBaseline, true); assert.equal(functionGasSnapshotsCheck.comparison.added.length, 0); assert.equal(functionGasSnapshotsCheck.comparison.removed.length, 0); assert.equal(functionGasSnapshotsCheck.comparison.changed.length, 0); - // --snapshot-check is read-only: it must not bootstrap the baseline. + // --snapshot-check is read-only: it must not bootstrap the snapshot. const snapshotPath = getFunctionGasSnapshotsPath(tmp.path); assert.equal(await exists(snapshotPath), false); }); @@ -330,7 +331,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { }); describe("snapshot cheatcodes", () => { - it("should not write snapshot cheatcodes on first run (no existing file)", async () => { + it("should fail on first run when there are no stored snapshot cheatcodes", async () => { const suiteResults = [ createSuiteResult("MyContract", [ createTestResultWithSnapshots([ @@ -347,13 +348,14 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { suiteResults, ); - assert.equal(snapshotCheatcodesCheck.passed, true); + // Checking with no stored snapshots is a mistake: it must fail. + assert.equal(snapshotCheatcodesCheck.passed, false); assert.equal(snapshotCheatcodesCheck.noBaseline, true); assert.equal(snapshotCheatcodesCheck.comparison.added.length, 0); assert.equal(snapshotCheatcodesCheck.comparison.removed.length, 0); assert.equal(snapshotCheatcodesCheck.comparison.changed.length, 0); - // --snapshot-check is read-only: it must not bootstrap the baseline. + // --snapshot-check is read-only: it must not bootstrap the snapshots. const cheatcodePath = getSnapshotCheatcodesPath( tmp.path, "TestGroup.json", @@ -691,10 +693,10 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { assert.match(text, /To update snapshots, run your tests with --snapshot/); }); - it("should log no-baseline message when there's no function gas baseline", () => { + it("should fail and log message when there's no function gas snapshot", () => { const result = { functionGasSnapshotsCheck: { - passed: true, + passed: false, comparison: { added: [], removed: [], @@ -713,14 +715,14 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { logSnapshotCheckResult(result, logger); const text = getLoggerOutput(); - assert.match(text, /Snapshot check passed/); + assert.match(text, /Snapshot check failed/); assert.match( text, - /Function gas snapshots: no baseline found\. Run your tests with --snapshot to create one\./, + /Function gas snapshots: no snapshot found\. Run your tests with --snapshot to create one\./, ); }); - it("should log no-baseline message when there's no snapshot cheatcodes baseline", () => { + it("should fail and log message when there are no stored snapshot cheatcodes", () => { const result = { functionGasSnapshotsCheck: { passed: true, @@ -728,7 +730,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { noBaseline: false, }, snapshotCheatcodesCheck: { - passed: true, + passed: false, comparison: { added: [], removed: [], @@ -742,10 +744,10 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { logSnapshotCheckResult(result, logger); const text = getLoggerOutput(); - assert.match(text, /Snapshot check passed/); + assert.match(text, /Snapshot check failed/); assert.match( text, - /Snapshot cheatcodes: no baseline found\. Run your tests with --snapshot to create one\./, + /Snapshot cheatcodes: no snapshots found\. Run your tests with --snapshot to create one\./, ); }); @@ -809,7 +811,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { assert.match(text, /Function gas snapshots: 1 added/); assert.match( text, - /1 function\(s\) produced by this run are not in the baseline:/, + /1 function\(s\) produced by this run are not in the snapshot:/, ); assert.match(text, /\+ MyContract#testB \(gas: 20000\)/); }); @@ -846,7 +848,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { assert.match(text, /Snapshot cheatcodes: 1 added/); assert.match( text, - /1 snapshot\(s\) produced by this run are not in the baseline:/, + /1 snapshot\(s\) produced by this run are not in the snapshot:/, ); assert.match(text, /\+ TestGroup#test-entry: 42/); }); @@ -985,7 +987,7 @@ describe("solidity-test/task-action (override in gas-analytics/index)", () => { Function gas snapshots: 1 added - 1 function(s) produced by this run are not in the baseline: + 1 function(s) produced by this run are not in the snapshot: + NewContract#testA() (gas: 7500) `; @@ -1079,7 +1081,7 @@ Function gas snapshots: 1 removed Function gas snapshots: 2 added, 1 removed - 2 function(s) produced by this run are not in the baseline: + 2 function(s) produced by this run are not in the snapshot: + ContractA#testNew() (gas: 10000) + ContractB#testFuzz(uint256) (runs: 256, μ: 15000, ~: 14500) @@ -1090,32 +1092,6 @@ Function gas snapshots: 2 added, 1 removed assert.equal(text, expected); }); - it("function gas no baseline", () => { - const result = { - functionGasSnapshotsCheck: { - passed: true, - comparison: { added: [], removed: [], changed: [] }, - noBaseline: true, - }, - snapshotCheatcodesCheck: { - passed: true, - comparison: { added: [], removed: [], changed: [] }, - noBaseline: false, - renamedGroups: [], - }, - }; - - logSnapshotCheckResult(result, logger); - - const text = getLoggerOutput(); - const expected = `Snapshot check passed - -Function gas snapshots: no baseline found. Run your tests with --snapshot to create one. -`; - - assert.equal(text, expected); - }); - it("cheatcodes added", () => { const result = { functionGasSnapshotsCheck: { @@ -1152,7 +1128,7 @@ Function gas snapshots: no baseline found. Run your tests with --snapshot to cre Snapshot cheatcodes: 1 added - 1 snapshot(s) produced by this run are not in the baseline: + 1 snapshot(s) produced by this run are not in the snapshot: + TestGroup#test-entry: 42 `; @@ -1249,7 +1225,7 @@ Snapshot cheatcodes: 1 removed Snapshot cheatcodes: 1 added, 2 removed - 1 snapshot(s) produced by this run are not in the baseline: + 1 snapshot(s) produced by this run are not in the snapshot: + TestGroup#test-entry: 42 2 stored snapshot(s) were not produced by this run: @@ -1259,6 +1235,36 @@ Snapshot cheatcodes: 1 added, 2 removed assert.equal(text, expected); }); + }); + + describe("failure cases", () => { + it("function gas no baseline", () => { + const result = { + functionGasSnapshotsCheck: { + passed: false, + comparison: { added: [], removed: [], changed: [] }, + noBaseline: true, + }, + snapshotCheatcodesCheck: { + passed: true, + comparison: { added: [], removed: [], changed: [] }, + noBaseline: false, + renamedGroups: [], + }, + }; + + logSnapshotCheckResult(result, logger); + + const text = getLoggerOutput(); + const expected = `Snapshot check failed + +Function gas snapshots: no snapshot found. Run your tests with --snapshot to create one. + +To update snapshots, run your tests with --snapshot +`; + + assert.equal(text, expected); + }); it("cheatcodes no baseline", () => { const result = { @@ -1268,7 +1274,7 @@ Snapshot cheatcodes: 1 added, 2 removed noBaseline: false, }, snapshotCheatcodesCheck: { - passed: true, + passed: false, comparison: { added: [], removed: [], changed: [] }, noBaseline: true, renamedGroups: [], @@ -1278,9 +1284,11 @@ Snapshot cheatcodes: 1 added, 2 removed logSnapshotCheckResult(result, logger); const text = getLoggerOutput(); - const expected = `Snapshot check passed + const expected = `Snapshot check failed + +Snapshot cheatcodes: no snapshots found. Run your tests with --snapshot to create one. -Snapshot cheatcodes: no baseline found. Run your tests with --snapshot to create one. +To update snapshots, run your tests with --snapshot `; assert.equal(text, expected); @@ -1289,12 +1297,12 @@ Snapshot cheatcodes: no baseline found. Run your tests with --snapshot to create it("both function gas and cheatcodes no baseline", () => { const result = { functionGasSnapshotsCheck: { - passed: true, + passed: false, comparison: { added: [], removed: [], changed: [] }, noBaseline: true, }, snapshotCheatcodesCheck: { - passed: true, + passed: false, comparison: { added: [], removed: [], changed: [] }, noBaseline: true, renamedGroups: [], @@ -1304,18 +1312,18 @@ Snapshot cheatcodes: no baseline found. Run your tests with --snapshot to create logSnapshotCheckResult(result, logger); const text = getLoggerOutput(); - const expected = `Snapshot check passed + const expected = `Snapshot check failed + +Function gas snapshots: no snapshot found. Run your tests with --snapshot to create one. -Function gas snapshots: no baseline found. Run your tests with --snapshot to create one. +Snapshot cheatcodes: no snapshots found. Run your tests with --snapshot to create one. -Snapshot cheatcodes: no baseline found. Run your tests with --snapshot to create one. +To update snapshots, run your tests with --snapshot `; assert.equal(text, expected); }); - }); - describe("failure cases", () => { it("function gas changed", () => { const result = { functionGasSnapshotsCheck: { @@ -1539,7 +1547,7 @@ Function gas snapshots: 2 changed, 1 added, 1 removed Expected (~): 20000 Actual (~): 18000 (-10.00%, Δ-2000) - 1 function(s) produced by this run are not in the baseline: + 1 function(s) produced by this run are not in the snapshot: + ContractA#testNew() (gas: 12000) 1 stored function(s) were not produced by this run: @@ -1606,7 +1614,7 @@ Snapshot cheatcodes: 1 changed, 1 added, 1 removed Expected: 42 Actual: 100 (+138.10%, Δ+58) - 1 snapshot(s) produced by this run are not in the baseline: + 1 snapshot(s) produced by this run are not in the snapshot: + GroupA#new-entry: 256 1 stored snapshot(s) were not produced by this run: @@ -1694,7 +1702,7 @@ Function gas snapshots: 1 changed, 1 added, 1 removed Expected (gas): 5000 Actual (gas): 6000 (+20.00%, Δ+1000) - 1 function(s) produced by this run are not in the baseline: + 1 function(s) produced by this run are not in the snapshot: + NewContract#testA() (gas: 7500) 1 stored function(s) were not produced by this run: @@ -1707,7 +1715,7 @@ Snapshot cheatcodes: 1 changed, 1 added, 1 removed Expected: 42 Actual: 100 (+138.10%, Δ+58) - 1 snapshot(s) produced by this run are not in the baseline: + 1 snapshot(s) produced by this run are not in the snapshot: + GroupA#new-entry: 256 1 stored snapshot(s) were not produced by this run: @@ -1769,7 +1777,7 @@ To update snapshots, run your tests with --snapshot Function gas snapshots: 1 added, 1 removed - 1 function(s) produced by this run are not in the baseline: + 1 function(s) produced by this run are not in the snapshot: + NewContract#testA() (gas: 7500) 1 stored function(s) were not produced by this run: @@ -1846,7 +1854,7 @@ Function gas snapshots: 1 changed Snapshot cheatcodes: 1 added, 1 removed - 1 snapshot(s) produced by this run are not in the baseline: + 1 snapshot(s) produced by this run are not in the snapshot: + GroupA#new-entry: 256 1 stored snapshot(s) were not produced by this run: @@ -1861,7 +1869,7 @@ To update snapshots, run your tests with --snapshot it("function gas no baseline, cheatcodes changed", () => { const result = { functionGasSnapshotsCheck: { - passed: true, + passed: false, comparison: { added: [], removed: [], @@ -1894,7 +1902,7 @@ To update snapshots, run your tests with --snapshot const text = getLoggerOutput(); const expected = `Snapshot check failed -Function gas snapshots: no baseline found. Run your tests with --snapshot to create one. +Function gas snapshots: no snapshot found. Run your tests with --snapshot to create one. Snapshot cheatcodes: 1 changed @@ -1930,7 +1938,7 @@ To update snapshots, run your tests with --snapshot noBaseline: false, }, snapshotCheatcodesCheck: { - passed: true, + passed: false, comparison: { added: [], removed: [], @@ -1953,7 +1961,7 @@ Function gas snapshots: 1 changed Expected (gas): 5000 Actual (gas): 6000 (+20.00%, Δ+1000) -Snapshot cheatcodes: no baseline found. Run your tests with --snapshot to create one. +Snapshot cheatcodes: no snapshots found. Run your tests with --snapshot to create one. To update snapshots, run your tests with --snapshot `; @@ -1962,5 +1970,128 @@ To update snapshots, run your tests with --snapshot }); }); }); + + describe("filtered runs (--grep or specific files)", () => { + it("should not report added/removed function gas snapshots", () => { + const result = { + functionGasSnapshotsCheck: { + passed: true, + comparison: { + added: [ + { + contractNameOrFqn: "NewContract", + functionSig: "testNew()", + gasUsage: { kind: "standard" as const, gas: 7500n }, + }, + ], + removed: [ + { + contractNameOrFqn: "OldContract", + functionSig: "testOld()", + gasUsage: { kind: "standard" as const, gas: 3000n }, + }, + ], + changed: [], + }, + noBaseline: false, + }, + snapshotCheatcodesCheck: { + passed: true, + comparison: { added: [], removed: [], changed: [] }, + noBaseline: false, + renamedGroups: [], + }, + }; + + logSnapshotCheckResult(result, logger, true); + + const text = getLoggerOutput(); + // Nothing but the header: added/removed are suppressed on filtered runs. + assert.equal(text, "Snapshot check passed"); + }); + + it("should not report added/removed snapshot cheatcodes", () => { + const result = { + functionGasSnapshotsCheck: { + passed: true, + comparison: { added: [], removed: [], changed: [] }, + noBaseline: false, + }, + snapshotCheatcodesCheck: { + passed: true, + comparison: { + added: [{ group: "GroupA", name: "new-entry", value: "256" }], + removed: [{ group: "GroupB", name: "old-entry", value: "128" }], + changed: [], + }, + noBaseline: false, + renamedGroups: [], + }, + }; + + logSnapshotCheckResult(result, logger, true); + + const text = getLoggerOutput(); + assert.equal(text, "Snapshot check passed"); + }); + + it("should still report changed values while hiding added/removed", () => { + const result = { + functionGasSnapshotsCheck: { + passed: false, + comparison: { + added: [ + { + contractNameOrFqn: "NewContract", + functionSig: "testNew()", + gasUsage: { kind: "standard" as const, gas: 7500n }, + }, + ], + removed: [ + { + contractNameOrFqn: "OldContract", + functionSig: "testOld()", + gasUsage: { kind: "standard" as const, gas: 3000n }, + }, + ], + changed: [ + { + contractNameOrFqn: "MyContract", + functionSig: "testFunc()", + kind: "standard" as const, + expected: 5000, + actual: 6000, + source: "contracts/MyContract.sol", + }, + ], + }, + noBaseline: false, + }, + snapshotCheatcodesCheck: { + passed: true, + comparison: { added: [], removed: [], changed: [] }, + noBaseline: false, + renamedGroups: [], + }, + }; + + logSnapshotCheckResult(result, logger, true); + + const text = getLoggerOutput(); + const expected = `Snapshot check failed + +Function gas snapshots: 1 changed + + MyContract#testFunc() + (in contracts/MyContract.sol) + Expected (gas): 5000 + Actual (gas): 6000 (+20.00%, Δ+1000) + +To update snapshots, run your tests with --snapshot +`; + + assert.equal(text, expected); + }); + }); }); }); From a52bd4cbca32b45264a9e798753bba9ba0bceace Mon Sep 17 00:00:00 2001 From: Luis Schaab Date: Mon, 22 Jun 2026 14:13:17 +0000 Subject: [PATCH 7/7] chore: update changeset --- .changeset/quiet-melons-wave.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/quiet-melons-wave.md b/.changeset/quiet-melons-wave.md index d752589fcdc..306d38098e7 100644 --- a/.changeset/quiet-melons-wave.md +++ b/.changeset/quiet-melons-wave.md @@ -3,4 +3,4 @@ "hardhat": patch --- -Make `hardhat test solidity --snapshot-check` read-only so it no longer rewrites or deletes `.gas-snapshot` or `snapshots/*.json` files; it now reports differences, fails when a stored gas value changed, and suggests `--snapshot` when there's no baseline. +Make `hardhat test solidity --snapshot-check` read-only. It no longer rewrites or deletes `.gas-snapshot` or `snapshots/*.json` files, and instead reports differences. The command now fails when a stored gas value differs from the current one, or when no baseline exists, with a hint to run `--snapshot`.