fix: make solidity --snapshot-check read-only

.changeset/quiet-melons-wave.md

@@ -0,0 +1,6 @@
+---
+# docs: https://github.com/NomicFoundation/hardhat-website/pull/278
+"hardhat": patch
+---
+
+Make `hardhat test solidity --snapshot-check` read-only. It no longer rewrites or deletes `.gas-snapshot` or `snapshots/*.json` files, and instead reports differences. The command now fails when a stored gas value differs from the current one, or when no baseline exists, with a hint to run `--snapshot`.

packages/hardhat/src/internal/builtin-plugins/gas-analytics/function-gas-snapshots.ts

@@ -64,7 +64,7 @@ export interface FunctionGasSnapshotChange {
 export interface FunctionGasSnapshotCheckResult {
   passed: boolean;
   comparison: FunctionGasSnapshotComparison;
-  written: boolean;
+  noBaseline: boolean;
 }
 
 export function getFunctionGasSnapshotsPath(basePath: string): string {
@@ -323,16 +323,17 @@ export async function checkFunctionGasSnapshots(
     previousFunctionGasSnapshots = await readFunctionGasSnapshots(basePath);
   } catch (error) {
     if (error instanceof FileNotFoundError) {
-      await writeFunctionGasSnapshots(basePath, functionGasSnapshots);
-
+      // Running a check without a stored snapshot is a mistake: fail so it's
+      // caught, but only when this run actually produced something to check.
+      const noBaseline = functionGasSnapshots.length > 0;
       return {
-        passed: true,
+        passed: !noBaseline,
         comparison: {
           added: [],
           removed: [],
           changed: [],
         },
-        written: true,
+        noBaseline,
       };
     }
 
@@ -344,67 +345,64 @@ export async function checkFunctionGasSnapshots(
     functionGasSnapshots,
   );
 
-  // Update snapshots when functions are added or removed (but not changed)
-  const hasAddedOrRemoved =
-    comparison.added.length > 0 || comparison.removed.length > 0;
-  if (comparison.changed.length === 0 && hasAddedOrRemoved) {
-    await writeFunctionGasSnapshots(basePath, functionGasSnapshots);
-  }
-
   return {
     passed: comparison.changed.length === 0,
     comparison,
-    written: hasAddedOrRemoved,
+    noBaseline: false,
   };
 }
 
 export function logFunctionGasSnapshotsSection(
   result: FunctionGasSnapshotCheckResult,
   logger: typeof console.log = console.log,
+  isFiltered = false,
 ): void {
-  const { comparison, written } = result;
+  const { comparison, noBaseline } = result;
   const changedLength = comparison.changed.length;
-  const addedLength = comparison.added.length;
-  const removedLength = comparison.removed.length;
   const hasChanges = changedLength > 0;
+  // On a filtered run (--grep or specific files), added and missing snapshots
+  // are mostly artifacts of the filter rather than real differences, so we
+  // don't report them.
+  const addedLength = isFiltered ? 0 : comparison.added.length;
+  const removedLength = isFiltered ? 0 : comparison.removed.length;
   const hasAdded = addedLength > 0;
   const hasRemoved = removedLength > 0;
   const hasAnyDifferences = hasChanges || hasAdded || hasRemoved;
-  const isFirstTimeWrite = written && !hasAnyDifferences;
 
   // Nothing to report
-  if (!isFirstTimeWrite && !hasAnyDifferences) {
+  if (!noBaseline && !hasAnyDifferences) {
     return;
   }
 
-  logger(
-    formatSectionHeader("Function gas snapshots", {
-      changedLength,
-      addedLength,
-      removedLength,
-    }),
-  );
-
-  if (isFirstTimeWrite) {
-    logger();
+  if (noBaseline) {
     logger(
       styleText(
-        "green",
-        "  No existing snapshots found. Function gas snapshots written successfully",
+        "yellow",
+        "Function gas snapshots: no snapshot found. Run your tests with --snapshot to create one.",
       ),
     );
     logger();
     return;
   }
 
+  logger(
+    formatSectionHeader("Function gas snapshots", {
+      changedLength,
+      addedLength,
+      removedLength,
+    }),
+  );
+
   if (hasChanges) {
     logger();
     printFunctionGasSnapshotChanges(comparison.changed, logger);
   }
 
   if (hasAdded) {
     logger();
-    logger(`  Added ${comparison.added.length} function(s):`);
+    logger(
+      `  ${comparison.added.length} function(s) produced by this run are not in the snapshot:`,
+    );
     const addedLines = stringifyFunctionGasSnapshots(comparison.added).split(
       "\n",
     );
@@ -415,7 +413,9 @@ export function logFunctionGasSnapshotsSection(
 
   if (hasRemoved) {
     logger();
-    logger(`  Removed ${comparison.removed.length} function(s):`);
+    logger(
+      `  ${comparison.removed.length} stored function(s) were not produced by this run:`,
+    );
     const removedLines = stringifyFunctionGasSnapshots(
       comparison.removed,
     ).split("\n");

packages/hardhat/src/internal/builtin-plugins/gas-analytics/snapshot-cheatcodes.ts

@@ -78,7 +78,7 @@ export interface SnapshotCheatcodesComparison {
 export interface SnapshotCheatcodesCheckResult {
   passed: boolean;
   comparison: SnapshotCheatcodesComparison;
-  written: boolean;
+  noBaseline: boolean;
   renamedGroups: RenamedSnapshotGroup[];
 }
 
@@ -363,20 +363,17 @@ export async function checkSnapshotCheatcodes(
     previousSnapshotCheatcodes = await readSnapshotCheatcodes(basePath);
   } catch (error) {
     if (error instanceof FileNotFoundError) {
-      // Only write if there are cheatcodes to save
-      const written = snapshotCheatcodes.size > 0;
-      if (written) {
-        await writeSnapshotCheatcodes(basePath, snapshotCheatcodes);
-      }
-
+      // Running a check without stored snapshots is a mistake: fail so it's
+      // caught, but only when this run actually produced something to check.
+      const noBaseline = snapshotCheatcodes.size > 0;
       return {
-        passed: true,
+        passed: !noBaseline,
         comparison: {
           added: [],
           removed: [],
           changed: [],
         },
-        written,
+        noBaseline,
         renamedGroups,
       };
     }
@@ -389,68 +386,65 @@ export async function checkSnapshotCheatcodes(
     snapshotCheatcodes,
   );
 
-  // Update snapshots when functions are added or removed (but not changed)
-  const hasAddedOrRemoved =
-    comparison.added.length > 0 || comparison.removed.length > 0;
-  if (comparison.changed.length === 0 && hasAddedOrRemoved) {
-    await writeSnapshotCheatcodes(basePath, snapshotCheatcodes);
-  }
-
   return {
     passed: comparison.changed.length === 0,
     comparison,
-    written: hasAddedOrRemoved,
+    noBaseline: false,
     renamedGroups,
   };
 }
 
 export function logSnapshotCheatcodesSection(
   result: SnapshotCheatcodesCheckResult,
   logger: typeof console.log = console.log,
+  isFiltered = false,
 ): void {
-  const { comparison, written } = result;
+  const { comparison, noBaseline } = result;
   const changedLength = comparison.changed.length;
-  const addedLength = comparison.added.length;
-  const removedLength = comparison.removed.length;
   const hasChanges = changedLength > 0;
+  // On a filtered run (--grep or specific files), added and missing snapshots
+  // are mostly artifacts of the filter rather than real differences, so we
+  // don't report them.
+  const addedLength = isFiltered ? 0 : comparison.added.length;
+  const removedLength = isFiltered ? 0 : comparison.removed.length;
   const hasAdded = addedLength > 0;
   const hasRemoved = removedLength > 0;
   const hasAnyDifferences = hasChanges || hasAdded || hasRemoved;
-  const isFirstTimeWrite = written && !hasAnyDifferences;
 
   // Nothing to report
-  if (!isFirstTimeWrite && !hasAnyDifferences) {
+  if (!noBaseline && !hasAnyDifferences) {
     return;
   }
 
-  logger(
-    formatSectionHeader("Snapshot cheatcodes", {
-      changedLength,
-      addedLength,
-      removedLength,
-    }),
-  );
-
-  if (isFirstTimeWrite) {
-    logger();
+  if (noBaseline) {
     logger(
       styleText(
-        "green",
-        "  No existing snapshots found. Snapshot cheatcodes written successfully",
+        "yellow",
+        "Snapshot cheatcodes: no snapshots found. Run your tests with --snapshot to create one.",
       ),
     );
     logger();
     return;
   }
 
+  logger(
+    formatSectionHeader("Snapshot cheatcodes", {
+      changedLength,
+      addedLength,
+      removedLength,
+    }),
+  );
+
   if (hasChanges) {
     logger();
     printSnapshotCheatcodeChanges(comparison.changed, logger);
   }
 
   if (hasAdded) {
     logger();
-    logger(`  Added ${comparison.added.length} snapshot(s):`);
+    logger(
+      `  ${comparison.added.length} snapshot(s) produced by this run are not in the snapshot:`,
+    );
     const addedLines = stringifySnapshotCheatcodes(comparison.added).split(
       "\n",
     );
@@ -461,7 +455,9 @@ export function logSnapshotCheatcodesSection(
 
   if (hasRemoved) {
     logger();
-    logger(`  Removed ${comparison.removed.length} snapshot(s):`);
+    logger(
+      `  ${comparison.removed.length} stored snapshot(s) were not produced by this run:`,
+    );
     const removedLines = stringifySnapshotCheatcodes(comparison.removed).split(
       "\n",
     );

packages/hardhat/src/internal/builtin-plugins/gas-analytics/tasks/solidity-test/task-action.ts

@@ -31,6 +31,9 @@ import {
 interface GasAnalyticsTestActionArguments {
   snapshot: boolean;
   snapshotCheck: boolean;
+  // Forwarded from the base `test solidity` task; used to detect scoped runs.
+  grep?: string;
+  testFiles?: string[];
 }
 
 export interface SnapshotResult {
@@ -74,7 +77,11 @@ const runSolidityTests: TaskOverrideActionFunction<
       rootPath,
       suiteResults,
     );
-    logSnapshotCheckResult(snapshotCheckResult);
+    // On a scoped run, added/missing snapshots are expected, so don't report
+    // them (they'd be noise from the filter, not real differences).
+    const isFiltered =
+      args.grep !== undefined || (args.testFiles?.length ?? 0) > 0;
+    logSnapshotCheckResult(snapshotCheckResult, console.log, isFiltered);
     snapshotCheckPassed =
       snapshotCheckResult.functionGasSnapshotsCheck.passed &&
       snapshotCheckResult.snapshotCheatcodesCheck.passed;
@@ -141,38 +148,44 @@ export async function handleSnapshotCheck(
 export function logSnapshotCheckResult(
   { functionGasSnapshotsCheck, snapshotCheatcodesCheck }: SnapshotCheckResult,
   logger: typeof console.log = console.log,
+  isFiltered = false,
 ): void {
   logger(
     functionGasSnapshotsCheck.passed && snapshotCheatcodesCheck.passed
       ? styleText("green", "Snapshot check passed")
       : styleText("red", "Snapshot check failed"),
   );
 
+  // On a filtered run, added/removed aren't reported, so they don't count
+  // towards a section having output (which drives the spacing below).
+  const showAddedRemoved = !isFiltered;
   const functionGasHasOutput =
-    functionGasSnapshotsCheck.written ||
+    functionGasSnapshotsCheck.noBaseline ||
     functionGasSnapshotsCheck.comparison.changed.length > 0 ||
-    functionGasSnapshotsCheck.comparison.added.length > 0 ||
-    functionGasSnapshotsCheck.comparison.removed.length > 0;
+    (showAddedRemoved &&
+      (functionGasSnapshotsCheck.comparison.added.length > 0 ||
+        functionGasSnapshotsCheck.comparison.removed.length > 0));
   const snapshotCheatcodesHasOutput =
-    snapshotCheatcodesCheck.written ||
+    snapshotCheatcodesCheck.noBaseline ||
     snapshotCheatcodesCheck.comparison.changed.length > 0 ||
-    snapshotCheatcodesCheck.comparison.added.length > 0 ||
-    snapshotCheatcodesCheck.comparison.removed.length > 0;
+    (showAddedRemoved &&
+      (snapshotCheatcodesCheck.comparison.added.length > 0 ||
+        snapshotCheatcodesCheck.comparison.removed.length > 0));
 
   // Add an extra newline if function gas snapshots have output
   if (functionGasHasOutput) {
     logger();
   }
 
-  logFunctionGasSnapshotsSection(functionGasSnapshotsCheck, logger);
+  logFunctionGasSnapshotsSection(functionGasSnapshotsCheck, logger, isFiltered);
 
   // Add an extra newline if only snapshot cheatcodes have output
   // (logFunctionGasSnapshotsSection adds one if it has output)
   if (!functionGasHasOutput && snapshotCheatcodesHasOutput) {
     logger();
   }
 
-  logSnapshotCheatcodesSection(snapshotCheatcodesCheck, logger);
+  logSnapshotCheatcodesSection(snapshotCheatcodesCheck, logger, isFiltered);
 
   // Add an extra newline if only the rename warnings have output
   // (logSnapshotCheatcodesSection adds one if it has output)