Perf/prefetch lists

packages/safe-chain/src/main.js

@@ -8,6 +8,7 @@ import { getLogFile } from "./config/settings.js";
 import { createSafeChainProxy } from "./registryProxy/registryProxy.js";
 import chalk from "chalk";
 import { getAuditStats } from "./scanning/audit/index.js";
+import { prefetchLists } from "./scanning/prefetchLists.js";
 import {
   initializeFileLogger,
   closeFileLogger,
@@ -56,7 +57,11 @@ export async function main(args) {
     // themselves — nested installs inside those scripts are re-intercepted by the
     // shims in PATH. Not starting the proxy prevents HTTPS_PROXY from leaking
     // into lifecycle-script child processes (vitest, native binaries, nock, etc.).
-    if (getPackageManager().commandNeedsProxy(args)) {
+    const packageManager = getPackageManager();
+    if (packageManager.commandNeedsProxy(args)) {
+      if (packageManager.commandInstallsPackages?.(args)) {
+        prefetchLists();
+      }
       await proxy.startServer();
     }
 

packages/safe-chain/src/packagemanager/bun/createBunPackageManager.js

@@ -4,6 +4,7 @@ import { reportCommandExecutionFailure } from "../_shared/commandErrors.js";
 
 // bun commands that only execute scripts; they never download packages.
 const BUN_LIFECYCLE_COMMANDS = new Set(["run", "test"]);
+const BUN_INSTALL_COMMANDS = new Set(["install", "i", "add", "a", "update"]);
 
 /**
  * @returns {import("../currentPackageManager.js").PackageManager}
@@ -21,6 +22,10 @@ export function createBunPackageManager() {
       const command = args.find((arg) => !arg.startsWith("-"))?.toLowerCase();
       return !command || !BUN_LIFECYCLE_COMMANDS.has(command);
     },
+    commandInstallsPackages(args) {
+      const command = args.find((arg) => !arg.startsWith("-"))?.toLowerCase();
+      return command !== undefined && BUN_INSTALL_COMMANDS.has(command);
+    },
   };
 }
 

packages/safe-chain/src/packagemanager/commandInstallsPackages.spec.js

@@ -0,0 +1,44 @@
+import { describe, it } from "node:test";
+import assert from "node:assert";
+import { createNpmPackageManager } from "./npm/createPackageManager.js";
+import { createPnpmPackageManager } from "./pnpm/createPackageManager.js";
+import { createYarnPackageManager } from "./yarn/createPackageManager.js";
+import { createBunPackageManager } from "./bun/createBunPackageManager.js";
+
+describe("commandInstallsPackages", () => {
+  const cases = {
+    npm: {
+      packageManager: createNpmPackageManager(),
+      installs: [["install"], ["i", "lodash"], ["ci"], ["clean-install"], ["update"], ["dedupe"]],
+      doesNotInstall: [[], ["view", "lodash"], ["config", "get", "registry"], ["ls"], ["run", "build"], ["exec", "foo"]],
+    },
+    pnpm: {
+      packageManager: createPnpmPackageManager(),
+      installs: [[], ["install"], ["i"], ["--filter", "pkg", "add", "lodash"], ["up"]],
+      doesNotInstall: [["view", "lodash"], ["run", "build"], ["dlx", "foo"], ["publish"]],
+    },
+    yarn: {
+      packageManager: createYarnPackageManager(),
+      installs: [[], ["install"], ["--immutable"], ["add", "lodash"], ["up"]],
+      doesNotInstall: [["info", "lodash"], ["run", "build"], ["dlx", "foo"]],
+    },
+    bun: {
+      packageManager: createBunPackageManager(),
+      installs: [["install"], ["i"], ["add", "lodash"], ["update"]],
+      doesNotInstall: [[], ["run", "build"], ["x", "foo"], ["pm", "ls"]],
+    },
+  };
+
+  for (const [name, { packageManager, installs, doesNotInstall }] of Object.entries(cases)) {
+    for (const args of installs) {
+      it(`is true for: ${name} ${args.join(" ")}`, () => {
+        assert.equal(packageManager.commandInstallsPackages?.(args), true);
+      });
+    }
+    for (const args of doesNotInstall) {
+      it(`is false for: ${name} ${args.join(" ")}`, () => {
+        assert.equal(packageManager.commandInstallsPackages?.(args), false);
+      });
+    }
+  }
+});

packages/safe-chain/src/packagemanager/currentPackageManager.js

@@ -38,6 +38,7 @@ const state = {
  * @property {(args: string[]) => boolean} isSupportedCommand
  * @property {(args: string[]) => Promise<GetDependencyUpdatesResult[]> | GetDependencyUpdatesResult[]} getDependencyUpdatesForCommand
  * @property {(args: string[]) => boolean} commandNeedsProxy - Returns true when the command downloads packages and needs the MITM proxy. Returns false for lifecycle-script commands (run/test/start/etc.) so their child processes don't inherit HTTPS_PROXY.
+ * @property {(args: string[]) => boolean} [commandInstallsPackages] - Returns true when the command installs dependencies, so the lists are prefetched.
  */
 
 /**

packages/safe-chain/src/packagemanager/npm/createPackageManager.js

@@ -12,6 +12,14 @@ import {
 // themselves. Nested npm install/ci/etc. calls within those scripts are
 // re-intercepted by the shims in PATH.
 const NPM_LIFECYCLE_COMMANDS = new Set(["run", "start", "stop", "restart", "test"]);
+const NPM_INSTALL_COMMANDS = new Set([
+  "install",
+  "ci",
+  "install-test",
+  "install-ci-test",
+  "update",
+  "dedupe",
+]);
 
 /**
  * @returns {import("../currentPackageManager.js").PackageManager}
@@ -51,6 +59,10 @@ export function createNpmPackageManager() {
       const command = getNpmCommandForArgs(args);
       return command === null || !NPM_LIFECYCLE_COMMANDS.has(command);
     },
+    commandInstallsPackages(args) {
+      const command = getNpmCommandForArgs(args);
+      return command !== null && NPM_INSTALL_COMMANDS.has(command);
+    },
   };
 }
 

packages/safe-chain/src/packagemanager/pnpm/createPackageManager.js

@@ -11,6 +11,8 @@ const PNPM_LIFECYCLE_COMMANDS = new Set(["run", "exec", "node", "test", "start",
 // certificate (#596). Installs from lifecycle scripts are still caught by the shims.
 const PNPM_NO_PROXY_COMMANDS = new Set([...PNPM_LIFECYCLE_COMMANDS, "publish", "stage"]);
 
+const PNPM_INSTALL_COMMANDS = new Set(["install", "i", "add", "update", "up", "upgrade"]);
+
 // Global flags whose value is a separate argument, which must be skipped when
 // looking for the subcommand. eg: pnpm --dir ./packages/foo publish
 const PNPM_GLOBAL_FLAGS_WITH_VALUE = new Set([
@@ -69,6 +71,10 @@ export function createPnpmPackageManager() {
       const command = findPnpmSubcommand(args);
       return !command || !PNPM_NO_PROXY_COMMANDS.has(command);
     },
+    commandInstallsPackages(args) {
+      const command = findPnpmSubcommand(args);
+      return !command || PNPM_INSTALL_COMMANDS.has(command);
+    },
   };
 }
 

packages/safe-chain/src/packagemanager/yarn/createPackageManager.js

@@ -3,6 +3,7 @@ import { runYarnCommand } from "./runYarnCommand.js";
 
 // yarn commands that only execute scripts and never download packages.
 const YARN_LIFECYCLE_COMMANDS = new Set(["run", "node"]);
+const YARN_INSTALL_COMMANDS = new Set(["install", "add", "up", "upgrade"]);
 
 const scanner = commandArgumentScanner();
 
@@ -25,6 +26,10 @@ export function createYarnPackageManager() {
       const command = args.find((arg) => !arg.startsWith("-"))?.toLowerCase();
       return !command || !YARN_LIFECYCLE_COMMANDS.has(command);
     },
+    commandInstallsPackages(args) {
+      const command = args.find((arg) => !arg.startsWith("-"))?.toLowerCase();
+      return !command || YARN_INSTALL_COMMANDS.has(command);
+    },
   };
 }
 

packages/safe-chain/src/scanning/prefetchLists.js

@@ -0,0 +1,16 @@
+import { skipMinimumPackageAge } from "../config/settings.js";
+import { openMalwareDatabase } from "./malwareDatabase.js";
+import { openNewPackagesDatabase } from "./newPackagesListCache.js";
+import { openSafePatchesDatabase } from "./safePatchesListCache.js";
+
+/**
+ * Starts loading the lists the proxy consults so they overlap with the package
+ * manager starting up. A failed load is retried by the request that needs it.
+ */
+export function prefetchLists() {
+  void openSafePatchesDatabase();
+  openMalwareDatabase().catch(() => {});
+  if (!skipMinimumPackageAge()) {
+    void openNewPackagesDatabase();
+  }
+}

packages/safe-chain/src/scanning/prefetchLists.spec.js

@@ -0,0 +1,83 @@
+import { describe, it, mock, beforeEach } from "node:test";
+import assert from "node:assert";
+
+let skipMinimumPackageAgeSetting = false;
+let malwareResult = Promise.resolve({});
+let opened = [];
+
+mock.module("../config/settings.js", {
+  namedExports: {
+    skipMinimumPackageAge: () => skipMinimumPackageAgeSetting,
+  },
+});
+
+mock.module("./malwareDatabase.js", {
+  namedExports: {
+    openMalwareDatabase: () => {
+      opened.push("malware");
+      return malwareResult;
+    },
+  },
+});
+
+mock.module("./newPackagesListCache.js", {
+  namedExports: {
+    openNewPackagesDatabase: () => {
+      opened.push("newPackages");
+      return Promise.resolve({});
+    },
+  },
+});
+
+mock.module("./safePatchesListCache.js", {
+  namedExports: {
+    openSafePatchesDatabase: () => {
+      opened.push("safePatches");
+      return Promise.resolve({});
+    },
+  },
+});
+
+const { prefetchLists } = await import("./prefetchLists.js");
+
+describe("prefetchLists", () => {
+  beforeEach(() => {
+    skipMinimumPackageAgeSetting = false;
+    malwareResult = Promise.resolve({});
+    opened = [];
+  });
+
+  it("starts loading the malware, new packages and safe patches lists", () => {
+    prefetchLists();
+
+    assert.deepStrictEqual(opened.sort(), [
+      "malware",
+      "newPackages",
+      "safePatches",
+    ]);
+  });
+
+  it("skips the new packages list when the minimum age check is disabled", () => {
+    skipMinimumPackageAgeSetting = true;
+
+    prefetchLists();
+
+    assert.deepStrictEqual(opened.sort(), ["malware", "safePatches"]);
+  });
+
+  it("does not leave an unhandled rejection when the malware list fails to load", async () => {
+    const unhandled = [];
+    const onUnhandled = (reason) => unhandled.push(reason);
+    process.on("unhandledRejection", onUnhandled);
+    try {
+      malwareResult = Promise.reject(new Error("offline"));
+
+      prefetchLists();
+      await new Promise((resolve) => setImmediate(resolve));
+
+      assert.deepStrictEqual(unhandled, []);
+    } finally {
+      process.off("unhandledRejection", onUnhandled);
+    }
+  });
+});