feat(core): Smaller .yarn/cache size (granular supportedArchitectures)

.yarn/versions/4d4fccf6.yml

@@ -0,0 +1,34 @@
+releases:
+  "@yarnpkg/builder": minor
+  "@yarnpkg/cli": minor
+  "@yarnpkg/core": minor
+  "@yarnpkg/doctor": minor
+  "@yarnpkg/extensions": minor
+  "@yarnpkg/nm": minor
+  "@yarnpkg/plugin-catalog": minor
+  "@yarnpkg/plugin-compat": minor
+  "@yarnpkg/plugin-constraints": minor
+  "@yarnpkg/plugin-dlx": minor
+  "@yarnpkg/plugin-essentials": minor
+  "@yarnpkg/plugin-exec": minor
+  "@yarnpkg/plugin-file": minor
+  "@yarnpkg/plugin-git": minor
+  "@yarnpkg/plugin-github": minor
+  "@yarnpkg/plugin-http": minor
+  "@yarnpkg/plugin-init": minor
+  "@yarnpkg/plugin-interactive-tools": minor
+  "@yarnpkg/plugin-jsr": minor
+  "@yarnpkg/plugin-link": minor
+  "@yarnpkg/plugin-nm": minor
+  "@yarnpkg/plugin-npm": minor
+  "@yarnpkg/plugin-npm-cli": minor
+  "@yarnpkg/plugin-pack": minor
+  "@yarnpkg/plugin-patch": minor
+  "@yarnpkg/plugin-pnp": minor
+  "@yarnpkg/plugin-pnpm": minor
+  "@yarnpkg/plugin-stage": minor
+  "@yarnpkg/plugin-typescript": minor
+  "@yarnpkg/plugin-version": minor
+  "@yarnpkg/plugin-workspace-tools": minor
+  "@yarnpkg/pnpify": minor
+  "@yarnpkg/sdks": minor

CHANGELOG.md

@@ -8,6 +8,7 @@ Yarn now accepts sponsors! Please take a look at our [OpenCollective](https://op
 Features in `master` can be tried out by running `yarn set version from sources` in your project.
 :::
 
+- `supportedArchitectures` now accepts a list of `{os, cpu, libc}` entries, letting you describe the exact architectures you want to support rather than the cross product of every field. Useful to keep zero-install caches as small as possible.
 - Fixes `preferInteractive` forcing interactive mode in non-TTY environments.
 - `node-modules` linker now honors user-defined symlinks for `<workspace>/node_modules` directories
 - `node-modules` linker supports hoisting into inner workspaces that are parents of other workspaces

packages/acceptance-tests/pkg-tests-specs/sources/features/prunedNativeDeps.test.ts

@@ -166,6 +166,145 @@ describe(`Features`, () => {
       }]);
     }));
 
+    it(`should only fetch the exact architectures listed when using the list form`, makeTemporaryEnv({
+      dependencies: {
+        [`optional-native`]: `1.0.0`,
+      },
+    }, async ({path, run}) => {
+      // The matrix form would fetch the cross product of both entries (and
+      // thus also fetch native-foo-x86 and native-bar-x64); the list form only
+      // fetches packages compatible with one of the entries taken as a whole.
+      await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+        supportedArchitectures: [{
+          os: `foo`,
+          cpu: `x64`,
+          libc: `glibc`,
+        }, {
+          os: `bar`,
+          cpu: `x86`,
+          libc: `musl`,
+        }],
+      });
+
+      const recording = await startRegistryRecording(async () => {
+        await run(`install`);
+      });
+
+      const tarballRequests = recording.filter(request => {
+        return request.type === RequestType.PackageTarball;
+      }).sort((a, b) => {
+        const aJson = JSON.stringify(a);
+        const bJson = JSON.stringify(b);
+        return aJson < bJson ? -1 : aJson > bJson ? 1 : 0;
+      });
+
+      expect(tarballRequests).toEqual([{
+        type: RequestType.PackageTarball,
+        localName: `native-foo-x64`,
+        version: `1.0.0`,
+      }, {
+        type: RequestType.PackageTarball,
+        localName: `native-libc-glibc`,
+        version: `1.0.0`,
+      }, {
+        type: RequestType.PackageTarball,
+        localName: `native-libc-musl`,
+        version: `1.0.0`,
+      }, {
+        type: RequestType.PackageTarball,
+        localName: `optional-native`,
+        version: `1.0.0`,
+      }]);
+    }));
+
+    it(`should support the list form with multiple values and nulls inside a single entry`, makeTemporaryEnv({
+      dependencies: {
+        [`optional-native`]: `1.0.0`,
+      },
+    }, async ({path, run, source}) => {
+      await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+        supportedArchitectures: [{
+          os: `foo`,
+          cpu: [`x64`, `x86`],
+          libc: null,
+        }],
+      });
+
+      const recording = await startRegistryRecording(async () => {
+        await run(`install`);
+      });
+
+      const tarballRequests = recording.filter(request => {
+        return request.type === RequestType.PackageTarball;
+      }).sort((a, b) => {
+        const aJson = JSON.stringify(a);
+        const bJson = JSON.stringify(b);
+        return aJson < bJson ? -1 : aJson > bJson ? 1 : 0;
+      });
+
+      expect(tarballRequests).toEqual([{
+        type: RequestType.PackageTarball,
+        localName: `native-foo-x64`,
+        version: `1.0.0`,
+      }, {
+        type: RequestType.PackageTarball,
+        localName: `native-foo-x86`,
+        version: `1.0.0`,
+      }, {
+        type: RequestType.PackageTarball,
+        localName: `native-libc-glibc`,
+        version: `1.0.0`,
+      }, {
+        type: RequestType.PackageTarball,
+        localName: `native-libc-musl`,
+        version: `1.0.0`,
+      }, {
+        type: RequestType.PackageTarball,
+        localName: `optional-native`,
+        version: `1.0.0`,
+      }]);
+    }));
+
+    it(`should treat a list with a single entry just like the legacy object form`, makeTemporaryEnv({
+      dependencies: {
+        [`optional-native`]: `1.0.0`,
+      },
+    }, async ({path, run, source}) => {
+      await xfs.writeJsonPromise(ppath.join(path, Filename.rc), {
+        supportedArchitectures: [{
+          os: [`foo`],
+          cpu: [`x64`],
+          libc: [`glibc`],
+        }],
+      });
+
+      const recording = await startRegistryRecording(async () => {
+        await run(`install`);
+      });
+
+      const tarballRequests = recording.filter(request => {
+        return request.type === RequestType.PackageTarball;
+      }).sort((a, b) => {
+        const aJson = JSON.stringify(a);
+        const bJson = JSON.stringify(b);
+        return aJson < bJson ? -1 : aJson > bJson ? 1 : 0;
+      });
+
+      expect(tarballRequests).toEqual([{
+        type: RequestType.PackageTarball,
+        localName: `native-foo-x64`,
+        version: `1.0.0`,
+      }, {
+        type: RequestType.PackageTarball,
+        localName: `native-libc-glibc`,
+        version: `1.0.0`,
+      }, {
+        type: RequestType.PackageTarball,
+        localName: `optional-native`,
+        version: `1.0.0`,
+      }]);
+    }));
+
     it(`should produce a stable lockfile, regardless of the architecture`, makeTemporaryEnv({
       dependencies: {
         [`optional-native`]: `1.0.0`,

packages/yarnpkg-core/sources/Configuration.ts

@@ -374,25 +374,26 @@ export const coreDefinitions: {[coreSettingName: string]: SettingsDefinition} =
     default: true,
   },
   supportedArchitectures: {
-    description: `Architectures that Yarn will fetch and inject into the resolver`,
+    description: `Architectures that Yarn will fetch and inject into the resolver. Can either be a single entry (fields combinatorially merged together) or a list of entries (entries are matched independently)`,
     type: SettingsType.SHAPE,
+    isArray: true,
     properties: {
       os: {
-        description: `Array of supported process.platform strings, or null to target them all`,
+        description: `Supported process.platform string (or array of strings), or null to target them all`,
         type: SettingsType.STRING,
         isArray: true,
         isNullable: true,
         default: [`current`],
       },
       cpu: {
-        description: `Array of supported process.arch strings, or null to target them all`,
+        description: `Supported process.arch string (or array of strings), or null to target them all`,
         type: SettingsType.STRING,
         isArray: true,
         isNullable: true,
         default: [`current`],
       },
       libc: {
-        description: `Array of supported libc libraries, or null to target them all`,
+        description: `Supported libc library (or array of libraries), or null to target them all`,
         type: SettingsType.STRING,
         isArray: true,
         isNullable: true,
@@ -685,7 +686,7 @@ export interface ConfigurationValueMap {
   defaultLanguageName: string;
   defaultProtocol: string;
   enableTransparentWorkspaces: boolean;
-  supportedArchitectures: miscUtils.ToMapValue<SupportedArchitectures>;
+  supportedArchitectures: Array<miscUtils.ToMapValue<SupportedArchitectures>>;
 
   enableMirror: boolean;
   enableNetwork: boolean;
@@ -779,7 +780,13 @@ function parseValue(configuration: Configuration, path: string, valueBase: unkno
   const value = configUtils.getValue(valueBase);
 
   if (definition.isArray || (definition.type === SettingsType.ANY && Array.isArray(value))) {
+    if ((value === null || value === `null`) && `isNullable` in definition && definition.isNullable)
+      return null;
+
     if (!Array.isArray(value)) {
+      if (definition.type === SettingsType.SHAPE || definition.type === SettingsType.MAP)
+        return [parseSingleValue(configuration, `${path}[0]`, valueBase, definition, folder)];
+
       return String(value).split(/,/).map(segment => {
         return parseSingleValue(configuration, path, segment, definition, folder);
       });
@@ -1810,23 +1817,31 @@ export class Configuration {
     return linkers;
   }
 
-  getSupportedArchitectures(): nodeUtils.ArchitectureSet {
+  getSupportedArchitectures(): nodeUtils.ArchitectureSetList {
     const architecture = nodeUtils.getArchitecture();
     const supportedArchitectures = this.get(`supportedArchitectures`);
 
-    let os = supportedArchitectures.get(`os`);
-    if (os !== null)
-      os = os.map(value => value === `current` ? architecture.os : value);
+    // An empty list would mean "no architecture at all", which is never what
+    // the user wants; we fallback on the current architecture instead (which
+    // is also what happens when the setting isn't set at all).
+    if (supportedArchitectures.length === 0)
+      return [nodeUtils.getArchitectureSet()];
 
-    let cpu = supportedArchitectures.get(`cpu`);
-    if (cpu !== null)
-      cpu = cpu.map(value => value === `current` ? architecture.cpu : value);
+    return supportedArchitectures.map(entry => {
+      let os = entry.get(`os`);
+      if (os !== null)
+        os = os.map(value => value === `current` ? architecture.os : value);
 
-    let libc = supportedArchitectures.get(`libc`);
-    if (libc !== null)
-      libc = miscUtils.mapAndFilter(libc, value => value === `current` ? architecture.libc ?? miscUtils.mapAndFilter.skip : value);
+      let cpu = entry.get(`cpu`);
+      if (cpu !== null)
+        cpu = cpu.map(value => value === `current` ? architecture.cpu : value);
 
-    return {os, cpu, libc};
+      let libc = entry.get(`libc`);
+      if (libc !== null)
+        libc = miscUtils.mapAndFilter(libc, value => value === `current` ? architecture.libc ?? miscUtils.mapAndFilter.skip : value);
+
+      return {os, cpu, libc};
+    });
   }
 
   isInteractive({interactive, stdout}: {interactive?: boolean, stdout: Writable}): boolean {

packages/yarnpkg-core/sources/nodeUtils.ts

@@ -79,6 +79,8 @@ export type ArchitectureSet = {
   libc: Array<string> | null;
 };
 
+export type ArchitectureSetList = Array<ArchitectureSet>;
+
 let architecture: Architecture | undefined;
 let architectureSet: ArchitectureSet | undefined;
 

packages/yarnpkg-core/sources/structUtils.ts

@@ -914,16 +914,20 @@ export function isPackageInRange(pkg: Package, range: Descriptor[`range`]) {
 /**
  * Returns whether the given package is compatible with the specified environment.
  */
-export function isPackageCompatible(pkg: Package, architectures: nodeUtils.ArchitectureSet) {
+export function isPackageCompatible(pkg: Package, architectures: nodeUtils.ArchitectureSet | nodeUtils.ArchitectureSetList) {
   if (!pkg.conditions)
     return true;
 
-  return conditionParser(pkg.conditions, specifier => {
+  const architectureList = Array.isArray(architectures)
+    ? architectures
+    : [architectures];
+
+  return architectureList.some(architecture => conditionParser(pkg.conditions!, specifier => {
     const [, name, value] = specifier.match(CONDITION_REGEX)!;
-    const supported = architectures[name as keyof typeof architectures];
+    const supported = architecture[name as keyof typeof architecture];
 
     return supported ? supported.includes(value) : true;
-  });
+  }));
 }
 
 export function allPeerRequests(root: PeerRequestNode | PeerRequirementNode): Iterable<PeerRequestNode> {

packages/yarnpkg-core/tests/Configuration.test.ts

@@ -80,6 +80,76 @@ describe(`Configuration`, () => {
     });
   });
 
+  describe(`supportedArchitectures`, () => {
+    const getSupportedArchitectures = async (value: any) => {
+      return await initializeConfiguration({supportedArchitectures: value}, async dir => {
+        const configuration = await Configuration.find(dir, null);
+        return configuration.getSupportedArchitectures();
+      });
+    };
+
+    it(`should support the legacy object form, as a single entry`, async () => {
+      await expect(getSupportedArchitectures({
+        os: [`darwin`, `linux`],
+        cpu: [`arm64`, `x64`],
+        libc: [`glibc`],
+      })).resolves.toEqual([{
+        os: [`darwin`, `linux`],
+        cpu: [`arm64`, `x64`],
+        libc: [`glibc`],
+      }]);
+    });
+
+    it(`should support a list of entries`, async () => {
+      await expect(getSupportedArchitectures([{
+        os: `darwin`,
+        cpu: `arm64`,
+        libc: `musl`,
+      }, {
+        os: `linux`,
+        cpu: `x64`,
+        libc: `glibc`,
+      }])).resolves.toEqual([{
+        os: [`darwin`],
+        cpu: [`arm64`],
+        libc: [`musl`],
+      }, {
+        os: [`linux`],
+        cpu: [`x64`],
+        libc: [`glibc`],
+      }]);
+    });
+
+    it(`should default the fields that aren't set on an entry`, async () => {
+      const [entry] = await getSupportedArchitectures([{
+        os: `linux`,
+      }]);
+
+      expect(entry.os).toEqual([`linux`]);
+      expect(entry.cpu).toEqual([process.arch]);
+    });
+
+    it(`should preserve null fields inside a list entry (targeting all values)`, async () => {
+      await expect(getSupportedArchitectures([{
+        os: `foo`,
+        cpu: [`x64`, `x86`],
+        libc: null,
+      }])).resolves.toEqual([{
+        os: [`foo`],
+        cpu: [`x64`, `x86`],
+        libc: null,
+      }]);
+    });
+
+    it(`should fallback on the current architecture when the list is empty`, async () => {
+      await expect(getSupportedArchitectures([])).resolves.toEqual([{
+        os: [process.platform],
+        cpu: [process.arch],
+        libc: expect.anything(),
+      }]);
+    });
+  });
+
   describe(`Environment interpolation`, () => {
     it(`should replace env variables`, async () => {
       process.env.ENV_AUTH_TOKEN = `AAA-BBB-CCC`;