diff --git a/components/legacy/scope/scope.ts b/components/legacy/scope/scope.ts index 7d20437c4c31..aecda8fe6333 100644 --- a/components/legacy/scope/scope.ts +++ b/components/legacy/scope/scope.ts @@ -760,8 +760,11 @@ once done, to continue working, please run "bit cc"` this.objects.scopeJson = scopeJson; } - public async getDependenciesGraphByComponentIds(componentIds: ComponentID[]): Promise { - if (!isFeatureEnabled(DEPS_GRAPH)) return undefined; + public async getDependenciesGraphByComponentIds( + componentIds: ComponentID[], + options?: { ignoreFeatureToggle?: boolean } + ): Promise { + if (!options?.ignoreFeatureToggle && !isFeatureEnabled(DEPS_GRAPH)) return undefined; let allGraph: DependenciesGraph | undefined; await pMapPool( componentIds, diff --git a/contrib/claude-skill-bit-cli/CLI_REFERENCE.md b/contrib/claude-skill-bit-cli/CLI_REFERENCE.md index 33cbb860dba9..7acb65fc1ef0 100644 --- a/contrib/claude-skill-bit-cli/CLI_REFERENCE.md +++ b/contrib/claude-skill-bit-cli/CLI_REFERENCE.md @@ -439,7 +439,7 @@ Flags: --name , --generator , --standalone, --no-package install workspace dependencies installs workspace dependencies and prepares the workspace for development. when packages are specified, adds them to workspace.jsonc policy and installs. when no packages specified, installs existing dependencies. automatically imports components, compiles components, links to node_modules, and writes config files. -Flags: --type [lifecycleType], --update, --save-prefix [savePrefix], --skip-dedupe, --skip-import, --skip-compile, --skip-write-config-files, --add-missing-deps, --skip-unavailable, --add-missing-peers, --recurring-install, --no-optional [noOptional], --lockfile-only, --allow-scripts [pkgNames], --disallow-scripts [pkgNames] +Flags: --type [lifecycleType], --update, --save-prefix [savePrefix], --skip-dedupe, --skip-import, --skip-compile, --skip-write-config-files, --add-missing-deps, --skip-unavailable, --add-missing-peers, --recurring-install, --no-optional [noOptional], --lockfile-only, --restore, --allow-scripts [pkgNames], --disallow-scripts [pkgNames] ## bit internalize [component-pattern] diff --git a/e2e/harmony/deps-graph.e2e.ts b/e2e/harmony/deps-graph.e2e.ts index 5cfa70f6d51d..ae91aadaa7da 100644 --- a/e2e/harmony/deps-graph.e2e.ts +++ b/e2e/harmony/deps-graph.e2e.ts @@ -363,4 +363,118 @@ export default new ${className}(); expect(lockfileAfterSecondImport.packages).to.not.have.property('@pnpm.e2e/foo@100.1.0'); }); }); + describe('bit install --restore rebuilds the lockfile from workspace component graphs', function () { + let randomStr: string; + let lockfileAfterRestore: any; + before(async () => { + randomStr = generateRandomStr(4); + const name = `@ci/${randomStr}.{name}`; + helper.scopeHelper.setWorkspaceWithRemoteScope(); + npmCiRegistry = new NpmCiRegistry(helper); + npmCiRegistry.configureCustomNameInPackageJsonHarmony(name); + await npmCiRegistry.init(); + npmCiRegistry.setRegistry(); + helper.env.setCustomNewEnv( + undefined, + undefined, + { policy: { peers: [] } }, + false, + 'custom-env/env', + 'custom-env/env' + ); + helper.fs.createFile('comp1', 'comp1.js', 'require("@pnpm.e2e/foo"); // eslint-disable-line'); + helper.command.addComponent('comp1'); + helper.extensions.addExtensionToVariant('comp1', `${helper.scopes.remote}/custom-env/env`, {}); + helper.fs.createFile('comp2', 'comp2.js', 'require("@pnpm.e2e/bar"); // eslint-disable-line'); + helper.command.addComponent('comp2'); + helper.extensions.addExtensionToVariant('comp2', `${helper.scopes.remote}/custom-env/env`, {}); + helper.extensions.workspaceJsonc.addKeyValToDependencyResolver('rootComponents', true); + await addDistTag({ package: '@pnpm.e2e/foo', version: '100.0.0', distTag: 'latest' }); + await addDistTag({ package: '@pnpm.e2e/bar', version: '100.0.0', distTag: 'latest' }); + helper.command.install('--add-missing-deps'); + helper.command.tagAllComponents('--skip-tests'); + helper.command.export(); + + helper.scopeHelper.reInitWorkspace(); + helper.scopeHelper.addRemoteScope(); + npmCiRegistry.setRegistry(); + helper.extensions.workspaceJsonc.addKeyValToDependencyResolver('rootComponents', true); + helper.command.import(`${helper.scopes.remote}/comp1@latest ${helper.scopes.remote}/comp2@latest`); + + await addDistTag({ package: '@pnpm.e2e/foo', version: '100.1.0', distTag: 'latest' }); + await addDistTag({ package: '@pnpm.e2e/bar', version: '100.1.0', distTag: 'latest' }); + helper.fs.deletePath('pnpm-lock.yaml'); + helper.fs.deletePath('node_modules'); + helper.command.runCmd('bit install --restore'); + lockfileAfterRestore = yaml.load(fs.readFileSync(path.join(helper.scopes.localPath, 'pnpm-lock.yaml'), 'utf8')); + }); + after(() => { + npmCiRegistry.destroy(); + helper.scopeHelper.destroy(); + }); + it('should mark the regenerated lockfile as restoredFromModel', () => { + expect(lockfileAfterRestore.bit.restoredFromModel).to.eq(true); + }); + it('should keep both components locked to the versions stored in their graphs', () => { + expect(lockfileAfterRestore.packages).to.have.property('@pnpm.e2e/foo@100.0.0'); + expect(lockfileAfterRestore.packages).to.have.property('@pnpm.e2e/bar@100.0.0'); + expect(lockfileAfterRestore.packages).to.not.have.property('@pnpm.e2e/foo@100.1.0'); + expect(lockfileAfterRestore.packages).to.not.have.property('@pnpm.e2e/bar@100.1.0'); + }); + }); + describe('bit install --restore works when the DEPS_GRAPH feature toggle is disabled', function () { + let randomStr: string; + let lockfileAfterRestore: any; + before(async () => { + randomStr = generateRandomStr(4); + const name = `@ci/${randomStr}.{name}`; + helper.scopeHelper.setWorkspaceWithRemoteScope(); + npmCiRegistry = new NpmCiRegistry(helper); + npmCiRegistry.configureCustomNameInPackageJsonHarmony(name); + await npmCiRegistry.init(); + npmCiRegistry.setRegistry(); + helper.env.setCustomNewEnv( + undefined, + undefined, + { policy: { peers: [] } }, + false, + 'custom-env/env', + 'custom-env/env' + ); + helper.fs.createFile('comp1', 'comp1.js', 'require("@pnpm.e2e/foo"); // eslint-disable-line'); + helper.command.addComponent('comp1'); + helper.extensions.addExtensionToVariant('comp1', `${helper.scopes.remote}/custom-env/env`, {}); + helper.extensions.workspaceJsonc.addKeyValToDependencyResolver('rootComponents', true); + await addDistTag({ package: '@pnpm.e2e/foo', version: '100.0.0', distTag: 'latest' }); + helper.command.install('--add-missing-deps'); + helper.command.tagAllComponents('--skip-tests'); + helper.command.export(); + + helper.scopeHelper.reInitWorkspace(); + helper.scopeHelper.addRemoteScope(); + npmCiRegistry.setRegistry(); + helper.extensions.workspaceJsonc.addKeyValToDependencyResolver('rootComponents', true); + + helper.command.resetFeatures(); + try { + helper.command.import(`${helper.scopes.remote}/comp1@latest`); + await addDistTag({ package: '@pnpm.e2e/foo', version: '100.1.0', distTag: 'latest' }); + helper.fs.deletePath('pnpm-lock.yaml'); + helper.fs.deletePath('node_modules'); + helper.command.runCmd('bit install --restore'); + lockfileAfterRestore = yaml.load(fs.readFileSync(path.join(helper.scopes.localPath, 'pnpm-lock.yaml'), 'utf8')); + } finally { + helper.command.setFeatures(DEPS_GRAPH); + } + }); + after(() => { + npmCiRegistry.destroy(); + helper.scopeHelper.destroy(); + }); + it('should still restore the lockfile from the stored graph', () => { + expect(lockfileAfterRestore.bit.restoredFromModel).to.eq(true); + expect(lockfileAfterRestore.packages).to.have.property('@pnpm.e2e/foo@100.0.0'); + expect(lockfileAfterRestore.packages).to.not.have.property('@pnpm.e2e/foo@100.1.0'); + }); + }); }); diff --git a/scopes/dependencies/dependency-resolver/package-manager.ts b/scopes/dependencies/dependency-resolver/package-manager.ts index 7999a3ded39e..6b8a4262b871 100644 --- a/scopes/dependencies/dependency-resolver/package-manager.ts +++ b/scopes/dependencies/dependency-resolver/package-manager.ts @@ -181,6 +181,8 @@ export type PackageManagerInstallOptions = { dependenciesGraph?: DependenciesGraph; + failOnDependenciesGraphError?: boolean; + forcedHarmonyVersion?: string; /** @@ -228,6 +230,8 @@ export interface PackageManager { * Name of the package manager */ name: string; + + readonly supportsDependencyGraphRestoration?: boolean; /** * install dependencies * @param componentDirectoryMap diff --git a/scopes/dependencies/pnpm/pnpm.package-manager.spec.ts b/scopes/dependencies/pnpm/pnpm.package-manager.spec.ts index a0a9cc300447..e12971bd742f 100644 --- a/scopes/dependencies/pnpm/pnpm.package-manager.spec.ts +++ b/scopes/dependencies/pnpm/pnpm.package-manager.spec.ts @@ -24,10 +24,46 @@ describe('PnpmPackageManager.getNetworkConfig', () => { }); }); +describe('PnpmPackageManager.install', () => { + it('rethrows dependency graph conversion errors when strict restoration is requested', async () => { + const packageManager = createPackageManager({}); + const restoreError = new Error('failed to restore lockfile'); + packageManager.dependenciesGraphToLockfile = async () => { + throw restoreError; + }; + + let thrown: unknown; + try { + await packageManager.install( + { + rootDir: '/tmp/workspace', + manifests: {}, + componentDirectoryMap: {} as any, + }, + { + dependenciesGraph: {} as any, + rootComponents: true, + failOnDependenciesGraphError: true, + } + ); + } catch (error) { + thrown = error; + } + + expect(thrown).to.equal(restoreError); + }); +}); + function createPackageManager(config: Partial) { const packageManager = new PnpmPackageManager( - {} as any, - {} as any, + { + getRegistries: async () => undefined, + getProxyConfig: async () => undefined, + getNetworkConfig: async () => undefined, + } as any, + { + error: () => {}, + } as any, { getCurrentUser: async () => ({ username: 'test-user' }), } as any diff --git a/scopes/dependencies/pnpm/pnpm.package-manager.ts b/scopes/dependencies/pnpm/pnpm.package-manager.ts index 2c4c7df3c6cc..2d54ec178588 100644 --- a/scopes/dependencies/pnpm/pnpm.package-manager.ts +++ b/scopes/dependencies/pnpm/pnpm.package-manager.ts @@ -12,7 +12,6 @@ import type { CalcDepsGraphOptions, } from '@teambit/dependency-resolver'; import { Registries, Registry } from '@teambit/pkg.entities.registry'; -import { DEPS_GRAPH, isFeatureEnabled } from '@teambit/harmony.modules.feature-toggle'; import type { Logger } from '@teambit/logger'; import { type LockfileFile } from '@pnpm/lockfile.types'; import { memoize, omit } from 'lodash'; @@ -70,6 +69,7 @@ function loadNodeApi(): typeof NodeApi { export class PnpmPackageManager implements PackageManager { readonly name = 'pnpm'; + readonly supportsDependencyGraphRestoration = true; readonly modulesManifestCache: Map = new Map(); private username: string; @@ -153,7 +153,6 @@ export class PnpmPackageManager implements PackageManager { const { config } = await this.readConfig(installOptions.packageManagerConfigRootDir); if ( installOptions.dependenciesGraph && - isFeatureEnabled(DEPS_GRAPH) && (installOptions.rootComponents || installOptions.rootComponentsForCapsules) ) { try { @@ -166,8 +165,8 @@ export class PnpmPackageManager implements PackageManager { cacheDir: config.cacheDir, }); } catch (error) { - // If the lockfile could not be created for some reason, it will be created later during installation. this.logger.error((error as Error).message); + if (installOptions.failOnDependenciesGraphError) throw error; } } diff --git a/scopes/dependencies/yarn/yarn.package-manager.ts b/scopes/dependencies/yarn/yarn.package-manager.ts index f9f6e2fb095c..4662a2203619 100644 --- a/scopes/dependencies/yarn/yarn.package-manager.ts +++ b/scopes/dependencies/yarn/yarn.package-manager.ts @@ -44,6 +44,7 @@ import { createRootComponentsDir } from './create-root-components-dir'; export class YarnPackageManager implements PackageManager { readonly name = 'yarn'; + readonly supportsDependencyGraphRestoration = false; constructor( private depResolver: DependencyResolverMain, diff --git a/scopes/harmony/cli-reference/cli-reference.json b/scopes/harmony/cli-reference/cli-reference.json index 2530d6e57b44..3aaa7f0a17c5 100644 --- a/scopes/harmony/cli-reference/cli-reference.json +++ b/scopes/harmony/cli-reference/cli-reference.json @@ -2416,6 +2416,11 @@ "lockfile-only", "dependencies are not written to node_modules. Only the lockfile is updated" ], + [ + "", + "restore", + "reconstruct the lockfile from each workspace component's stored dependency graph before installing" + ], [ "", "allow-scripts [pkgNames]", diff --git a/scopes/harmony/cli-reference/cli-reference.mdx b/scopes/harmony/cli-reference/cli-reference.mdx index 2774f5b76c3e..068517722632 100644 --- a/scopes/harmony/cli-reference/cli-reference.mdx +++ b/scopes/harmony/cli-reference/cli-reference.mdx @@ -1289,6 +1289,7 @@ automatically imports components, compiles components, links to node_modules, an | `--recurring-install` | | automatically run install again if there are non loaded old envs in your workspace | | `--no-optional [noOptional]` | | do not install optional dependencies (works with pnpm only) | | `--lockfile-only` | | dependencies are not written to node_modules. Only the lockfile is updated | +| `--restore` | | reconstruct the lockfile from each workspace component's stored dependency graph before installing | | `--allow-scripts [pkgNames]` | | a comma separated list of package names that are allowed to run installation scripts | | `--disallow-scripts [pkgNames]` | | a comma separated list of package names that are NOT allowed to run installation scripts | diff --git a/scopes/scope/scope/scope.main.runtime.ts b/scopes/scope/scope/scope.main.runtime.ts index 6f71ec4d61f9..b9bb7c83c73e 100644 --- a/scopes/scope/scope/scope.main.runtime.ts +++ b/scopes/scope/scope/scope.main.runtime.ts @@ -1530,8 +1530,11 @@ export class ScopeMain implements ComponentFactory { return scope; } - public getDependenciesGraphByComponentIds(componentIds: ComponentID[]): Promise { - return this.legacyScope.getDependenciesGraphByComponentIds(componentIds); + public getDependenciesGraphByComponentIds( + componentIds: ComponentID[], + options?: { ignoreFeatureToggle?: boolean } + ): Promise { + return this.legacyScope.getDependenciesGraphByComponentIds(componentIds, options); } } diff --git a/scopes/workspace/install/exceptions/index.ts b/scopes/workspace/install/exceptions/index.ts index 7cfeee5d060e..ea7a46ddf636 100644 --- a/scopes/workspace/install/exceptions/index.ts +++ b/scopes/workspace/install/exceptions/index.ts @@ -1,3 +1,4 @@ export { DependencyTypeNotSupportedInPolicy } from './dependency-type-not-supported-in-policy'; export { UnpublishedComponentDependency } from './unpublished-component-dependency'; export type { UnpublishedSnapDependency } from './unpublished-component-dependency'; +export { RestoreNotSupportedByPackageManager } from './restore-not-supported-by-package-manager'; diff --git a/scopes/workspace/install/exceptions/restore-not-supported-by-package-manager.ts b/scopes/workspace/install/exceptions/restore-not-supported-by-package-manager.ts new file mode 100644 index 000000000000..0f6e1371b57b --- /dev/null +++ b/scopes/workspace/install/exceptions/restore-not-supported-by-package-manager.ts @@ -0,0 +1,7 @@ +import { BitError } from '@teambit/bit-error'; + +export class RestoreNotSupportedByPackageManager extends BitError { + constructor(packageManagerName: string) { + super(`the --restore option is not supported by package manager "${packageManagerName}"`); + } +} diff --git a/scopes/workspace/install/install.cmd.tsx b/scopes/workspace/install/install.cmd.tsx index 62701786e2da..9e9929e85853 100644 --- a/scopes/workspace/install/install.cmd.tsx +++ b/scopes/workspace/install/install.cmd.tsx @@ -22,6 +22,7 @@ type InstallCmdOptions = { noOptional: boolean; recurringInstall: boolean; lockfileOnly: boolean; + restore: boolean; allowScripts?: string; disallowScripts?: string; }; @@ -69,6 +70,11 @@ automatically imports components, compiles components, links to node_modules, an ], ['', 'no-optional [noOptional]', 'do not install optional dependencies (works with pnpm only)'], ['', 'lockfile-only', 'dependencies are not written to node_modules. Only the lockfile is updated'], + [ + '', + 'restore', + "reconstruct the lockfile from each workspace component's stored dependency graph before installing", + ], [ '', 'allow-scripts [pkgNames]', @@ -142,6 +148,7 @@ automatically imports components, compiles components, links to node_modules, an updateAll: options.update, recurringInstall: options.recurringInstall, lockfileOnly: options.lockfileOnly, + restoreFromDependenciesGraph: options.restore, showExternalPackageManagerPrompt: true, allowScripts: this._parseAllowScriptsFlags(options.allowScripts, options.disallowScripts), }; diff --git a/scopes/workspace/install/install.main.runtime.ts b/scopes/workspace/install/install.main.runtime.ts index 972117259ddf..2d1434b83b6a 100644 --- a/scopes/workspace/install/install.main.runtime.ts +++ b/scopes/workspace/install/install.main.runtime.ts @@ -5,7 +5,7 @@ import { getRootComponentDir, linkPkgsToRootComponents } from '@teambit/workspac import type { CompilerMain } from '@teambit/compiler'; import { CompilerAspect, CompilationInitiator } from '@teambit/compiler'; import type { CLIMain, CommandList } from '@teambit/cli'; -import { CLIAspect, MainRuntime } from '@teambit/cli'; +import { CLIAspect, MainRuntime, formatWarningSummary } from '@teambit/cli'; import chalk from 'chalk'; import yesno from 'yesno'; import type { Workspace } from '@teambit/workspace'; @@ -43,6 +43,7 @@ import type { WorkspaceDependencyLifecycleType, DependencyResolverMain, DependencyInstaller, + PackageManager, PackageManagerInstallOptions, WorkspacePolicyEntry, LinkingOptions, @@ -52,7 +53,11 @@ import type { WorkspacePolicy, UpdatedComponent, } from '@teambit/dependency-resolver'; -import { DependencyResolverAspect, ComponentDependency, ensureHoistedDependencyResolution } from '@teambit/dependency-resolver'; +import { + DependencyResolverAspect, + ComponentDependency, + ensureHoistedDependencyResolution, +} from '@teambit/dependency-resolver'; import type { WorkspaceConfigFilesMain } from '@teambit/workspace-config-files'; import { WorkspaceConfigFilesAspect } from '@teambit/workspace-config-files'; import type { Logger, LoggerMain } from '@teambit/logger'; @@ -68,7 +73,11 @@ import { BundlerAspect } from '@teambit/bundler'; import type { UiMain } from '@teambit/ui'; import { UIAspect } from '@teambit/ui'; import { EXTERNAL_PM_POSTINSTALL_SCRIPT } from '@teambit/host-initializer'; -import { DependencyTypeNotSupportedInPolicy, UnpublishedComponentDependency } from './exceptions'; +import { + DependencyTypeNotSupportedInPolicy, + RestoreNotSupportedByPackageManager, + UnpublishedComponentDependency, +} from './exceptions'; import type { UnpublishedSnapDependency } from './exceptions'; import { InstallAspect } from './install.aspect'; import { pickOutdatedPkgs } from './pick-outdated-pkgs'; @@ -110,6 +119,7 @@ export type WorkspaceInstallOptions = { writeConfigFiles?: boolean; skipPrune?: boolean; dependenciesGraph?: DependenciesGraph; + restoreFromDependenciesGraph?: boolean; allowScripts?: Record; }; @@ -357,6 +367,7 @@ export class InstallMain { await this.dependencyResolver.persistConfig('update allowScripts configuration'); } const pm = this.dependencyResolver.getPackageManager(); + this.ensurePackageManagerSupportsRestore(options, pm); this.logger.console( `installing dependencies in workspace using ${pm?.name} (${chalk.cyan( this.dependencyResolver.packageManagerName @@ -394,11 +405,13 @@ export class InstallMain { } ); + const dependenciesGraph = await this.resolveDependenciesGraph(options, { hasRootComponents }); const pmInstallOptions: PackageManagerInstallOptions = { ...calcManifestsOpts, autoInstallPeers: this.dependencyResolver.config.autoInstallPeers, dedupePeers: this.dependencyResolver.config.dedupePeers, - dependenciesGraph: options?.dependenciesGraph, + dependenciesGraph, + failOnDependenciesGraphError: options?.restoreFromDependenciesGraph, includeOptionalDeps: options?.includeOptionalDeps, neverBuiltDependencies: this.dependencyResolver.config.neverBuiltDependencies, allowScripts: this.dependencyResolver.getAllowedScripts(), @@ -539,6 +552,42 @@ export class InstallMain { return nonLoadedEnvs.length > 0; } + private ensurePackageManagerSupportsRestore( + options: ModulesInstallOptions | undefined, + packageManager: PackageManager | undefined + ) { + if (options?.restoreFromDependenciesGraph && !packageManager?.supportsDependencyGraphRestoration) { + throw new RestoreNotSupportedByPackageManager(packageManager?.name ?? this.dependencyResolver.packageManagerName); + } + } + + private async resolveDependenciesGraph( + options: ModulesInstallOptions | undefined, + context: { hasRootComponents: boolean } + ): Promise { + if (options?.dependenciesGraph) return options.dependenciesGraph; + if (!options?.restoreFromDependenciesGraph) return undefined; + if (!context.hasRootComponents) { + this.logger.console( + formatWarningSummary( + '--restore requires "rootComponents: true" in the dependency-resolver config; falling back to a regular install.' + ) + ); + return undefined; + } + const graph = await this.workspace.scope.getDependenciesGraphByComponentIds(this.workspace.listIds(), { + ignoreFeatureToggle: true, + }); + if (!graph) { + this.logger.console( + formatWarningSummary( + '--restore was requested but no workspace component has a stored dependency graph. Falling back to a regular install.' + ) + ); + } + return graph; + } + /** * Called only when the package manager install failed. A "No matching version found" failure usually points * at a component dependency that resolves to a snap which was never published — its build failed or hasn't