diff --git a/scopes/toolbox/fs/hard-link-directory/hard-link-directory.spec.ts b/scopes/toolbox/fs/hard-link-directory/hard-link-directory.spec.ts index f78638bc411f..a2cb9fbeb85c 100644 --- a/scopes/toolbox/fs/hard-link-directory/hard-link-directory.spec.ts +++ b/scopes/toolbox/fs/hard-link-directory/hard-link-directory.spec.ts @@ -130,3 +130,150 @@ describe('hardLinkDirectory()', () => { expect(fs.readdirSync(dest2Dir)).to.deep.equal([]); }); }); + +function findQuarantined(parentDir: string, originalName: string): string | undefined { + const quarantineDir = fs.readdirSync(parentDir).find((entry) => entry.startsWith(`${originalName}.bit-stray-`)); + if (!quarantineDir) return undefined; + const quarantined = path.join(parentDir, quarantineDir, originalName); + try { + fs.lstatSync(quarantined); + return quarantined; + } catch { + return undefined; + } +} + +it('recover when an ancestor of the destination subdirectory is a regular file', async () => { + const tempDir = globalBitTempDir(); + const srcDir = path.join(tempDir, 'source'); + const destDir = path.join(tempDir, 'dest'); + + fs.mkdirpSync(srcDir); + fs.mkdirpSync(path.join(srcDir, '@scope', 'pkg')); + fs.writeFileSync(path.join(srcDir, '@scope/pkg/file.txt'), 'Hello World'); + + // Simulate a corrupted node_modules layout: '@scope' exists as a regular file + // where a directory is expected. This is the shape of the ENOTDIR mkdir failure + // seen during 'bit install' post-install linking into '.bit_roots'. + fs.mkdirpSync(destDir); + fs.writeFileSync(path.join(destDir, '@scope'), 'stray file'); + + await hardLinkDirectory(srcDir, [destDir]); + + expect(fs.readFileSync(path.join(destDir, '@scope/pkg/file.txt'), 'utf8')).to.equal('Hello World'); + // The stray entry must be preserved (renamed, not deleted) so the user can recover it. + const quarantined = findQuarantined(destDir, '@scope'); + expect(quarantined).to.not.equal(undefined); + expect(fs.readFileSync(quarantined!, 'utf8')).to.equal('stray file'); +}); + +it('recover when the exact destination subdirectory exists as a regular file', async () => { + const tempDir = globalBitTempDir(); + const srcDir = path.join(tempDir, 'source'); + const destDir = path.join(tempDir, 'dest'); + + fs.mkdirpSync(srcDir); + fs.mkdirpSync(path.join(srcDir, 'subdir')); + fs.writeFileSync(path.join(srcDir, 'subdir/file.txt'), 'Hello World'); + + fs.mkdirpSync(destDir); + fs.writeFileSync(path.join(destDir, 'subdir'), 'stray file'); + + await hardLinkDirectory(srcDir, [destDir]); + + expect(fs.readFileSync(path.join(destDir, 'subdir/file.txt'), 'utf8')).to.equal('Hello World'); + const quarantined = findQuarantined(destDir, 'subdir'); + expect(quarantined).to.not.equal(undefined); + expect(fs.readFileSync(quarantined!, 'utf8')).to.equal('stray file'); +}); + +it('recover when an ancestor of the destination subdirectory is a dangling symlink', async () => { + const tempDir = globalBitTempDir(); + const srcDir = path.join(tempDir, 'source'); + const destDir = path.join(tempDir, 'dest'); + + fs.mkdirpSync(srcDir); + fs.mkdirpSync(path.join(srcDir, '@scope', 'pkg')); + fs.writeFileSync(path.join(srcDir, '@scope/pkg/file.txt'), 'Hello World'); + + fs.mkdirpSync(destDir); + // Dangling symlink at '@scope' — points to a non-existent target. lstat reports it + // as a symlink (not a directory), so mkdir(@scope/pkg) fails with ENOENT through it. + const offender = path.join(destDir, '@scope'); + fs.symlinkSync(path.join(tempDir, 'does-not-exist'), offender, 'junction'); + const originalTarget = fs.readlinkSync(offender); + + await hardLinkDirectory(srcDir, [destDir]); + + expect(fs.readFileSync(path.join(destDir, '@scope/pkg/file.txt'), 'utf8')).to.equal('Hello World'); + // The dangling symlink itself must be preserved as a symlink at the quarantined name. + const quarantined = findQuarantined(destDir, '@scope'); + expect(quarantined).to.not.equal(undefined); + expect(fs.lstatSync(quarantined!).isSymbolicLink()).to.equal(true); + expect(fs.readlinkSync(quarantined!)).to.equal(originalTarget); +}); + +it('does not overwrite an existing quarantine file', async () => { + const tempDir = globalBitTempDir(); + const srcDir = path.join(tempDir, 'source'); + const destDir = path.join(tempDir, 'dest'); + const offender = path.join(destDir, 'subdir'); + const timestamp = 123456789; + const existingQuarantine = `${offender}.bit-stray-${timestamp}`; + + fs.mkdirpSync(path.join(srcDir, 'subdir')); + fs.writeFileSync(path.join(srcDir, 'subdir/file.txt'), 'Hello World'); + fs.mkdirpSync(destDir); + fs.writeFileSync(offender, 'new stray file'); + fs.writeFileSync(existingQuarantine, 'previous stray file'); + + const originalDateNow = Date.now; + Date.now = () => timestamp; + try { + await hardLinkDirectory(srcDir, [destDir]); + } finally { + Date.now = originalDateNow; + } + + expect(fs.readFileSync(existingQuarantine, 'utf8')).to.equal('previous stray file'); + expect(fs.readFileSync(path.join(`${existingQuarantine}-1`, 'subdir'), 'utf8')).to.equal('new stray file'); + expect(fs.readFileSync(path.join(destDir, 'subdir/file.txt'), 'utf8')).to.equal('Hello World'); +}); + +it('recovers when concurrent calls encounter the same non-directory entry', async () => { + const tempDir = globalBitTempDir(); + const srcDir = path.join(tempDir, 'source'); + const destDir = path.join(tempDir, 'dest'); + const offender = path.join(destDir, 'subdir'); + + fs.mkdirpSync(path.join(srcDir, 'subdir')); + fs.writeFileSync(path.join(srcDir, 'subdir/file.txt'), 'Hello World'); + fs.mkdirpSync(destDir); + fs.writeFileSync(offender, 'stray file'); + + const originalRename = fs.rename; + let offenderRenames = 0; + let releaseRenames!: () => void; + const bothRenamesStarted = new Promise((resolve) => { + releaseRenames = resolve; + }); + (fs as any).rename = async (src: string, dest: string) => { + if (src !== offender) return originalRename(src, dest); + offenderRenames += 1; + if (offenderRenames === 2) releaseRenames(); + await bothRenamesStarted; + return originalRename(src, dest); + }; + + try { + await Promise.all([hardLinkDirectory(srcDir, [destDir]), hardLinkDirectory(srcDir, [destDir])]); + } finally { + (fs as any).rename = originalRename; + } + + expect(offenderRenames).to.equal(2); + expect(fs.readFileSync(path.join(destDir, 'subdir/file.txt'), 'utf8')).to.equal('Hello World'); + const quarantineDirs = fs.readdirSync(destDir).filter((entry) => entry.startsWith('subdir.bit-stray-')); + expect(quarantineDirs).to.have.lengthOf(1); + expect(fs.readFileSync(path.join(destDir, quarantineDirs[0], 'subdir'), 'utf8')).to.equal('stray file'); +}); diff --git a/scopes/toolbox/fs/hard-link-directory/hard-link-directory.ts b/scopes/toolbox/fs/hard-link-directory/hard-link-directory.ts index faf7a017c3c0..3cb970113667 100644 --- a/scopes/toolbox/fs/hard-link-directory/hard-link-directory.ts +++ b/scopes/toolbox/fs/hard-link-directory/hard-link-directory.ts @@ -2,6 +2,10 @@ import path from 'path'; import fs from 'fs-extra'; import symlinkDir from 'symlink-dir'; import resolveLinkTarget from 'resolve-link-target'; +import { formatWarningSummary } from '@teambit/cli'; +import { getConfig } from '@teambit/config-store'; +import { CFG_NO_WARNINGS } from '@teambit/legacy.constants'; +import { logger } from '@teambit/legacy.logger'; /** * Hard link all files from a directory to several target directories. @@ -20,11 +24,7 @@ export async function hardLinkDirectory(src: string, destDirs: string[]) { const destSubdirs = await Promise.all( destDirs.map(async (destDir) => { const destSubdir = path.join(destDir, file.name); - try { - await fs.mkdir(destSubdir, { recursive: true }); - } catch (err: any) { - if (err.code !== 'EEXIST') throw err; - } + await ensureDir(destSubdir); return destSubdir; }) ); @@ -36,9 +36,9 @@ export async function hardLinkDirectory(src: string, destDirs: string[]) { let srcStats: fs.Stats; try { srcStats = await fs.stat(srcFile); - } catch (err: any) { + } catch (err) { // if the link is broken, ignore it - if (err.code === 'ENOENT') return; + if (errnoCode(err) === 'ENOENT') return; throw err; } if (srcStats.isDirectory()) { @@ -56,8 +56,8 @@ export async function hardLinkDirectory(src: string, destDirs: string[]) { const destFile = path.join(destDir, file.name); try { await linkFile(srcFile, destFile); - } catch (err: any) { - if (err.code === 'ENOENT') { + } catch (err) { + if (errnoCode(err) === 'ENOENT') { // broken symlinks are skipped return; } @@ -72,18 +72,19 @@ export async function hardLinkDirectory(src: string, destDirs: string[]) { async function linkFile(srcFile: string, destFile: string) { try { await fs.link(srcFile, destFile); - } catch (err: any) { - if (err.code === 'ENOENT') { - await fs.mkdir(path.dirname(destFile), { recursive: true }); + } catch (err) { + const code = errnoCode(err); + if (code === 'ENOENT' || code === 'ENOTDIR') { + await ensureDir(path.dirname(destFile)); await linkFileIfNotExists(srcFile, destFile); return; } - if (err.code === 'EXDEV') { + if (code === 'EXDEV') { // hard links can't cross devices (e.g. bind mounts or overlayfs on CI), fall back to copying await fs.copyFile(srcFile, destFile); return; } - if (err.code !== 'EEXIST') { + if (code !== 'EEXIST') { throw err; } } @@ -92,13 +93,121 @@ async function linkFile(srcFile: string, destFile: string) { async function linkFileIfNotExists(srcFile: string, destFile: string) { try { await fs.link(srcFile, destFile); - } catch (err: any) { - if (err.code === 'EXDEV') { + } catch (err) { + const code = errnoCode(err); + if (code === 'EXDEV') { await fs.copyFile(srcFile, destFile); return; } - if (err.code !== 'EEXIST') { + if (code !== 'EEXIST') { throw err; } } } + +/** + * Like `fs.mkdir(dir, { recursive: true })`, but recovers from a corrupted node_modules + * tree where some ancestor of `dir` exists as a regular file or a non-directory symlink + * (which causes `mkdir` to throw `ENOTDIR` or `ENOENT` through a broken symlink). The + * blocking entry is moved aside (not deleted — the offender could be high up the tree + * and we don't want to discard the user's data) and `mkdir` is retried. + */ +async function ensureDir(dir: string) { + let mkdirError: unknown; + for (let attempt = 0; attempt < 10; attempt++) { + try { + await fs.mkdir(dir, { recursive: true }); + return; + } catch (err) { + mkdirError = err; + } + + // ENOTDIR: a regular file blocks the path. EEXIST: leaf already exists as a non-directory + // (rare with recursive: true). ENOENT: a dangling symlink in the path can't be traversed. + const code = errnoCode(mkdirError); + if (code !== 'ENOTDIR' && code !== 'EEXIST' && code !== 'ENOENT') throw mkdirError; + const offender = await findNonDirectoryAncestor(dir); + if (offender == null) { + // EEXIST with a directory already at `dir` is benign — recursive mkdir normally + // swallows it, but be defensive against races. + if (code === 'EEXIST') return; + // Another worker may have already moved the offender. Retry mkdir against the new state. + continue; + } + const quarantined = await quarantineStrayEntry(offender); + // Another worker already quarantined this entry. Retry mkdir against the new state. + if (quarantined == null) continue; + const msg = + `non-directory entry at ${offender} blocked link target ${dir}; ` + + `moved aside to ${quarantined} so the install could continue. inspect or delete it manually if it isn't expected.`; + logger.warn(msg); + printRecoveryWarning(msg); + } + throw mkdirError; +} + +/** + * Reserve a unique sibling directory, then atomically move `offender` into it. The empty, + * exclusively created directory makes the rename no-clobber without recreating files or + * symlinks, so Windows junction types and all other entry metadata remain intact. + * On the rare chance the directory name already exists (e.g. a previous recovery in the + * same millisecond, or a leftover from a prior failed run), keep bumping a counter. + * Returns undefined when another worker already moved the offender. + */ +async function quarantineStrayEntry(offender: string): Promise { + const base = `${offender}.bit-stray-${Date.now()}`; + let quarantineDir = base; + for (let i = 1; ; i++) { + try { + await fs.mkdir(quarantineDir, { mode: 0o700 }); + } catch (err) { + if (errnoCode(err) !== 'EEXIST') throw err; + quarantineDir = `${base}-${i}`; + continue; + } + + const quarantined = path.join(quarantineDir, path.basename(offender)); + try { + await fs.rename(offender, quarantined); + return quarantined; + } catch (err) { + // Only remove the directory if it is still empty. Never recursively remove it: after a + // successful rename it contains user data, even if an unusual filesystem reports an error. + await fs.rmdir(quarantineDir).catch(() => undefined); + if (errnoCode(err) === 'ENOENT') return undefined; + throw err; + } + } +} + +function printRecoveryWarning(msg: string) { + if (getConfig(CFG_NO_WARNINGS) === 'true' || !logger.shouldWriteToConsole) return; + logger.console(formatWarningSummary(msg), 'warn'); +} + +/** + * Walk up from `dir` until we find an existing path component. If that component is not + * a directory, return it (it's the entry blocking `mkdir`). Otherwise return null. + */ +async function findNonDirectoryAncestor(dir: string): Promise { + let current = dir; + while (current && path.dirname(current) !== current) { + let stat: fs.Stats; + try { + stat = await fs.lstat(current); + } catch (err) { + const code = errnoCode(err); + if (code === 'ENOENT' || code === 'ENOTDIR') { + current = path.dirname(current); + continue; + } + throw err; + } + return stat.isDirectory() ? null : current; + } + return null; +} + +function errnoCode(err: unknown): string | undefined { + return (err as NodeJS.ErrnoException | undefined)?.code; +}