From 677f1b420f4dff8a31ddc6aff7e3977ed1f03b2a Mon Sep 17 00:00:00 2001 From: PaulHan Date: Mon, 22 Apr 2024 23:13:36 +0800 Subject: [PATCH 1/3] feat: support isolated folder for monorepo --- examples/api/example-skipIsolated.ts | 14 +++++ fixtures/lib1/file2.c | 29 ++++++++++ fixtures/lib1/file_1.js | 55 ++++++++++++++++++ fixtures/lib2/file_2.js | 56 +++++++++++++++++++ .../core/src/interfaces/options.interface.ts | 1 + packages/finder/src/in-files-detector.ts | 6 +- packages/finder/src/validators/index.ts | 1 + .../src/validators/skip-isolated.validator.ts | 43 ++++++++++++++ packages/jscpd/README.md | 29 +++++++++- packages/jscpd/__tests__/options.test.ts | 44 +++++++++++++++ packages/jscpd/src/init/cli.ts | 1 + packages/jscpd/src/options.ts | 1 + 12 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 examples/api/example-skipIsolated.ts create mode 100644 fixtures/lib1/file2.c create mode 100644 fixtures/lib1/file_1.js create mode 100644 fixtures/lib2/file_2.js create mode 100644 packages/finder/src/validators/skip-isolated.validator.ts diff --git a/examples/api/example-skipIsolated.ts b/examples/api/example-skipIsolated.ts new file mode 100644 index 00000000..4d4eb6a0 --- /dev/null +++ b/examples/api/example-skipIsolated.ts @@ -0,0 +1,14 @@ +import {detectClones} from "jscpd"; + +(async () => { + const clones = await detectClones({ + path: [ + __dirname + '/../fixtures' + ], + skipIsolated: [ + ['packages/businessA', 'packages/businessB', 'packages/businessC'], + ], + silent: true + }); + console.log(clones); +})() diff --git a/fixtures/lib1/file2.c b/fixtures/lib1/file2.c new file mode 100644 index 00000000..25e333b1 --- /dev/null +++ b/fixtures/lib1/file2.c @@ -0,0 +1,29 @@ +/* + * Copy the size of snapshot frame "sn" to frame "fr". Do the same for all + * following frames and children. + * Returns a pointer to the old current window, or NULL. + */ +static win_T *restore_snapshot_rec(frame_T *sn, frame_T *fr) +{ + win_T *wp = NULL; + win_T *wp2; + + fr->fr_height = sn->fr_height; + fr->fr_width = sn->fr_width; + if (fr->fr_layout == FR_LEAF) { + frame_new_height(fr, fr->fr_height, FALSE, FALSE); + frame_new_width(fr, fr->fr_width, FALSE, FALSE); + wp = sn->fr_win; + } + win_T *wp = NULL; + win_T *wp2; + + fr->fr_height = sn->fr_height; + fr->fr_width = sn->fr_width; + if (fr->fr_layout == FR_LEAF) { + frame_new_height(fr, fr->fr_height, FALSE, FALSE); + frame_new_width(fr, fr->fr_width, FALSE, FALSE); + wp = sn->fr_win; + } + return wp; +} diff --git a/fixtures/lib1/file_1.js b/fixtures/lib1/file_1.js new file mode 100644 index 00000000..91afd38e --- /dev/null +++ b/fixtures/lib1/file_1.js @@ -0,0 +1,55 @@ +/** +*12312 +*/ +function utf8_encode ( str_data ) { + // Encodes an ISO-8859-1 string to UTF-8 + // + // + original by: Webtoolkit.info (http://www.webtoolkit.info/) + str_data = str_data.replace(/\r\n/g,"\n"); + var utftext = ""; + + for (var n = 0; n < str_data.length; n++) { + var c = str_data.charCodeAt(n); + if (c < 128) { + utftext += String.fromCharCode(c); + } else if((c > 127) && (c < 2048)) { + utftext += String.fromCharCode((c >> 6) | 192); + utftext += String.fromCharCode((c & 63) | 128); + } else { + utftext += String.fromCharCode((c >> 12) | 224); + utftext += String.fromCharCode(((c >> 6) & 63) | 128); + utftext += String.fromCharCode((c & 63) | 128); + } + } + + return utftext; +} + +module.exports = function (store) { + function getset (name, value) { + var node = vars.store; + var keys = name.split('.'); + keys.slice(0,-1).forEach(function (k) { + if (node[k] === undefined) node[k] = {}; + node = node[k] + }); + var key = keys[keys.length - 1]; + if (arguments.length == 1) { + return node[key]; + } + else { + return node[key] = value; + } + } + + var vars = { + get : function (name) { + return getset(name); + }, + set : function (name, value) { + return getset(name, value); + }, + store : store || {}, + }; + return vars; +}; \ No newline at end of file diff --git a/fixtures/lib2/file_2.js b/fixtures/lib2/file_2.js new file mode 100644 index 00000000..c64e76cf --- /dev/null +++ b/fixtures/lib2/file_2.js @@ -0,0 +1,56 @@ +module.exports = function (store) { + function getset (name, value) { + var node = vars.store; + var keys = name.split('.'); + keys.slice(0,-1).forEach(function (k) { + if (node[k] === undefined) node[k] = {}; + node = node[k] + }); + var key = keys[keys.length - 1]; + if (arguments.length == 1) { + return node[key]; + } + else { + return node[key] = value; + } + } + + var vars = { + get : function (name) { + return getset(name); + }, + set : function (name, value) { + return getset(name, value); + }, + store : store || {}, + }; + return vars; +}; +module.exports = function (store) { + function getset (name, value) { + var node = vars.store; + var keys = name.split('.'); + keys.slice(0,-1).forEach(function (k) { + if (node[k] === undefined) node[k] = {}; + node = node[k] + }); + var key = keys[keys.length - 1]; + if (arguments.length == 1) { + return node[key]; + } + else { + return node[key] = value; + } + } + + var vars = { + get : function (name) { + return getset(name); + }, + set : function (name, value) { + return getset(name, value); + }, + store : store || {}, + }; + return vars; +}; \ No newline at end of file diff --git a/packages/core/src/interfaces/options.interface.ts b/packages/core/src/interfaces/options.interface.ts index 91cf9ed2..bd9a55f4 100644 --- a/packages/core/src/interfaces/options.interface.ts +++ b/packages/core/src/interfaces/options.interface.ts @@ -27,6 +27,7 @@ export interface IOptions { absolute?: boolean; noSymlinks?: boolean; skipLocal?: boolean; + skipIsolated?: string[][]; ignoreCase?: boolean; gitignore?: boolean; // eslint-disable-next-line @typescript-eslint/no-explicit-any diff --git a/packages/finder/src/in-files-detector.ts b/packages/finder/src/in-files-detector.ts index 7277c005..02b50694 100644 --- a/packages/finder/src/in-files-detector.ts +++ b/packages/finder/src/in-files-detector.ts @@ -13,7 +13,7 @@ import { } from '@jscpd/core'; import {getFormatByFile} from '@jscpd/tokenizer'; import {EntryWithContent, IHook, IReporter} from './interfaces'; -import {SkipLocalValidator} from './validators'; +import {SkipLocalValidator, SkipIsolatedValidator} from './validators'; export class InFilesDetector { @@ -55,6 +55,10 @@ export class InFilesDetector { validators.push(new SkipLocalValidator()); } + if (options.skipIsolated) { + validators.push(new SkipIsolatedValidator()); + } + const detector = new Detector(this.tokenizer, store, validators, options); this.subscribes.forEach((listener: ISubscriber) => { diff --git a/packages/finder/src/validators/index.ts b/packages/finder/src/validators/index.ts index 634a197f..2f00458d 100644 --- a/packages/finder/src/validators/index.ts +++ b/packages/finder/src/validators/index.ts @@ -1 +1,2 @@ export * from './skip-local.validator'; +export * from './skip-isolated.validator'; diff --git a/packages/finder/src/validators/skip-isolated.validator.ts b/packages/finder/src/validators/skip-isolated.validator.ts new file mode 100644 index 00000000..60accd8f --- /dev/null +++ b/packages/finder/src/validators/skip-isolated.validator.ts @@ -0,0 +1,43 @@ +import {getOption, IClone, ICloneValidator, IOptions, IValidationResult} from '@jscpd/core'; +import {isAbsolute, relative} from "path"; + +export class SkipIsolatedValidator implements ICloneValidator { + isRelativeMemoMap = new Map(); + + + validate(clone: IClone, options: IOptions): IValidationResult { + const status = !this.shouldSkipClone(clone, options); + return { + status, + clone, + message: [ + `Sources of duplication located in isolated folder (${clone.duplicationA.sourceId}, ${clone.duplicationB.sourceId})` + ] + }; + } + + public shouldSkipClone(clone: IClone, options: IOptions): boolean { + const skipIsolatedPathList: string[][] = getOption('skipIsolated', options); + return skipIsolatedPathList.some( + (dirList) => { + const relA = dirList.find(dir => this.isRelativeMemo(clone.duplicationA.sourceId, dir)); + const relB = dirList.find(dir => this.isRelativeMemo(clone.duplicationB.sourceId, dir)); + return relA && relB && relA !== relB + } + ); + } + + private isRelativeMemo(file: string, dir: string) { + const memoKey = `${file},${dir}`; + if (this.isRelativeMemoMap.has(memoKey)) return this.isRelativeMemoMap.get(memoKey); + const isRel = SkipIsolatedValidator.isRelative(file, dir) + this.isRelativeMemoMap.set(memoKey, isRel); + return isRel; + } + + private static isRelative(file: string, path: string): boolean { + const rel = relative(path, file); + return rel !== '' && !rel.startsWith('..') && !isAbsolute(rel); + } + +} diff --git a/packages/jscpd/README.md b/packages/jscpd/README.md index 01e7e7fe..ac7e7415 100644 --- a/packages/jscpd/README.md +++ b/packages/jscpd/README.md @@ -92,9 +92,9 @@ Minimal block size of code in tokens. The block of code less than `min-tokens` w - Cli options: `--min-tokens`, `-k` - Type: **number** - Default: **50** - + *This option is called ``minTokens`` in the config file.* - + ### Min Lines Minimal block size of code in lines. The block of code less than `min-lines` will be skipped. @@ -243,6 +243,31 @@ will detect clones in separate folders only, clones from same folder will be ski - Type: **boolean** - Default: **false** +### Skip Isolated +Use for skip detect duplications in isolated folder. + +Example: +```bash +jscpd . --skipLocal "packages/businessA|packages/businessB,libs/businessA|libs|businessB" +``` +clones from the isolated folder will be skipped. + +```text +monorepo/ +├─ .monorepo.config.json +├─ node_modules/ +├─ packages/ +│ ├─ businessA/ // for TEAM A only +│ ├─ businessB/ // for TEAM B only +├─ globals/ +├─ infra / +``` + +- Cli options: `--skipIsolated` +- Type: **string[][]** + + + ### Formats Extensions Define the list of formats with file extensions. Available over [150 formats](../../supported_formats.md). diff --git a/packages/jscpd/__tests__/options.test.ts b/packages/jscpd/__tests__/options.test.ts index 890ce658..3c357a73 100644 --- a/packages/jscpd/__tests__/options.test.ts +++ b/packages/jscpd/__tests__/options.test.ts @@ -4,6 +4,7 @@ import {IClone} from '@jscpd/core'; import {jscpd, detectClones} from '../src'; import {bold, yellow} from 'colors/safe'; import sinon = require('sinon'); +import * as fs from "fs"; const pathToFixtures = __dirname + '/../../../fixtures'; @@ -126,6 +127,49 @@ describe('jscpd options', () => { }); }); + describe('skip isolated', () => { + const folder1Path = pathToFixtures + '/folder1'; + const folder2Path = pathToFixtures + '/folder2'; + const lib1Path = pathToFixtures + '/lib1'; + const lib2Path = pathToFixtures + '/lib2'; + + it('should not skip clone if it is located in isolated folder without --skipIsolated option', async () => { + const clones: IClone[] = await jscpd([ + '', '', + folder1Path, + folder2Path, + lib1Path, + lib2Path, + ]); + // lib2 file_2.js lib2 file_2.js + // lib1 file_1.js lib2 file_2.js + // lib1 file2.c lib1 file2.c + // folder2 file_2.js lib2 file_2.js + // folder1 file_1.js lib1 file_1.js + // folder1 file2.c lib1 file2.c + expect(clones.length).to.equal(6); + }); + + it('should skip clone if it is located in isolated folder with --skipIsolated option', async () => { + const clones: IClone[] = await jscpd([ + '', '', + folder1Path, + folder2Path, + lib1Path, + lib2Path, + '--skipIsolated', + // folder1Path is isolated with folder2Path, lib1Path is isolated with lib2Path + `${folder1Path}|${folder2Path},${lib1Path}|${lib2Path}` + ]); + // lib2 file_2.js lib2 file_2.js + // lib1 file2.c lib1 file2.c + // folder2 file_2.js lib2 file_2.js + // folder1 file_1.js lib1 file_1.js + // folder1 file2.c lib1 file2.c + expect(clones.length).to.equal(5); + }); + }); + describe('silent', () => { it('should not print more information about detection process', async () => { await jscpd(['', '', fileWithClones, '--silent']); diff --git a/packages/jscpd/src/init/cli.ts b/packages/jscpd/src/init/cli.ts index b92bc2c5..2a98e188 100644 --- a/packages/jscpd/src/init/cli.ts +++ b/packages/jscpd/src/init/cli.ts @@ -50,6 +50,7 @@ export function initCli(packageJson, argv: string[]): Command { .option('-v, --verbose', 'show full information during detection process') .option('--list', 'show list of total supported formats') .option('--skipLocal', 'skip duplicates in local folders, just detect cross folders duplications') + .option('--skipIsolated [string]', 'skip duplicates cross from isolated folders for monorepo (e.g. packages/a|packages/b,lib/a|lib/b)') .option('--exitCode [number]', 'exit code to use when code duplications are detected') cli.parse(argv); diff --git a/packages/jscpd/src/options.ts b/packages/jscpd/src/options.ts index ef7a2bb9..f5357b08 100644 --- a/packages/jscpd/src/options.ts +++ b/packages/jscpd/src/options.ts @@ -27,6 +27,7 @@ const convertCliToOptions = (cli: Command): Partial => { absolute: cli.absolute, noSymlinks: cli.noSymlinks, skipLocal: cli.skipLocal, + skipIsolated: cli.skipIsolated?.split(',')?.map(s => s.split('|')), ignoreCase: cli.ignoreCase, gitignore: cli.gitignore, exitCode: cli.exitCode, From 102eb7e1a685979d1caf95725435df0ed0e63adb Mon Sep 17 00:00:00 2001 From: PaulHan Date: Mon, 22 Apr 2024 23:23:31 +0800 Subject: [PATCH 2/3] feat: remove typo --- packages/jscpd/__tests__/options.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/jscpd/__tests__/options.test.ts b/packages/jscpd/__tests__/options.test.ts index 3c357a73..cf309d81 100644 --- a/packages/jscpd/__tests__/options.test.ts +++ b/packages/jscpd/__tests__/options.test.ts @@ -4,7 +4,6 @@ import {IClone} from '@jscpd/core'; import {jscpd, detectClones} from '../src'; import {bold, yellow} from 'colors/safe'; import sinon = require('sinon'); -import * as fs from "fs"; const pathToFixtures = __dirname + '/../../../fixtures'; From 3c90dcc7973980463b307134e7667afff5974a49 Mon Sep 17 00:00:00 2001 From: PaulHan Date: Fri, 17 May 2024 13:18:05 +0800 Subject: [PATCH 3/3] feat: skip check optimize --- packages/finder/src/validators/skip-isolated.validator.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/finder/src/validators/skip-isolated.validator.ts b/packages/finder/src/validators/skip-isolated.validator.ts index 60accd8f..2d302723 100644 --- a/packages/finder/src/validators/skip-isolated.validator.ts +++ b/packages/finder/src/validators/skip-isolated.validator.ts @@ -21,8 +21,11 @@ export class SkipIsolatedValidator implements ICloneValidator { return skipIsolatedPathList.some( (dirList) => { const relA = dirList.find(dir => this.isRelativeMemo(clone.duplicationA.sourceId, dir)); + if (!relA) { + return false; + } const relB = dirList.find(dir => this.isRelativeMemo(clone.duplicationB.sourceId, dir)); - return relA && relB && relA !== relB + return relB && relA !== relB; } ); }