Skip to content

Commit 07c2fbb

Browse files
rob-gordonclaude
andauthored
refactor(parse): replace the strip-comments fork with an in-repo scanner (#62)
* refactor(parse): add stripComments, a scanner that matches the strip-comments fork The test uses the fork as its oracle over 43 edge cases, a seeded fuzz corpus of 20,000 inputs, and generated documents. A second test checks that the line count never changes, which parse relies on. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * refactor(parse): call stripComments and drop the strip-comments packages parse now uses the in-repo scanner. @tone-row/strip-comments moves to devDependencies, pinned to 2.0.6, as the test oracle. strip-comments and @types/strip-comments had no importers and are removed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * refactor(parse): drop no-op conditions from the stripComments escape and quote checks A backslash before a line terminator, and a quote at index 0, give the same output either way. Four edge cases now pin quirks that only the fuzz corpus covered before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: remove the stale package-level bun.lockb Bun resolves the workspace from the root bun.lockb, which CI installs with --frozen-lockfile. This copy hasn't changed since the switch to bun and still listed the removed strip-comments packages. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 94faadb commit 07c2fbb

6 files changed

Lines changed: 182 additions & 8 deletions

File tree

‎bun.lockb‎

-768 Bytes
Binary file not shown.

‎graph-selector/bun.lockb‎

-117 KB
Binary file not shown.

‎graph-selector/package.json‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
"author": "",
3737
"license": "ISC",
3838
"devDependencies": {
39+
"@tone-row/strip-comments": "2.0.6",
3940
"@typescript-eslint/eslint-plugin": "^6.7.3",
4041
"@typescript-eslint/parser": "^6.7.3",
4142
"eslint": "^8.50.0",
@@ -47,13 +48,10 @@
4748
"vitest": "^0.34.5"
4849
},
4950
"dependencies": {
50-
"@types/strip-comments": "^2.0.2",
51-
"@tone-row/strip-comments": "^2.0.6",
5251
"@types/cytoscape": "^3.19.11",
5352
"html-entities": "^2.4.0",
5453
"monaco-editor": "^0.43.0",
55-
"pkgroll": "^2.4.1",
56-
"strip-comments": "^2.0.1"
54+
"pkgroll": "^2.4.1"
5755
},
5856
"browserslist": [
5957
"defaults"

‎graph-selector/src/parse.ts‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@ import { getEdgeBreakIndex, getFeaturesIndex } from "./regexps";
33

44
import { getFeatureData } from "./getFeatureData";
55
import { matchAndRemovePointers } from "./matchAndRemovePointers";
6-
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
7-
// @ts-ignore
8-
import { strip } from "@tone-row/strip-comments";
6+
import { stripComments } from "./stripComments";
97
import { ParseError } from "./ParseError";
108
import { getIndentSize } from "./getIndentSize";
119

@@ -35,7 +33,7 @@ export function parse(text: string): Graph {
3533
text = text.replace(/(https?:)\/\//g, "$1\\/\\/");
3634

3735
// break into lines
38-
let lines = strip(text, { preserveNewlines: true }).split(/\n/g);
36+
let lines = stripComments(text).split(/\n/g);
3937

4038
// unescape backslashes in newlines
4139
lines = lines.map((line: string) => line.replace(/\\n/g, "\n"));
Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
import { describe, expect, test } from "vitest";
2+
// @ts-expect-error the fork's package.json "exports" hides its type declarations
3+
import { strip } from "@tone-row/strip-comments";
4+
import { stripComments } from "./stripComments";
5+
import { realisticDoc, rng } from "../bench/generate";
6+
7+
const fork = (text: string): string => strip(text, { preserveNewlines: true });
8+
const escapeUrls = (text: string) => text.replace(/(https?:)\/\//g, "$1\\/\\/");
9+
10+
const EDGE_CASES = [
11+
"a // line comment",
12+
"// whole line\nb",
13+
"a\n // indented\nb",
14+
"a /* inline */ b",
15+
"a /* multi\nline\ncomment */\nb",
16+
"a /* unclosed\nb\nc",
17+
"a */ b",
18+
"/* a /* nested */ still? */ b",
19+
"/**/ after",
20+
"/***/ after",
21+
"/*! protected */ x",
22+
"//! protected",
23+
"x /*a*/\ny",
24+
"see https:\\/\\/example.com // c",
25+
'[url="http:\\/\\/x.com"] a',
26+
"ftp://host/path",
27+
'"quoted // not comment" x',
28+
"'single // hidden' x",
29+
"`tick // hidden` x",
30+
'say "open // c\nnext // d\nclose" e',
31+
"don't // c\nwon't",
32+
"a'b // c'",
33+
'"" // c',
34+
'""x" // c',
35+
'esc \\"q // c',
36+
'"a\\" // b" // c',
37+
"a \\/\\/ b",
38+
"a \\// b",
39+
"a \\\\// b",
40+
"# hash",
41+
"<!-- html -->",
42+
"-- sql",
43+
"a\r\nb // c\r\nd",
44+
"a // c\u2028b",
45+
"a /* x */\nb /* y */ c",
46+
"label \\n newline // c",
47+
"{ a // c\n}",
48+
"a: b // c",
49+
'"unterminated // c',
50+
'/* "a*/" b */ c',
51+
"/* \\*/ x */ y",
52+
'"""// c"',
53+
'\u00e9"a // b"',
54+
'"a\x01b" // c"',
55+
'"a\\" b // c',
56+
"a // c\u2029b",
57+
"_'x // y'",
58+
];
59+
60+
const TOKENS = [
61+
...["/", "*", "\\", '"', "'", "`", "\n", "\r", "\u2028", "\u2029", "\x01"],
62+
...["a", "_", "\u00e9", "\u{1F600}", " ", "!", ":"],
63+
...["/*", "*/", "//", "/**", "/*!", "*/\n", '\\"', "\\*", '"""', "\\\n"],
64+
];
65+
66+
function fuzzCorpus(count: number, seed: number) {
67+
const r = rng(seed);
68+
return Array.from({ length: count }, () => {
69+
const length = 1 + Math.floor(r() * 40);
70+
return Array.from({ length }, () => TOKENS[Math.floor(r() * TOKENS.length)]).join("");
71+
});
72+
}
73+
74+
const corpus = fuzzCorpus(20000, 7);
75+
76+
function mismatches(inputs: string[]) {
77+
return inputs
78+
.map((input) => ({ input, expected: fork(input), actual: stripComments(input) }))
79+
.filter(({ expected, actual }) => expected !== actual)
80+
.slice(0, 5);
81+
}
82+
83+
describe("stripComments matches @tone-row/strip-comments", () => {
84+
test.each(EDGE_CASES)("%j", (input) => {
85+
expect(stripComments(input)).toBe(fork(input));
86+
});
87+
88+
test("seeded fuzz corpus", () => {
89+
expect(mismatches(corpus)).toEqual([]);
90+
});
91+
92+
test("realistic documents, raw and with URLs escaped as parse does", () => {
93+
const docs = [50, 500, 2000].flatMap((lines) =>
94+
[1, 2, 3, 4, 5].map((seed) => realisticDoc(lines, seed)),
95+
);
96+
expect(mismatches([...docs, ...docs.map(escapeUrls)])).toEqual([]);
97+
});
98+
});
99+
100+
test("stripComments keeps the line count, which parse relies on for line numbers", () => {
101+
const lineCount = (text: string) => text.split("\n").length;
102+
const changed = [...EDGE_CASES, ...corpus].filter(
103+
(input) => lineCount(stripComments(input)) !== lineCount(input),
104+
);
105+
expect(changed.slice(0, 5)).toEqual([]);
106+
});
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
const isWordChar = (c: string) => /\w/.test(c);
2+
const isLineTerminator = (c: string) =>
3+
c === "\n" || c === "\r" || c === "\u2028" || c === "\u2029";
4+
const newlinesIn = (s: string) => s.replace(/[^\n]/g, "");
5+
6+
/**
7+
* Mirrors the fork's lazy /^(['"`])((?:\\\1|[^\1])+?)(\1)/: the body needs at least one char,
8+
* `[^\1]` is the octal class "not \x01", and on failure the match backtracks to the last
9+
* escaped quote.
10+
*/
11+
function findClosingQuote(text: string, open: number): number {
12+
const quote = text.charAt(open);
13+
let lastEscapedQuote = -1;
14+
for (let j = open + 1; ; j++) {
15+
if (j > open + 1 && text.charAt(j) === quote) return j;
16+
if (j >= text.length || text.charAt(j) === "\x01") return lastEscapedQuote;
17+
if (text.charAt(j) === "\\" && text.charAt(j + 1) === quote) {
18+
lastEscapedQuote = j + 1;
19+
j++;
20+
}
21+
}
22+
}
23+
24+
export function stripComments(text: string): string {
25+
let out = "";
26+
let keptFrom = 0;
27+
let depth = 0;
28+
let i = 0;
29+
while (i < text.length) {
30+
const c = text.charAt(i);
31+
const next = text.charAt(i + 1);
32+
if (c === "\\") {
33+
i += 2;
34+
continue;
35+
}
36+
if (depth === 0 && (c === '"' || c === "'" || c === "`") && !isWordChar(text.charAt(i - 1))) {
37+
const close = findClosingQuote(text, i);
38+
if (close !== -1) {
39+
i = close + 1;
40+
continue;
41+
}
42+
}
43+
if (c === "/" && next === "*") {
44+
if (depth === 0) {
45+
out += text.slice(keptFrom, i);
46+
keptFrom = i;
47+
}
48+
depth++;
49+
i += 2;
50+
// The fork's opener is /^\/\*\*?/, so `/**/` never closes and swallows the rest of the document.
51+
if (text.charAt(i) === "*") i++;
52+
continue;
53+
}
54+
if (depth > 0 && c === "*" && next === "/") {
55+
i += 2;
56+
if (--depth === 0) {
57+
out += newlinesIn(text.slice(keptFrom, i));
58+
keptFrom = i;
59+
}
60+
continue;
61+
}
62+
if (depth === 0 && c === "/" && next === "/") {
63+
out += text.slice(keptFrom, i);
64+
while (i < text.length && !isLineTerminator(text.charAt(i))) i++;
65+
keptFrom = i;
66+
continue;
67+
}
68+
i++;
69+
}
70+
const rest = text.slice(keptFrom);
71+
return out + (depth > 0 ? newlinesIn(rest) : rest);
72+
}

0 commit comments

Comments
 (0)