diff --git a/src/analyze.ts b/src/analyze.ts index d07d378..fb91957 100644 --- a/src/analyze.ts +++ b/src/analyze.ts @@ -430,14 +430,15 @@ export function analyze(text: string, options: AnalyzeOptions = {}): AnalysisRes // are symbols, not letters (category So): circled "Ⓐⓓⓜⓘⓝ", parenthesized, // squared. A run of >=2 such glyphs whose NFKC form is an ASCII word is the // same disguised-word attack as fullwidth/math styling (which tokenize and are - // handled per-token). Runs of styled DIGITS (①②③ -> 123) fold to non-letters - // and are ignored, keeping legitimate enclosed numbering out. + // handled per-token). { let runStart = -1; let runCount = 0; const flush = (end: number) => { if (runStart >= 0 && runCount >= 2) { const fold = text.slice(runStart, end).normalize('NFKC').normalize('NFC'); + // Belt-and-braces: every character in the run already passed these two + // tests individually, so the concatenation cannot fail them. if (ASCII_PRINTABLE_RE.test(fold) && ASCII_LETTER_RE.test(fold)) { signals.confusable_word = true; words.push({ @@ -458,6 +459,8 @@ export function analyze(text: string, options: AnalyzeOptions = {}): AnalysisRes const width = cp > 0xffff ? 2 : 1; const ch = String.fromCodePoint(cp); const nf = ch.normalize('NFKC'); + // Styled DIGITS (①②③ -> 123) fail ASCII_LETTER_RE and so never join a + // run, keeping legitimate enclosed numbering out. const styledLetter = !TOKEN_CHAR_RE.test(ch) && nf !== ch && diff --git a/test/analyze.test.ts b/test/analyze.test.ts index 032b8be..795cbea 100644 --- a/test/analyze.test.ts +++ b/test/analyze.test.ts @@ -77,6 +77,16 @@ describe('analyze — spoofed content', () => { expect(hot!.signals).toContain('confusable_word'); }); + it('flags Unicode non-characters as illegal code points', () => { + // Two disjoint ranges reach the same verdict: the BMP block U+FDD0–U+FDEF + // and the U+xxFFFE/U+xxFFFF pair that closes every plane. + for (const text of ['hi ﷐ there', 'hi ￾ there', 'hi \u{1fffe} there']) { + const r = analyze(text); + expect(r.signals.illegal).toBe(true); + expect(r.normalized).toBe('hi there'); + } + }); + it('reports codepoint-accurate word offsets', () => { const r = analyze(CUSTOMER_SAMPLE); for (const w of r.words) { @@ -109,6 +119,20 @@ describe('analyze — legitimate content', () => { expect(r.normalized).toBe(text); }); + it('does not call a styled word confusable unless the whole word folds to ASCII', () => { + // "𝗉𝗋é" has two styled letters that individually fold to ASCII, but the + // token folds to "pré" — still non-ASCII, so it is not a disguised word. + const r = analyze('𝗉𝗋é'); + expect(r.signals.confusable_word).toBe(false); + }); + + it('ignores runs of enclosed digits, which fold to non-letters', () => { + // ①②③ → "123": a styled run, but not a disguised WORD. Legitimate + // enclosed numbering must survive untouched. + const r = analyze('see item ①②③ below'); + expect(r.signals.confusable_word).toBe(false); + }); + it('treats expected scripts as legitimate for whole words', () => { // Without expectedScripts this could look suspicious in Latin-dominant // text; with them, whole Cyrillic words are the sender's normal traffic.