Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions src/analyze.ts
Original file line number Diff line number Diff line change
Expand Up @@ -430,14 +430,15 @@ export function analyze(text: string, options: AnalyzeOptions = {}): AnalysisRes
// are symbols, not letters (category So): circled "Ⓐⓓⓜⓘⓝ", parenthesized,
// squared. A run of >=2 such glyphs whose NFKC form is an ASCII word is the
// same disguised-word attack as fullwidth/math styling (which tokenize and are
// handled per-token). Runs of styled DIGITS (①②③ -> 123) fold to non-letters
// and are ignored, keeping legitimate enclosed numbering out.
// handled per-token).
{
let runStart = -1;
let runCount = 0;
const flush = (end: number) => {
if (runStart >= 0 && runCount >= 2) {
const fold = text.slice(runStart, end).normalize('NFKC').normalize('NFC');
// Belt-and-braces: every character in the run already passed these two
// tests individually, so the concatenation cannot fail them.
if (ASCII_PRINTABLE_RE.test(fold) && ASCII_LETTER_RE.test(fold)) {
signals.confusable_word = true;
words.push({
Expand All @@ -458,6 +459,8 @@ export function analyze(text: string, options: AnalyzeOptions = {}): AnalysisRes
const width = cp > 0xffff ? 2 : 1;
const ch = String.fromCodePoint(cp);
const nf = ch.normalize('NFKC');
// Styled DIGITS (①②③ -> 123) fail ASCII_LETTER_RE and so never join a
// run, keeping legitimate enclosed numbering out.
const styledLetter =
!TOKEN_CHAR_RE.test(ch) &&
nf !== ch &&
Expand Down
24 changes: 24 additions & 0 deletions test/analyze.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,16 @@ describe('analyze — spoofed content', () => {
expect(hot!.signals).toContain('confusable_word');
});

it('flags Unicode non-characters as illegal code points', () => {
// Two disjoint ranges reach the same verdict: the BMP block U+FDD0–U+FDEF
// and the U+xxFFFE/U+xxFFFF pair that closes every plane.
for (const text of ['hi ﷐ there', 'hi ￾ there', 'hi \u{1fffe} there']) {
const r = analyze(text);
expect(r.signals.illegal).toBe(true);
expect(r.normalized).toBe('hi there');
}
});

it('reports codepoint-accurate word offsets', () => {
const r = analyze(CUSTOMER_SAMPLE);
for (const w of r.words) {
Expand Down Expand Up @@ -109,6 +119,20 @@ describe('analyze — legitimate content', () => {
expect(r.normalized).toBe(text);
});

it('does not call a styled word confusable unless the whole word folds to ASCII', () => {
// "𝗉𝗋é" has two styled letters that individually fold to ASCII, but the
// token folds to "pré" — still non-ASCII, so it is not a disguised word.
const r = analyze('𝗉𝗋é');
expect(r.signals.confusable_word).toBe(false);
});

it('ignores runs of enclosed digits, which fold to non-letters', () => {
// ①②③ → "123": a styled run, but not a disguised WORD. Legitimate
// enclosed numbering must survive untouched.
const r = analyze('see item ①②③ below');
expect(r.signals.confusable_word).toBe(false);
});

it('treats expected scripts as legitimate for whole words', () => {
// Without expectedScripts this could look suspicious in Latin-dominant
// text; with them, whole Cyrillic words are the sender's normal traffic.
Expand Down