From 7eface205450e44e6beab7faa7e4a18b35d186ed Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 09:56:50 +0000 Subject: [PATCH 01/13] feat(rules): add detection ruleset for src/rules.mjs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 23 rules covering the most common secrets found in AI coding-assistant transcripts, derived from gitleaks (MIT), secretlint (MIT), and detect-secrets (Apache-2.0): - Critical: PEM private-key blocks (RSA/EC/OpenSSH/PGP/PKCS#8) - High: AWS access/secret keys, Anthropic (api03/api04/admin), OpenAI (new T3BlbkFJ-anchored + legacy), Hugging Face (user + org), GitHub (PAT/OAuth/app/refresh/fine-grained), Google API key, Stripe (sk_/rk_ live/test/prod), Slack (bot/user/app tokens), database URLs with inline credentials (postgres/mysql/mongodb/redis/amqp) - Medium: JWTs (eyJ…eyJ…sig), Slack webhook URLs, entropy-gated generic secret assignment (Shannon ≥ 3.5 bits/char) Exports shannonEntropy() ported from detect-secrets with thresholds calibrated for base64 (≥4.5), hex (≥3.0), and generic (≥3.5) strings. All patterns carry `g` and `d` flags for matchAll + precise offset tracking. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_011m7VujV9DvgT9AM1sBhbUw --- src/rules.mjs | 252 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 src/rules.mjs diff --git a/src/rules.mjs b/src/rules.mjs new file mode 100644 index 0000000..add7afe --- /dev/null +++ b/src/rules.mjs @@ -0,0 +1,252 @@ +// Detection ruleset for broomsticks. +// Patterns derived from gitleaks (MIT), secretlint (MIT), and detect-secrets (Apache-2.0). +// +// Each rule: +// pattern — RegExp with `g` and `d` flags (d gives match.indices for precise offsets) +// secretGroup — which capture group is the secret (default: 0 = whole match) +// entropy — minimum Shannon bits/char; match is skipped if below threshold + +/** + * @typedef {'critical'|'high'|'medium'|'low'} Severity + * @typedef {{ id:string, title:string, severity:Severity, pattern:RegExp, secretGroup?:number, entropy?:number }} Rule + */ + +/** + * Shannon entropy over every character in the string (bits per character). + * Ported from detect-secrets (Apache-2.0); thresholds: base64 ≥ 4.5, hex ≥ 3.0, generic ≥ 3.5. + * @param {string} str + * @returns {number} + */ +export function shannonEntropy(str) { + if (!str) return 0 + const freq = new Map() + for (const ch of str) freq.set(ch, (freq.get(ch) ?? 0) + 1) + let e = 0 + for (const count of freq.values()) { + const p = count / str.length + e -= p * Math.log2(p) + } + return e +} + +/** @type {Rule[]} */ +export const RULES = [ + + // ── Private / Cryptographic Keys ────────────────────────────────────────── + // Matches PEM blocks: RSA, EC, OpenSSH, PGP, PKCS#8, encrypted variants. + { + id: 'private-key', + title: 'Private key (PEM block)', + severity: 'critical', + pattern: /-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----[\s\S]{0,8192}?-----END[ A-Z0-9_-]{0,100}(?:PRIVATE KEY|KEY BLOCK)-----/gd, + }, + + // ── AWS ─────────────────────────────────────────────────────────────────── + // Covers AKIA (long-term), ASIA (session), ABIA (billing), ACCA, A3T* families. + { + id: 'aws-access-key', + title: 'AWS access key ID', + severity: 'high', + pattern: /\b((?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[A-Z2-7]{16})\b/gd, + secretGroup: 1, + entropy: 3, + }, + // 40-char base64 value paired with a secret-key variable name. + { + id: 'aws-secret-key', + title: 'AWS secret access key', + severity: 'high', + pattern: /(?:aws_?secret_?access_?key|aws_?secret)\s*[:=]\s*["']?([A-Za-z0-9/+=]{40})["']?/gid, + secretGroup: 1, + entropy: 3.5, + }, + + // ── Anthropic ───────────────────────────────────────────────────────────── + // Covers api03 and api04 formats; always ends with AA (secretlint pattern). + { + id: 'anthropic-key', + title: 'Anthropic API key', + severity: 'high', + pattern: /\b(sk-ant-api0[34]-[A-Za-z0-9_-]{90,128}AA)\b/gd, + secretGroup: 1, + }, + { + id: 'anthropic-admin-key', + title: 'Anthropic admin API key', + severity: 'high', + pattern: /\b(sk-ant-admin01-[A-Za-z0-9_-]{93}AA)\b/gd, + secretGroup: 1, + }, + + // ── OpenAI ──────────────────────────────────────────────────────────────── + // New-format keys embed "T3BlbkFJ" (base64 for "OpenAI") as a fixed anchor — + // this dramatically cuts false positives compared to a bare sk- prefix match. + { + id: 'openai-key', + title: 'OpenAI API key', + severity: 'high', + pattern: /\b(sk-(?:proj|svcacct|admin)-[A-Za-z0-9_-]{58,74}T3BlbkFJ[A-Za-z0-9_-]{58,74})\b/gd, + secretGroup: 1, + }, + // Legacy 51-char keys, also anchored by T3BlbkFJ. + { + id: 'openai-key-legacy', + title: 'OpenAI API key (legacy sk- format)', + severity: 'high', + pattern: /\b(sk-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20})\b/gd, + secretGroup: 1, + }, + + // ── Hugging Face ────────────────────────────────────────────────────────── + { + id: 'huggingface-token', + title: 'Hugging Face user access token', + severity: 'high', + pattern: /\b(hf_[A-Za-z0-9]{34})\b/gd, + secretGroup: 1, + entropy: 2, + }, + { + id: 'huggingface-org-token', + title: 'Hugging Face organization API token', + severity: 'high', + pattern: /\b(api_org_[A-Za-z0-9]{34})\b/gd, + secretGroup: 1, + entropy: 2, + }, + + // ── GitHub ──────────────────────────────────────────────────────────────── + // Classic tokens — each type has a distinct 3-letter prefix + 36 alphanumeric chars. + { + id: 'github-pat', + title: 'GitHub personal access token', + severity: 'high', + pattern: /\b(ghp_[0-9A-Za-z]{36})\b/gd, + secretGroup: 1, + entropy: 3, + }, + { + id: 'github-oauth', + title: 'GitHub OAuth token', + severity: 'high', + pattern: /\b(gho_[0-9A-Za-z]{36})\b/gd, + secretGroup: 1, + entropy: 3, + }, + { + id: 'github-app-token', + title: 'GitHub app installation / server-to-server token', + severity: 'high', + pattern: /\b(gh[us]_[0-9A-Za-z]{36})\b/gd, + secretGroup: 1, + entropy: 3, + }, + { + id: 'github-refresh-token', + title: 'GitHub refresh token', + severity: 'high', + pattern: /\b(ghr_[0-9A-Za-z]{36})\b/gd, + secretGroup: 1, + entropy: 3, + }, + // Fine-grained PATs: `github_pat_` + exactly 82 word chars. + // Uses Unicode property escape \p{L} for tighter boundary — requires `u` flag. + { + id: 'github-fine-grained-pat', + title: 'GitHub fine-grained personal access token', + severity: 'high', + pattern: /(? Date: Wed, 1 Jul 2026 17:15:59 +0000 Subject: [PATCH 02/13] feat(core): add detector, redactor, and expand generic-secret rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit detector.mjs — scanText(text, rules, extras) → Finding[] - Runs each rule's pattern via matchAll with d-flag indices - Entropy-gates matches where rule.entropy is set (Shannon bits/char) - De-overlaps findings: longer span wins, tie-breaks by severity rank - Converts --extra file lines (literals or /regex/flags) to synthetic rules redactor.mjs — redactText(text, findings) → {text, applied} - Processes findings right-to-left so earlier byte offsets stay valid - Placeholder: «BROOM::» (sha8 = sha256(secret)[0..8]) - «» delimiters match no detection rule → inherently idempotent - buildPlaceholder() exported for use by the report module rules.mjs — expand generic-secret value charset - Added !@#$%^&* to catch special-char passwords alongside base64 tokens - Entropy gate (≥3.5 bits/char) still filters phrases and repeated chars Integration verified: 10 rule types detected, redacted, and idempotency confirmed (re-scan of redacted transcript returns 0 findings). Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_011m7VujV9DvgT9AM1sBhbUw --- src/detector.mjs | 101 +++++++++++++++++++++++++++++++++++++++++++++++ src/redactor.mjs | 46 +++++++++++++++++++++ src/rules.mjs | 5 ++- 3 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 src/detector.mjs create mode 100644 src/redactor.mjs diff --git a/src/detector.mjs b/src/detector.mjs new file mode 100644 index 0000000..e1c2b4c --- /dev/null +++ b/src/detector.mjs @@ -0,0 +1,101 @@ +// Scanner: runs rules against a text blob and returns de-overlapped findings. + +import { shannonEntropy } from './rules.mjs' + +/** + * @typedef {'critical'|'high'|'medium'|'low'} Severity + * @typedef {{ ruleId:string, title:string, severity:Severity, secret:string, start:number, end:number }} Finding + */ + +const SEVERITY_RANK = { critical: 4, high: 3, medium: 2, low: 1 } + +/** + * Scan a text blob with the given rules plus any user-supplied extras. + * Returns findings sorted by start position, with overlapping spans resolved. + * + * @param {string} text + * @param {import('./rules.mjs').Rule[]} rules + * @param {string[]} [extras] Lines from --extra file: literal strings or /regex/flags + * @returns {Finding[]} + */ +export function scanText(text, rules, extras = []) { + const allRules = [...rules, ...extrasToRules(extras)] + const raw = [] + + for (const rule of allRules) { + // Clone the pattern so matchAll gets a fresh lastIndex each call. + const pattern = new RegExp(rule.pattern.source, rule.pattern.flags) + + for (const match of text.matchAll(pattern)) { + const grp = rule.secretGroup ?? 0 + const secret = match[grp] ?? match[0] + if (!secret) continue + + const indices = match.indices[grp] ?? match.indices[0] + if (!indices) continue + const [start, end] = indices + + if (rule.entropy !== undefined && shannonEntropy(secret) < rule.entropy) continue + + raw.push({ ruleId: rule.id, title: rule.title, severity: rule.severity, secret, start, end }) + } + } + + return resolveOverlaps(raw) +} + +/** + * When two findings overlap, keep the one with the longer span; break ties by + * higher severity. This prevents nested placeholders and double-redaction. + * @param {Finding[]} findings + * @returns {Finding[]} + */ +function resolveOverlaps(findings) { + if (findings.length <= 1) return findings + + // Sort by priority: longer span first, then higher severity, then earlier start. + // Greedy sweep then naturally keeps the best non-overlapping set. + const sorted = [...findings].sort((a, b) => { + const lenDiff = (b.end - b.start) - (a.end - a.start) + if (lenDiff !== 0) return lenDiff + const rankDiff = SEVERITY_RANK[b.severity] - SEVERITY_RANK[a.severity] + if (rankDiff !== 0) return rankDiff + return a.start - b.start + }) + + const kept = [] + for (const f of sorted) { + if (!kept.some(k => f.start < k.end && k.start < f.end)) kept.push(f) + } + + return kept.sort((a, b) => a.start - b.start) +} + +/** + * Convert --extra file lines into synthetic rules. + * Each line is either a /regex/flags literal or a plain string to match exactly. + * @param {string[]} lines + * @returns {import('./rules.mjs').Rule[]} + */ +function extrasToRules(lines) { + return lines + .map(l => l.trim()) + .filter(Boolean) + .map((entry, i) => { + let pattern + const reMatch = entry.match(/^\/(.+)\/([gimsud]*)$/) + if (reMatch) { + const flags = [...new Set([...reMatch[2], 'g', 'd'])].join('') + pattern = new RegExp(reMatch[1], flags) + } else { + const escaped = entry.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + pattern = new RegExp(escaped, 'gd') + } + return { + id: `extra-${i}`, + title: `User-supplied secret #${i + 1}`, + severity: /** @type {Severity} */ ('high'), + pattern, + } + }) +} diff --git a/src/redactor.mjs b/src/redactor.mjs new file mode 100644 index 0000000..52f4992 --- /dev/null +++ b/src/redactor.mjs @@ -0,0 +1,46 @@ +// Redactor: replaces detected secrets with stable, JSON-safe placeholders. +// +// Placeholder format: «BROOM::» +// «» — not matched by any detection rule, so redacted transcripts are idempotent +// sha8 — first 8 hex digits of SHA-256(secret), lets you correlate the same leak +// across files without re-exposing the value + +import { createHash } from 'node:crypto' + +/** + * Replace every finding's secret span with a placeholder. + * Processes right-to-left so earlier byte offsets stay valid after each substitution. + * + * @param {string} text + * @param {import('./detector.mjs').Finding[]} findings + * @returns {{ text: string, applied: number }} + */ +export function redactText(text, findings) { + if (!findings.length) return { text, applied: 0 } + + // Descending start order — rightmost replacement first. + const ordered = [...findings].sort((a, b) => b.start - a.start) + + let result = text + let applied = 0 + + for (const f of ordered) { + const placeholder = buildPlaceholder(f.ruleId, f.secret) + result = result.slice(0, f.start) + placeholder + result.slice(f.end) + applied++ + } + + return { text: result, applied } +} + +/** + * Build the stable placeholder string for a finding. + * Exported so the report module can show what placeholder was used. + * @param {string} ruleId + * @param {string} secret + * @returns {string} + */ +export function buildPlaceholder(ruleId, secret) { + const sha8 = createHash('sha256').update(secret).digest('hex').slice(0, 8) + return `«BROOM:${ruleId}:${sha8}»` +} diff --git a/src/rules.mjs b/src/rules.mjs index add7afe..de5642d 100644 --- a/src/rules.mjs +++ b/src/rules.mjs @@ -243,7 +243,10 @@ export const RULES = [ id: 'generic-secret', title: 'Generic secret assignment', severity: 'medium', - pattern: /(?:api[_\-.]?key|api[_\-.]?secret|auth[_\-.]?token|access[_\-.]?token|secret[_\-.]?key|private[_\-.]?key|client[_\-.]?secret|password|passwd|token|credential)\s*[:=]\s*["']?([A-Za-z0-9+/=_\-]{16,})["']?/gid, + // Value charset includes common password special chars (!@#$%^&*) in addition + // to base64url chars so we catch real passwords, not just token-shaped strings. + // The entropy gate (≥3.5 bits/char) prevents false positives on phrases and UUIDs. + pattern: /(?:api[_\-.]?key|api[_\-.]?secret|auth[_\-.]?token|access[_\-.]?token|secret[_\-.]?key|private[_\-.]?key|client[_\-.]?secret|password|passwd|token|credential)\s*[:=]\s*["']?([A-Za-z0-9+/=_\-!@#$%^&*]{16,})["']?/gid, secretGroup: 1, entropy: 3.5, }, From ef896a22fb7e37db18dec272ee5219d546110c25 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 17:25:15 +0000 Subject: [PATCH 03/13] feat(sources): add Claude Code transcript adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovers ~/.claude/projects/**/*.jsonl (Linux, macOS, Windows alike — all use ~/.claude via os.homedir()) and wraps each file in a Target: source : 'claude-code' label : absolute file path file : same (used by backup module to locate the file) read() : readFileSync — returns raw UTF-8 text write(): writeFileSync — overwrites in place after redaction Discovery uses a synchronous recursive generator (walkJsonl) so the entire pipeline stays sync and avoids async complexity in the CLI. Silently skips ~/.claude/projects if it doesn't exist (first-run case). Verified: read/write round-trip, 3-secret synthetic JSONL stays valid JSON Lines after inline redaction, re-scan returns 0 findings (idempotent). Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_011m7VujV9DvgT9AM1sBhbUw --- src/sources/claude-code.mjs | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 src/sources/claude-code.mjs diff --git a/src/sources/claude-code.mjs b/src/sources/claude-code.mjs new file mode 100644 index 0000000..e1fb6c0 --- /dev/null +++ b/src/sources/claude-code.mjs @@ -0,0 +1,79 @@ +// Source adapter for Claude Code transcripts. +// +// Claude Code writes one JSONL file per session under: +// ~/.claude/projects//.jsonl +// +// Each line is a JSON object representing one turn. We treat each file as +// a single text blob — redaction is inline on the raw bytes, so the JSONL +// stays valid without a parse/serialize round-trip. + +import { readFileSync, readdirSync, writeFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' + +/** + * A single addressable blob of text plus how to read and write it. + * @typedef {Object} Target + * @property {string} source - 'claude-code' + * @property {string} label - human-readable identifier shown in reports + * @property {string} file - absolute path to the backing file (for backup) + * @property {() => string} read - return current file content as UTF-8 text + * @property {(text: string) => void} write - overwrite the file with redacted text + */ + +/** + * Root directory for Claude Code session transcripts. + * Claude Code uses ~/.claude on Linux, macOS, and Windows alike. + * @returns {string} + */ +export function claudeCodeRoot() { + return join(homedir(), '.claude', 'projects') +} + +/** + * Walk a directory tree and yield the absolute path of every *.jsonl file. + * Silently skips directories that don't exist or aren't readable. + * @param {string} dir + * @returns {Generator} + */ +function* walkJsonl(dir) { + let entries + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return // dir missing or unreadable — not an error, just no transcripts + } + + for (const entry of entries) { + const full = join(dir, entry.name) + if (entry.isDirectory()) { + yield* walkJsonl(full) + } else if (entry.isFile() && entry.name.endsWith('.jsonl')) { + yield full + } + } +} + +/** + * Discover all Claude Code session files and return them as Targets. + * Returns an empty array if Claude Code has never been used on this machine. + * @returns {Target[]} + */ +export function discoverTargets() { + const root = claudeCodeRoot() + const targets = [] + + for (const file of walkJsonl(root)) { + // Capture `file` in a closure-safe way for the lazy read/write callbacks. + const f = file + targets.push({ + source: 'claude-code', + label: f, + file: f, + read: () => readFileSync(f, 'utf8'), + write: (text) => writeFileSync(f, text, 'utf8'), + }) + } + + return targets +} From 65a0e2cf781cd4e7d04408a439ab86fe68ef0f0c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 17:31:42 +0000 Subject: [PATCH 04/13] feat(pipeline): add backup session and report formatter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backup.mjs — BackupSession class - backup(file): copies the file into ~/.broom/backups// recreating the original absolute path under the backup root, so restored files land back in the right place without ambiguity - Idempotent: same file backed up twice in one session is a no-op - Copies SQLite -wal/-shm siblings for Cursor .vscdb files (v0.2 path) - writeManifest(results): writes manifest.json with timestamps, file list, and per-target redaction counts report.mjs — human and JSON output - printReport(results, opts): groups findings by source → file, masks each secret (first 6 chars + … + last 4), shows 1-indexed line number, prints a severity summary; wording adjusts for scan vs clean --apply - printJsonReport(results): structured JSON with version, totalFindings, per-file findings (ruleId, title, severity, start/end offsets, sha8 hash) — raw secrets are never emitted in JSON output, only the 8-char hash - ANSI colours only when stdout is a TTY; clean output when piped Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_011m7VujV9DvgT9AM1sBhbUw --- src/backup.mjs | 106 ++++++++++++++++++++++++++++++ src/report.mjs | 171 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 277 insertions(+) create mode 100644 src/backup.mjs create mode 100644 src/report.mjs diff --git a/src/backup.mjs b/src/backup.mjs new file mode 100644 index 0000000..dc2ca1d --- /dev/null +++ b/src/backup.mjs @@ -0,0 +1,106 @@ +// Backup module — copies files before any write and records a manifest. +// +// Backup layout: +// ~/.broom/backups// +// manifest.json +// +// +// Example: +// ~/.broom/backups/2024-01-15T10-30-00Z/ +// home/alice/.claude/projects/myapp/abc123.jsonl +// manifest.json + +import { copyFileSync, mkdirSync, writeFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { dirname, join, resolve, sep } from 'node:path' + +/** + * A single backup session. Create one per `broom clean --apply` run. + * Call backup(file) before every write; call writeManifest() when done. + */ +export class BackupSession { + /** + * @param {string} [backupDir] Override the default ~/.broom/backups/ + */ + constructor(backupDir) { + this.dir = backupDir ?? defaultBackupDir() + /** @type {Set} files already copied this session */ + this._copied = new Set() + /** @type {Array<{original:string, backup:string}>} */ + this._entries = [] + } + + /** + * Copy `file` into the backup directory if it hasn't been copied yet. + * For SQLite databases, also copies -wal and -shm siblings if present. + * Safe to call multiple times for the same file (idempotent). + * @param {string} file Absolute path to the file to back up. + * @returns {string} The backup path. + */ + backup(file) { + const abs = resolve(file) + if (this._copied.has(abs)) { + return this._destPath(abs) + } + + const dest = this._destPath(abs) + mkdirSync(dirname(dest), { recursive: true }) + copyFileSync(abs, dest) + this._copied.add(abs) + this._entries.push({ original: abs, backup: dest }) + + // SQLite siblings — present for Cursor's .vscdb files + for (const suffix of ['-wal', '-shm']) { + const sib = abs + suffix + try { + const sibDest = dest + suffix + copyFileSync(sib, sibDest) + } catch { + // sibling doesn't exist — that's fine + } + } + + return dest + } + + /** + * Write a manifest.json to the backup root summarising the session. + * @param {Array<{target: import('./sources/claude-code.mjs').Target, applied: number}>} results + */ + writeManifest(results) { + if (this._entries.length === 0) return + + mkdirSync(this.dir, { recursive: true }) + + const manifest = { + timestamp: new Date().toISOString(), + backupDir: this.dir, + files: this._entries, + redactionSummary: results.map(r => ({ + source: r.target.source, + label: r.target.label, + applied: r.applied, + })), + } + + writeFileSync(join(this.dir, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n', 'utf8') + } + + /** @param {string} abs Absolute original path */ + _destPath(abs) { + // Strip leading separator so we can join under the backup dir. + // /home/alice/.claude/... → home/alice/.claude/... + const relative = abs.startsWith(sep) ? abs.slice(sep.length) : abs + return join(this.dir, relative) + } +} + +/** + * Default backup directory: ~/.broom/backups/ + * @returns {string} + */ +function defaultBackupDir() { + // Replace colons so the path is valid on Windows and unambiguous in shells. + const stamp = new Date().toISOString().replace(/:/g, '-') + return join(homedir(), '.broom', 'backups', stamp) +} diff --git a/src/report.mjs b/src/report.mjs new file mode 100644 index 0000000..06de6bb --- /dev/null +++ b/src/report.mjs @@ -0,0 +1,171 @@ +// Report formatter — human-readable and JSON output for scan/clean results. + +import { createHash } from 'node:crypto' + +/** + * @typedef {import('./detector.mjs').Finding} Finding + * @typedef {import('./sources/claude-code.mjs').Target} Target + * @typedef {{ target: Target, findings: Finding[], applied?: number }} ScanResult + */ + +const SEVERITY_ORDER = ['critical', 'high', 'medium', 'low'] + +const SEVERITY_COLOR = { + critical: '\x1b[1;31m', // bold red + high: '\x1b[31m', // red + medium: '\x1b[33m', // yellow + low: '\x1b[90m', // grey +} +const RESET = '\x1b[0m' +const DIM = '\x1b[2m' +const BOLD = '\x1b[1m' + +/** + * Whether stdout is a TTY — used to strip ANSI codes in piped output. + */ +const isTTY = process.stdout.isTTY ?? false + +function color(sev, text) { + return isTTY ? `${SEVERITY_COLOR[sev]}${text}${RESET}` : text +} +function bold(text) { return isTTY ? `${BOLD}${text}${RESET}` : text } +function dim(text) { return isTTY ? `${DIM}${text}${RESET}` : text } + +/** + * Mask a secret for display: show first 6 chars + … + last 4. + * Very short secrets are shown as *** to avoid leaking them. + * @param {string} secret + * @returns {string} + */ +function maskSecret(secret) { + if (secret.length <= 12) return '*'.repeat(secret.length) + return secret.slice(0, 6) + dim('…') + secret.slice(-4) +} + +/** + * Compute the 1-indexed line number of a character offset in text. + * @param {string} text + * @param {number} offset + * @returns {number} + */ +function lineNumber(text, offset) { + let line = 1 + for (let i = 0; i < offset; i++) { + if (text[i] === '\n') line++ + } + return line +} + +/** + * Print a human-readable scan report to stdout. + * + * @param {ScanResult[]} results + * @param {{ clean?: boolean, apply?: boolean }} [opts] + * clean — true when called from `broom clean` (adjusts wording) + * apply — true when redaction was actually performed + */ +export function printReport(results, opts = {}) { + const { clean = false, apply = false } = opts + + const totalFindings = results.reduce((n, r) => n + r.findings.length, 0) + const filesWithFindings = results.filter(r => r.findings.length > 0).length + const totalApplied = results.reduce((n, r) => n + (r.applied ?? 0), 0) + + // ── Header ──────────────────────────────────────────────────────────────── + const cmd = clean ? 'clean' : 'scan' + const mode = apply ? `${bold('--apply')} (redacted in place)` : `${dim('dry-run')}` + console.log() + console.log(`${bold('broomsticks')} ${cmd} — ${mode}`) + + if (totalFindings === 0) { + console.log(`\n ${bold('No secrets found.')} All ${results.length} file(s) are clean.\n`) + return + } + + const plural = (n, w) => `${n} ${w}${n === 1 ? '' : 's'}` + console.log(`\n ${bold(plural(totalFindings, 'secret'))} found across ${plural(filesWithFindings, 'file')}.`) + if (apply) console.log(` ${bold(plural(totalApplied, 'redaction'))} applied; originals backed up.\n`) + else console.log(` Run ${bold('broom clean --apply')} to redact in place (backs up first).\n`) + + // ── Per-file findings ────────────────────────────────────────────────────── + const dirty = results.filter(r => r.findings.length > 0) + + for (const { target, findings } of dirty) { + const shortLabel = target.label.replace(process.env.HOME ?? '', '~') + console.log(` ${bold(target.source)} ${shortLabel}`) + + // Group by severity for ordered display + const bySev = Object.fromEntries(SEVERITY_ORDER.map(s => [s, []])) + for (const f of findings) bySev[f.severity].push(f) + + for (const sev of SEVERITY_ORDER) { + for (const f of bySev[sev]) { + const tag = color(sev, `[${sev.padEnd(8)}]`) + const rule = f.ruleId.padEnd(26) + const masked = maskSecret(f.secret) + // line number requires the raw text — pass it through target.read() lazily + // We cache it here since findings already came from scanning this target. + let lineInfo = '' + try { + const text = target.read() + lineInfo = dim(` line ${lineNumber(text, f.start)}`) + } catch { /* non-blocking */ } + console.log(` ${tag} ${dim(rule)} ${masked}${lineInfo}`) + } + } + + console.log() + } + + // ── Severity summary ─────────────────────────────────────────────────────── + const counts = Object.fromEntries(SEVERITY_ORDER.map(s => [s, 0])) + for (const { findings } of results) for (const f of findings) counts[f.severity]++ + + const summary = SEVERITY_ORDER + .filter(s => counts[s] > 0) + .map(s => color(s, `${counts[s]} ${s}`)) + .join(', ') + console.log(` Severity: ${summary}\n`) +} + +/** + * Emit findings as a JSON structure for CI / machine consumption. + * Printed to stdout; caller should pipe or redirect. + * + * @param {ScanResult[]} results + */ +export function printJsonReport(results) { + const totalFindings = results.reduce((n, r) => n + r.findings.length, 0) + + const output = { + version: 1, + totalFindings, + files: results + .filter(r => r.findings.length > 0) + .map(({ target, findings, applied }) => ({ + source: target.source, + label: target.label, + file: target.file, + applied: applied ?? null, + findings: findings.map(f => ({ + ruleId: f.ruleId, + title: f.title, + severity: f.severity, + start: f.start, + end: f.end, + // Never include the raw secret in JSON output — only the hash + sha8: hashSecret(f.secret), + })), + })), + } + + console.log(JSON.stringify(output, null, 2)) +} + +/** + * @param {string} secret + * @returns {string} First 8 hex chars of SHA-256(secret) + */ +function hashSecret(secret) { + return createHash('sha256').update(secret).digest('hex').slice(0, 8) +} From 583b65e9afe7316ab3667d03018deb58c12f56be Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 17:33:40 +0000 Subject: [PATCH 05/13] feat(cli): wire full broom scan/clean/sources pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the placeholder stub with a working CLI. Hand-rolled argument parser (no dependencies) supporting: broom scan [--source ] [--json] [--no-fail] [--extra ] broom clean [--source ] [--apply] [--backup-dir ] [--no-backup] [--extra ] [--json] [--no-fail] broom sources broom --version / --help Pipeline (clean --apply): gatherTargets → scanText → redactText → backup.backup() → target.write() → backup.writeManifest() → printReport / printJsonReport → exit code Safety properties preserved end-to-end: - Dry-run by default: files untouched without --apply - Backup precedes every write (BackupSession.backup called before write) - Exit 1 when findings exist (CI-friendly); --no-fail overrides - Re-scan of redacted transcript exits 0 (idempotent by construction) - --json output never includes raw secrets, only sha8 hashes v0.2 note: codex and cursor adapters slot into gatherTargets() when ready. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_011m7VujV9DvgT9AM1sBhbUw --- bin/broom.mjs | 189 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 174 insertions(+), 15 deletions(-) diff --git a/bin/broom.mjs b/bin/broom.mjs index b183b7a..bb955b9 100644 --- a/bin/broom.mjs +++ b/bin/broom.mjs @@ -1,29 +1,188 @@ #!/usr/bin/env node -// broomsticks — placeholder release to reserve the npm name while the scanner -// is built out. See PLAN.md for the design and roadmap. No secrets are read, -// written, or transmitted by this stub; it only prints this notice. +// broomsticks — sweep leaked secrets out of AI coding-assistant transcripts. +// Zero runtime dependencies; reads/writes nothing without explicit --apply. + import { readFileSync } from 'node:fs' import { fileURLToPath } from 'node:url' import { dirname, join } from 'node:path' +import { RULES } from '../src/rules.mjs' +import { scanText } from '../src/detector.mjs' +import { redactText } from '../src/redactor.mjs' +import { BackupSession } from '../src/backup.mjs' +import { printReport, printJsonReport } from '../src/report.mjs' +import { discoverTargets as claudeCodeTargets } from '../src/sources/claude-code.mjs' + +// ── Package metadata ────────────────────────────────────────────────────────── const pkgPath = join(dirname(fileURLToPath(import.meta.url)), '..', 'package.json') const pkg = JSON.parse(readFileSync(pkgPath, 'utf8')) -const args = process.argv.slice(2) -if (args.includes('--version') || args.includes('-v')) { - console.log(pkg.version) +// ── Argument parsing (no dependencies — hand-rolled) ───────────────────────── +const argv = process.argv.slice(2) + +function flag(name) { + return argv.includes(name) +} + +function option(name) { + const i = argv.indexOf(name) + return i !== -1 ? argv[i + 1] : undefined +} + +function options(name) { + const vals = [] + for (let i = 0; i < argv.length; i++) { + if (argv[i] === name && argv[i + 1]) vals.push(argv[i + 1]) + } + return vals +} + +// ── Top-level flags ─────────────────────────────────────────────────────────── +if (flag('--version') || flag('-v')) { console.log(pkg.version); process.exit(0) } +if (flag('--help') || flag('-h') || argv.length === 0) { printHelp(); process.exit(0) } + +const command = argv[0] +if (!['scan', 'clean', 'sources'].includes(command)) { + console.error(`broom: unknown command '${command}'. Try 'broom --help'.`) + process.exit(1) +} + +// ── Shared options ──────────────────────────────────────────────────────────── +const sources = options('--source') // [] means all +const extraFile = option('--extra') +const jsonOut = flag('--json') +const noFail = flag('--no-fail') +const apply = flag('--apply') +const backupDir = option('--backup-dir') +const noBackup = flag('--no-backup') + +// ── Load extra secrets from --extra file ────────────────────────────────────── +let extras = [] +if (extraFile) { + try { + extras = readFileSync(extraFile, 'utf8').split('\n') + } catch (e) { + console.error(`broom: cannot read --extra file '${extraFile}': ${e.message}`) + process.exit(1) + } +} + +// ── `broom sources` ─────────────────────────────────────────────────────────── +if (command === 'sources') { + const allTargets = gatherTargets(sources) + console.log(`\n Discovered sources:\n`) + if (allTargets.length === 0) { + console.log(' (none found — no supported AI assistant appears to be installed)\n') + } else { + const bySrc = {} + for (const t of allTargets) (bySrc[t.source] ??= []).push(t) + for (const [src, ts] of Object.entries(bySrc)) { + console.log(` ${src} (${ts.length} file${ts.length === 1 ? '' : 's'})`) + for (const t of ts) console.log(` ${t.label}`) + console.log() + } + } process.exit(0) } -console.log(`broomsticks v${pkg.version} — sweep secrets out of AI coding-assistant transcripts +// ── `broom scan` / `broom clean` ───────────────────────────────────────────── +const targets = gatherTargets(sources) + +if (targets.length === 0) { + console.error('\n broom: no transcript files found. Have you used Claude Code, Codex, or Cursor?\n') + process.exit(noFail ? 0 : 1) +} + +const isClean = command === 'clean' +const backup = (!noBackup && apply) ? new BackupSession(backupDir) : null - This is an early placeholder release. The scanner is not implemented yet — - this command only prints this notice (it reads/writes/sends nothing). +const scanResults = [] - Planned commands: - broom scan scan Claude Code / Codex / Cursor transcripts for secrets - broom clean --apply redact found secrets in place (backs up first) +for (const target of targets) { + let text + try { + text = target.read() + } catch (e) { + console.error(`broom: cannot read '${target.label}': ${e.message}`) + continue + } - Design & roadmap: https://github.com/digitaldrreamer/broomsticks/blob/main/PLAN.md - Follow progress: https://github.com/digitaldrreamer/broomsticks`) -process.exit(0) + const findings = scanText(text, RULES, extras) + + let applied = 0 + + if (findings.length > 0 && isClean && apply) { + // Back up before the first write to this file + if (backup) backup.backup(target.file) + + const { text: redacted, applied: n } = redactText(text, findings) + try { + target.write(redacted) + applied = n + } catch (e) { + console.error(`broom: cannot write '${target.label}': ${e.message}`) + } + } + + scanResults.push({ target, findings, applied: isClean ? applied : undefined }) +} + +// ── Write manifest after all writes complete ────────────────────────────────── +if (backup) backup.writeManifest(scanResults) + +// ── Output ──────────────────────────────────────────────────────────────────── +if (jsonOut) { + printJsonReport(scanResults) +} else { + printReport(scanResults, { clean: isClean, apply }) +} + +const totalFindings = scanResults.reduce((n, r) => n + r.findings.length, 0) +process.exit(totalFindings > 0 && !noFail ? 1 : 0) + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +/** + * Collect Targets from all enabled source adapters. + * @param {string[]} filter If non-empty, only include these source ids. + * @returns {import('../src/sources/claude-code.mjs').Target[]} + */ +function gatherTargets(filter) { + const all = [ + ...claudeCodeTargets(), + // codex and cursor adapters added in v0.2 + ] + if (!filter.length) return all + return all.filter(t => filter.includes(t.source)) +} + +function printHelp() { + console.log(` +broomsticks v${pkg.version} — sweep secrets out of AI coding-assistant transcripts + +USAGE + broom scan [options] find secrets (read-only, exits 1 if found) + broom clean [options] preview redactions (dry-run by default) + broom clean --apply [options] redact in place — backs up first + broom sources list discovered transcript files + broom --version + +OPTIONS + --source Restrict to one source (repeatable): claude-code + --apply Perform redaction (clean only; dry-run without it) + --backup-dir Where to write backups (default: ~/.broom/backups//) + --no-backup Skip backup — strongly discouraged + --extra File of additional secrets to redact (one per line, + plain strings or /regex/flags) + --json Machine-readable JSON output + --no-fail Exit 0 even when secrets are found (CI override) + --help, -h Show this help + --version, -v Print version + +EXAMPLES + npx broomsticks scan + broom scan --source claude-code --json | jq '.totalFindings' + broom clean --apply + broom clean --apply --extra ./leaked-keys.txt +`) +} From 41bd4930cb06c8d71161e0aafda85e6808f21d16 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 17:48:04 +0000 Subject: [PATCH 06/13] feat(sources): add Codex and Cursor adapters (v0.2 sources) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codex.mjs Discovers ~/.codex/history.jsonl + ~/.codex/sessions/**/*.jsonl. Same JSONL read/overwrite pattern as the Claude Code adapter. cursor.mjs Discovers state.vscdb files under the OS-appropriate Cursor User dir: Linux ~/.config/Cursor/User/{globalStorage,workspaceStorage/**/} macOS ~/Library/Application Support/Cursor/User/{...} Windows %APPDATA%\Cursor\User\{...} Uses node:sqlite (DatabaseSync, built-in since Node 22.5) — zero extra deps. Scans ItemTable and cursorDiskKV tables; filters rows by key pattern /chat|composer|aiService|aichat|prompt|cursorai/i so unrelated VS Code state is never touched. One Target per matching row; .file points to the .vscdb so BackupSession copies the whole database file once regardless of how many rows are targeted. read() / write() each open a fresh connection so concurrent access is safe and we always see the latest on-disk value. bin/broom.mjs Wire codexTargets() + cursorTargets() into gatherTargets(); update --source help text to list all three ids (claude-code | codex | cursor). All three sources tested end-to-end: broom sources, broom scan, --source filter, and scan --json all confirmed working across Claude Code + Codex + Cursor in a single run. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_011m7VujV9DvgT9AM1sBhbUw --- bin/broom.mjs | 8 +- src/sources/codex.mjs | 76 +++++++++++++++++++ src/sources/cursor.mjs | 164 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 src/sources/codex.mjs create mode 100644 src/sources/cursor.mjs diff --git a/bin/broom.mjs b/bin/broom.mjs index bb955b9..2ae4ea1 100644 --- a/bin/broom.mjs +++ b/bin/broom.mjs @@ -12,6 +12,8 @@ import { redactText } from '../src/redactor.mjs' import { BackupSession } from '../src/backup.mjs' import { printReport, printJsonReport } from '../src/report.mjs' import { discoverTargets as claudeCodeTargets } from '../src/sources/claude-code.mjs' +import { discoverTargets as codexTargets } from '../src/sources/codex.mjs' +import { discoverTargets as cursorTargets } from '../src/sources/cursor.mjs' // ── Package metadata ────────────────────────────────────────────────────────── const pkgPath = join(dirname(fileURLToPath(import.meta.url)), '..', 'package.json') @@ -150,7 +152,8 @@ process.exit(totalFindings > 0 && !noFail ? 1 : 0) function gatherTargets(filter) { const all = [ ...claudeCodeTargets(), - // codex and cursor adapters added in v0.2 + ...codexTargets(), + ...cursorTargets(), ] if (!filter.length) return all return all.filter(t => filter.includes(t.source)) @@ -168,7 +171,8 @@ USAGE broom --version OPTIONS - --source Restrict to one source (repeatable): claude-code + --source Restrict to one source (repeatable): + claude-code | codex | cursor --apply Perform redaction (clean only; dry-run without it) --backup-dir Where to write backups (default: ~/.broom/backups//) --no-backup Skip backup — strongly discouraged diff --git a/src/sources/codex.mjs b/src/sources/codex.mjs new file mode 100644 index 0000000..4c3ca06 --- /dev/null +++ b/src/sources/codex.mjs @@ -0,0 +1,76 @@ +// Source adapter for OpenAI Codex CLI transcripts. +// +// Codex writes: +// ~/.codex/history.jsonl — global command history +// ~/.codex/sessions/**/*.jsonl — per-session conversation logs + +import { readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' + +/** + * Root directory for Codex data. + * @returns {string} + */ +export function codexRoot() { + return join(homedir(), '.codex') +} + +/** + * Recursively yield every *.jsonl file under a directory. + * Silently skips missing or unreadable directories. + * @param {string} dir + * @returns {Generator} + */ +function* walkJsonl(dir) { + let entries + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + const full = join(dir, entry.name) + if (entry.isDirectory()) yield* walkJsonl(full) + else if (entry.isFile() && entry.name.endsWith('.jsonl')) yield full + } +} + +/** + * Discover all Codex transcript files and return them as Targets. + * Returns an empty array if Codex has not been used on this machine. + * @returns {import('./claude-code.mjs').Target[]} + */ +export function discoverTargets() { + const root = codexRoot() + const targets = [] + + // history.jsonl sits directly in ~/.codex + const history = join(root, 'history.jsonl') + try { + statSync(history) + targets.push(makeTarget(history)) + } catch { /* file absent */ } + + // Per-session files live under ~/.codex/sessions/** + for (const file of walkJsonl(join(root, 'sessions'))) { + targets.push(makeTarget(file)) + } + + return targets +} + +/** + * @param {string} file + * @returns {import('./claude-code.mjs').Target} + */ +function makeTarget(file) { + const f = file + return { + source: 'codex', + label: f, + file: f, + read: () => readFileSync(f, 'utf8'), + write: (text) => writeFileSync(f, text, 'utf8'), + } +} diff --git a/src/sources/cursor.mjs b/src/sources/cursor.mjs new file mode 100644 index 0000000..d056722 --- /dev/null +++ b/src/sources/cursor.mjs @@ -0,0 +1,164 @@ +// Source adapter for Cursor AI editor transcripts. +// +// Cursor stores conversation state in SQLite databases: +// Linux: ~/.config/Cursor/User/{globalStorage,workspaceStorage/**/}state.vscdb +// macOS: ~/Library/Application Support/Cursor/User/{...}/state.vscdb +// Windows: %APPDATA%\Cursor\User\{...}\state.vscdb +// +// Each database has two tables we care about: +// ItemTable — main VS Code key-value store +// cursorDiskKV — Cursor-specific overflow store +// +// We scan rows whose key matches /chat|composer|aiService|aichat|prompt|cursorai/i +// and return one Target per row (keyed on table+key so writes hit the right row). +// The `.file` property points to the .vscdb so it gets backed up once per DB. + +import { readdirSync, existsSync } from 'node:fs' +import { homedir, platform } from 'node:os' +import { join } from 'node:path' +import { DatabaseSync } from 'node:sqlite' + +/** Keys matching this pattern carry AI conversation content worth scanning. */ +const AI_KEY_RE = /chat|composer|aiService|aichat|prompt|cursorai/i + +/** Tables present in Cursor's state.vscdb */ +const TABLES = ['ItemTable', 'cursorDiskKV'] + +/** + * OS-specific root for Cursor's User data directory. + * @returns {string} + */ +export function cursorUserRoot() { + const p = platform() + if (p === 'darwin') { + return join(homedir(), 'Library', 'Application Support', 'Cursor', 'User') + } + if (p === 'win32') { + return join(process.env.APPDATA ?? join(homedir(), 'AppData', 'Roaming'), 'Cursor', 'User') + } + // Linux / other + return join(homedir(), '.config', 'Cursor', 'User') +} + +/** + * Recursively yield every state.vscdb file under a directory. + * @param {string} dir + * @returns {Generator} + */ +function* walkVscdb(dir) { + let entries + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + const full = join(dir, entry.name) + if (entry.isDirectory()) yield* walkVscdb(full) + else if (entry.isFile() && entry.name === 'state.vscdb') yield full + } +} + +/** + * Open a SQLite database read-only and return all AI-related rows from the + * known tables. Returns [] if the DB is unreadable or has no matching tables. + * + * @param {string} dbPath + * @returns {Array<{table:string, key:string, value:string}>} + */ +function readAiRows(dbPath) { + let db + try { + db = new DatabaseSync(dbPath, { open: true }) + } catch { + return [] + } + + const rows = [] + + // Check which tables actually exist before querying + let existingTables + try { + existingTables = new Set( + db.prepare("SELECT name FROM sqlite_master WHERE type='table'") + .all() + .map(r => r.name) + ) + } catch { + db.close() + return [] + } + + for (const table of TABLES) { + if (!existingTables.has(table)) continue + try { + const all = db.prepare(`SELECT key, value FROM [${table}]`).all() + for (const row of all) { + if (typeof row.key === 'string' && AI_KEY_RE.test(row.key)) { + // value may be stored as Buffer (BLOB) or string — normalise to string + const value = Buffer.isBuffer(row.value) + ? row.value.toString('utf8') + : String(row.value ?? '') + rows.push({ table, key: row.key, value }) + } + } + } catch { + // table unreadable — skip + } + } + + db.close() + return rows +} + +/** + * Discover all Cursor AI conversation rows and return them as Targets. + * One Target per DB row; multiple Targets may share the same `.file`. + * Returns [] if Cursor is not installed or has no conversation history. + * + * @returns {import('./claude-code.mjs').Target[]} + */ +export function discoverTargets() { + const userRoot = cursorUserRoot() + const targets = [] + + for (const dbPath of walkVscdb(userRoot)) { + const rows = readAiRows(dbPath) + + for (const { table, key, value } of rows) { + // Capture loop vars in closure-safe consts + const db = dbPath + const tbl = table + const k = key + + targets.push({ + source: 'cursor', + label: `${db}:${tbl}:${k}`, + file: db, // backup module copies the .vscdb file (once per file) + + read() { + // Re-open DB on each read so we always get the current value + const conn = new DatabaseSync(db, { open: true }) + try { + const row = conn.prepare(`SELECT value FROM [${tbl}] WHERE key = ?`).get(k) + if (!row) return '' + return Buffer.isBuffer(row.value) ? row.value.toString('utf8') : String(row.value ?? '') + } finally { + conn.close() + } + }, + + write(text) { + const conn = new DatabaseSync(db, { open: true }) + try { + conn.prepare(`UPDATE [${tbl}] SET value = ? WHERE key = ?`).run(text, k) + } finally { + conn.close() + } + }, + }) + } + } + + return targets +} From 86085ae5872968285a9c29a6184c6fc709bc2665 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 18:20:24 +0000 Subject: [PATCH 07/13] feat: allowlist, broom install command, and Claude Code skill/hook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - src/allowlist.mjs: DEMO_SECRETS with built-in false-positive literals (AWS doc keys, generic placeholders); loadAllowlist() merges built-ins with ~/.broom/allowlist.txt; isAllowlisted() checks literals and /regex/flags patterns. Stripe test keys handled via regex in DEFAULT_ALLOWLIST_CONTENT rather than as a literal to avoid secret-scanning tool false positives. - src/install.mjs: runInstall() seeds the allowlist file, writes broom-sweep SKILL.md to ~/.claude/skills/, installs a Stop hook at ~/.claude/hooks/stop-broom.mjs, and registers it in ~/.claude/settings.json - skills/broom-sweep/SKILL.md: bundled skill (npx-skills compatible); uses dynamic context injection to preview dry-run output; messaging frames local-transcript-only leaks as fully remediated by redaction — no rotation needed if the secret hasn't left the local machine - bin/broom.mjs: adds `broom install` (--yes for CI), --allowlist / --no-allowlist flags, first-run nudge when skill is not yet installed, allowlist filtering applied after scanText - package.json: adds src/ and skills/ to published files list Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_011m7VujV9DvgT9AM1sBhbUw --- bin/broom.mjs | 45 +++++++-- package.json | 2 + skills/broom-sweep/SKILL.md | 20 ++++ src/allowlist.mjs | 135 ++++++++++++++++++++++++++ src/install.mjs | 189 ++++++++++++++++++++++++++++++++++++ 5 files changed, 382 insertions(+), 9 deletions(-) create mode 100644 skills/broom-sweep/SKILL.md create mode 100644 src/allowlist.mjs create mode 100644 src/install.mjs diff --git a/bin/broom.mjs b/bin/broom.mjs index 2ae4ea1..23286bf 100644 --- a/bin/broom.mjs +++ b/bin/broom.mjs @@ -11,9 +11,11 @@ import { scanText } from '../src/detector.mjs' import { redactText } from '../src/redactor.mjs' import { BackupSession } from '../src/backup.mjs' import { printReport, printJsonReport } from '../src/report.mjs' +import { loadAllowlist, isAllowlisted } from '../src/allowlist.mjs' import { discoverTargets as claudeCodeTargets } from '../src/sources/claude-code.mjs' import { discoverTargets as codexTargets } from '../src/sources/codex.mjs' import { discoverTargets as cursorTargets } from '../src/sources/cursor.mjs' +import { runInstall, isInstalled } from '../src/install.mjs' // ── Package metadata ────────────────────────────────────────────────────────── const pkgPath = join(dirname(fileURLToPath(import.meta.url)), '..', 'package.json') @@ -44,19 +46,27 @@ if (flag('--version') || flag('-v')) { console.log(pkg.version); process.exit(0) if (flag('--help') || flag('-h') || argv.length === 0) { printHelp(); process.exit(0) } const command = argv[0] -if (!['scan', 'clean', 'sources'].includes(command)) { +if (!['scan', 'clean', 'sources', 'install'].includes(command)) { console.error(`broom: unknown command '${command}'. Try 'broom --help'.`) process.exit(1) } +// ── `broom install` ─────────────────────────────────────────────────────────── +if (command === 'install') { + await runInstall({ yes: flag('--yes') }) + process.exit(0) +} + // ── Shared options ──────────────────────────────────────────────────────────── -const sources = options('--source') // [] means all -const extraFile = option('--extra') -const jsonOut = flag('--json') -const noFail = flag('--no-fail') -const apply = flag('--apply') -const backupDir = option('--backup-dir') -const noBackup = flag('--no-backup') +const sources = options('--source') // [] means all +const extraFile = option('--extra') +const allowFile = option('--allowlist') +const jsonOut = flag('--json') +const noFail = flag('--no-fail') +const apply = flag('--apply') +const backupDir = option('--backup-dir') +const noBackup = flag('--no-backup') +const noAllowlist = flag('--no-allowlist') // ── Load extra secrets from --extra file ────────────────────────────────────── let extras = [] @@ -69,6 +79,15 @@ if (extraFile) { } } +// ── Load allowlist ──────────────────────────────────────────────────────────── +const allowlist = noAllowlist ? null : loadAllowlist(allowFile) + +// ── First-run nudge (scan/clean only, TTY only, not yet installed) ──────────── +if (!isInstalled() && process.stdout.isTTY && !jsonOut) { + console.log('\n Tip: run `broom install` to add a Claude Code skill + Stop hook that') + console.log(' automatically sweeps secrets after each Claude turn.\n') +} + // ── `broom sources` ─────────────────────────────────────────────────────────── if (command === 'sources') { const allTargets = gatherTargets(sources) @@ -109,7 +128,10 @@ for (const target of targets) { continue } - const findings = scanText(text, RULES, extras) + const raw = scanText(text, RULES, extras) + const findings = allowlist + ? raw.filter(f => !isAllowlisted(f.secret, allowlist)) + : raw let applied = 0 @@ -168,6 +190,7 @@ USAGE broom clean [options] preview redactions (dry-run by default) broom clean --apply [options] redact in place — backs up first broom sources list discovered transcript files + broom install install Claude Code skill + Stop hook broom --version OPTIONS @@ -178,8 +201,11 @@ OPTIONS --no-backup Skip backup — strongly discouraged --extra File of additional secrets to redact (one per line, plain strings or /regex/flags) + --allowlist Custom allowlist file (default: ~/.broom/allowlist.txt) + --no-allowlist Disable allowlist suppression (report all findings) --json Machine-readable JSON output --no-fail Exit 0 even when secrets are found (CI override) + --yes Skip confirmation prompts (install only) --help, -h Show this help --version, -v Print version @@ -188,5 +214,6 @@ EXAMPLES broom scan --source claude-code --json | jq '.totalFindings' broom clean --apply broom clean --apply --extra ./leaked-keys.txt + broom install `) } diff --git a/package.json b/package.json index b66d140..3d8abad 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,8 @@ }, "files": [ "bin", + "src", + "skills", "README.md", "PLAN.md", "LICENSE" diff --git a/skills/broom-sweep/SKILL.md b/skills/broom-sweep/SKILL.md new file mode 100644 index 0000000..07950fb --- /dev/null +++ b/skills/broom-sweep/SKILL.md @@ -0,0 +1,20 @@ +--- +name: broom-sweep +description: Scan and clean leaked secrets from AI assistant transcripts. Use when broomsticks reports secrets detected in transcript files, or when the user asks to sweep, scan, or redact secrets from their AI chat history. +allowed-tools: Bash(broom *) Bash(npx broomsticks *) +--- + +## Dry-run preview + +!`broom clean 2>&1` + +## Your task + +Present the dry-run output above to the user. If nothing was found, tell the user their transcripts are clean and no action is needed. + +If secrets were found: +1. Explain what was detected and where (which transcript files) +2. Reassure the user: if these secrets exist only in local AI transcript files and have not been shared, synced to a third-party service, or committed to a repository, redacting the transcripts is complete remediation — no credential rotation is needed +3. Ask the user to confirm before applying redactions +4. If they confirm, run `broom clean --apply` +5. Report how many redactions were applied and confirm that originals are backed up under `~/.broom/backups/` diff --git a/src/allowlist.mjs b/src/allowlist.mjs new file mode 100644 index 0000000..7cb8620 --- /dev/null +++ b/src/allowlist.mjs @@ -0,0 +1,135 @@ +// Allowlist — suppresses known false positives from scan findings. +// +// Sources (merged at scan time): +// 1. Built-in DEMO_SECRETS — well-known placeholder values from docs/tutorials +// 2. User file at ~/.broom/allowlist.txt (one entry per line) +// Lines starting with # are comments; /regex/flags entries compile to RegExp. +// +// A finding is suppressed when its secret matches any entry exactly (string) +// or fully (regex). Only the secret value is tested, not the rule or context. + +import { readFileSync, existsSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' + +/** + * Built-in demo / documentation secrets that are safe to suppress. + * These appear in AWS docs, GitHub tutorials, etc. and frequently end up in + * transcripts from "why doesn't this work?" conversations. + * + * Note: Stripe test keys are handled via regex in DEFAULT_ALLOWLIST_CONTENT + * rather than a literal here to avoid triggering secret-scanning tools on this + * source file itself. + */ +export const DEMO_SECRETS = [ + // AWS — official documentation placeholder credentials + 'AKIAIOSFODNN7EXAMPLE', + 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', + // GitHub — placeholder PAT used in docs + 'ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', + // Generic placeholder shapes that appear in .env.example files + 'your_api_key_here', + 'your-secret-key', + 'changeme', + 'replace_me', + 'INSERT_API_KEY_HERE', +] + +/** + * Default path for the user allowlist file. + * @returns {string} + */ +export function allowlistPath() { + return join(homedir(), '.broom', 'allowlist.txt') +} + +/** + * Default content written to the allowlist file on first run. + * Users can add their own entries below the built-in block. + */ +export const DEFAULT_ALLOWLIST_CONTENT = `# broomsticks allowlist — one entry per line. +# Lines starting with # are comments. +# Use /regex/flags syntax for pattern matching (e.g. /sk_test_[A-Za-z0-9]+/). +# +# ── Built-in: well-known documentation placeholder values ────────────────────── +# These appear in AWS, Stripe, GitHub, and other official docs and tutorials. +# Remove any line here if you want broomsticks to flag that value in your sessions. + +# AWS documentation example credentials (https://docs.aws.amazon.com) +AKIAIOSFODNN7EXAMPLE +wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + +# Stripe test-mode keys (not live; safe to allow unless you want to catch them) +/sk_test_[A-Za-z0-9]{10,99}/ + +# Generic placeholder values that show up in .env.example files +your_api_key_here +your-secret-key +changeme +replace_me +INSERT_API_KEY_HERE + +# ── Your entries ────────────────────────────────────────────────────────────── +# Add your own known-safe values below this line. + +` + +/** + * @typedef {{ literals: Set, patterns: RegExp[] }} Allowlist + */ + +/** + * Parse a list of text lines into a compiled Allowlist. + * @param {string[]} lines + * @returns {Allowlist} + */ +export function parseAllowlist(lines) { + const literals = new Set() + const patterns = [] + + for (const raw of lines) { + const line = raw.trim() + if (!line || line.startsWith('#')) continue + + const reMatch = line.match(/^\/(.+)\/([gimsud]*)$/) + if (reMatch) { + try { patterns.push(new RegExp(reMatch[1], reMatch[2])) } catch { /* bad regex — skip */ } + } else { + literals.add(line) + } + } + + return { literals, patterns } +} + +/** + * Load the combined allowlist: built-in demo secrets + user file (if present). + * @param {string} [file] Override the default allowlist path. + * @returns {Allowlist} + */ +export function loadAllowlist(file) { + const lines = [...DEMO_SECRETS] + + const path = file ?? allowlistPath() + if (existsSync(path)) { + try { + lines.push(...readFileSync(path, 'utf8').split('\n')) + } catch { /* unreadable — use built-ins only */ } + } + + return parseAllowlist(lines) +} + +/** + * Return true if `secret` is covered by the allowlist and should be suppressed. + * @param {string} secret + * @param {Allowlist} allowlist + * @returns {boolean} + */ +export function isAllowlisted(secret, allowlist) { + if (allowlist.literals.has(secret)) return true + return allowlist.patterns.some(re => { + re.lastIndex = 0 + return re.test(secret) + }) +} diff --git a/src/install.mjs b/src/install.mjs new file mode 100644 index 0000000..5c78971 --- /dev/null +++ b/src/install.mjs @@ -0,0 +1,189 @@ +// First-run onboarding: seed allowlist, install Claude Code skill + Stop hook. +// +// What gets installed: +// ~/.broom/allowlist.txt — false-positive suppression list +// ~/.claude/skills/broom-sweep/SKILL.md — teaches Claude to sweep secrets +// ~/.claude/hooks/stop-broom.mjs — scans after each Claude turn +// ~/.claude/settings.json (hooks entry) — registers the Stop hook + +import { mkdirSync, writeFileSync, existsSync, readFileSync, chmodSync } from 'node:fs' +import { homedir } from 'node:os' +import { join, dirname } from 'node:path' +import { createInterface } from 'node:readline' +import { DEFAULT_ALLOWLIST_CONTENT, allowlistPath } from './allowlist.mjs' + +// ── Embedded assets ─────────────────────────────────────────────────────────── + +const SKILL_MD = `--- +name: broom-sweep +description: Scan and clean leaked secrets from AI assistant transcripts. Use when broomsticks reports secrets detected in transcript files, or when the user asks to sweep, scan, or redact secrets from their AI chat history. +allowed-tools: Bash(broom *) Bash(npx broomsticks *) +--- + +## Dry-run preview + +\`\`\`! +broom clean 2>&1 +\`\`\` + +## Your task + +Present the dry-run output above to the user. If nothing was found, tell the user their transcripts are clean and no action is needed. + +If secrets were found: +1. Explain what was detected and where (which transcript files) +2. Reassure the user: if these secrets exist only in local AI transcript files and have not been shared, synced to a third-party service, or committed to a repository, redacting the transcripts is complete remediation — no credential rotation is needed +3. Ask the user to confirm before applying redactions +4. If they confirm, run \`broom clean --apply\` +5. Report how many redactions were applied and confirm that originals are backed up under ~/.broom/backups/ +` + +// Stop hook: a standalone Node ESM script Claude Code runs after each turn. +// Outputs a JSON block decision when secrets are found, which blocks Claude +// from ending the turn and injects the reason into the next model pass. +const HOOK_SCRIPT = `#!/usr/bin/env node +// broomsticks Stop hook — silent scan after each Claude turn. +// Outputs JSON block decision when secrets are found in transcript files. +import { execSync } from 'node:child_process' + +function scan() { + for (const cmd of ['broom scan --json --no-fail', 'npx broomsticks scan --json --no-fail']) { + try { + return execSync(cmd, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }) + } catch { /* command not found or scan itself errored */ } + } + return null +} + +const output = scan() +if (!output) process.exit(0) + +let count = 0 +try { count = JSON.parse(output).totalFindings ?? 0 } catch { process.exit(0) } + +if (count > 0) { + process.stdout.write(JSON.stringify({ + decision: 'block', + reason: \`broomsticks detected \${count} secret(s) in your local AI transcript files. Invoke /broom-sweep to preview and redact them. Always ask the user for confirmation before applying.\`, + }) + '\\n') +} +` + +// ── Paths ───────────────────────────────────────────────────────────────────── + +export function skillPath() { return join(homedir(), '.claude', 'skills', 'broom-sweep', 'SKILL.md') } +export function hookPath() { return join(homedir(), '.claude', 'hooks', 'stop-broom.mjs') } +export function settingsPath() { return join(homedir(), '.claude', 'settings.json') } + +/** True when the skill + hook are already installed. */ +export function isInstalled() { + return existsSync(skillPath()) && existsSync(hookPath()) +} + +// ── Interactive prompt ──────────────────────────────────────────────────────── + +/** + * Ask a yes/no question on stdin/stdout. + * Resolves true for 'y'/'yes'/'' (default yes), false otherwise. + * @param {string} question + * @returns {Promise} + */ +function askYesNo(question) { + return new Promise(resolve => { + const rl = createInterface({ input: process.stdin, output: process.stdout }) + rl.question(question, answer => { + rl.close() + const a = answer.trim().toLowerCase() + resolve(a === '' || a === 'y' || a === 'yes') + }) + }) +} + +// ── Install ─────────────────────────────────────────────────────────────────── + +/** + * Run the onboarding flow. + * + * @param {{ yes?: boolean, silent?: boolean }} opts + * yes — skip confirmation prompt (CI / --yes flag) + * silent — suppress all output except errors + * @returns {Promise} List of actions taken. + */ +export async function runInstall({ yes = false, silent = false } = {}) { + const log = silent ? () => {} : (...a) => console.log(...a) + + log(` + broomsticks — first-run setup + + This will install: + 1. ~/.broom/allowlist.txt — false-positive suppression list + 2. ~/.claude/skills/broom-sweep/SKILL.md — teaches Claude to sweep secrets + 3. ~/.claude/hooks/stop-broom.mjs — scans after each Claude turn + 4. ~/.claude/settings.json (Stop hook entry) — registers the hook +`) + + const proceed = yes || !process.stdin.isTTY + ? true + : await askYesNo(' Proceed? [Y/n] ') + + if (!proceed) { + log('\n Setup cancelled.\n') + return [] + } + + const done = [] + + // 1. Allowlist + const listPath = allowlistPath() + if (!existsSync(listPath)) { + mkdirSync(dirname(listPath), { recursive: true }) + writeFileSync(listPath, DEFAULT_ALLOWLIST_CONTENT, 'utf8') + done.push(`created ${listPath}`) + } else { + done.push(`exists ${listPath} (unchanged)`) + } + + // 2. Skill + const sp = skillPath() + mkdirSync(dirname(sp), { recursive: true }) + writeFileSync(sp, SKILL_MD, 'utf8') + done.push(`installed ${sp}`) + + // 3. Hook script + const hp = hookPath() + mkdirSync(dirname(hp), { recursive: true }) + writeFileSync(hp, HOOK_SCRIPT, 'utf8') + chmodSync(hp, 0o755) + done.push(`installed ${hp}`) + + // 4. Register hook in settings.json + const sp2 = settingsPath() + let settings = {} + if (existsSync(sp2)) { + try { settings = JSON.parse(readFileSync(sp2, 'utf8')) } catch { /* corrupt — start fresh */ } + } + settings.hooks ??= {} + settings.hooks.Stop ??= [] + + const alreadyWired = settings.hooks.Stop.some(entry => + entry.hooks?.some(h => typeof h.command === 'string' && h.command.includes('stop-broom')) + ) + if (!alreadyWired) { + settings.hooks.Stop.push({ hooks: [{ type: 'command', command: hp }] }) + mkdirSync(dirname(sp2), { recursive: true }) + writeFileSync(sp2, JSON.stringify(settings, null, 2) + '\n', 'utf8') + done.push(`updated ${sp2}`) + } else { + done.push(`exists ${sp2} (hook already registered)`) + } + + log() + for (const line of done) log(` ✓ ${line}`) + log(` + All done. Restart Claude Code (or run /reload-plugins) for the skill to take effect. + The Stop hook will scan your transcripts silently after each turn and prompt + Claude to invoke /broom-sweep if secrets are found. +`) + + return done +} From 8023f03f9bbfbe8e880dcf74de29422b50285d41 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 18:23:08 +0000 Subject: [PATCH 08/13] chore: set executable bit on bin/broom.mjs Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_011m7VujV9DvgT9AM1sBhbUw --- bin/broom.mjs | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bin/broom.mjs diff --git a/bin/broom.mjs b/bin/broom.mjs old mode 100644 new mode 100755 From ccc812fbc20feb200e3728d540d9b5fb6a611f7d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 18:45:07 +0000 Subject: [PATCH 09/13] =?UTF-8?q?feat:=20add=20broom=20proxy=20=E2=80=94?= =?UTF-8?q?=20universal=20redacting=20proxy=20for=20all=20AI=20tools?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/proxy.mjs: - HTTP server (node:http / node:https, zero new deps) listening on 127.0.0.1:7777 by default - Routes: POST /v1/messages → api.anthropic.com (Anthropic) POST /v1/chat/completions → api.openai.com (OpenAI-compatible) - Outgoing requests: deep-walk all JSON string values, run through scanText + allowlist filter, replace secrets with «BROOM:ruleId:sha8» placeholders; vault stores sha8 → realSecret in-memory for session - Incoming responses: same redaction on non-streaming JSON; for SSE, buffers the full stream, accumulates text deltas per content-block index, redacts the complete string, re-emits a synthetic SSE stream with collapsed deltas — handles both Anthropic and OpenAI SSE formats - delete accept-encoding before forwarding so upstream returns plaintext - installProxyEnv(): appends ANTHROPIC_BASE_URL / OPENAI_BASE_URL to ~/.zshrc, ~/.bashrc, ~/.bash_profile, ~/.profile (idempotent) bin/broom.mjs: - `broom proxy [--port N] [--verbose]` — start the proxy - `broom proxy --install` — write env vars to shell profiles Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_011m7VujV9DvgT9AM1sBhbUw --- bin/broom.mjs | 47 +++++++- src/proxy.mjs | 318 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 src/proxy.mjs diff --git a/bin/broom.mjs b/bin/broom.mjs index 23286bf..f7781ff 100755 --- a/bin/broom.mjs +++ b/bin/broom.mjs @@ -16,6 +16,7 @@ import { discoverTargets as claudeCodeTargets } from '../src/sources/claude-code import { discoverTargets as codexTargets } from '../src/sources/codex.mjs' import { discoverTargets as cursorTargets } from '../src/sources/cursor.mjs' import { runInstall, isInstalled } from '../src/install.mjs' +import { startProxy, installProxyEnv } from '../src/proxy.mjs' // ── Package metadata ────────────────────────────────────────────────────────── const pkgPath = join(dirname(fileURLToPath(import.meta.url)), '..', 'package.json') @@ -46,7 +47,7 @@ if (flag('--version') || flag('-v')) { console.log(pkg.version); process.exit(0) if (flag('--help') || flag('-h') || argv.length === 0) { printHelp(); process.exit(0) } const command = argv[0] -if (!['scan', 'clean', 'sources', 'install'].includes(command)) { +if (!['scan', 'clean', 'sources', 'install', 'proxy'].includes(command)) { console.error(`broom: unknown command '${command}'. Try 'broom --help'.`) process.exit(1) } @@ -57,6 +58,46 @@ if (command === 'install') { process.exit(0) } +// ── `broom proxy` ──────────────────────────────────────────────────────────── +if (command === 'proxy') { + const port = parseInt(option('--port') ?? '7777', 10) + const verbose = flag('--verbose') + + if (flag('--install')) { + const updated = installProxyEnv(port) + if (updated.length === 0) { + console.log('\n broom proxy: env vars already present in all shell init files.\n') + } else { + console.log('\n broom proxy: added env vars to:') + for (const f of updated) console.log(` ${f}`) + console.log(`\n Open a new terminal (or run: source ~/.zshrc) then start the proxy:\n`) + console.log(` broom proxy\n`) + } + process.exit(0) + } + + const server = await startProxy({ port, verbose, allowlistFile: option('--allowlist') }) + console.log(` + broom proxy listening on http://127.0.0.1:${port} + + Point your AI tools at this proxy: + export ANTHROPIC_BASE_URL=http://127.0.0.1:${port} + export OPENAI_BASE_URL=http://127.0.0.1:${port} + + Or run \`broom proxy --install\` to add these permanently to your shell. + + Routes: + POST /v1/messages → api.anthropic.com (Claude Code, Aider) + POST /v1/chat/completions → api.openai.com (Codex, OpenAI-compatible) + + Press Ctrl-C to stop. +`) + + process.on('SIGINT', () => { server.close(); process.exit(0) }) + process.on('SIGTERM', () => { server.close(); process.exit(0) }) + // Server holds the event loop open — no further code needed +} + // ── Shared options ──────────────────────────────────────────────────────────── const sources = options('--source') // [] means all const extraFile = option('--extra') @@ -191,6 +232,8 @@ USAGE broom clean --apply [options] redact in place — backs up first broom sources list discovered transcript files broom install install Claude Code skill + Stop hook + broom proxy [options] start local redacting proxy for all AI tools + broom proxy --install add ANTHROPIC_BASE_URL / OPENAI_BASE_URL to shell broom --version OPTIONS @@ -205,6 +248,8 @@ OPTIONS --no-allowlist Disable allowlist suppression (report all findings) --json Machine-readable JSON output --no-fail Exit 0 even when secrets are found (CI override) + --port Proxy port (default: 7777) + --verbose Log redaction counts to stderr (proxy only) --yes Skip confirmation prompts (install only) --help, -h Show this help --version, -v Print version diff --git a/src/proxy.mjs b/src/proxy.mjs new file mode 100644 index 0000000..b4dc0de --- /dev/null +++ b/src/proxy.mjs @@ -0,0 +1,318 @@ +// broomsticks local proxy. +// +// Sits between any AI coding client and the upstream LLM API. On every +// outgoing request it redacts secrets from user messages before they reach +// the model. On every incoming response it redacts any secrets the model may +// have echoed back. Real values are held in an in-memory vault (sha8 → secret) +// so a future hook or shell wrapper can expand them before tool execution. +// +// Routing (detected from request path): +// POST /v1/messages → api.anthropic.com (Anthropic) +// POST /v1/chat/completions → api.openai.com (OpenAI-compatible) +// +// Both streaming (SSE) and non-streaming responses are handled. Streaming +// responses are buffered in full before redaction — a secret that straddles +// two SSE chunks cannot be safely found otherwise. The client receives a +// valid synthetic SSE stream with collapsed text deltas. + +import { createServer } from 'node:http' +import { request as tlsReq } from 'node:https' +import { createHash } from 'node:crypto' +import { existsSync, readFileSync, appendFileSync } from 'node:fs' +import { homedir } from 'node:os' +import { join } from 'node:path' + +import { RULES } from './rules.mjs' +import { scanText } from './detector.mjs' +import { redactText } from './redactor.mjs' +import { loadAllowlist, isAllowlisted } from './allowlist.mjs' + +// ── Routing ─────────────────────────────────────────────────────────────────── + +const ROUTES = { + '/v1/messages': 'api.anthropic.com', + '/v1/chat/completions': 'api.openai.com', +} + +function resolveUpstream(path) { + for (const [prefix, host] of Object.entries(ROUTES)) { + if (path === prefix || path.startsWith(prefix + '?')) return host + } + return null +} + +// ── Vault helpers ───────────────────────────────────────────────────────────── + +function sha8(secret) { + return createHash('sha256').update(secret).digest('hex').slice(0, 8) +} + +// ── Redaction ───────────────────────────────────────────────────────────────── + +function redactStr(text, allowlist, vault) { + const raw = scanText(text, RULES) + const findings = allowlist ? raw.filter(f => !isAllowlisted(f.secret, allowlist)) : raw + if (findings.length === 0) return text + for (const f of findings) vault.set(sha8(f.secret), f.secret) + return redactText(text, findings).text +} + +/** Recursively walk a parsed JSON value and redact every string leaf. */ +function redactValue(v, allowlist, vault) { + if (typeof v === 'string') return redactStr(v, allowlist, vault) + if (Array.isArray(v)) return v.map(x => redactValue(x, allowlist, vault)) + if (v !== null && typeof v === 'object') { + const out = {} + for (const [k, val] of Object.entries(v)) out[k] = redactValue(val, allowlist, vault) + return out + } + return v +} + +// ── Stream utilities ────────────────────────────────────────────────────────── + +function collect(stream) { + return new Promise((resolve, reject) => { + const chunks = [] + stream.on('data', c => chunks.push(c)) + stream.on('end', () => resolve(Buffer.concat(chunks))) + stream.on('error', reject) + }) +} + +// ── SSE redaction ───────────────────────────────────────────────────────────── +// +// Strategy: accumulate the full text for each content block index across all +// delta events, redact the complete string, then re-emit one synthetic delta +// per block with the full redacted text. All non-delta events pass through. + +function redactAnthropicSSE(lines, allowlist, vault) { + const accumulated = new Map() // blockIndex → full text + for (const line of lines) { + if (!line.startsWith('data:')) continue + const raw = line.slice(5).trim() + if (!raw || raw === '[DONE]') continue + let ev; try { ev = JSON.parse(raw) } catch { continue } + if (ev.type === 'content_block_delta' && ev.delta?.type === 'text_delta') { + const i = ev.index ?? 0 + accumulated.set(i, (accumulated.get(i) ?? '') + (ev.delta.text ?? '')) + } + } + + const redacted = new Map() + for (const [i, text] of accumulated) redacted.set(i, redactStr(text, allowlist, vault)) + + const emitted = new Set() + const out = [] + for (const line of lines) { + if (!line.startsWith('data:')) { out.push(line); continue } + const raw = line.slice(5).trim() + if (!raw || raw === '[DONE]') { out.push(line); continue } + let ev; try { ev = JSON.parse(raw) } catch { out.push(line); continue } + + if (ev.type === 'content_block_delta' && ev.delta?.type === 'text_delta') { + const i = ev.index ?? 0 + if (!emitted.has(i)) { + emitted.add(i) + out.push('data: ' + JSON.stringify({ + ...ev, delta: { ...ev.delta, text: redacted.get(i) ?? '' }, + })) + } + // drop subsequent deltas — text already emitted above + } else { + out.push(line) + } + } + return out +} + +function redactOpenAISSE(lines, allowlist, vault) { + const accumulated = new Map() // choiceIndex → full content string + for (const line of lines) { + if (!line.startsWith('data:')) continue + const raw = line.slice(5).trim() + if (!raw || raw === '[DONE]') continue + let ev; try { ev = JSON.parse(raw) } catch { continue } + for (const c of ev.choices ?? []) { + if (typeof c.delta?.content === 'string') { + const i = c.index ?? 0 + accumulated.set(i, (accumulated.get(i) ?? '') + c.delta.content) + } + } + } + + const redacted = new Map() + for (const [i, text] of accumulated) redacted.set(i, redactStr(text, allowlist, vault)) + + const emitted = new Set() + const out = [] + for (const line of lines) { + if (!line.startsWith('data:')) { out.push(line); continue } + const raw = line.slice(5).trim() + if (!raw || raw === '[DONE]') { out.push(line); continue } + let ev; try { ev = JSON.parse(raw) } catch { out.push(line); continue } + + const hasContent = (ev.choices ?? []).some(c => typeof c.delta?.content === 'string') + if (!hasContent) { out.push(line); continue } + + const newChoices = (ev.choices ?? []).map(c => { + if (typeof c.delta?.content !== 'string') return c + const i = c.index ?? 0 + if (emitted.has(i)) return { ...c, delta: { ...c.delta, content: '' } } + emitted.add(i) + return { ...c, delta: { ...c.delta, content: redacted.get(i) ?? '' } } + }) + out.push('data: ' + JSON.stringify({ ...ev, choices: newChoices })) + } + return out +} + +// ── Upstream forwarding ─────────────────────────────────────────────────────── + +function forwardRequest(upstreamHost, req, body) { + return new Promise((resolve, reject) => { + const headers = { ...req.headers, host: upstreamHost } + headers['content-length'] = String(body.length) + delete headers['accept-encoding'] // need plaintext to scan + + const upReq = tlsReq( + { host: upstreamHost, port: 443, path: req.url, method: req.method, headers }, + resolve, + ) + upReq.on('error', reject) + upReq.end(body) + }) +} + +// ── Core handler ────────────────────────────────────────────────────────────── + +async function handleRequest(req, res, allowlist, vault, verbose) { + const upstreamHost = resolveUpstream(req.url ?? '/') + + if (!upstreamHost) { + res.writeHead(404, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ + error: { message: `broom proxy: no route for ${req.method} ${req.url}` }, + })) + return + } + + // ── Redact outgoing request body ────────────────────────────────────────── + const rawBody = await collect(req) + let outBody = rawBody + const vaultBefore = vault.size + + if (req.headers['content-type']?.includes('application/json')) { + try { + const json = JSON.parse(rawBody.toString('utf8')) + const cleaned = redactValue(json, allowlist, vault) + outBody = Buffer.from(JSON.stringify(cleaned), 'utf8') + } catch { /* non-JSON — forward as-is */ } + } + + if (verbose && vault.size > vaultBefore) { + console.error(` ← request: redacted ${vault.size - vaultBefore} secret(s) (vault size: ${vault.size})`) + } + + // ── Forward to upstream ─────────────────────────────────────────────────── + const upRes = await forwardRequest(upstreamHost, req, outBody) + const upBody = await collect(upRes) + + // ── Redact incoming response body ───────────────────────────────────────── + const isSSE = upRes.headers['content-type']?.includes('text/event-stream') + const vaultMid = vault.size + let responseBody + + if (isSSE) { + const lines = upBody.toString('utf8').split('\n') + const isAnthropic = upstreamHost.includes('anthropic') + const cleanedLines = isAnthropic + ? redactAnthropicSSE(lines, allowlist, vault) + : redactOpenAISSE(lines, allowlist, vault) + responseBody = Buffer.from(cleanedLines.join('\n'), 'utf8') + } else { + try { + const json = JSON.parse(upBody.toString('utf8')) + const cleaned = redactValue(json, allowlist, vault) + responseBody = Buffer.from(JSON.stringify(cleaned), 'utf8') + } catch { + responseBody = upBody + } + } + + if (verbose && vault.size > vaultMid) { + console.error(` → response: redacted ${vault.size - vaultMid} secret(s) from model output`) + } + + // ── Write response ──────────────────────────────────────────────────────── + const outHeaders = { ...upRes.headers, 'content-length': String(responseBody.length) } + res.writeHead(upRes.statusCode ?? 200, outHeaders) + res.end(responseBody) +} + +// ── Shell profile installation ──────────────────────────────────────────────── + +const PROFILE_CANDIDATES = ['.zshrc', '.bashrc', '.bash_profile', '.profile'] + +const ENV_BLOCK = (port) => ` +# ── broomsticks proxy ──────────────────────────────────────────────────────── +# Route AI API calls through the local broomsticks proxy so secrets are +# redacted before they reach the model. Start with: broom proxy +export ANTHROPIC_BASE_URL=http://127.0.0.1:${port} +export OPENAI_BASE_URL=http://127.0.0.1:${port} +# ──────────────────────────────────────────────────────────────────────────── +` + +const BROOM_MARKER = '# ── broomsticks proxy' + +/** + * Append proxy env-var exports to every shell init file found in $HOME. + * Idempotent: skips files that already contain the marker. + * + * @param {number} port + * @returns {string[]} paths that were updated + */ +export function installProxyEnv(port = 7777) { + const home = homedir() + const updated = [] + + for (const name of PROFILE_CANDIDATES) { + const file = join(home, name) + if (!existsSync(file)) continue + const contents = readFileSync(file, 'utf8') + if (contents.includes(BROOM_MARKER)) continue // already installed + appendFileSync(file, ENV_BLOCK(port), 'utf8') + updated.push(file) + } + + return updated +} + +// ── Public entry point ──────────────────────────────────────────────────────── + +/** + * Start the broomsticks proxy. + * + * @param {{ port?: number, verbose?: boolean, allowlistFile?: string }} opts + * @returns {Promise} + */ +export function startProxy({ port = 7777, verbose = false, allowlistFile } = {}) { + const allowlist = loadAllowlist(allowlistFile) + const vault = new Map() + + const server = createServer(async (req, res) => { + try { + await handleRequest(req, res, allowlist, vault, verbose) + } catch (err) { + if (!res.headersSent) { + res.writeHead(502, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ error: { message: 'broom proxy: ' + err.message } })) + } + } + }) + + return new Promise((resolve, reject) => { + server.on('error', reject) + server.listen(port, '127.0.0.1', () => resolve(server)) + }) +} From 16a58a05a332db1e40c0dfbe62ea775e159d5def Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 22:18:22 +0000 Subject: [PATCH 10/13] feat: proxy daemon installation (launchd / systemd) broom proxy --install --daemon registers the proxy as a login-persistent daemon so it starts automatically after reboot: - macOS: ~/Library/LaunchAgents/com.broomsticks.proxy.plist (launchctl) - Linux: ~/.config/systemd/user/broom-proxy.service (systemd --user) KeepAlive / Restart=on-failure so the proxy self-heals on crash. Logs written to ~/.broom/proxy.log. broom proxy --install (without --daemon) keeps current behaviour: writes env vars to shell profiles and suggests --daemon as the next step. broom proxy --uninstall unloads/disables and removes the daemon file. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_011m7VujV9DvgT9AM1sBhbUw --- bin/broom.mjs | 37 +++++++++++++-- src/proxy.mjs | 121 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 152 insertions(+), 6 deletions(-) diff --git a/bin/broom.mjs b/bin/broom.mjs index f7781ff..48d24c9 100755 --- a/bin/broom.mjs +++ b/bin/broom.mjs @@ -16,7 +16,7 @@ import { discoverTargets as claudeCodeTargets } from '../src/sources/claude-code import { discoverTargets as codexTargets } from '../src/sources/codex.mjs' import { discoverTargets as cursorTargets } from '../src/sources/cursor.mjs' import { runInstall, isInstalled } from '../src/install.mjs' -import { startProxy, installProxyEnv } from '../src/proxy.mjs' +import { startProxy, installProxyEnv, installDaemon, uninstallDaemon } from '../src/proxy.mjs' // ── Package metadata ────────────────────────────────────────────────────────── const pkgPath = join(dirname(fileURLToPath(import.meta.url)), '..', 'package.json') @@ -63,6 +63,15 @@ if (command === 'proxy') { const port = parseInt(option('--port') ?? '7777', 10) const verbose = flag('--verbose') + if (flag('--uninstall')) { + const removed = uninstallDaemon() + console.log(removed + ? '\n broom proxy: daemon removed. Env vars in your shell profile still point to\n the proxy — remove them manually or they will silently fail to connect.\n' + : '\n broom proxy: no daemon found to remove.\n' + ) + process.exit(0) + } + if (flag('--install')) { const updated = installProxyEnv(port) if (updated.length === 0) { @@ -70,8 +79,28 @@ if (command === 'proxy') { } else { console.log('\n broom proxy: added env vars to:') for (const f of updated) console.log(` ${f}`) - console.log(`\n Open a new terminal (or run: source ~/.zshrc) then start the proxy:\n`) - console.log(` broom proxy\n`) + } + + if (flag('--daemon')) { + try { + const { path, platform } = installDaemon({ + port, + nodeBin: process.execPath, + broomBin: process.argv[1], + }) + console.log(`\n broom proxy: daemon installed (${platform})`) + console.log(` ${path}`) + console.log('\n The proxy will start automatically at login and restart on failure.') + console.log(' Logs: ~/.broom/proxy.log') + console.log(' To remove: broom proxy --uninstall\n') + } catch (err) { + console.error('\n broom proxy --daemon failed:', err.message) + console.error(' Start the proxy manually: broom proxy\n') + process.exit(1) + } + } else { + console.log(`\n Open a new terminal (or: source ~/.zshrc) then run:\n broom proxy\n`) + console.log(` To start automatically at login:\n broom proxy --install --daemon\n`) } process.exit(0) } @@ -234,6 +263,8 @@ USAGE broom install install Claude Code skill + Stop hook broom proxy [options] start local redacting proxy for all AI tools broom proxy --install add ANTHROPIC_BASE_URL / OPENAI_BASE_URL to shell + broom proxy --install --daemon also register as a login-persistent daemon + broom proxy --uninstall remove the daemon (macOS / Linux) broom --version OPTIONS diff --git a/src/proxy.mjs b/src/proxy.mjs index b4dc0de..0a09d42 100644 --- a/src/proxy.mjs +++ b/src/proxy.mjs @@ -18,9 +18,10 @@ import { createServer } from 'node:http' import { request as tlsReq } from 'node:https' import { createHash } from 'node:crypto' -import { existsSync, readFileSync, appendFileSync } from 'node:fs' -import { homedir } from 'node:os' -import { join } from 'node:path' +import { existsSync, readFileSync, appendFileSync, mkdirSync, writeFileSync, unlinkSync } from 'node:fs' +import { homedir } from 'node:os' +import { join, dirname } from 'node:path' +import { execFileSync } from 'node:child_process' import { RULES } from './rules.mjs' import { scanText } from './detector.mjs' @@ -288,6 +289,120 @@ export function installProxyEnv(port = 7777) { return updated } +// ── Daemon installation ─────────────────────────────────────────────────────── + +function launchdPlist(nodeBin, broomBin, port) { + const logFile = join(homedir(), '.broom', 'proxy.log') + return ` + + + + Label + com.broomsticks.proxy + ProgramArguments + + ${nodeBin} + ${broomBin} + proxy + --port + ${port} + + RunAtLoad + + KeepAlive + + StandardOutPath + ${logFile} + StandardErrorPath + ${logFile} + + +` +} + +function systemdService(nodeBin, broomBin, port) { + return `[Unit] +Description=broomsticks secret-redacting proxy +After=network.target + +[Service] +ExecStart=${nodeBin} ${broomBin} proxy --port ${port} +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=default.target +` +} + +/** + * Install the proxy as a login-persistent daemon. + * - macOS : ~/Library/LaunchAgents/com.broomsticks.proxy.plist (launchd) + * - Linux : ~/.config/systemd/user/broom-proxy.service (systemd) + * + * @param {{ port?: number, nodeBin: string, broomBin: string }} opts + * @returns {{ path: string, platform: string, alreadyRunning: boolean }} + */ +export function installDaemon({ port = 7777, nodeBin, broomBin }) { + const home = homedir() + mkdirSync(join(home, '.broom'), { recursive: true }) + + if (process.platform === 'darwin') { + const agentsDir = join(home, 'Library', 'LaunchAgents') + const plistPath = join(agentsDir, 'com.broomsticks.proxy.plist') + mkdirSync(agentsDir, { recursive: true }) + writeFileSync(plistPath, launchdPlist(nodeBin, broomBin, port), 'utf8') + + // Unload first in case a stale copy is already registered + try { execFileSync('launchctl', ['unload', plistPath], { stdio: 'ignore' }) } catch {} + execFileSync('launchctl', ['load', '-w', plistPath]) + + return { path: plistPath, platform: 'macos' } + } + + if (process.platform === 'linux') { + const serviceDir = join(home, '.config', 'systemd', 'user') + const servicePath = join(serviceDir, 'broom-proxy.service') + mkdirSync(serviceDir, { recursive: true }) + writeFileSync(servicePath, systemdService(nodeBin, broomBin, port), 'utf8') + + execFileSync('systemctl', ['--user', 'daemon-reload'], { stdio: 'ignore' }) + execFileSync('systemctl', ['--user', 'enable', 'broom-proxy'], { stdio: 'ignore' }) + execFileSync('systemctl', ['--user', 'start', 'broom-proxy'], { stdio: 'ignore' }) + + return { path: servicePath, platform: 'linux' } + } + + throw new Error(`Daemon installation is not supported on ${process.platform}. Run \`broom proxy\` manually or add it to your startup scripts.`) +} + +/** + * Remove the daemon installed by installDaemon. + */ +export function uninstallDaemon() { + const home = homedir() + + if (process.platform === 'darwin') { + const plistPath = join(home, 'Library', 'LaunchAgents', 'com.broomsticks.proxy.plist') + if (!existsSync(plistPath)) return false + try { execFileSync('launchctl', ['unload', plistPath], { stdio: 'ignore' }) } catch {} + try { unlinkSync(plistPath) } catch {} + return true + } + + if (process.platform === 'linux') { + try { execFileSync('systemctl', ['--user', 'stop', 'broom-proxy'], { stdio: 'ignore' }) } catch {} + try { execFileSync('systemctl', ['--user', 'disable', 'broom-proxy'], { stdio: 'ignore' }) } catch {} + const servicePath = join(home, '.config', 'systemd', 'user', 'broom-proxy.service') + try { unlinkSync(servicePath) } catch {} + try { execFileSync('systemctl', ['--user', 'daemon-reload'], { stdio: 'ignore' }) } catch {} + return true + } + + return false +} + // ── Public entry point ──────────────────────────────────────────────────────── /** From e0193e91f82b8bced13575ff1b92f3c3d61b2819 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Jul 2026 22:30:17 +0000 Subject: [PATCH 11/13] fix: apply all valid PR review comments proxy.mjs: - delete transfer-encoding from forwarded request headers (chunked + content-length together violates HTTP and triggers 400 from Cloudflare) - delete transfer-encoding from response headers before sending fixed- length buffer to client - guard both SSE redaction loops against JSON.parse returning null (would throw TypeError on ev.type / ev.choices access) - wrap shell profile read/append in try-catch (EACCES should not crash CLI) - uninstallDaemon on Linux: check existsSync before calling systemctl, return false when no daemon is installed (was always returning true) report.mjs: - read target text once per target, not once per finding inside the inner loop (for Cursor/SQLite this was re-opening the DB for every secret found) install.mjs: - prepend `node` to the hook command on Windows where shebangs are not processed natively cursor.mjs: - open SQLite in readOnly:true for both discoverTargets and target.read() to avoid write-lock conflicts when Cursor is running bin/broom.mjs: - validate --port is a number between 1 and 65535 before starting proxy Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_011m7VujV9DvgT9AM1sBhbUw --- bin/broom.mjs | 7 ++++++- src/install.mjs | 3 ++- src/proxy.mjs | 22 +++++++++++++++------- src/report.mjs | 17 +++++++---------- src/sources/cursor.mjs | 6 +++--- 5 files changed, 33 insertions(+), 22 deletions(-) diff --git a/bin/broom.mjs b/bin/broom.mjs index 48d24c9..109be6e 100755 --- a/bin/broom.mjs +++ b/bin/broom.mjs @@ -60,7 +60,12 @@ if (command === 'install') { // ── `broom proxy` ──────────────────────────────────────────────────────────── if (command === 'proxy') { - const port = parseInt(option('--port') ?? '7777', 10) + const portStr = option('--port') ?? '7777' + const port = parseInt(portStr, 10) + if (isNaN(port) || port < 1 || port > 65535) { + console.error(`broom: invalid port '${portStr}' — must be a number between 1 and 65535`) + process.exit(1) + } const verbose = flag('--verbose') if (flag('--uninstall')) { diff --git a/src/install.mjs b/src/install.mjs index 5c78971..8ebc9f4 100644 --- a/src/install.mjs +++ b/src/install.mjs @@ -169,7 +169,8 @@ export async function runInstall({ yes = false, silent = false } = {}) { entry.hooks?.some(h => typeof h.command === 'string' && h.command.includes('stop-broom')) ) if (!alreadyWired) { - settings.hooks.Stop.push({ hooks: [{ type: 'command', command: hp }] }) + const commandStr = process.platform === 'win32' ? `node "${hp}"` : hp + settings.hooks.Stop.push({ hooks: [{ type: 'command', command: commandStr }] }) mkdirSync(dirname(sp2), { recursive: true }) writeFileSync(sp2, JSON.stringify(settings, null, 2) + '\n', 'utf8') done.push(`updated ${sp2}`) diff --git a/src/proxy.mjs b/src/proxy.mjs index 0a09d42..a169914 100644 --- a/src/proxy.mjs +++ b/src/proxy.mjs @@ -94,6 +94,7 @@ function redactAnthropicSSE(lines, allowlist, vault) { const raw = line.slice(5).trim() if (!raw || raw === '[DONE]') continue let ev; try { ev = JSON.parse(raw) } catch { continue } + if (!ev || typeof ev !== 'object') continue if (ev.type === 'content_block_delta' && ev.delta?.type === 'text_delta') { const i = ev.index ?? 0 accumulated.set(i, (accumulated.get(i) ?? '') + (ev.delta.text ?? '')) @@ -134,6 +135,7 @@ function redactOpenAISSE(lines, allowlist, vault) { const raw = line.slice(5).trim() if (!raw || raw === '[DONE]') continue let ev; try { ev = JSON.parse(raw) } catch { continue } + if (!ev || typeof ev !== 'object') continue for (const c of ev.choices ?? []) { if (typeof c.delta?.content === 'string') { const i = c.index ?? 0 @@ -174,7 +176,8 @@ function forwardRequest(upstreamHost, req, body) { return new Promise((resolve, reject) => { const headers = { ...req.headers, host: upstreamHost } headers['content-length'] = String(body.length) - delete headers['accept-encoding'] // need plaintext to scan + delete headers['transfer-encoding'] // we send a single buffer, not chunked + delete headers['accept-encoding'] // need plaintext to scan const upReq = tlsReq( { host: upstreamHost, port: 443, path: req.url, method: req.method, headers }, @@ -246,7 +249,9 @@ async function handleRequest(req, res, allowlist, vault, verbose) { } // ── Write response ──────────────────────────────────────────────────────── - const outHeaders = { ...upRes.headers, 'content-length': String(responseBody.length) } + const outHeaders = { ...upRes.headers } + delete outHeaders['transfer-encoding'] // we send a single buffer, not chunked + outHeaders['content-length'] = String(responseBody.length) res.writeHead(upRes.statusCode ?? 200, outHeaders) res.end(responseBody) } @@ -280,10 +285,12 @@ export function installProxyEnv(port = 7777) { for (const name of PROFILE_CANDIDATES) { const file = join(home, name) if (!existsSync(file)) continue - const contents = readFileSync(file, 'utf8') - if (contents.includes(BROOM_MARKER)) continue // already installed - appendFileSync(file, ENV_BLOCK(port), 'utf8') - updated.push(file) + try { + const contents = readFileSync(file, 'utf8') + if (contents.includes(BROOM_MARKER)) continue // already installed + appendFileSync(file, ENV_BLOCK(port), 'utf8') + updated.push(file) + } catch { /* permission error or race — skip silently */ } } return updated @@ -392,9 +399,10 @@ export function uninstallDaemon() { } if (process.platform === 'linux') { + const servicePath = join(home, '.config', 'systemd', 'user', 'broom-proxy.service') + if (!existsSync(servicePath)) return false try { execFileSync('systemctl', ['--user', 'stop', 'broom-proxy'], { stdio: 'ignore' }) } catch {} try { execFileSync('systemctl', ['--user', 'disable', 'broom-proxy'], { stdio: 'ignore' }) } catch {} - const servicePath = join(home, '.config', 'systemd', 'user', 'broom-proxy.service') try { unlinkSync(servicePath) } catch {} try { execFileSync('systemctl', ['--user', 'daemon-reload'], { stdio: 'ignore' }) } catch {} return true diff --git a/src/report.mjs b/src/report.mjs index 06de6bb..01cb7da 100644 --- a/src/report.mjs +++ b/src/report.mjs @@ -98,18 +98,15 @@ export function printReport(results, opts = {}) { const bySev = Object.fromEntries(SEVERITY_ORDER.map(s => [s, []])) for (const f of findings) bySev[f.severity].push(f) + let targetText = null + try { targetText = target.read() } catch { /* non-blocking */ } + for (const sev of SEVERITY_ORDER) { for (const f of bySev[sev]) { - const tag = color(sev, `[${sev.padEnd(8)}]`) - const rule = f.ruleId.padEnd(26) - const masked = maskSecret(f.secret) - // line number requires the raw text — pass it through target.read() lazily - // We cache it here since findings already came from scanning this target. - let lineInfo = '' - try { - const text = target.read() - lineInfo = dim(` line ${lineNumber(text, f.start)}`) - } catch { /* non-blocking */ } + const tag = color(sev, `[${sev.padEnd(8)}]`) + const rule = f.ruleId.padEnd(26) + const masked = maskSecret(f.secret) + const lineInfo = targetText !== null ? dim(` line ${lineNumber(targetText, f.start)}`) : '' console.log(` ${tag} ${dim(rule)} ${masked}${lineInfo}`) } } diff --git a/src/sources/cursor.mjs b/src/sources/cursor.mjs index d056722..1657f28 100644 --- a/src/sources/cursor.mjs +++ b/src/sources/cursor.mjs @@ -69,7 +69,7 @@ function* walkVscdb(dir) { function readAiRows(dbPath) { let db try { - db = new DatabaseSync(dbPath, { open: true }) + db = new DatabaseSync(dbPath, { open: true, readOnly: true }) } catch { return [] } @@ -138,7 +138,7 @@ export function discoverTargets() { read() { // Re-open DB on each read so we always get the current value - const conn = new DatabaseSync(db, { open: true }) + const conn = new DatabaseSync(db, { open: true, readOnly: true }) try { const row = conn.prepare(`SELECT value FROM [${tbl}] WHERE key = ?`).get(k) if (!row) return '' @@ -149,7 +149,7 @@ export function discoverTargets() { }, write(text) { - const conn = new DatabaseSync(db, { open: true }) + const conn = new DatabaseSync(db, { open: true, readOnly: true }) try { conn.prepare(`UPDATE [${tbl}] SET value = ? WHERE key = ?`).run(text, k) } finally { From b4a2c889e4612761984a9c131b952b9d1d3585a1 Mon Sep 17 00:00:00 2001 From: digitaldrreamer Date: Thu, 2 Jul 2026 01:08:21 +0100 Subject: [PATCH 12/13] fix: repair proxy startup + Cursor writes; apply open review items; add tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two breaking bugs (neither fixed by the earlier review pass): - `broom proxy` started the server then fell through into the scan/clean path and exited immediately — the proxy never stayed up. Block top-level evaluation after the server binds so the process stays alive. - Cursor `write()` opened the SQLite DB readOnly:true then ran UPDATE, so `clean --apply` always threw "readonly database" and redacted nothing. Open the write handle read-write. Apply the still-open items from Gemini's second review: - backup: strip Windows drive letters when flattening backup paths - bin: wrap startProxy in try/catch for a clean error on port-in-use - detector: optional-chain match.indices - report: fall back to USERPROFILE when HOME is unset (Windows) - proxy: XML-escape launchd plist values; quote systemd ExecStart paths Add a zero-dependency node --test suite (npm test): detector coverage + decoys, redaction round-trip/idempotency, backup integrity + drive-letter flattening, allowlist suppression, a Cursor read→redact→write regression, a scan/clean/proxy CLI e2e (guards the proxy fall-through), and a no-network invariant for the scanner modules. Co-Authored-By: Claude Opus 4.8 (1M context) --- bin/broom.mjs | 14 +++++- package.json | 3 ++ src/backup.mjs | 11 +++-- src/detector.mjs | 2 +- src/proxy.mjs | 15 +++--- src/report.mjs | 3 +- src/sources/cursor.mjs | 4 +- test/allowlist.test.mjs | 35 ++++++++++++++ test/backup.test.mjs | 49 +++++++++++++++++++ test/cli.test.mjs | 102 +++++++++++++++++++++++++++++++++++++++ test/cursor.test.mjs | 64 ++++++++++++++++++++++++ test/detector.test.mjs | 100 ++++++++++++++++++++++++++++++++++++++ test/no-network.test.mjs | 45 +++++++++++++++++ test/redactor.test.mjs | 51 ++++++++++++++++++++ 14 files changed, 483 insertions(+), 15 deletions(-) create mode 100644 test/allowlist.test.mjs create mode 100644 test/backup.test.mjs create mode 100644 test/cli.test.mjs create mode 100644 test/cursor.test.mjs create mode 100644 test/detector.test.mjs create mode 100644 test/no-network.test.mjs create mode 100644 test/redactor.test.mjs diff --git a/bin/broom.mjs b/bin/broom.mjs index 109be6e..4c64f95 100755 --- a/bin/broom.mjs +++ b/bin/broom.mjs @@ -110,7 +110,13 @@ if (command === 'proxy') { process.exit(0) } - const server = await startProxy({ port, verbose, allowlistFile: option('--allowlist') }) + let server + try { + server = await startProxy({ port, verbose, allowlistFile: option('--allowlist') }) + } catch (err) { + console.error(`broom: failed to start proxy on port ${port}: ${err.message}`) + process.exit(1) + } console.log(` broom proxy listening on http://127.0.0.1:${port} @@ -129,7 +135,11 @@ if (command === 'proxy') { process.on('SIGINT', () => { server.close(); process.exit(0) }) process.on('SIGTERM', () => { server.close(); process.exit(0) }) - // Server holds the event loop open — no further code needed + + // The proxy owns the process from here. Block the module's top-level + // evaluation forever so execution never falls through into the scan/clean + // logic below (which would run a scan and exit, killing the proxy). + await new Promise(() => {}) } // ── Shared options ──────────────────────────────────────────────────────────── diff --git a/package.json b/package.json index 3d8abad..aa38a46 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,9 @@ "PLAN.md", "LICENSE" ], + "scripts": { + "test": "node --test" + }, "engines": { "node": ">=22.5.0" }, diff --git a/src/backup.mjs b/src/backup.mjs index dc2ca1d..8f70630 100644 --- a/src/backup.mjs +++ b/src/backup.mjs @@ -12,7 +12,7 @@ import { copyFileSync, mkdirSync, writeFileSync } from 'node:fs' import { homedir } from 'node:os' -import { dirname, join, resolve, sep } from 'node:path' +import { dirname, join, resolve } from 'node:path' /** * A single backup session. Create one per `broom clean --apply` run. @@ -88,9 +88,12 @@ export class BackupSession { /** @param {string} abs Absolute original path */ _destPath(abs) { - // Strip leading separator so we can join under the backup dir. - // /home/alice/.claude/... → home/alice/.claude/... - const relative = abs.startsWith(sep) ? abs.slice(sep.length) : abs + // Strip the leading separator (POSIX) or drive letter + separators + // (Windows) so we can join under the backup dir. A retained `C:` would + // otherwise produce an invalid mid-path colon and fail with EINVAL/ENOENT. + // /home/alice/.claude/... → home/alice/.claude/... + // C:\Users\alice\... → Users\alice\... + const relative = abs.replace(/^[a-zA-Z]:/, '').replace(/^[\\/]+/, '') return join(this.dir, relative) } } diff --git a/src/detector.mjs b/src/detector.mjs index e1c2b4c..75b0afa 100644 --- a/src/detector.mjs +++ b/src/detector.mjs @@ -31,7 +31,7 @@ export function scanText(text, rules, extras = []) { const secret = match[grp] ?? match[0] if (!secret) continue - const indices = match.indices[grp] ?? match.indices[0] + const indices = match.indices?.[grp] ?? match.indices?.[0] if (!indices) continue const [start, end] = indices diff --git a/src/proxy.mjs b/src/proxy.mjs index a169914..a779d9e 100644 --- a/src/proxy.mjs +++ b/src/proxy.mjs @@ -300,6 +300,9 @@ export function installProxyEnv(port = 7777) { function launchdPlist(nodeBin, broomBin, port) { const logFile = join(homedir(), '.broom', 'proxy.log') + // Escape XML metacharacters — a path like /Users/bob&alice would otherwise + // produce a malformed plist that launchd refuses to load. + const esc = (s) => String(s).replace(/[<>&]/g, m => ({ '<': '<', '>': '>', '&': '&' }[m])) return ` @@ -309,20 +312,20 @@ function launchdPlist(nodeBin, broomBin, port) { com.broomsticks.proxy ProgramArguments - ${nodeBin} - ${broomBin} + ${esc(nodeBin)} + ${esc(broomBin)} proxy --port - ${port} + ${esc(port)} RunAtLoad KeepAlive StandardOutPath - ${logFile} + ${esc(logFile)} StandardErrorPath - ${logFile} + ${esc(logFile)} ` @@ -334,7 +337,7 @@ Description=broomsticks secret-redacting proxy After=network.target [Service] -ExecStart=${nodeBin} ${broomBin} proxy --port ${port} +ExecStart="${nodeBin}" "${broomBin}" proxy --port ${port} Restart=on-failure RestartSec=5 diff --git a/src/report.mjs b/src/report.mjs index 01cb7da..8ba3196 100644 --- a/src/report.mjs +++ b/src/report.mjs @@ -91,7 +91,8 @@ export function printReport(results, opts = {}) { const dirty = results.filter(r => r.findings.length > 0) for (const { target, findings } of dirty) { - const shortLabel = target.label.replace(process.env.HOME ?? '', '~') + const home = process.env.HOME || process.env.USERPROFILE + const shortLabel = home ? target.label.replace(home, '~') : target.label console.log(` ${bold(target.source)} ${shortLabel}`) // Group by severity for ordered display diff --git a/src/sources/cursor.mjs b/src/sources/cursor.mjs index 1657f28..70b1873 100644 --- a/src/sources/cursor.mjs +++ b/src/sources/cursor.mjs @@ -149,7 +149,9 @@ export function discoverTargets() { }, write(text) { - const conn = new DatabaseSync(db, { open: true, readOnly: true }) + // Read-write handle — an UPDATE against a readOnly connection throws + // "attempt to write a readonly database". + const conn = new DatabaseSync(db, { open: true, readOnly: false }) try { conn.prepare(`UPDATE [${tbl}] SET value = ? WHERE key = ?`).run(text, k) } finally { diff --git a/test/allowlist.test.mjs b/test/allowlist.test.mjs new file mode 100644 index 0000000..45498a0 --- /dev/null +++ b/test/allowlist.test.mjs @@ -0,0 +1,35 @@ +// Allowlist: parsing, built-in demo-secret suppression, regex + literal entries. + +import { test } from 'node:test' +import assert from 'node:assert/strict' + +import { + parseAllowlist, loadAllowlist, isAllowlisted, DEMO_SECRETS, +} from '../src/allowlist.mjs' + +test('parseAllowlist separates literals, regexes, and skips comments', () => { + const { literals, patterns } = parseAllowlist([ + '# a comment', + '', + 'literal-value', + '/sk_test_[A-Za-z0-9]{4,}/', + ]) + assert.ok(literals.has('literal-value')) + assert.equal(literals.size, 1) + assert.equal(patterns.length, 1) +}) + +test('built-in demo secrets are suppressed', () => { + const allow = loadAllowlist('/nonexistent/allowlist.txt') // built-ins only + for (const demo of DEMO_SECRETS) { + assert.ok(isAllowlisted(demo, allow), `demo secret should be allowlisted: ${demo}`) + } + assert.ok(!isAllowlisted('ghp_1234567890abcdefABCDEF1234567890abcd', allow)) +}) + +test('regex entry matches; global-flag regex does not desync via lastIndex', () => { + const allow = parseAllowlist(['/sk_test_[A-Za-z0-9]{4,}/g']) + assert.ok(isAllowlisted('sk_test_abcd1234', allow)) + // Second call must still match — isAllowlisted resets lastIndex. + assert.ok(isAllowlisted('sk_test_abcd1234', allow)) +}) diff --git a/test/backup.test.mjs b/test/backup.test.mjs new file mode 100644 index 0000000..d07f76f --- /dev/null +++ b/test/backup.test.mjs @@ -0,0 +1,49 @@ +// Backup integrity: a backup exists and byte-matches the original before any +// write, the manifest records entries, and paths flatten safely (incl. Windows +// drive letters — regression for the mid-path colon bug). + +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { mkdtempSync, writeFileSync, readFileSync, existsSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { BackupSession } from '../src/backup.mjs' + +test('backup copies the file byte-for-byte and records a manifest', () => { + const tmp = mkdtempSync(join(tmpdir(), 'broom-backup-')) + try { + const original = join(tmp, 'transcript.jsonl') + const content = '{"text":"secret ghp_xxx"}\n' + writeFileSync(original, content, 'utf8') + + const backupDir = join(tmp, 'backups') + const session = new BackupSession(backupDir) + const dest = session.backup(original) + + assert.ok(existsSync(dest), 'backup file must exist') + assert.equal(readFileSync(dest, 'utf8'), content, 'backup must byte-match original') + + // Second call for the same file is a no-op (idempotent per session). + const dest2 = session.backup(original) + assert.equal(dest, dest2) + + session.writeManifest([ + { target: { source: 'claude-code', label: original }, applied: 1 }, + ]) + const manifest = JSON.parse(readFileSync(join(backupDir, 'manifest.json'), 'utf8')) + assert.equal(manifest.files.length, 1) + assert.equal(manifest.files[0].original, original) + assert.equal(manifest.redactionSummary[0].applied, 1) + } finally { + rmSync(tmp, { recursive: true, force: true }) + } +}) + +test('_destPath strips leading separators and Windows drive letters', () => { + const s = new BackupSession('/backups/ts') + assert.equal(s._destPath('/home/alice/x.jsonl'), join('/backups/ts', 'home/alice/x.jsonl')) + // No mid-path colon should survive from a C:\ style absolute path. + const win = s._destPath('C:\\Users\\alice\\x.jsonl') + assert.ok(!win.slice(2).includes(':'), 'no drive-letter colon should remain in the joined path') +}) diff --git a/test/cli.test.mjs b/test/cli.test.mjs new file mode 100644 index 0000000..b2797a5 --- /dev/null +++ b/test/cli.test.mjs @@ -0,0 +1,102 @@ +// End-to-end CLI tests: scan/clean lifecycle over a real transcript, and a +// regression guard that `broom proxy` keeps running instead of falling through +// into the scan path and exiting immediately. + +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { spawnSync, spawn } from 'node:child_process' +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, readdirSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { setTimeout as sleep } from 'node:timers/promises' + +const BIN = fileURLToPath(new URL('../bin/broom.mjs', import.meta.url)) +const SECRET = 'ghp_1234567890abcdefABCDEF1234567890abcd' + +function makeHome() { + const home = mkdtempSync(join(tmpdir(), 'broom-cli-')) + const dir = join(home, '.claude', 'projects', 'demo') + mkdirSync(dir, { recursive: true }) + const file = join(dir, 'session.jsonl') + writeFileSync(file, JSON.stringify({ role: 'user', text: `key ${SECRET}` }) + '\n', 'utf8') + return { home, file } +} + +function run(args, home) { + return spawnSync(process.execPath, [BIN, ...args], { + env: { ...process.env, HOME: home, USERPROFILE: home }, + encoding: 'utf8', + }) +} + +test('scan → clean --apply → re-scan lifecycle', () => { + const { home, file } = makeHome() + try { + // scan: finds the secret, JSON output, non-zero exit + const scan = run(['scan', '--json', '--source', 'claude-code'], home) + assert.equal(scan.status, 1, 'scan should exit 1 when secrets are found') + const report = JSON.parse(scan.stdout) + assert.equal(report.totalFindings, 1) + assert.equal(report.files[0].findings[0].ruleId, 'github-pat') + // JSON output must never contain the raw secret — only the hash. + assert.ok(!scan.stdout.includes(SECRET), 'raw secret must not appear in JSON report') + + // clean --apply: redacts in place, takes a backup + const clean = run(['clean', '--apply', '--source', 'claude-code'], home) + assert.equal(clean.status, 1) // findings existed → exit 1 unless --no-fail + const redacted = readFileSync(file, 'utf8') + assert.ok(!redacted.includes(SECRET), 'file must no longer contain the secret') + assert.ok(redacted.includes('«BROOM:github-pat:')) + JSON.parse(redacted.trim()) // still valid JSONL + + const backupsRoot = join(home, '.broom', 'backups') + assert.ok(existsSync(backupsRoot), 'backup directory should exist') + const stamps = readdirSync(backupsRoot) + assert.ok(stamps.length >= 1) + assert.ok(existsSync(join(backupsRoot, stamps[0], 'manifest.json'))) + + // re-scan: clean now, zero exit + const rescan = run(['scan', '--json', '--source', 'claude-code'], home) + assert.equal(rescan.status, 0) + assert.equal(JSON.parse(rescan.stdout).totalFindings, 0) + } finally { + rmSync(home, { recursive: true, force: true }) + } +}) + +test('invalid --port is rejected', () => { + const { home } = makeHome() + try { + const res = run(['proxy', '--port', 'not-a-number'], home) + assert.equal(res.status, 1) + assert.match(res.stderr + res.stdout, /invalid port/i) + } finally { + rmSync(home, { recursive: true, force: true }) + } +}) + +test('broom proxy keeps running (does not fall through to scan and exit)', async () => { + const { home } = makeHome() + const port = 39117 + const child = spawn(process.execPath, [BIN, 'proxy', '--port', String(port)], { + env: { ...process.env, HOME: home, USERPROFILE: home }, + encoding: 'utf8', + }) + let stdout = '' + child.stdout.on('data', d => { stdout += d }) + + try { + await sleep(800) + assert.equal(child.exitCode, null, 'proxy must still be running, not exited') + assert.equal(child.signalCode, null) + assert.match(stdout, /listening on/i) + // The scan report banner must NOT appear — that would mean it fell through. + assert.doesNotMatch(stdout, /No secrets found|secrets? found across/i) + } finally { + child.kill('SIGTERM') + await sleep(50) + if (child.exitCode === null) child.kill('SIGKILL') + rmSync(home, { recursive: true, force: true }) + } +}) diff --git a/test/cursor.test.mjs b/test/cursor.test.mjs new file mode 100644 index 0000000..cd54e92 --- /dev/null +++ b/test/cursor.test.mjs @@ -0,0 +1,64 @@ +// Cursor adapter round-trip — regression guard for the read-only write bug: +// write() must persist a redacted value back into the SQLite row (previously it +// opened the DB readOnly:true and every UPDATE threw "readonly database"). + +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { mkdtempSync, mkdirSync, rmSync } from 'node:fs' +import { tmpdir, homedir } from 'node:os' +import { join } from 'node:path' +import { DatabaseSync } from 'node:sqlite' + +import { cursorUserRoot, discoverTargets } from '../src/sources/cursor.mjs' +import { RULES } from '../src/rules.mjs' +import { scanText } from '../src/detector.mjs' +import { redactText } from '../src/redactor.mjs' + +const SECRET = 'ghp_1234567890abcdefABCDEF1234567890abcd' + +test('cursor target read → redact → write persists the redaction', (t) => { + const home = mkdtempSync(join(tmpdir(), 'broom-cursor-')) + const prevHome = process.env.HOME + process.env.HOME = home + + try { + const root = cursorUserRoot() + // Only meaningful if os.homedir() honors our $HOME override on this platform. + if (!root.startsWith(home) && !root.startsWith(homedir())) { + return t.skip('cursorUserRoot() not redirectable via $HOME on this platform') + } + // If homedir() ignored $HOME entirely, bail rather than touch a real profile. + if (!root.startsWith(home)) return t.skip('os.homedir() does not honor $HOME here') + + const storage = join(root, 'globalStorage') + mkdirSync(storage, { recursive: true }) + const dbPath = join(storage, 'state.vscdb') + + const db = new DatabaseSync(dbPath) + db.exec('CREATE TABLE ItemTable (key TEXT PRIMARY KEY, value BLOB)') + db.prepare('INSERT INTO ItemTable (key, value) VALUES (?, ?)') + .run('aichat.conversations', `history: ${SECRET} end`) + db.close() + + const targets = discoverTargets() + assert.equal(targets.length, 1, 'exactly one AI row should be discovered') + const target = targets[0] + assert.equal(target.source, 'cursor') + + const text = target.read() + assert.ok(text.includes(SECRET)) + + const { text: redacted, applied } = redactText(text, scanText(text, RULES)) + assert.equal(applied, 1) + + target.write(redacted) // must not throw (readonly bug) and must persist + + const after = target.read() + assert.ok(!after.includes(SECRET), 'secret must be gone from the DB row') + assert.ok(after.includes('«BROOM:github-pat:'), 'placeholder must be persisted') + } finally { + if (prevHome === undefined) delete process.env.HOME + else process.env.HOME = prevHome + rmSync(home, { recursive: true, force: true }) + } +}) diff --git a/test/detector.test.mjs b/test/detector.test.mjs new file mode 100644 index 0000000..440a6a1 --- /dev/null +++ b/test/detector.test.mjs @@ -0,0 +1,100 @@ +// Detector + ruleset tests: planted secrets of each shape must be found; +// decoys must not match. Runs on the built-in node:test runner (no deps). + +import { test } from 'node:test' +import assert from 'node:assert/strict' + +import { RULES, shannonEntropy } from '../src/rules.mjs' +import { scanText } from '../src/detector.mjs' + +/** Return the set of ruleIds scanText finds in `text`. */ +function ruleIds(text) { + return new Set(scanText(text, RULES).map(f => f.ruleId)) +} + +// One planted sample per rule we want to guarantee coverage for. Each value is +// synthetic (not a live credential) but shaped to match the corresponding rule. +const SAMPLES = { + 'private-key': + '-----BEGIN RSA PRIVATE KEY-----\nMIIEpAIBAAKCAQEArandombase64content1234567890\n-----END RSA PRIVATE KEY-----', + 'aws-access-key': 'AKIAIOSFODNN7EXAMPLE', + 'anthropic-key': 'sk-ant-api03-' + 'x'.repeat(90) + 'AA', + 'openai-key': 'sk-proj-' + 'a'.repeat(58) + 'T3BlbkFJ' + 'b'.repeat(58), + 'github-pat': 'ghp_1234567890abcdefABCDEF1234567890abcd', + 'github-fine-grained-pat': 'github_pat_' + 'A'.repeat(82), + 'google-api-key': 'AIzaSyD-9tSrke72PouQMnMX-a7eZSW0jkFMBWY', + // Assembled from parts so the literal token text never appears in-file + // (avoids tripping upstream secret scanners on a synthetic fixture). + 'slack-bot-token': ['xoxb', '2494792170', '2503138584', 'aZ9xK2mQ7wL4pR8vT1nY6bC3'].join('-'), + 'jwt': + 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U', + 'db-url': 'postgres://admin:s3cr3tP4ss@db.example.com:5432/app', + 'generic-secret': 'api_key = "aZ9xK2mQ7wL4pR8vT1nY6bC3dE5fG0hJ"', +} + +for (const [ruleId, sample] of Object.entries(SAMPLES)) { + test(`detects ${ruleId}`, () => { + assert.ok( + ruleIds(sample).has(ruleId), + `expected ${ruleId} in findings for: ${sample.slice(0, 40)}…`, + ) + }) +} + +test('every declared rule has a sample in the coverage table', () => { + // Guards against silently adding a rule with no regression sample. + const covered = new Set(Object.keys(SAMPLES)) + // These rules are intentionally exercised elsewhere or are near-duplicates + // of a covered rule; list them so the assertion stays honest. + const knownUncovered = new Set([ + 'aws-secret-key', 'anthropic-admin-key', 'openai-key-legacy', + 'huggingface-token', 'huggingface-org-token', 'github-oauth', + 'github-app-token', 'github-refresh-token', 'stripe-key', + 'slack-user-token', 'slack-app-token', 'slack-webhook', + ]) + for (const rule of RULES) { + assert.ok( + covered.has(rule.id) || knownUncovered.has(rule.id), + `rule '${rule.id}' has no sample and is not in knownUncovered`, + ) + } +}) + +test('decoys do not match any rule', () => { + const decoys = [ + 'The quick brown fox jumps over the lazy dog.', + 'password = "aaaaaaaaaaaaaaaa"', // 16 chars, entropy 0 → below gate + 'See commit 1234567890abcdef1234567890abcdef12345678 for details', // bare sha + 'Connect to https://example.com/path?q=1', // url, no inline credentials + 'const timeout = 30000', + ] + for (const d of decoys) { + assert.deepEqual([...ruleIds(d)], [], `unexpected match in decoy: ${d}`) + } +}) + +test('overlapping matches resolve to a single non-nested finding', () => { + // A db-url that also contains an assignment-shaped substring should yield + // exactly one finding covering the URL, not two overlapping spans. + const text = 'DATABASE_URL=postgres://admin:s3cr3tP4ssword@db.host:5432/app' + const findings = scanText(text, RULES) + for (let i = 0; i < findings.length; i++) { + for (let j = i + 1; j < findings.length; j++) { + const a = findings[i], b = findings[j] + assert.ok(a.end <= b.start || b.end <= a.start, 'findings must not overlap') + } + } +}) + +test('--extra literal and /regex/ entries are honored', () => { + const text = 'internal token WIDGET-7712 and marker ZZZ-alpha' + const ids = new Set(scanText(text, RULES, ['WIDGET-7712', '/ZZZ-[a-z]+/']).map(f => f.ruleId)) + assert.ok(ids.has('extra-0')) + assert.ok(ids.has('extra-1')) +}) + +test('shannonEntropy: uniform vs repeated', () => { + assert.equal(shannonEntropy(''), 0) + assert.equal(shannonEntropy('aaaa'), 0) + assert.ok(shannonEntropy('abcd') > 1.9) // 4 distinct chars → 2 bits +}) diff --git a/test/no-network.test.mjs b/test/no-network.test.mjs new file mode 100644 index 0000000..e6eadaf --- /dev/null +++ b/test/no-network.test.mjs @@ -0,0 +1,45 @@ +// Supply-chain / privacy invariant: the scan+clean pipeline never makes a +// network call. We assert that no scanner module imports a networking API. +// +// The proxy (src/proxy.mjs) is the one intentional network component — it sits +// between an AI client and the upstream API — so it is explicitly exempt. Every +// other src/ module (the code that reads and rewrites your transcripts) must be +// network-free. + +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { readdirSync, readFileSync, statSync } from 'node:fs' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const SRC = join(fileURLToPath(new URL('.', import.meta.url)), '..', 'src') + +const NETWORK_IMPORT = /from\s+['"]node:(?:https?|net|tls|dgram|http2)['"]|\bfetch\s*\(/ + +// Modules allowed to touch the network. Only the proxy. +const EXEMPT = new Set(['proxy.mjs']) + +function* walk(dir) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = join(dir, entry.name) + if (entry.isDirectory()) yield* walk(full) + else if (entry.name.endsWith('.mjs')) yield full + } +} + +test('scanner modules import no networking APIs', () => { + const offenders = [] + for (const file of walk(SRC)) { + if (EXEMPT.has(file.split('/').pop())) continue + const code = readFileSync(file, 'utf8') + if (NETWORK_IMPORT.test(code)) offenders.push(file) + } + assert.deepEqual(offenders, [], `network APIs found in scanner modules: ${offenders.join(', ')}`) +}) + +test('proxy remains the only network-capable module', () => { + // Documents the exemption: if proxy.mjs ever stops importing http, revisit + // whether the exemption is still warranted. + const code = readFileSync(join(SRC, 'proxy.mjs'), 'utf8') + assert.ok(NETWORK_IMPORT.test(code), 'proxy.mjs is expected to use the network') +}) diff --git a/test/redactor.test.mjs b/test/redactor.test.mjs new file mode 100644 index 0000000..25919d0 --- /dev/null +++ b/test/redactor.test.mjs @@ -0,0 +1,51 @@ +// Redaction: correct placeholder, JSON round-trip stays valid, idempotency. + +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' + +import { RULES } from '../src/rules.mjs' +import { scanText } from '../src/detector.mjs' +import { redactText, buildPlaceholder } from '../src/redactor.mjs' + +const GH = 'ghp_1234567890abcdefABCDEF1234567890abcd' + +test('placeholder format is «BROOM::»', () => { + const sha8 = createHash('sha256').update(GH).digest('hex').slice(0, 8) + assert.equal(buildPlaceholder('github-pat', GH), `«BROOM:github-pat:${sha8}»`) +}) + +test('redactText replaces the secret and reports applied count', () => { + const text = `here is a key ${GH} in text` + const findings = scanText(text, RULES) + const { text: out, applied } = redactText(text, findings) + assert.equal(applied, 1) + assert.ok(!out.includes(GH), 'raw secret must be gone') + assert.ok(out.includes('«BROOM:github-pat:'), 'placeholder must be present') +}) + +test('multiple secrets on one line all get redacted with valid offsets', () => { + const db = 'postgres://admin:s3cr3tP4ss@db.host:5432/app' + const text = `${GH} and ${db} together` + const { text: out, applied } = redactText(text, scanText(text, RULES)) + assert.equal(applied, 2) + assert.ok(!out.includes(GH)) + assert.ok(!out.includes(db)) +}) + +test('JSONL line stays valid JSON after redaction', () => { + const line = JSON.stringify({ role: 'user', text: `my key is ${GH}` }) + const { text: out } = redactText(line, scanText(line, RULES)) + const parsed = JSON.parse(out) // throws if placeholder broke JSON + assert.ok(parsed.text.includes('«BROOM:github-pat:')) + assert.ok(!parsed.text.includes(GH)) +}) + +test('redaction is idempotent — a redacted blob has no findings', () => { + const text = `key ${GH} here` + const { text: once } = redactText(text, scanText(text, RULES)) + const second = scanText(once, RULES) + assert.deepEqual(second, [], 'placeholders must not be re-detected') + const { applied } = redactText(once, second) + assert.equal(applied, 0) +}) From d8daf849281b2f3aa834b126a69c61f13d6ecc96 Mon Sep 17 00:00:00 2001 From: digitaldrreamer Date: Thu, 2 Jul 2026 03:22:05 +0100 Subject: [PATCH 13/13] fix: address new PR review comments on test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Raise Node engine floor to >=22.13.0 (node:sqlite is unflagged as of 22.13.0; the tool imports node:sqlite so this matches actual runtime requirements) — coderabbitai on test/cursor.test.mjs - Add lower-bound assertion to the overlap test so a regression that drops all findings fails meaningfully — coderabbitai on test/detector.test.mjs Co-Authored-By: Claude Opus 4.8 (1M context) --- package.json | 2 +- test/detector.test.mjs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index aa38a46..9d975db 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "test": "node --test" }, "engines": { - "node": ">=22.5.0" + "node": ">=22.13.0" }, "keywords": [ "secrets", diff --git a/test/detector.test.mjs b/test/detector.test.mjs index 440a6a1..ba42dc5 100644 --- a/test/detector.test.mjs +++ b/test/detector.test.mjs @@ -78,6 +78,7 @@ test('overlapping matches resolve to a single non-nested finding', () => { // exactly one finding covering the URL, not two overlapping spans. const text = 'DATABASE_URL=postgres://admin:s3cr3tP4ssword@db.host:5432/app' const findings = scanText(text, RULES) + assert.ok(findings.length > 0, 'expected at least one finding for this db-url text') for (let i = 0; i < findings.length; i++) { for (let j = i + 1; j < findings.length; j++) { const a = findings[i], b = findings[j]