feat(dedup): opt-in near-duplicate observation dedup — Tier-0 auto-merge + Tier-1 candidates (#3038)

docs/public/configuration.mdx

@@ -66,6 +66,33 @@ Use [LiteLLM Gateway](configuration/litellm-gateway) when you want `CLAUDE_MEM_P
 | `CLAUDE_MEM_PYTHON_VERSION`   | `3.13`                          | Python version for chroma-mcp         |
 | `CLAUDE_CODE_PATH`            | _(auto-detect)_                 | Path to Claude Code CLI (for Windows) |
 
+### Near-Duplicate Deduplication
+
+Opt-in (off by default) dedup of observations whose titles describe the same recurring
+work but are worded slightly differently, so `content_hash` (byte-identical only) misses them.
+Two deterministic tiers:
+
+- **Tier 0 — exact-normalized-title** → safe silent auto-merge: a new observation whose
+  title matches an existing one after lowercasing / whitespace-collapse / punctuation-strip
+  is collapsed onto the existing row and its `occurrence_count` is bumped (cross-session).
+- **Tier 1 — IDF-weighted similarity** → *review-only* near-duplicate **candidates** are
+  recorded in `observation_dedup_candidates` (never auto-merged), so distinct work that
+  differs only in one rare token (`rdlp-api` vs `rdlp-plugin`) is never silently destroyed.
+
+| Setting                              | Default | Description                           |
+|--------------------------------------|---------|---------------------------------------|
+| `CLAUDE_MEM_DEDUP_ENABLED`           | `false` | Master switch. Off = byte-identical legacy behavior |
+| `CLAUDE_MEM_DEDUP_COSINE_THRESHOLD`  | `0.80`  | Min IDF-weighted cosine for a Tier-1 candidate |
+| `CLAUDE_MEM_DEDUP_IDF_VETO_DF`       | `10`    | A title token in ≤ N project rows is "discriminating" and vetoes a merge |
+| `CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS` | `2`     | Require ≥ N shared tokens before scoring (kills sparse-vector noise) |
+| `CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS`  | `10`    | Cold-start gate: skip Tier-1 below N docs/project (IDF unreliable) |
+| `CLAUDE_MEM_DEDUP_MAX_SCAN`          | `2000`  | Per-insert Tier-1 candidate-scan cap |
+| `CLAUDE_MEM_DEDUP_MAX_BACKFILL_ROWS` | `50000` | `dedup-scan` skips a project larger than this (memory safety) |
+
+After enabling on an existing database, run a one-time scan to backfill the IDF model and
+surface accumulated near-duplicates: `POST /api/dedup/scan` (worker, localhost-only).
+Review candidates via `GET /api/dedup/candidates?project=<name>`.
+
 ## Model Configuration
 
 Configure which Claude model compresses your observations (only applies when `CLAUDE_MEM_PROVIDER=claude`).

src/services/dedup/idf.ts

@@ -0,0 +1,24 @@
+/**
+ * Inverse document frequency for near-duplicate detection (#3038).
+ *
+ * IDF is what lets the dedup separate "near-identical but differ in a rare,
+ * discriminating token" (e.g. `rdlp-api` vs `rdlp-plugin`) from a true reword:
+ * a rare token carries high weight, so a difference in it dominates both the
+ * TF-IDF cosine and the IDF-veto. This is the Salton/Fellegi-Sunter insight —
+ * disagreement on a rare field is strong evidence of non-match.
+ *
+ * Pure math only; the document-frequency table is maintained by the store layer.
+ */
+
+/**
+ * Smoothed IDF: `log(1 + N / (df + 0.5))`.
+ * The `+0.5` smoothing keeps `df = 0` (token unseen in corpus) finite and maximal.
+ */
+export function idf(df: number, n: number): number {
+  return Math.log(1 + n / (df + 0.5));
+}
+
+/** Build a reusable `token -> idf` function from a df lookup and corpus size N. */
+export function buildIdfFn(dfLookup: (token: string) => number, n: number): (token: string) => number {
+  return (token: string) => idf(dfLookup(token), n);
+}

src/services/dedup/idfVeto.ts

@@ -0,0 +1,29 @@
+/**
+ * IDF-veto — the discriminating-token gate for near-duplicate detection (#3038).
+ *
+ * Two titles may be lexically near-identical yet describe DISTINCT work because
+ * they differ in one high-information token (`rdlp-api` vs `rdlp-plugin`,
+ * `ffmpeg-7.1` vs `6.1`). This is Fellegi-Sunter's "blocking key": disagreement
+ * on a rare field (low collision probability `u`) is strong evidence of NON-match.
+ *
+ * The veto fires when ANY token in the symmetric difference of the two token sets
+ * has `idf > thetaIdf` — i.e. a rare, discriminating token is present on exactly
+ * one side. A fired veto vetoes the merge regardless of cosine similarity.
+ *
+ * Known limitation (by design): when the discriminator is a COMMON token
+ * (`code` vs `security` review), its IDF is low and the veto does not fire.
+ * Lexical methods cannot separate those; that residual is the Branch-2
+ * LLM-adjudication tier's job. See tests/dedup/idf-veto.test.ts.
+ */
+export function vetoFires(
+  a: string[],
+  b: string[],
+  idfFn: (t: string) => number,
+  thetaIdf: number
+): boolean {
+  const setA = new Set(a);
+  const setB = new Set(b);
+  for (const t of setA) if (!setB.has(t) && idfFn(t) > thetaIdf) return true;
+  for (const t of setB) if (!setA.has(t) && idfFn(t) > thetaIdf) return true;
+  return false;
+}

src/services/dedup/nearDuplicate.ts

@@ -0,0 +1,76 @@
+/**
+ * Near-duplicate tier decision for #3038.
+ *
+ * Two safe tiers, both deterministic:
+ *   - Tier-0 "exact": titles equal after normalization → safe silent auto-merge.
+ *   - Tier-1 "candidate": IDF-weighted cosine >= threshold AND the IDF-veto does
+ *     NOT fire → a near-duplicate CANDIDATE (persisted for review / future
+ *     LLM adjudication), never a silent merge.
+ * Everything else → "none".
+ *
+ * Validated against a real 7,651-observation DB: simhash-over-narrative was
+ * useless (false-positive dominated); IDF-cosine + veto separates true rewords
+ * from distinct-but-similar titles that differ in a rare token.
+ */
+import { normalizeTitle, tokenizeWs } from './normalize.js';
+import { tfidfCosine } from './tfidfCosine.js';
+import { vetoFires } from './idfVeto.js';
+
+export type DedupTier = 'exact' | 'candidate' | 'none';
+export type DedupMethod = 'exact' | 'idf_cosine' | 'none';
+
+export interface ClassifyThresholds {
+  /** Minimum IDF-weighted cosine for a Tier-1 candidate. */
+  cosineThreshold: number;
+  /** A symmetric-difference token with idf above this vetoes the merge. */
+  vetoThetaIdf: number;
+  /**
+   * Minimum count of shared tokens required before computing cosine. Short titles
+   * make cosine "jumpy" — a single shared rare token can dominate it to ~1.0 even
+   * when the titles are otherwise disjoint (sparse-vector noise). Defaults to 2.
+   */
+  minSharedTokens?: number;
+}
+
+export interface PairClassification {
+  tier: DedupTier;
+  method: DedupMethod;
+  score: number;
+}
+
+/** Runtime configuration surfaced via settings (resolved to thresholds by the store). */
+export interface FuzzyDedupConfig {
+  enabled: boolean;
+  cosineThreshold: number;
+  /** A token appearing in <= idfVetoDf records is "discriminating". */
+  idfVetoDf: number;
+}
+
+export function classifyPair(
+  a: string | null | undefined,
+  b: string | null | undefined,
+  idfFn: (t: string) => number,
+  thresholds: ClassifyThresholds
+): PairClassification {
+  // Tier-0 requires a NON-EMPTY normal form: null/empty/whitespace/punctuation-only/
+  // emoji-only titles all normalize to '' and must NOT collapse into each other
+  // (that would silently auto-merge distinct observations — data loss). The Tier-1
+  // fall-through is already safe for these (empty tokens → cosine 0 → 'none').
+  const normA = normalizeTitle(a);
+  if (normA !== '' && normA === normalizeTitle(b)) {
+    return { tier: 'exact', method: 'exact', score: 1 };
+  }
+  const ta = tokenizeWs(a);
+  const tb = tokenizeWs(b);
+  // Sparse-vector noise guard: require >=N shared tokens before trusting cosine.
+  const minShared = thresholds.minSharedTokens ?? 2;
+  const setB = new Set(tb);
+  let shared = 0;
+  for (const t of new Set(ta)) if (setB.has(t)) shared++;
+  if (shared < minShared) return { tier: 'none', method: 'none', score: 0 };
+  const score = tfidfCosine(ta, tb, idfFn);
+  if (score >= thresholds.cosineThreshold && !vetoFires(ta, tb, idfFn, thresholds.vetoThetaIdf)) {
+    return { tier: 'candidate', method: 'idf_cosine', score };
+  }
+  return { tier: 'none', method: 'none', score };
+}

src/services/dedup/normalize.ts

@@ -0,0 +1,29 @@
+/**
+ * Text normalization + tokenization for near-duplicate detection (#3038).
+ *
+ * Two deliberately-different transforms:
+ *  - `normalizeTitle` strips punctuation → used for Tier-0 exact-normalized-title
+ *    matching (case/whitespace/punctuation-insensitive equality).
+ *  - `tokenizeWs` splits on whitespace ONLY (keeping `rdlp-api`, `ffmpeg-7.1.conf`,
+ *    `download.rs` as single tokens) → used for IDF weighting and the IDF-veto,
+ *    where a compound identifier must stay one rare token. Splitting it would make
+ *    `api` look common and break the discriminating-token veto.
+ */
+
+/** Lowercase, replace non-alphanumeric (Unicode-aware) with a space, collapse runs, trim. */
+export function normalizeTitle(s: string | null | undefined): string {
+  return (s ?? '')
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]/gu, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+/** Whitespace-only split + lowercase; preserves compound identifiers as single tokens. */
+export function tokenizeWs(s: string | null | undefined): string[] {
+  return (s ?? '')
+    .toLowerCase()
+    .trim()
+    .split(/\s+/)
+    .filter(Boolean);
+}

src/services/dedup/tfidfCosine.ts

@@ -0,0 +1,34 @@
+/**
+ * IDF-weighted TF-IDF cosine similarity for near-duplicate detection (#3038).
+ *
+ * Each title is a bag of tokens weighted by IDF (tf=1 — titles are short, ~5-12
+ * tokens, so term repetition is negligible). Cosine of the two IDF vectors.
+ *
+ * Why this and not plain string/token similarity: a rare discriminating token
+ * present on one side only (`rdlp-api` vs `rdlp-plugin`, `ffmpeg-7.1` vs `6.1`)
+ * has a large IDF, so it inflates that side's norm while contributing nothing to
+ * the dot product — pulling cosine down. Plain Levenshtein/token-sort sees high
+ * character overlap and wrongly scores these ~0.9.
+ */
+
+function idfVector(tokens: string[], idfFn: (t: string) => number): Map<string, number> {
+  const v = new Map<string, number>();
+  for (const t of new Set(tokens)) v.set(t, idfFn(t));
+  return v;
+}
+
+export function tfidfCosine(a: string[], b: string[], idfFn: (t: string) => number): number {
+  const va = idfVector(a, idfFn);
+  const vb = idfVector(b, idfFn);
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (const [t, w] of va) {
+    normA += w * w;
+    const wb = vb.get(t);
+    if (wb !== undefined) dot += w * wb;
+  }
+  for (const [, w] of vb) normB += w * w;
+  if (normA === 0 || normB === 0) return 0;
+  return dot / Math.sqrt(normA * normB);
+}