From db542c3ca215983961bd6c31f30c51b663da8cb3 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 00:38:06 +0200
Subject: [PATCH 01/19] feat(dedup): add normalizeTitle + whitespace tokenizer
 for #3038

Tier-0 exact-normalized-title normalization (punctuation/case/ws-insensitive)
and a whitespace-only tokenizer that preserves compound identifiers
(rdlp-api, ffmpeg-7.1.conf) as single tokens for IDF-veto correctness.
---
 src/services/dedup/normalize.ts | 29 +++++++++++++++++++
 tests/dedup/normalize.test.ts   | 50 +++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 src/services/dedup/normalize.ts
 create mode 100644 tests/dedup/normalize.test.ts

diff --git a/src/services/dedup/normalize.ts b/src/services/dedup/normalize.ts
new file mode 100644
index 0000000000..bc42e5fa98
--- /dev/null
+++ b/src/services/dedup/normalize.ts
@@ -0,0 +1,29 @@
+/**
+ * Text normalization + tokenization for near-duplicate detection (#3038).
+ *
+ * Two deliberately-different transforms:
+ *  - `normalizeTitle` strips punctuation → used for Tier-0 exact-normalized-title
+ *    matching (case/whitespace/punctuation-insensitive equality).
+ *  - `tokenizeWs` splits on whitespace ONLY (keeping `rdlp-api`, `ffmpeg-7.1.conf`,
+ *    `download.rs` as single tokens) → used for IDF weighting and the IDF-veto,
+ *    where a compound identifier must stay one rare token. Splitting it would make
+ *    `api` look common and break the discriminating-token veto.
+ */
+
+/** Lowercase, replace non-alphanumeric (Unicode-aware) with a space, collapse runs, trim. */
+export function normalizeTitle(s: string | null | undefined): string {
+  return (s ?? '')
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]/gu, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+/** Whitespace-only split + lowercase; preserves compound identifiers as single tokens. */
+export function tokenizeWs(s: string | null | undefined): string[] {
+  return (s ?? '')
+    .toLowerCase()
+    .trim()
+    .split(/\s+/)
+    .filter(Boolean);
+}
diff --git a/tests/dedup/normalize.test.ts b/tests/dedup/normalize.test.ts
new file mode 100644
index 0000000000..8169cbed37
--- /dev/null
+++ b/tests/dedup/normalize.test.ts
@@ -0,0 +1,50 @@
+import { describe, it, expect } from 'bun:test';
+import { normalizeTitle, tokenizeWs } from '../../src/services/dedup/normalize.js';
+
+describe('normalizeTitle', () => {
+  it('lowercases, strips punctuation, and collapses whitespace', () => {
+    expect(normalizeTitle('Fixed   the   BUG!!!')).toBe('fixed the bug');
+  });
+
+  it('treats punctuation-separated variants as equal (for Tier-0 exact match)', () => {
+    expect(normalizeTitle('On-Demand Checkpoint.')).toBe(normalizeTitle('on demand checkpoint'));
+  });
+
+  it('is idempotent', () => {
+    const once = normalizeTitle('Hardened On-Demand Checkpoint!');
+    expect(normalizeTitle(once)).toBe(once);
+  });
+
+  it('distinguishes genuinely different titles (negative)', () => {
+    expect(normalizeTitle('Added the relevance_count column'))
+      .not.toBe(normalizeTitle('Removed the relevance_count column'));
+  });
+
+  it('handles null/empty', () => {
+    expect(normalizeTitle(null)).toBe('');
+    expect(normalizeTitle('')).toBe('');
+    expect(normalizeTitle('   ')).toBe('');
+  });
+});
+
+describe('tokenizeWs', () => {
+  it('splits on whitespace only and lowercases', () => {
+    expect(tokenizeWs('Added rdlp-redact to rdlp-api')).toEqual(['added', 'rdlp-redact', 'to', 'rdlp-api']);
+  });
+
+  it('PRESERVES compound identifiers as single tokens (veto correctness)', () => {
+    // If these split, idf("api") would be common and the IDF-veto would fail to fire.
+    expect(tokenizeWs('Pinned versions in ffmpeg-7.1.conf')).toContain('ffmpeg-7.1.conf');
+    expect(tokenizeWs('clippy violation in download.rs')).toContain('download.rs');
+  });
+
+  it('keeps version/identifier tokens distinct (negative)', () => {
+    expect(tokenizeWs('ffmpeg-7.1.conf')).not.toEqual(tokenizeWs('ffmpeg-6.1.conf'));
+  });
+
+  it('returns [] for empty/whitespace/null', () => {
+    expect(tokenizeWs('')).toEqual([]);
+    expect(tokenizeWs('   ')).toEqual([]);
+    expect(tokenizeWs(null)).toEqual([]);
+  });
+});

From fd0df07d72810a9d0b432113d03711b8340f07a2 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 00:39:17 +0200
Subject: [PATCH 02/19] feat(dedup): add smoothed IDF math for #3038

idf(df,N)=log(1+N/(df+0.5)) + buildIdfFn; rare tokens weigh high so a
difference in a discriminating token dominates cosine + veto.
---
 src/services/dedup/idf.ts | 24 ++++++++++++++++++++++++
 tests/dedup/idf.test.ts   | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 57 insertions(+)
 create mode 100644 src/services/dedup/idf.ts
 create mode 100644 tests/dedup/idf.test.ts

diff --git a/src/services/dedup/idf.ts b/src/services/dedup/idf.ts
new file mode 100644
index 0000000000..480627acc2
--- /dev/null
+++ b/src/services/dedup/idf.ts
@@ -0,0 +1,24 @@
+/**
+ * Inverse document frequency for near-duplicate detection (#3038).
+ *
+ * IDF is what lets the dedup separate "near-identical but differ in a rare,
+ * discriminating token" (e.g. `rdlp-api` vs `rdlp-plugin`) from a true reword:
+ * a rare token carries high weight, so a difference in it dominates both the
+ * TF-IDF cosine and the IDF-veto. This is the Salton/Fellegi-Sunter insight —
+ * disagreement on a rare field is strong evidence of non-match.
+ *
+ * Pure math only; the document-frequency table is maintained by the store layer.
+ */
+
+/**
+ * Smoothed IDF: `log(1 + N / (df + 0.5))`.
+ * The `+0.5` smoothing keeps `df = 0` (token unseen in corpus) finite and maximal.
+ */
+export function idf(df: number, n: number): number {
+  return Math.log(1 + n / (df + 0.5));
+}
+
+/** Build a reusable `token -> idf` function from a df lookup and corpus size N. */
+export function buildIdfFn(dfLookup: (token: string) => number, n: number): (token: string) => number {
+  return (token: string) => idf(dfLookup(token), n);
+}
diff --git a/tests/dedup/idf.test.ts b/tests/dedup/idf.test.ts
new file mode 100644
index 0000000000..b7a0d4516a
--- /dev/null
+++ b/tests/dedup/idf.test.ts
@@ -0,0 +1,33 @@
+import { describe, it, expect } from 'bun:test';
+import { idf, buildIdfFn } from '../../src/services/dedup/idf.js';
+
+describe('idf', () => {
+  it('gives a rare token a higher score than a common token', () => {
+    expect(idf(1, 1000)).toBeGreaterThan(idf(900, 1000));
+  });
+
+  it('is monotonically non-increasing in df', () => {
+    expect(idf(1, 1000)).toBeGreaterThanOrEqual(idf(2, 1000));
+    expect(idf(10, 1000)).toBeGreaterThanOrEqual(idf(100, 1000));
+  });
+
+  it('smooths df=0 without dividing by zero (highest score)', () => {
+    const z = idf(0, 1000);
+    expect(Number.isFinite(z)).toBe(true);
+    expect(z).toBeGreaterThan(idf(1, 1000));
+  });
+
+  it('a token present in every record has near-zero discriminating power', () => {
+    // df == N: log(1 + N/(N+0.5)) ≈ log(2) — low, and below a typical veto threshold.
+    expect(idf(1000, 1000)).toBeLessThan(idf(10, 1000));
+  });
+});
+
+describe('buildIdfFn', () => {
+  it('builds an idf function backed by a df lookup + corpus size', () => {
+    const df = new Map<string, number>([['the', 900], ['rdlp-api', 2]]);
+    const fn = buildIdfFn((t) => df.get(t) ?? 0, 1000);
+    expect(fn('rdlp-api')).toBeGreaterThan(fn('the'));
+    expect(fn('never-seen')).toBeGreaterThan(fn('rdlp-api')); // df=0 → highest
+  });
+});

From 1a0e7a72d236374f45d7f3da641d6db673d038c5 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 00:40:22 +0200
Subject: [PATCH 03/19] feat(dedup): add IDF-weighted TF-IDF cosine for #3038

Rare discriminating tokens inflate the norm without contributing to the
dot product, pulling cosine below threshold for distinct-but-similar
titles (rdlp-api vs rdlp-plugin) that plain token-sort wrongly scores ~0.9.
---
 src/services/dedup/tfidfCosine.ts | 34 ++++++++++++++++++++++++
 tests/dedup/tfidf-cosine.test.ts  | 44 +++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 src/services/dedup/tfidfCosine.ts
 create mode 100644 tests/dedup/tfidf-cosine.test.ts

diff --git a/src/services/dedup/tfidfCosine.ts b/src/services/dedup/tfidfCosine.ts
new file mode 100644
index 0000000000..8c9b880195
--- /dev/null
+++ b/src/services/dedup/tfidfCosine.ts
@@ -0,0 +1,34 @@
+/**
+ * IDF-weighted TF-IDF cosine similarity for near-duplicate detection (#3038).
+ *
+ * Each title is a bag of tokens weighted by IDF (tf=1 — titles are short, ~5-12
+ * tokens, so term repetition is negligible). Cosine of the two IDF vectors.
+ *
+ * Why this and not plain string/token similarity: a rare discriminating token
+ * present on one side only (`rdlp-api` vs `rdlp-plugin`, `ffmpeg-7.1` vs `6.1`)
+ * has a large IDF, so it inflates that side's norm while contributing nothing to
+ * the dot product — pulling cosine down. Plain Levenshtein/token-sort sees high
+ * character overlap and wrongly scores these ~0.9.
+ */
+
+function idfVector(tokens: string[], idfFn: (t: string) => number): Map<string, number> {
+  const v = new Map<string, number>();
+  for (const t of new Set(tokens)) v.set(t, idfFn(t));
+  return v;
+}
+
+export function tfidfCosine(a: string[], b: string[], idfFn: (t: string) => number): number {
+  const va = idfVector(a, idfFn);
+  const vb = idfVector(b, idfFn);
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (const [t, w] of va) {
+    normA += w * w;
+    const wb = vb.get(t);
+    if (wb !== undefined) dot += w * wb;
+  }
+  for (const [, w] of vb) normB += w * w;
+  if (normA === 0 || normB === 0) return 0;
+  return dot / Math.sqrt(normA * normB);
+}
diff --git a/tests/dedup/tfidf-cosine.test.ts b/tests/dedup/tfidf-cosine.test.ts
new file mode 100644
index 0000000000..703d186940
--- /dev/null
+++ b/tests/dedup/tfidf-cosine.test.ts
@@ -0,0 +1,44 @@
+import { describe, it, expect } from 'bun:test';
+import { tfidfCosine } from '../../src/services/dedup/tfidfCosine.js';
+import { buildIdfFn } from '../../src/services/dedup/idf.js';
+import { tokenizeWs } from '../../src/services/dedup/normalize.js';
+
+// Corpus model: shared words common (low idf), discriminating identifiers rare (high idf).
+const DF = new Map<string, number>([
+  ['added', 500], ['rdlp-redact', 50], ['dependency', 300], ['to', 900], ['crate', 200],
+  ['rdlp-api', 3], ['rdlp-plugin', 3],
+]);
+const idfFn = buildIdfFn((t) => DF.get(t) ?? 0, 1000);
+const tc = (a: string, b: string) => tfidfCosine(tokenizeWs(a), tokenizeWs(b), idfFn);
+
+describe('tfidfCosine', () => {
+  it('scores identical strings 1.0', () => {
+    expect(tc('Added rdlp-redact to rdlp-api crate', 'Added rdlp-redact to rdlp-api crate')).toBeCloseTo(1, 5);
+  });
+
+  it('scores a pure word-reorder ~1.0 (same token set)', () => {
+    expect(tc('rdlp-redact dependency added', 'added rdlp-redact dependency')).toBeCloseTo(1, 5);
+  });
+
+  it('scores LOW when the only difference is a rare discriminating token', () => {
+    // "rdlp-api" vs "rdlp-plugin": high-idf tokens present on each side, absent from the
+    // intersection → they inflate both norms but contribute nothing to the dot product.
+    expect(tc('Added rdlp-redact dependency to rdlp-api crate',
+              'Added rdlp-redact dependency to rdlp-plugin crate')).toBeLessThan(0.78);
+  });
+
+  it('scores 0 for disjoint token sets', () => {
+    expect(tc('added rdlp-redact', 'dependency crate')).toBe(0);
+  });
+
+  it('is symmetric', () => {
+    const a = 'Added rdlp-redact dependency to rdlp-api crate';
+    const b = 'Added rdlp-redact dependency to rdlp-plugin crate';
+    expect(tc(a, b)).toBeCloseTo(tc(b, a), 10);
+  });
+
+  it('returns 0 when either side is empty', () => {
+    expect(tc('', 'added crate')).toBe(0);
+    expect(tc('added crate', '')).toBe(0);
+  });
+});

From 2ed65dd8f627f3fc654e04d283db71c0f4066c98 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 00:41:21 +0200
Subject: [PATCH 04/19] feat(dedup): add IDF-veto discriminating-token gate for
 #3038

Fellegi-Sunter blocking-key: a rare token present on only one side vetoes
the merge regardless of cosine. Test documents the known limitation that
common-token discriminators (code vs security) need the Branch-2 LLM tier.
---
 src/services/dedup/idfVeto.ts | 29 +++++++++++++++++++++
 tests/dedup/idf-veto.test.ts  | 47 +++++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+)
 create mode 100644 src/services/dedup/idfVeto.ts
 create mode 100644 tests/dedup/idf-veto.test.ts

diff --git a/src/services/dedup/idfVeto.ts b/src/services/dedup/idfVeto.ts
new file mode 100644
index 0000000000..3c9c4e3232
--- /dev/null
+++ b/src/services/dedup/idfVeto.ts
@@ -0,0 +1,29 @@
+/**
+ * IDF-veto — the discriminating-token gate for near-duplicate detection (#3038).
+ *
+ * Two titles may be lexically near-identical yet describe DISTINCT work because
+ * they differ in one high-information token (`rdlp-api` vs `rdlp-plugin`,
+ * `ffmpeg-7.1` vs `6.1`). This is Fellegi-Sunter's "blocking key": disagreement
+ * on a rare field (low collision probability `u`) is strong evidence of NON-match.
+ *
+ * The veto fires when ANY token in the symmetric difference of the two token sets
+ * has `idf > thetaIdf` — i.e. a rare, discriminating token is present on exactly
+ * one side. A fired veto vetoes the merge regardless of cosine similarity.
+ *
+ * Known limitation (by design): when the discriminator is a COMMON token
+ * (`code` vs `security` review), its IDF is low and the veto does not fire.
+ * Lexical methods cannot separate those; that residual is the Branch-2
+ * LLM-adjudication tier's job. See tests/dedup/idf-veto.test.ts.
+ */
+export function vetoFires(
+  a: string[],
+  b: string[],
+  idfFn: (t: string) => number,
+  thetaIdf: number
+): boolean {
+  const setA = new Set(a);
+  const setB = new Set(b);
+  for (const t of setA) if (!setB.has(t) && idfFn(t) > thetaIdf) return true;
+  for (const t of setB) if (!setA.has(t) && idfFn(t) > thetaIdf) return true;
+  return false;
+}
diff --git a/tests/dedup/idf-veto.test.ts b/tests/dedup/idf-veto.test.ts
new file mode 100644
index 0000000000..16fe9376a4
--- /dev/null
+++ b/tests/dedup/idf-veto.test.ts
@@ -0,0 +1,47 @@
+import { describe, it, expect } from 'bun:test';
+import { vetoFires } from '../../src/services/dedup/idfVeto.js';
+import { buildIdfFn, idf } from '../../src/services/dedup/idf.js';
+import { tokenizeWs } from '../../src/services/dedup/normalize.js';
+
+const N = 1000;
+const DF = new Map<string, number>([
+  // common (low idf) — never discriminating
+  ['added', 500], ['dependency', 300], ['to', 900], ['crate', 200], ['versions', 400],
+  ['review', 400], ['approval', 150], ['for', 900], ['in', 900], ['pinned', 250],
+  ['code', 250], ['security', 180], // common words that happen to be discriminating IN CONTEXT
+  // rare (high idf) — discriminating identifiers
+  ['rdlp-api', 3], ['rdlp-plugin', 3], ['ffmpeg-7.1.conf', 1], ['ffmpeg-6.1.conf', 1],
+  ['countycode', 4], ['municipalitynumber', 4],
+]);
+const idfFn = buildIdfFn((t) => DF.get(t) ?? 0, N);
+const THETA = idf(10, N); // "token in <= ~10 records is discriminating"
+const veto = (a: string, b: string) => vetoFires(tokenizeWs(a), tokenizeWs(b), idfFn, THETA);
+
+describe('vetoFires', () => {
+  it('fires when the difference is a rare identifier (rdlp-api vs rdlp-plugin)', () => {
+    expect(veto('Added dependency to rdlp-api crate', 'Added dependency to rdlp-plugin crate')).toBe(true);
+  });
+
+  it('fires when the difference is a rare version token (ffmpeg-7.1 vs 6.1)', () => {
+    expect(veto('Pinned versions in ffmpeg-7.1.conf', 'Pinned versions in ffmpeg-6.1.conf')).toBe(true);
+  });
+
+  it('fires on distinct rare value-objects (CountyCode vs MunicipalityNumber)', () => {
+    expect(veto('CountyCode record value object', 'MunicipalityNumber record value object')).toBe(true);
+  });
+
+  it('does NOT fire for a pure word-reorder (empty symmetric difference)', () => {
+    expect(veto('added dependency crate', 'crate added dependency')).toBe(false);
+  });
+
+  it('does NOT fire when sides are identical', () => {
+    expect(veto('added rdlp-api crate', 'added rdlp-api crate')).toBe(false);
+  });
+
+  it('KNOWN LIMITATION: does NOT fire when the discriminator is a COMMON token (code vs security)', () => {
+    // "code"/"security" are common (low idf) yet here they ARE the distinction.
+    // Lexical methods cannot catch this — it is precisely the residual that the
+    // Branch-2 LLM-adjudication tier is designed to handle. Documented, not a bug.
+    expect(veto('Code review approval for archive', 'Security review approval for archive')).toBe(false);
+  });
+});

From d8d8e7f32a7b100ca5ede8deffc7b017291b3163 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 00:45:47 +0200
Subject: [PATCH 05/19] feat(dedup): add tier-decision (classifyPair) + golden
 fixture for #3038

Tier-0 exact-normalized-title (safe auto-merge) / Tier-1 candidate
(cosine>=threshold AND !veto). Golden fixture encodes the real-DB-validated
cases incl. the common-token-discriminator limitation (code/security never
auto-merged) and the true recurring-dup that IS flagged.
---
 src/services/dedup/nearDuplicate.ts | 59 +++++++++++++++++++++++++++++
 tests/dedup/fixtures/golden.json    | 42 ++++++++++++++++++++
 tests/dedup/near-duplicate.test.ts  | 43 +++++++++++++++++++++
 3 files changed, 144 insertions(+)
 create mode 100644 src/services/dedup/nearDuplicate.ts
 create mode 100644 tests/dedup/fixtures/golden.json
 create mode 100644 tests/dedup/near-duplicate.test.ts

diff --git a/src/services/dedup/nearDuplicate.ts b/src/services/dedup/nearDuplicate.ts
new file mode 100644
index 0000000000..e5cfdfd1db
--- /dev/null
+++ b/src/services/dedup/nearDuplicate.ts
@@ -0,0 +1,59 @@
+/**
+ * Near-duplicate tier decision for #3038.
+ *
+ * Two safe tiers, both deterministic:
+ *   - Tier-0 "exact": titles equal after normalization → safe silent auto-merge.
+ *   - Tier-1 "candidate": IDF-weighted cosine >= threshold AND the IDF-veto does
+ *     NOT fire → a near-duplicate CANDIDATE (persisted for review / future
+ *     LLM adjudication), never a silent merge.
+ * Everything else → "none".
+ *
+ * Validated against a real 7,651-observation DB: simhash-over-narrative was
+ * useless (false-positive dominated); IDF-cosine + veto separates true rewords
+ * from distinct-but-similar titles that differ in a rare token.
+ */
+import { normalizeTitle, tokenizeWs } from './normalize.js';
+import { tfidfCosine } from './tfidfCosine.js';
+import { vetoFires } from './idfVeto.js';
+
+export type DedupTier = 'exact' | 'candidate' | 'none';
+export type DedupMethod = 'exact' | 'idf_cosine' | 'none';
+
+export interface ClassifyThresholds {
+  /** Minimum IDF-weighted cosine for a Tier-1 candidate. */
+  cosineThreshold: number;
+  /** A symmetric-difference token with idf above this vetoes the merge. */
+  vetoThetaIdf: number;
+}
+
+export interface PairClassification {
+  tier: DedupTier;
+  method: DedupMethod;
+  score: number;
+}
+
+/** Runtime configuration surfaced via settings (resolved to thresholds by the store). */
+export interface FuzzyDedupConfig {
+  enabled: boolean;
+  cosineThreshold: number;
+  /** A token appearing in <= idfVetoDf records is "discriminating". */
+  idfVetoDf: number;
+}
+
+export function classifyPair(
+  a: string | null | undefined,
+  b: string | null | undefined,
+  idfFn: (t: string) => number,
+  thresholds: ClassifyThresholds
+): PairClassification {
+  if (normalizeTitle(a) === normalizeTitle(b)) {
+    return { tier: 'exact', method: 'exact', score: 1 };
+  }
+  const ta = tokenizeWs(a);
+  const tb = tokenizeWs(b);
+  const score = tfidfCosine(ta, tb, idfFn);
+  if (score >= thresholds.cosineThreshold && !vetoFires(ta, tb, idfFn, thresholds.vetoThetaIdf)) {
+    return { tier: 'candidate', method: 'idf_cosine', score };
+  }
+  return { tier: 'none', method: 'none', score };
+}
diff --git a/tests/dedup/fixtures/golden.json b/tests/dedup/fixtures/golden.json
new file mode 100644
index 0000000000..3ed05272bf
--- /dev/null
+++ b/tests/dedup/fixtures/golden.json
@@ -0,0 +1,42 @@
+{
+  "_comment": "Golden oracle for #3038 near-dup tiering. Cases derived from real-DB validation (spikes over a 7,651-observation claude-mem.db). corpus builds the IDF model; cases assert classifyPair tiers. cosineThreshold/vetoDf mirror the shipped defaults' intent.",
+  "cosineThreshold": 0.78,
+  "vetoDf": 2,
+  "corpus": [
+    "Hardened on-demand checkpoint against transcript read race",
+    "On-demand checkpoint hardened against transcript read race",
+    "Race condition fixed in checkpoint hook transcript reading",
+    "Added rdlp-redact dependency to rdlp-api crate",
+    "Added rdlp-redact dependency to rdlp-plugin crate",
+    "Pinned ninja and meson versions in ffmpeg-7.1.conf",
+    "Pinned ninja and meson versions in ffmpeg-6.1.conf",
+    "Code review approval for xeve archive",
+    "Security review approval for xeve archive",
+    "Erebus platform architecture settled and documented",
+    "Erebus platform architecture settled and fully documented",
+    "Added fuzzy dedup guard",
+    "Fixed worker respawn poison loop",
+    "Code review of the parser module",
+    "Security review of the parser module",
+    "Code review approval for the build",
+    "Security review approval for the build",
+    "Erebus platform documented in detail",
+    "Erebus architecture fully documented with docs",
+    "Added dependency to the build crate",
+    "Pinned versions in the build config",
+    "Settled the platform architecture docs",
+    "Fully documented the release with docs",
+    "Reviewed the parser module for the build",
+    "Worker respawn guard added to the loop"
+  ],
+  "cases": [
+    { "a": "Hardened on-demand checkpoint against transcript read race", "b": "On-demand checkpoint hardened against transcript read race", "tier": "candidate", "note": "reporter trio: pure word-reorder reword -> flagged" },
+    { "a": "Hardened on-demand checkpoint against transcript read race", "b": "Race condition fixed in checkpoint hook transcript reading", "tier": "none", "note": "reporter trio: semantic paraphrase -> lexical miss (Branch-2 LLM tier)" },
+    { "a": "Added rdlp-redact dependency to rdlp-api crate", "b": "Added rdlp-redact dependency to rdlp-plugin crate", "tier": "none", "note": "class-3: rare identifier discriminator vetoed (DISTINCT)" },
+    { "a": "Pinned ninja and meson versions in ffmpeg-7.1.conf", "b": "Pinned ninja and meson versions in ffmpeg-6.1.conf", "tier": "none", "note": "class-3: rare version token vetoed (DISTINCT)" },
+    { "a": "Code review approval for xeve archive", "b": "Security review approval for xeve archive", "tier": "candidate", "note": "common-token discriminator: Branch-1 flags candidate, never auto-merges; Branch-2 resolves" },
+    { "a": "Erebus platform architecture settled and documented", "b": "Erebus platform architecture settled and fully documented", "tier": "candidate", "note": "true recurring near-dup (#3038 accumulation pattern)" },
+    { "a": "On-Demand Checkpoint.", "b": "on demand checkpoint", "tier": "exact", "note": "Tier-0 punctuation/case variant -> safe auto-merge" },
+    { "a": "Added fuzzy dedup guard", "b": "Fixed worker respawn poison loop", "tier": "none", "note": "unrelated" }
+  ]
+}
diff --git a/tests/dedup/near-duplicate.test.ts b/tests/dedup/near-duplicate.test.ts
new file mode 100644
index 0000000000..0797d6fd73
--- /dev/null
+++ b/tests/dedup/near-duplicate.test.ts
@@ -0,0 +1,43 @@
+import { describe, it, expect } from 'bun:test';
+import golden from './fixtures/golden.json';
+import { classifyPair } from '../../src/services/dedup/nearDuplicate.js';
+import { buildIdfFn, idf } from '../../src/services/dedup/idf.js';
+import { tokenizeWs } from '../../src/services/dedup/normalize.js';
+
+// Build the IDF model from the fixture corpus, exactly as the store layer will.
+const N = golden.corpus.length;
+const df = new Map<string, number>();
+for (const title of golden.corpus) {
+  for (const t of new Set(tokenizeWs(title))) df.set(t, (df.get(t) ?? 0) + 1);
+}
+const idfFn = buildIdfFn((t) => df.get(t) ?? 0, N);
+const thresholds = {
+  cosineThreshold: golden.cosineThreshold,
+  vetoThetaIdf: idf(golden.vetoDf, N),
+};
+
+describe('golden corpus df preconditions (guards against miscalibration)', () => {
+  it('rare identifiers appear exactly once (df=1 -> discriminating)', () => {
+    for (const t of ['rdlp-api', 'rdlp-plugin', 'ffmpeg-7.1.conf', 'ffmpeg-6.1.conf']) {
+      expect(df.get(t)).toBe(1);
+    }
+  });
+  it('common-in-context discriminators are actually common (df>2 -> NOT vetoed)', () => {
+    expect(df.get('code')!).toBeGreaterThan(golden.vetoDf);
+    expect(df.get('security')!).toBeGreaterThan(golden.vetoDf);
+  });
+});
+
+describe('classifyPair against the golden fixture', () => {
+  for (const c of golden.cases) {
+    it(`${c.tier.toUpperCase()}: ${c.note}`, () => {
+      expect(classifyPair(c.a, c.b, idfFn, thresholds).tier).toBe(c.tier);
+    });
+  }
+
+  it('NEVER auto-merges (Tier-0 exact) a common-token discriminator pair', () => {
+    // The single most important data-loss guard: code/security must not be 'exact'.
+    const r = classifyPair('Code review approval for xeve archive', 'Security review approval for xeve archive', idfFn, thresholds);
+    expect(r.tier).not.toBe('exact');
+  });
+});

From afbfc3db9d9e3d9b3c29047553e74c9d71d3b6f4 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 00:52:20 +0200
Subject: [PATCH 06/19] fix(dedup): never Tier-0 auto-merge titles that
 normalize to empty (#3038)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Code review caught a data-loss defect: null/empty/whitespace/punctuation-only/
emoji-only titles all normalize to '' and would collapse into each other as
Tier-0 'exact' (silent merge of distinct observations). This project uses
emoji titles (🔵/✅), so it's real. Guard the exact branch on a non-empty
normal form; Tier-1 fall-through is already safe (empty → cosine 0 → none).

Adds 5 empty/symbolic negative guards, normalize empty-collapse preconditions,
and an N=0 empty-corpus cosine test (no NaN on first observation).
---
 src/services/dedup/nearDuplicate.ts |  7 ++++++-
 tests/dedup/near-duplicate.test.ts  | 22 ++++++++++++++++++++++
 tests/dedup/normalize.test.ts       |  8 ++++++++
 tests/dedup/tfidf-cosine.test.ts    |  5 +++++
 4 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/src/services/dedup/nearDuplicate.ts b/src/services/dedup/nearDuplicate.ts
index e5cfdfd1db..841dbc9cf8 100644
--- a/src/services/dedup/nearDuplicate.ts
+++ b/src/services/dedup/nearDuplicate.ts
@@ -46,7 +46,12 @@ export function classifyPair(
   idfFn: (t: string) => number,
   thresholds: ClassifyThresholds
 ): PairClassification {
-  if (normalizeTitle(a) === normalizeTitle(b)) {
+  // Tier-0 requires a NON-EMPTY normal form: null/empty/whitespace/punctuation-only/
+  // emoji-only titles all normalize to '' and must NOT collapse into each other
+  // (that would silently auto-merge distinct observations — data loss). The Tier-1
+  // fall-through is already safe for these (empty tokens → cosine 0 → 'none').
+  const normA = normalizeTitle(a);
+  if (normA !== '' && normA === normalizeTitle(b)) {
     return { tier: 'exact', method: 'exact', score: 1 };
   }
   const ta = tokenizeWs(a);
diff --git a/tests/dedup/near-duplicate.test.ts b/tests/dedup/near-duplicate.test.ts
index 0797d6fd73..63afdab59c 100644
--- a/tests/dedup/near-duplicate.test.ts
+++ b/tests/dedup/near-duplicate.test.ts
@@ -41,3 +41,25 @@ describe('classifyPair against the golden fixture', () => {
     expect(r.tier).not.toBe('exact');
   });
 });
+
+describe('classifyPair: empty/symbolic titles must NOT false-merge (data-loss guard)', () => {
+  // normalizeTitle('') === normalizeTitle('🔵') === normalizeTitle('...') === '' —
+  // distinct observations whose titles normalize to empty must NEVER be Tier-0 'exact'.
+  // (This project uses emoji titles like 🔵/✅, so this is real, not theoretical.)
+  const cases: [string | null | undefined, string | null | undefined][] = [
+    ['...', '!!!'],
+    [null, ''],
+    ['🎉🎉', '***'],
+    ['   ', undefined],
+    ['🔵', '✅'],
+  ];
+  for (const [a, b] of cases) {
+    it(`does not merge ${JSON.stringify(a)} with ${JSON.stringify(b)}`, () => {
+      expect(classifyPair(a, b, idfFn, thresholds).tier).not.toBe('exact');
+    });
+  }
+
+  it('still merges two genuinely-equal non-empty titles', () => {
+    expect(classifyPair('On-Demand Checkpoint.', 'on demand checkpoint', idfFn, thresholds).tier).toBe('exact');
+  });
+});
diff --git a/tests/dedup/normalize.test.ts b/tests/dedup/normalize.test.ts
index 8169cbed37..1fc421af75 100644
--- a/tests/dedup/normalize.test.ts
+++ b/tests/dedup/normalize.test.ts
@@ -25,6 +25,14 @@ describe('normalizeTitle', () => {
     expect(normalizeTitle('')).toBe('');
     expect(normalizeTitle('   ')).toBe('');
   });
+
+  it('collapses punctuation-only and emoji-only titles to empty (precondition for the Tier-0 empty-guard)', () => {
+    // These are the inputs that make the empty-normal-form data-loss guard load-bearing.
+    expect(normalizeTitle('!!!')).toBe('');
+    expect(normalizeTitle('...')).toBe('');
+    expect(normalizeTitle('🔵')).toBe('');
+    expect(normalizeTitle('🎉🎉')).toBe('');
+  });
 });
 
 describe('tokenizeWs', () => {
diff --git a/tests/dedup/tfidf-cosine.test.ts b/tests/dedup/tfidf-cosine.test.ts
index 703d186940..c3a47065c5 100644
--- a/tests/dedup/tfidf-cosine.test.ts
+++ b/tests/dedup/tfidf-cosine.test.ts
@@ -41,4 +41,9 @@ describe('tfidfCosine', () => {
     expect(tc('', 'added crate')).toBe(0);
     expect(tc('added crate', '')).toBe(0);
   });
+
+  it('returns 0 (no NaN) on an empty corpus (N=0, every idf=0) — first-observation case', () => {
+    const emptyIdf = buildIdfFn(() => 0, 0); // N=0 → idf=log(1)=0 for every token
+    expect(tfidfCosine(tokenizeWs('added crate'), tokenizeWs('added crate'), emptyIdf)).toBe(0);
+  });
 });

From 76f59aafa776bac8872e92367439969caf17fcc2 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 01:12:10 +0200
Subject: [PATCH 07/19] feat(dedup): require >=2 shared tokens before cosine
 (#3038)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Research Q-D: short-title cosine is jumpy — a single shared rare token can
dominate it to ~1.0 even for otherwise-disjoint titles. Gate Tier-1 on a
configurable minimum shared-token count (default 2) to kill sparse-vector
noise before scoring.
---
 src/services/dedup/nearDuplicate.ts | 12 ++++++++++++
 tests/dedup/near-duplicate.test.ts  | 24 ++++++++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/src/services/dedup/nearDuplicate.ts b/src/services/dedup/nearDuplicate.ts
index 841dbc9cf8..fd740324b1 100644
--- a/src/services/dedup/nearDuplicate.ts
+++ b/src/services/dedup/nearDuplicate.ts
@@ -24,6 +24,12 @@ export interface ClassifyThresholds {
   cosineThreshold: number;
   /** A symmetric-difference token with idf above this vetoes the merge. */
   vetoThetaIdf: number;
+  /**
+   * Minimum count of shared tokens required before computing cosine. Short titles
+   * make cosine "jumpy" — a single shared rare token can dominate it to ~1.0 even
+   * when the titles are otherwise disjoint (sparse-vector noise). Defaults to 2.
+   */
+  minSharedTokens?: number;
 }
 
 export interface PairClassification {
@@ -56,6 +62,12 @@ export function classifyPair(
   }
   const ta = tokenizeWs(a);
   const tb = tokenizeWs(b);
+  // Sparse-vector noise guard: require >=N shared tokens before trusting cosine.
+  const minShared = thresholds.minSharedTokens ?? 2;
+  const setB = new Set(tb);
+  let shared = 0;
+  for (const t of new Set(ta)) if (setB.has(t)) shared++;
+  if (shared < minShared) return { tier: 'none', method: 'none', score: 0 };
   const score = tfidfCosine(ta, tb, idfFn);
   if (score >= thresholds.cosineThreshold && !vetoFires(ta, tb, idfFn, thresholds.vetoThetaIdf)) {
     return { tier: 'candidate', method: 'idf_cosine', score };
diff --git a/tests/dedup/near-duplicate.test.ts b/tests/dedup/near-duplicate.test.ts
index 63afdab59c..013c803e1b 100644
--- a/tests/dedup/near-duplicate.test.ts
+++ b/tests/dedup/near-duplicate.test.ts
@@ -42,6 +42,30 @@ describe('classifyPair against the golden fixture', () => {
   });
 });
 
+describe('classifyPair: >=2 shared-token pre-filter (sparse-vector noise guard, research Q-D)', () => {
+  // One shared RARE token can dominate cosine to ~1.0 even when the titles are otherwise
+  // disjoint — a short-title false positive. Require >=2 shared non-trivial tokens first.
+  const DF2 = new Map<string, number>([
+    ['apifoo', 1], ['common1', 800], ['common2', 800], ['common3', 800], ['common4', 800],
+  ]);
+  const idf2 = buildIdfFn((t) => DF2.get(t) ?? 0, 1000);
+  const T = { cosineThreshold: 0.8, vetoThetaIdf: idf(10, 1000) };
+  const a = 'apifoo common1 common2';
+  const b = 'apifoo common3 common4';
+
+  it('rejects a pair sharing only ONE token even when cosine is high', () => {
+    expect(classifyPair(a, b, idf2, { ...T, minSharedTokens: 2 }).tier).toBe('none');
+  });
+
+  it('would otherwise classify it as candidate (proves the cosine was high — pre-filter is doing the work)', () => {
+    expect(classifyPair(a, b, idf2, { ...T, minSharedTokens: 1 }).tier).toBe('candidate');
+  });
+
+  it('defaults minSharedTokens to 2 when unspecified', () => {
+    expect(classifyPair(a, b, idf2, T).tier).toBe('none');
+  });
+});
+
 describe('classifyPair: empty/symbolic titles must NOT false-merge (data-loss guard)', () => {
   // normalizeTitle('') === normalizeTitle('🔵') === normalizeTitle('...') === '' —
   // distinct observations whose titles normalize to empty must NEVER be Tier-0 'exact'.

From 97b9ef7d151378c99394bf51f9ff1a8d5942ef98 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Thu, 25 Jun 2026 20:04:43 +0200
Subject: [PATCH 08/19] test(settings): isolate loadFromFile tests from all
 ambient default env keys
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The SettingsDefaultsManager `loadFromFile` edge-case tests assert
`loadFromFile(<empty|corrupt file>)` deepEquals `getAllDefaults()`. But
`loadFromFile` applies env overrides on top of file/defaults by default, while
`getAllDefaults()` returns pure defaults — so any settings-default key present
in `process.env` makes the two diverge.

The suite already stripped `CLAUDE_MEM_DATA_DIR` (pinned by the preload
tripwire) for this reason, but only that one key. On a contributor machine with
a running claude-mem install, other keys are exported too (e.g.
`CLAUDE_MEM_API_TIMEOUT_MS=120000`), which silently failed 9 of these tests
locally while CI — a clean env — stayed green.

Generalize the isolation: snapshot and delete EVERY `getAllDefaults()` key from
`process.env` in beforeEach, restore in afterEach. Robust to whichever
CLAUDE_MEM_* vars the host exports. No production code changes — `loadFromFile`'s
env-override behavior is correct and already covered by the "environment
variable overrides" describe block; this only fixes test isolation.

Before: full suite 2159 pass / 9 fail on a dev box exporting CLAUDE_MEM_* vars.
After: 2159 pass / 0 fail in the same env.
(cherry picked from commit 7ea4880d4540ee34516067e63e8c0b72fca98b21)
---
 .../shared/settings-defaults-manager.test.ts  | 32 ++++++++++++-------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/tests/shared/settings-defaults-manager.test.ts b/tests/shared/settings-defaults-manager.test.ts
index f065390b75..a656d8d987 100644
--- a/tests/shared/settings-defaults-manager.test.ts
+++ b/tests/shared/settings-defaults-manager.test.ts
@@ -8,27 +8,35 @@ import { SettingsDefaultsManager } from '../../src/shared/SettingsDefaultsManage
 describe('SettingsDefaultsManager', () => {
   let tempDir: string;
   let settingsPath: string;
-  let prevDataDirEnv: string | undefined;
+  let savedDefaultKeyEnv: Record<string, string | undefined>;
 
   beforeEach(() => {
     tempDir = join(tmpdir(), `settings-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
     mkdirSync(tempDir, { recursive: true });
     settingsPath = join(tempDir, 'settings.json');
 
-    // The preload tripwire (tests/preload.ts) pins CLAUDE_MEM_DATA_DIR for
-    // the whole run, and loadFromFile applies env overrides on top of file
-    // values — which would make every loadFromFile result diverge from
-    // getAllDefaults()'s hardcoded ~/.claude-mem default. These tests are
-    // about file > defaults behavior on an EXPLICIT settingsPath (no real
-    // data-dir I/O happens here), so drop the env override for their
-    // duration and restore it after.
-    prevDataDirEnv = process.env.CLAUDE_MEM_DATA_DIR;
-    delete process.env.CLAUDE_MEM_DATA_DIR;
+    // loadFromFile applies env overrides on top of file/defaults, so ANY
+    // settings-default key present in process.env makes its result diverge
+    // from getAllDefaults(). On a dev machine this is not just the
+    // CLAUDE_MEM_DATA_DIR pinned by the preload tripwire (tests/preload.ts) —
+    // a running claude-mem install also exports e.g. CLAUDE_MEM_API_TIMEOUT_MS,
+    // which silently broke these tests on contributor boxes while passing in a
+    // clean CI env. These tests cover file > defaults behavior on an EXPLICIT
+    // settingsPath (no real data-dir I/O), so strip EVERY default key from the
+    // env for their duration and restore after — robust to whichever
+    // CLAUDE_MEM_* vars the host happens to export.
+    savedDefaultKeyEnv = {};
+    for (const key of Object.keys(SettingsDefaultsManager.getAllDefaults())) {
+      savedDefaultKeyEnv[key] = process.env[key];
+      delete process.env[key];
+    }
   });
 
   afterEach(() => {
-    if (prevDataDirEnv === undefined) delete process.env.CLAUDE_MEM_DATA_DIR;
-    else process.env.CLAUDE_MEM_DATA_DIR = prevDataDirEnv;
+    for (const [key, value] of Object.entries(savedDefaultKeyEnv)) {
+      if (value === undefined) delete process.env[key];
+      else process.env[key] = value;
+    }
     try {
       rmSync(tempDir, { recursive: true, force: true });
     } catch {

From 94b4e18145c04e7f4150ecf28e409ba4e0a7c62c Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 01:17:15 +0200
Subject: [PATCH 09/19] feat(dedup): add CLAUDE_MEM_DEDUP_* settings (off by
 default) for #3038

Six opt-in knobs: ENABLED=false, COSINE_THRESHOLD=0.80 (empirical short-title
sweet spot), IDF_VETO_DF=10, MIN_SHARED_TOKENS=2, MIN_PROJECT_DOCS=10
(cold-start gate), MAX_SCAN=2000. Env/settings.json only (no viewer UI wiring
needed for a server-side off-by-default flag).
---
 src/shared/SettingsDefaultsManager.ts | 12 ++++++
 tests/dedup/config.test.ts            | 57 +++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 tests/dedup/config.test.ts

diff --git a/src/shared/SettingsDefaultsManager.ts b/src/shared/SettingsDefaultsManager.ts
index bb4013c62b..6c09292b40 100644
--- a/src/shared/SettingsDefaultsManager.ts
+++ b/src/shared/SettingsDefaultsManager.ts
@@ -89,6 +89,12 @@ export interface SettingsDefaults {
   CLAUDE_MEM_SERVER_BETA_URL: string;
   CLAUDE_MEM_SERVER_BETA_API_KEY: string;
   CLAUDE_MEM_SERVER_BETA_PROJECT_ID: string;
+  CLAUDE_MEM_DEDUP_ENABLED: string;                // #3038 — near-duplicate dedup (off by default; probabilistic)
+  CLAUDE_MEM_DEDUP_COSINE_THRESHOLD: string;       // Tier-1 IDF-cosine threshold (0.80 = empirical short-title sweet spot)
+  CLAUDE_MEM_DEDUP_IDF_VETO_DF: string;            // token in <= N project records is "discriminating" (vetoes a merge)
+  CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS: string;      // require >= N shared tokens before computing cosine (sparse-vector guard)
+  CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS: string;       // cold-start: skip fuzzy Tier-1 below N docs/project (IDF unreliable)
+  CLAUDE_MEM_DEDUP_MAX_SCAN: string;               // cap Tier-1 candidate scan per insert (logged when hit)
 }
 
 export class SettingsDefaultsManager {
@@ -177,6 +183,12 @@ export class SettingsDefaultsManager {
     CLAUDE_MEM_SERVER_BETA_URL: `http://127.0.0.1:${process.env.CLAUDE_MEM_SERVER_PORT ?? String(37877 + ((process.getuid?.() ?? 77) % 100))}`,  // Legacy server-beta runtime URL — UID-derived for multi-account isolation
     CLAUDE_MEM_SERVER_BETA_API_KEY: '',                     // Legacy local hook API key (read as fallback when CLAUDE_MEM_SERVER_API_KEY unset)
     CLAUDE_MEM_SERVER_BETA_PROJECT_ID: '',                  // Legacy Postgres project_id (read as fallback when CLAUDE_MEM_SERVER_PROJECT_ID unset)
+    CLAUDE_MEM_DEDUP_ENABLED: 'false',                      // #3038 — opt-in; Tier-0 exact auto-merge + Tier-1 review-only candidates
+    CLAUDE_MEM_DEDUP_COSINE_THRESHOLD: '0.80',             // empirical near-dup sweet spot for short titles (F1≈0.95)
+    CLAUDE_MEM_DEDUP_IDF_VETO_DF: '10',                   // token in <=10 project records vetoes the merge (Fellegi-Sunter blocking key)
+    CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS: '2',             // >=2 shared tokens before cosine (kills sparse-vector noise)
+    CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS: '10',            // cold-start gate: IDF unreliable below ~10 docs/project
+    CLAUDE_MEM_DEDUP_MAX_SCAN: '2000',                  // per-insert candidate-scan cap (logged when exceeded)
   };
 
   static getAllDefaults(): SettingsDefaults {
diff --git a/tests/dedup/config.test.ts b/tests/dedup/config.test.ts
new file mode 100644
index 0000000000..11721ef913
--- /dev/null
+++ b/tests/dedup/config.test.ts
@@ -0,0 +1,57 @@
+import { describe, it, expect, afterEach } from 'bun:test';
+import { SettingsDefaultsManager } from '../../src/shared/SettingsDefaultsManager.js';
+
+const DEDUP_KEYS = [
+  'CLAUDE_MEM_DEDUP_ENABLED',
+  'CLAUDE_MEM_DEDUP_COSINE_THRESHOLD',
+  'CLAUDE_MEM_DEDUP_IDF_VETO_DF',
+  'CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS',
+  'CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS',
+  'CLAUDE_MEM_DEDUP_MAX_SCAN',
+] as const;
+
+// Env isolation (lesson from #3056/#3058): never leak DEDUP overrides across tests.
+const saved: Record<string, string | undefined> = {};
+for (const k of DEDUP_KEYS) saved[k] = process.env[k];
+afterEach(() => {
+  for (const k of DEDUP_KEYS) {
+    if (saved[k] === undefined) delete process.env[k];
+    else process.env[k] = saved[k];
+  }
+});
+
+describe('dedup settings defaults', () => {
+  it('ships all DEDUP keys with the researched defaults (off by default)', () => {
+    for (const k of DEDUP_KEYS) delete process.env[k];
+    const d = SettingsDefaultsManager.getAllDefaults() as Record<string, string>;
+    expect(d.CLAUDE_MEM_DEDUP_ENABLED).toBe('false');
+    expect(d.CLAUDE_MEM_DEDUP_COSINE_THRESHOLD).toBe('0.80');
+    expect(d.CLAUDE_MEM_DEDUP_IDF_VETO_DF).toBe('10');
+    expect(d.CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS).toBe('2');
+    expect(d.CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS).toBe('10');
+    expect(d.CLAUDE_MEM_DEDUP_MAX_SCAN).toBe('2000');
+  });
+
+  it('is disabled by default via getBool', () => {
+    delete process.env.CLAUDE_MEM_DEDUP_ENABLED;
+    expect(SettingsDefaultsManager.getBool('CLAUDE_MEM_DEDUP_ENABLED' as never)).toBe(false);
+  });
+
+  it('parses integer knobs via getInt', () => {
+    delete process.env.CLAUDE_MEM_DEDUP_IDF_VETO_DF;
+    expect(SettingsDefaultsManager.getInt('CLAUDE_MEM_DEDUP_IDF_VETO_DF' as never)).toBe(10);
+    expect(SettingsDefaultsManager.getInt('CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS' as never)).toBe(10);
+  });
+
+  it('parses the cosine threshold as a float from get()', () => {
+    delete process.env.CLAUDE_MEM_DEDUP_COSINE_THRESHOLD;
+    expect(Number(SettingsDefaultsManager.get('CLAUDE_MEM_DEDUP_COSINE_THRESHOLD' as never))).toBeCloseTo(0.8, 5);
+  });
+
+  it('honors env overrides', () => {
+    process.env.CLAUDE_MEM_DEDUP_ENABLED = 'true';
+    process.env.CLAUDE_MEM_DEDUP_COSINE_THRESHOLD = '0.85';
+    expect(SettingsDefaultsManager.getBool('CLAUDE_MEM_DEDUP_ENABLED' as never)).toBe(true);
+    expect(Number(SettingsDefaultsManager.get('CLAUDE_MEM_DEDUP_COSINE_THRESHOLD' as never))).toBeCloseTo(0.85, 5);
+  });
+});

From a71f110e5c39804ed432e01be55983b193b4bf9f Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 01:21:30 +0200
Subject: [PATCH 10/19] =?UTF-8?q?feat(dedup):=20schema=20migration=20v33?=
 =?UTF-8?q?=20=E2=80=94=20occurrence=5Fcount=20+=20token=5Fdf=20+=20candid?=
 =?UTF-8?q?ates=20(#3038)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Pure-SQL migration (repo norm — no JS backfill in migrations): adds
observations.occurrence_count (default 1), token_df (per-project IDF model,
filled forward / rebuilt by dedup-scan), dedup_meta (cold-start + drift
tracking), and observation_dedup_candidates (Tier-1 review-only, mirrors
observation_feedback; UNIQUE(observation_id,duplicate_of_id) + method/status
CHECKs). schema.sql updated to match.
---
 src/services/sqlite/SessionStore.ts           | 59 +++++++++++++++
 src/services/sqlite/schema.sql                | 42 +++++++++++
 .../session-store-dedup-migration.test.ts     | 75 +++++++++++++++++++
 3 files changed, 176 insertions(+)
 create mode 100644 tests/sqlite/session-store-dedup-migration.test.ts

diff --git a/src/services/sqlite/SessionStore.ts b/src/services/sqlite/SessionStore.ts
index 8680747736..c34cc7b7ae 100644
--- a/src/services/sqlite/SessionStore.ts
+++ b/src/services/sqlite/SessionStore.ts
@@ -80,6 +80,7 @@ export class SessionStore {
     this.ensureSDKSessionsPlatformContentIdentity();
     this.ensureUserPromptsSessionDbId();
     this.ensurePendingMessagesSessionToolUniqueIndex();
+    this.addDedupTables();
   }
 
   private getIndexColumns(indexName: string): string[] {
@@ -127,6 +128,64 @@ export class SessionStore {
     return row?.id ?? null;
   }
 
+  // #3038 near-duplicate dedup: occurrence_count (Tier-0 merge bump), per-project
+  // token document-frequency (IDF model), dedup bookkeeping, and the Tier-1
+  // review-only candidates table. Pure DDL; the IDF model is filled forward on
+  // insert and (re)built by the opt-in dedup-scan, never a JS backfill here.
+  private addDedupTables(): void {
+    const applied = this.db.prepare('SELECT version FROM schema_versions WHERE version = ?').get(36) as SchemaVersion | undefined;
+
+    const obsCols = this.db.query('PRAGMA table_info(observations)').all() as TableColumnInfo[];
+    if (!obsCols.some(c => c.name === 'occurrence_count')) {
+      this.db.run('ALTER TABLE observations ADD COLUMN occurrence_count INTEGER NOT NULL DEFAULT 1');
+    }
+
+    this.db.run(`
+      CREATE TABLE IF NOT EXISTS token_df (
+        project TEXT    NOT NULL,
+        token   TEXT    NOT NULL,
+        df      INTEGER NOT NULL DEFAULT 0,
+        PRIMARY KEY (project, token)
+      )
+    `);
+
+    this.db.run(`
+      CREATE TABLE IF NOT EXISTS dedup_meta (
+        project                TEXT    PRIMARY KEY,
+        doc_count              INTEGER NOT NULL DEFAULT 0,
+        last_rebuild_doc_count INTEGER NOT NULL DEFAULT 0,
+        deleted_since_rebuild  INTEGER NOT NULL DEFAULT 0
+      )
+    `);
+
+    this.db.run(`
+      CREATE TABLE IF NOT EXISTS observation_dedup_candidates (
+        id               INTEGER PRIMARY KEY AUTOINCREMENT,
+        observation_id   INTEGER NOT NULL,
+        duplicate_of_id  INTEGER NOT NULL,
+        project          TEXT    NOT NULL,
+        method           TEXT    NOT NULL CHECK(method IN ('exact', 'idf_cosine')),
+        score            REAL    NOT NULL,
+        status           TEXT    NOT NULL DEFAULT 'pending'
+                                 CHECK(status IN ('pending', 'merged', 'distinct', 'dismissed')),
+        created_at       TEXT    NOT NULL,
+        created_at_epoch INTEGER NOT NULL,
+        metadata         TEXT,
+        FOREIGN KEY (observation_id)  REFERENCES observations(id) ON DELETE CASCADE,
+        FOREIGN KEY (duplicate_of_id) REFERENCES observations(id) ON DELETE CASCADE,
+        UNIQUE(observation_id, duplicate_of_id)
+      )
+    `);
+
+    this.db.run('CREATE INDEX IF NOT EXISTS idx_token_df_project ON token_df(project)');
+    this.db.run('CREATE INDEX IF NOT EXISTS idx_dedup_candidates_project ON observation_dedup_candidates(project, status)');
+    this.db.run('CREATE INDEX IF NOT EXISTS idx_dedup_candidates_obs ON observation_dedup_candidates(observation_id)');
+
+    if (!applied) {
+      this.db.prepare('INSERT OR IGNORE INTO schema_versions (version, applied_at) VALUES (?, ?)').run(36, new Date().toISOString());
+    }
+  }
+
   private dropWorkerPidColumn(): void {
     const applied = this.db.prepare('SELECT version FROM schema_versions WHERE version = ?').get(32) as SchemaVersion | undefined;
 
diff --git a/src/services/sqlite/schema.sql b/src/services/sqlite/schema.sql
index cf1fda9e08..75e7accbee 100644
--- a/src/services/sqlite/schema.sql
+++ b/src/services/sqlite/schema.sql
@@ -77,6 +77,7 @@ CREATE TABLE IF NOT EXISTS observations (
   merged_into_project  TEXT,
   generated_by_model   TEXT,
   metadata             TEXT,
+  occurrence_count     INTEGER NOT NULL DEFAULT 1,  -- #3038: bumped on a Tier-0 exact-normalized-title merge
   created_at           TEXT    NOT NULL,
   created_at_epoch     INTEGER NOT NULL,
   FOREIGN KEY(memory_session_id) REFERENCES sdk_sessions(memory_session_id)
@@ -187,3 +188,44 @@ CREATE TABLE IF NOT EXISTS observation_feedback (
 );
 CREATE INDEX IF NOT EXISTS idx_feedback_observation ON observation_feedback(observation_id);
 CREATE INDEX IF NOT EXISTS idx_feedback_signal      ON observation_feedback(signal_type);
+
+-- ─────────────────────────────────────────────────────────────────────
+-- #3038 near-duplicate dedup (opt-in, off by default).
+-- token_df: per-project token document-frequency = the IDF model. Filled
+--   forward on insert; (re)built by the opt-in dedup-scan, never a migration backfill.
+-- dedup_meta: per-project doc_count (cold-start gate) + drift-rebuild tracking.
+-- observation_dedup_candidates: Tier-1 review-only near-dup candidates (never auto-merged).
+-- ─────────────────────────────────────────────────────────────────────
+CREATE TABLE IF NOT EXISTS token_df (
+  project TEXT    NOT NULL,
+  token   TEXT    NOT NULL,
+  df      INTEGER NOT NULL DEFAULT 0,
+  PRIMARY KEY (project, token)
+);
+CREATE INDEX IF NOT EXISTS idx_token_df_project ON token_df(project);
+
+CREATE TABLE IF NOT EXISTS dedup_meta (
+  project                TEXT    PRIMARY KEY,
+  doc_count              INTEGER NOT NULL DEFAULT 0,
+  last_rebuild_doc_count INTEGER NOT NULL DEFAULT 0,
+  deleted_since_rebuild  INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE IF NOT EXISTS observation_dedup_candidates (
+  id               INTEGER PRIMARY KEY AUTOINCREMENT,
+  observation_id   INTEGER NOT NULL,
+  duplicate_of_id  INTEGER NOT NULL,
+  project          TEXT    NOT NULL,
+  method           TEXT    NOT NULL CHECK(method IN ('exact', 'idf_cosine')),
+  score            REAL    NOT NULL,
+  status           TEXT    NOT NULL DEFAULT 'pending'
+                           CHECK(status IN ('pending', 'merged', 'distinct', 'dismissed')),
+  created_at       TEXT    NOT NULL,
+  created_at_epoch INTEGER NOT NULL,
+  metadata         TEXT,
+  FOREIGN KEY (observation_id)  REFERENCES observations(id) ON DELETE CASCADE,
+  FOREIGN KEY (duplicate_of_id) REFERENCES observations(id) ON DELETE CASCADE,
+  UNIQUE(observation_id, duplicate_of_id)
+);
+CREATE INDEX IF NOT EXISTS idx_dedup_candidates_project ON observation_dedup_candidates(project, status);
+CREATE INDEX IF NOT EXISTS idx_dedup_candidates_obs     ON observation_dedup_candidates(observation_id);
diff --git a/tests/sqlite/session-store-dedup-migration.test.ts b/tests/sqlite/session-store-dedup-migration.test.ts
new file mode 100644
index 0000000000..4b424726de
--- /dev/null
+++ b/tests/sqlite/session-store-dedup-migration.test.ts
@@ -0,0 +1,75 @@
+import { describe, it, expect, beforeEach, afterEach } from 'bun:test';
+import { SessionStore } from '../../src/services/sqlite/SessionStore.js';
+
+function cols(store: any, t: string): string[] {
+  return (store.db.query(`PRAGMA table_info(${t})`).all() as { name: string }[]).map((c) => c.name);
+}
+function hasTable(store: any, n: string): boolean {
+  return !!store.db.query("SELECT name FROM sqlite_master WHERE type='table' AND name = ?").get(n);
+}
+const OBS = {
+  type: 'discovery', title: 'T', subtitle: null as string | null, facts: [] as string[],
+  narrative: 'N', concepts: [] as string[], files_read: [] as string[], files_modified: [] as string[],
+};
+
+describe('dedup schema migration (#3038)', () => {
+  let store: any;
+  beforeEach(() => { store = new SessionStore(':memory:'); });
+  afterEach(() => store.close());
+
+  function session(mem: string): string {
+    const id = store.createSDKSession(`content-${mem}`, 'project', 'prompt');
+    store.updateMemorySessionId(id, mem);
+    return mem;
+  }
+  function store1(mem: string, title: string) {
+    return store.storeObservation(mem, 'project', { ...OBS, title, narrative: `n-${title}` }, 1, 0, Date.now());
+  }
+
+  it('adds observations.occurrence_count defaulting to 1', () => {
+    expect(cols(store, 'observations')).toContain('occurrence_count');
+    const r = store1(session('m1'), 'Hello');
+    const row = store.db.prepare('SELECT occurrence_count FROM observations WHERE id = ?').get(r.id) as { occurrence_count: number };
+    expect(row.occurrence_count).toBe(1);
+  });
+
+  it('creates token_df / dedup_meta / observation_dedup_candidates with the expected columns', () => {
+    expect(hasTable(store, 'token_df')).toBe(true);
+    expect(cols(store, 'token_df')).toEqual(expect.arrayContaining(['project', 'token', 'df']));
+    expect(hasTable(store, 'dedup_meta')).toBe(true);
+    expect(cols(store, 'dedup_meta')).toEqual(expect.arrayContaining(['project', 'doc_count', 'last_rebuild_doc_count', 'deleted_since_rebuild']));
+    expect(hasTable(store, 'observation_dedup_candidates')).toBe(true);
+    expect(cols(store, 'observation_dedup_candidates')).toEqual(
+      expect.arrayContaining(['id', 'observation_id', 'duplicate_of_id', 'project', 'method', 'score', 'status', 'created_at', 'created_at_epoch', 'metadata'])
+    );
+  });
+
+  it('starts candidates empty and records schema version 36', () => {
+    expect((store.db.query('SELECT COUNT(*) c FROM observation_dedup_candidates').get() as { c: number }).c).toBe(0);
+    expect(!!store.db.query('SELECT version FROM schema_versions WHERE version = 36').get()).toBe(true);
+  });
+
+  it('is idempotent (re-running migrations on the same db does not throw)', () => {
+    expect(() => new SessionStore(store.db)).not.toThrow();
+  });
+
+  it('enforces UNIQUE(observation_id, duplicate_of_id) on candidates', () => {
+    const mem = session('m2');
+    const a = store1(mem, 'A');
+    const b = store1(mem, 'B');
+    const ins = () => store.db.prepare(
+      "INSERT INTO observation_dedup_candidates (observation_id, duplicate_of_id, project, method, score, status, created_at, created_at_epoch) VALUES (?,?,?,?,?,?,?,?)"
+    ).run(a.id, b.id, 'project', 'idf_cosine', 0.9, 'pending', new Date().toISOString(), Date.now());
+    ins();
+    expect(ins).toThrow();
+  });
+
+  it('rejects an invalid method/status via CHECK constraints', () => {
+    const mem = session('m3');
+    const a = store1(mem, 'A');
+    const b = store1(mem, 'B');
+    expect(() => store.db.prepare(
+      "INSERT INTO observation_dedup_candidates (observation_id, duplicate_of_id, project, method, score, status, created_at, created_at_epoch) VALUES (?,?,?,?,?,?,?,?)"
+    ).run(a.id, b.id, 'project', 'bogus_method', 0.9, 'pending', new Date().toISOString(), Date.now())).toThrow();
+  });
+});

From 1700d80f3d465e60828e1332a32e00bc4cabf1e3 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 01:22:53 +0200
Subject: [PATCH 11/19] feat(dedup): per-project token_df maintenance + IDF
 lookup + cold-start gate (#3038)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

dedup-store.ts (plain fns over Database, keeps SessionStore lean):
bumpTokenDf (forward DF/doc_count maintenance, unique tokens only),
buildProjectIdf (project-scoped idf + corpus size), isFuzzyReady
(cold-start gate, research Q-B). Called on real inserts only — a Tier-0
merge adds no document.
---
 src/services/sqlite/dedup-store.ts | 50 ++++++++++++++++++++++++++++++
 tests/sqlite/dedup-store.test.ts   | 49 +++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+)
 create mode 100644 src/services/sqlite/dedup-store.ts
 create mode 100644 tests/sqlite/dedup-store.test.ts

diff --git a/src/services/sqlite/dedup-store.ts b/src/services/sqlite/dedup-store.ts
new file mode 100644
index 0000000000..10cce5ffe6
--- /dev/null
+++ b/src/services/sqlite/dedup-store.ts
@@ -0,0 +1,50 @@
+/**
+ * Per-project IDF model maintenance for near-duplicate dedup (#3038).
+ *
+ * `token_df` is the document-frequency table (one row per project+token), and
+ * `dedup_meta.doc_count` is the per-project document count. Together they define
+ * IDF. Maintained FORWARD on each real observation insert (NOT on a Tier-0 merge,
+ * which adds no new document) and (re)built by the opt-in dedup-scan command.
+ *
+ * Kept as plain functions over a Database so they're unit-testable and keep the
+ * already-large SessionStore lean.
+ */
+import type { Database } from 'bun:sqlite';
+import { tokenizeWs } from '../dedup/normalize.js';
+import { buildIdfFn } from '../dedup/idf.js';
+
+/** Record one new document: +1 df for each UNIQUE title token, +1 project doc_count. */
+export function bumpTokenDf(db: Database, project: string, title: string | null | undefined): void {
+  const tokens = [...new Set(tokenizeWs(title))];
+  const dfStmt = db.prepare(
+    'INSERT INTO token_df (project, token, df) VALUES (?, ?, 1) ' +
+    'ON CONFLICT(project, token) DO UPDATE SET df = df + 1'
+  );
+  for (const t of tokens) dfStmt.run(project, t);
+  db.prepare(
+    'INSERT INTO dedup_meta (project, doc_count) VALUES (?, 1) ' +
+    'ON CONFLICT(project) DO UPDATE SET doc_count = doc_count + 1'
+  ).run(project);
+}
+
+/** Project document count (0 if the project has no dedup_meta row yet). */
+export function getProjectDocCount(db: Database, project: string): number {
+  const row = db.prepare('SELECT doc_count FROM dedup_meta WHERE project = ?').get(project) as { doc_count: number } | undefined;
+  return row?.doc_count ?? 0;
+}
+
+/** Build a project-scoped `token -> idf` function plus the corpus size, from token_df. */
+export function buildProjectIdf(db: Database, project: string): { idfFn: (t: string) => number; docCount: number } {
+  const docCount = getProjectDocCount(db, project);
+  const dfStmt = db.prepare('SELECT df FROM token_df WHERE project = ? AND token = ?');
+  const dfLookup = (token: string): number => {
+    const row = dfStmt.get(project, token) as { df: number } | undefined;
+    return row?.df ?? 0;
+  };
+  return { idfFn: buildIdfFn(dfLookup, docCount), docCount };
+}
+
+/** Cold-start gate: fuzzy Tier-1 is only trustworthy once the corpus is large enough. */
+export function isFuzzyReady(db: Database, project: string, minDocs: number): boolean {
+  return getProjectDocCount(db, project) >= minDocs;
+}
diff --git a/tests/sqlite/dedup-store.test.ts b/tests/sqlite/dedup-store.test.ts
new file mode 100644
index 0000000000..9829e5ad07
--- /dev/null
+++ b/tests/sqlite/dedup-store.test.ts
@@ -0,0 +1,49 @@
+import { describe, it, expect, beforeEach, afterEach } from 'bun:test';
+import { SessionStore } from '../../src/services/sqlite/SessionStore.js';
+import { bumpTokenDf, getProjectDocCount, buildProjectIdf, isFuzzyReady } from '../../src/services/sqlite/dedup-store.js';
+
+describe('dedup-store: token_df maintenance + IDF lookup (#3038)', () => {
+  let store: any;
+  beforeEach(() => { store = new SessionStore(':memory:'); });
+  afterEach(() => store.close());
+
+  it('bumps df per UNIQUE title token and increments project doc_count', () => {
+    bumpTokenDf(store.db, 'p', 'added rdlp-api crate crate'); // 'crate' twice -> counted once
+    const df = (t: string) => (store.db.prepare('SELECT df FROM token_df WHERE project=? AND token=?').get('p', t) as any)?.df ?? 0;
+    expect(df('added')).toBe(1);
+    expect(df('rdlp-api')).toBe(1);
+    expect(df('crate')).toBe(1);
+    expect(getProjectDocCount(store.db, 'p')).toBe(1);
+
+    bumpTokenDf(store.db, 'p', 'added plugin');
+    expect(df('added')).toBe(2);
+    expect(df('plugin')).toBe(1);
+    expect(getProjectDocCount(store.db, 'p')).toBe(2);
+  });
+
+  it('scopes df per project', () => {
+    bumpTokenDf(store.db, 'p1', 'shared token');
+    bumpTokenDf(store.db, 'p2', 'shared token');
+    const df = (proj: string, t: string) => (store.db.prepare('SELECT df FROM token_df WHERE project=? AND token=?').get(proj, t) as any)?.df ?? 0;
+    expect(df('p1', 'shared')).toBe(1);
+    expect(df('p2', 'shared')).toBe(1);
+    expect(getProjectDocCount(store.db, 'p1')).toBe(1);
+  });
+
+  it('buildProjectIdf weights a rare token above a common one', () => {
+    for (let i = 0; i < 20; i++) bumpTokenDf(store.db, 'p', `common token-${i}`); // 'common' in all 20
+    bumpTokenDf(store.db, 'p', 'common raretoken');                              // 'raretoken' in 1
+    const { idfFn, docCount } = buildProjectIdf(store.db, 'p');
+    expect(docCount).toBe(21);
+    expect(idfFn('raretoken')).toBeGreaterThan(idfFn('common'));
+    expect(idfFn('never-seen')).toBeGreaterThan(idfFn('raretoken')); // df=0 -> highest
+  });
+
+  it('isFuzzyReady gates on the cold-start minimum doc count', () => {
+    for (let i = 0; i < 9; i++) bumpTokenDf(store.db, 'p', `doc ${i}`);
+    expect(isFuzzyReady(store.db, 'p', 10)).toBe(false); // 9 < 10
+    bumpTokenDf(store.db, 'p', 'doc 9');
+    expect(isFuzzyReady(store.db, 'p', 10)).toBe(true);  // 10 >= 10
+    expect(isFuzzyReady(store.db, 'brand-new', 10)).toBe(false); // unknown project -> 0
+  });
+});

From c85203a3ebcbba4accc9710059b6afddb9f5664a Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 01:30:24 +0200
Subject: [PATCH 12/19] feat(dedup): indexed title_norm_key for O(1) unbounded
 Tier-0 (#3038)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Research: SQLite can't express \p{L}-aware normalization (ASCII-only lower(),
no regexp_replace, no bun:sqlite custom fns), so precompute the key in app code
and index it — the content_hash pattern. computeTitleNormKey = sha256(project +
normalizeTitle), NULL when title normalizes to empty (data-loss guard reused at
the persistence layer). findTier0Canonical does the O(1) lookup. NON-unique
index keeps dedup app-gated on the flag (off = byte-identical).
---
 src/services/sqlite/SessionStore.ts           |  7 ++++
 src/services/sqlite/dedup-store.ts            | 32 ++++++++++++++++++-
 src/services/sqlite/schema.sql                |  2 ++
 tests/sqlite/dedup-store.test.ts              | 22 ++++++++++++-
 .../session-store-dedup-migration.test.ts     |  6 ++++
 5 files changed, 67 insertions(+), 2 deletions(-)

diff --git a/src/services/sqlite/SessionStore.ts b/src/services/sqlite/SessionStore.ts
index c34cc7b7ae..5c9933ce47 100644
--- a/src/services/sqlite/SessionStore.ts
+++ b/src/services/sqlite/SessionStore.ts
@@ -139,6 +139,13 @@ export class SessionStore {
     if (!obsCols.some(c => c.name === 'occurrence_count')) {
       this.db.run('ALTER TABLE observations ADD COLUMN occurrence_count INTEGER NOT NULL DEFAULT 1');
     }
+    // Precomputed exact-normalized-title key for O(1) Tier-0 lookup (SQLite can't
+    // express the normalization itself). NON-unique index — dedup stays app-gated
+    // on CLAUDE_MEM_DEDUP_ENABLED so disabled = byte-identical legacy behavior.
+    if (!obsCols.some(c => c.name === 'title_norm_key')) {
+      this.db.run('ALTER TABLE observations ADD COLUMN title_norm_key TEXT');
+    }
+    this.db.run('CREATE INDEX IF NOT EXISTS idx_observations_title_norm ON observations(project, title_norm_key)');
 
     this.db.run(`
       CREATE TABLE IF NOT EXISTS token_df (
diff --git a/src/services/sqlite/dedup-store.ts b/src/services/sqlite/dedup-store.ts
index 10cce5ffe6..2251e02978 100644
--- a/src/services/sqlite/dedup-store.ts
+++ b/src/services/sqlite/dedup-store.ts
@@ -9,10 +9,40 @@
  * Kept as plain functions over a Database so they're unit-testable and keep the
  * already-large SessionStore lean.
  */
+import { createHash } from 'crypto';
 import type { Database } from 'bun:sqlite';
-import { tokenizeWs } from '../dedup/normalize.js';
+import { normalizeTitle, tokenizeWs } from '../dedup/normalize.js';
 import { buildIdfFn } from '../dedup/idf.js';
 
+/**
+ * Project-scoped exact-normalized-title key for O(1) Tier-0 lookup (#3038).
+ * SQLite cannot express `\p{L}`-aware normalization itself (ASCII-only lower(),
+ * no regexp_replace, no bun:sqlite custom functions), so we precompute the key
+ * in app code and store it in an indexed column — the same pattern as content_hash.
+ *
+ * Returns null when the title normalizes to empty (null/punctuation/emoji-only):
+ * those must NOT collapse into each other (data-loss guard), and a NULL key never
+ * matches via `title_norm_key = ?`.
+ */
+export function computeTitleNormKey(project: string, title: string | null | undefined): string | null {
+  const norm = normalizeTitle(title);
+  if (norm === '') return null;
+  return createHash('sha256').update(`${project}\x00${norm}`).digest('hex').slice(0, 32);
+}
+
+/** O(1) Tier-0 lookup: the existing canonical row for this (project, normalized-title), or null. */
+export function findTier0Canonical(
+  db: Database,
+  project: string,
+  normKey: string | null
+): { id: number; occurrence_count: number; created_at_epoch: number } | null {
+  if (normKey === null) return null;
+  return (db.prepare(
+    'SELECT id, occurrence_count, created_at_epoch FROM observations ' +
+    'WHERE project = ? AND title_norm_key = ? ORDER BY created_at_epoch ASC, id ASC LIMIT 1'
+  ).get(project, normKey) as { id: number; occurrence_count: number; created_at_epoch: number } | undefined) ?? null;
+}
+
 /** Record one new document: +1 df for each UNIQUE title token, +1 project doc_count. */
 export function bumpTokenDf(db: Database, project: string, title: string | null | undefined): void {
   const tokens = [...new Set(tokenizeWs(title))];
diff --git a/src/services/sqlite/schema.sql b/src/services/sqlite/schema.sql
index 75e7accbee..ef990db286 100644
--- a/src/services/sqlite/schema.sql
+++ b/src/services/sqlite/schema.sql
@@ -78,6 +78,7 @@ CREATE TABLE IF NOT EXISTS observations (
   generated_by_model   TEXT,
   metadata             TEXT,
   occurrence_count     INTEGER NOT NULL DEFAULT 1,  -- #3038: bumped on a Tier-0 exact-normalized-title merge
+  title_norm_key       TEXT,                        -- #3038: sha256(project + normalizeTitle); O(1) Tier-0 lookup (NULL when title normalizes to empty)
   created_at           TEXT    NOT NULL,
   created_at_epoch     INTEGER NOT NULL,
   FOREIGN KEY(memory_session_id) REFERENCES sdk_sessions(memory_session_id)
@@ -92,6 +93,7 @@ CREATE INDEX IF NOT EXISTS idx_observations_content_hash ON observations(content
 CREATE INDEX IF NOT EXISTS idx_observations_agent_type   ON observations(agent_type);
 CREATE INDEX IF NOT EXISTS idx_observations_agent_id     ON observations(agent_id);
 CREATE INDEX IF NOT EXISTS idx_observations_merged_into  ON observations(merged_into_project);
+CREATE INDEX IF NOT EXISTS idx_observations_title_norm   ON observations(project, title_norm_key);
 
 -- ─────────────────────────────────────────────────────────────────────
 -- session_summaries: one summary row per memory session.
diff --git a/tests/sqlite/dedup-store.test.ts b/tests/sqlite/dedup-store.test.ts
index 9829e5ad07..7cee226139 100644
--- a/tests/sqlite/dedup-store.test.ts
+++ b/tests/sqlite/dedup-store.test.ts
@@ -1,6 +1,6 @@
 import { describe, it, expect, beforeEach, afterEach } from 'bun:test';
 import { SessionStore } from '../../src/services/sqlite/SessionStore.js';
-import { bumpTokenDf, getProjectDocCount, buildProjectIdf, isFuzzyReady } from '../../src/services/sqlite/dedup-store.js';
+import { bumpTokenDf, getProjectDocCount, buildProjectIdf, isFuzzyReady, computeTitleNormKey, findTier0Canonical } from '../../src/services/sqlite/dedup-store.js';
 
 describe('dedup-store: token_df maintenance + IDF lookup (#3038)', () => {
   let store: any;
@@ -39,6 +39,26 @@ describe('dedup-store: token_df maintenance + IDF lookup (#3038)', () => {
     expect(idfFn('never-seen')).toBeGreaterThan(idfFn('raretoken')); // df=0 -> highest
   });
 
+  it('computeTitleNormKey: equal for normalization-equivalent titles, distinct per project, null for empty', () => {
+    expect(computeTitleNormKey('p', 'On-Demand Checkpoint.')).toBe(computeTitleNormKey('p', 'on demand checkpoint'));
+    expect(computeTitleNormKey('p1', 'same title')).not.toBe(computeTitleNormKey('p2', 'same title')); // project-scoped
+    expect(computeTitleNormKey('p', 'Added X')).not.toBe(computeTitleNormKey('p', 'Removed X'));
+    for (const empty of [null, '', '   ', '!!!', '🔵']) expect(computeTitleNormKey('p', empty)).toBeNull();
+  });
+
+  it('findTier0Canonical returns the oldest matching row, null on miss or null key', () => {
+    const id = store.createSDKSession('content-t0', 'project', 'prompt');
+    store.updateMemorySessionId(id, 'mem-t0');
+    const norm = computeTitleNormKey('project', 'Hardened Checkpoint!');
+    const o = { type: 'discovery', title: 'x', subtitle: null, facts: [], narrative: 'x', concepts: [], files_read: [], files_modified: [] };
+    store.db.prepare("UPDATE observations SET title_norm_key = ? WHERE id = ?")
+      .run(norm, store.storeObservation('mem-t0', 'project', o, 1, 0, Date.now()).id);
+    const hit = findTier0Canonical(store.db, 'project', norm);
+    expect(hit?.id).toBeGreaterThan(0);
+    expect(findTier0Canonical(store.db, 'project', computeTitleNormKey('project', 'totally different'))).toBeNull();
+    expect(findTier0Canonical(store.db, 'project', null)).toBeNull();
+  });
+
   it('isFuzzyReady gates on the cold-start minimum doc count', () => {
     for (let i = 0; i < 9; i++) bumpTokenDf(store.db, 'p', `doc ${i}`);
     expect(isFuzzyReady(store.db, 'p', 10)).toBe(false); // 9 < 10
diff --git a/tests/sqlite/session-store-dedup-migration.test.ts b/tests/sqlite/session-store-dedup-migration.test.ts
index 4b424726de..40099cb8bc 100644
--- a/tests/sqlite/session-store-dedup-migration.test.ts
+++ b/tests/sqlite/session-store-dedup-migration.test.ts
@@ -26,6 +26,12 @@ describe('dedup schema migration (#3038)', () => {
     return store.storeObservation(mem, 'project', { ...OBS, title, narrative: `n-${title}` }, 1, 0, Date.now());
   }
 
+  it('adds observations.title_norm_key + its composite index', () => {
+    expect(cols(store, 'observations')).toContain('title_norm_key');
+    const idx = store.db.query("SELECT name FROM sqlite_master WHERE type='index' AND name='idx_observations_title_norm'").get();
+    expect(!!idx).toBe(true);
+  });
+
   it('adds observations.occurrence_count defaulting to 1', () => {
     expect(cols(store, 'observations')).toContain('occurrence_count');
     const r = store1(session('m1'), 'Hello');

From 560f9ce4fa7cb95e2ac3cca3d4350dcd971404a7 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 01:37:12 +0200
Subject: [PATCH 13/19] feat(dedup): integrate Tier-0 merge + Tier-1 candidate
 persist into storeObservation(s) (#3038)

Both the single and batch (ResponseProcessor) write paths, fully gated on
CLAUDE_MEM_DEDUP_ENABLED (off = byte-identical legacy behavior):
- Tier-0: O(1) title_norm_key lookup -> bump occurrence_count + reuse id,
  cross-session and intra-batch (in-transaction visibility); mirrors the
  content_hash ON CONFLICT semantics.
- Forward token_df/doc_count maintenance on real inserts only.
- Tier-1 (>= MIN_PROJECT_DOCS): capped recent-window classifyPair scan ->
  persist review-only candidates (INSERT OR IGNORE). Full-corpus sweep is the
  upcoming dedup-scan.
7 integration tests (cross-session/cross-project/intra-batch/cold-start/disabled);
typecheck clean; 80 sqlite tests green.
---
 src/services/sqlite/SessionStore.ts           |  73 +++++++++++-
 src/services/sqlite/dedup-store.ts            |  59 +++++++++-
 .../session-store-dedup-integration.test.ts   | 106 ++++++++++++++++++
 3 files changed, 231 insertions(+), 7 deletions(-)
 create mode 100644 tests/sqlite/session-store-dedup-integration.test.ts

diff --git a/src/services/sqlite/SessionStore.ts b/src/services/sqlite/SessionStore.ts
index 5c9933ce47..19f4258478 100644
--- a/src/services/sqlite/SessionStore.ts
+++ b/src/services/sqlite/SessionStore.ts
@@ -13,6 +13,11 @@ import {
 } from '../../types/database.js';
 import type { ObservationSearchResult, SessionSummarySearchResult } from './types.js';
 import { computeObservationContentHash } from './observations/store.js';
+import { SettingsDefaultsManager } from '../../shared/SettingsDefaultsManager.js';
+import {
+  computeTitleNormKey, findTier0Canonical, bumpTokenDf, isFuzzyReady, recordTier1Candidates,
+  type DedupRuntimeConfig,
+} from './dedup-store.js';
 import { parseFileList } from './observations/files.js';
 import { DEFAULT_PLATFORM_SOURCE, normalizePlatformSource, sortPlatformSources } from '../../shared/platform-source.js';
 import { findRecentDuplicateUserPrompt as findRecentDuplicateUserPromptRecord } from './prompts/get.js';
@@ -2216,6 +2221,31 @@ export class SessionStore {
     return result?.prompt_text ?? null;
   }
 
+  // #3038 — resolve the dedup knobs from settings (cheap; off by default).
+  private dedupConfig(): DedupRuntimeConfig & { enabled: boolean; minProjectDocs: number } {
+    return {
+      enabled: SettingsDefaultsManager.getBool('CLAUDE_MEM_DEDUP_ENABLED'),
+      cosineThreshold: Number(SettingsDefaultsManager.get('CLAUDE_MEM_DEDUP_COSINE_THRESHOLD')),
+      idfVetoDf: SettingsDefaultsManager.getInt('CLAUDE_MEM_DEDUP_IDF_VETO_DF'),
+      minSharedTokens: SettingsDefaultsManager.getInt('CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS'),
+      maxScan: SettingsDefaultsManager.getInt('CLAUDE_MEM_DEDUP_MAX_SCAN'),
+      minProjectDocs: SettingsDefaultsManager.getInt('CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS'),
+    };
+  }
+
+  // Forward IDF maintenance + Tier-1 candidate scan for a freshly-inserted observation.
+  private maintainDedupOnInsert(
+    project: string,
+    obsId: number,
+    title: string | null | undefined,
+    dedup: DedupRuntimeConfig & { minProjectDocs: number }
+  ): void {
+    bumpTokenDf(this.db, project, title);
+    if (isFuzzyReady(this.db, project, dedup.minProjectDocs)) {
+      recordTier1Candidates(this.db, project, obsId, title, dedup);
+    }
+  }
+
   storeObservation(
     memorySessionId: string,
     project: string,
@@ -2241,13 +2271,26 @@ export class SessionStore {
     const timestampIso = new Date(timestampEpoch).toISOString();
 
     const contentHash = computeObservationContentHash(memorySessionId, observation.title, observation.narrative);
+    const dedup = this.dedupConfig();
+    const titleNormKey = computeTitleNormKey(project, observation.title);
+
+    // Tier-0 (#3038): an existing same-project row with an equal NORMALIZED title is a
+    // duplicate of any age — bump its occurrence_count and return it, no insert. Mirrors
+    // the content_hash ON CONFLICT DO NOTHING semantics but cross-session + normalization-aware.
+    if (dedup.enabled) {
+      const canonical = findTier0Canonical(this.db, project, titleNormKey);
+      if (canonical) {
+        this.db.prepare('UPDATE observations SET occurrence_count = occurrence_count + 1 WHERE id = ?').run(canonical.id);
+        return { id: canonical.id, createdAtEpoch: canonical.created_at_epoch };
+      }
+    }
 
     const stmt = this.db.prepare(`
       INSERT INTO observations
       (memory_session_id, project, type, title, subtitle, facts, narrative, concepts,
        files_read, files_modified, prompt_number, discovery_tokens, agent_type, agent_id, content_hash, created_at, created_at_epoch,
-       generated_by_model, metadata)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       generated_by_model, metadata, title_norm_key)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
       ON CONFLICT(memory_session_id, content_hash) DO NOTHING
       RETURNING id, created_at_epoch
     `);
@@ -2271,10 +2314,12 @@ export class SessionStore {
       timestampIso,
       timestampEpoch,
       generatedByModel || null,
-      observation.metadata ?? null
+      observation.metadata ?? null,
+      titleNormKey
     ) as { id: number; created_at_epoch: number } | null;
 
     if (inserted) {
+      if (dedup.enabled) this.maintainDedupOnInsert(project, inserted.id, observation.title, dedup);
       return { id: inserted.id, createdAtEpoch: inserted.created_at_epoch };
     }
 
@@ -2366,6 +2411,7 @@ export class SessionStore {
   ): { observationIds: number[]; summaryId: number | null; createdAtEpoch: number } {
     const timestampEpoch = overrideTimestampEpoch ?? Date.now();
     const timestampIso = new Date(timestampEpoch).toISOString();
+    const dedup = this.dedupConfig();
 
     const storeTx = this.db.transaction(() => {
       const observationIds: number[] = [];
@@ -2374,8 +2420,8 @@ export class SessionStore {
         INSERT INTO observations
         (memory_session_id, project, type, title, subtitle, facts, narrative, concepts,
          files_read, files_modified, prompt_number, discovery_tokens, agent_type, agent_id, content_hash, created_at, created_at_epoch,
-         generated_by_model)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+         generated_by_model, title_norm_key)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
         ON CONFLICT(memory_session_id, content_hash) DO NOTHING
         RETURNING id
       `);
@@ -2385,6 +2431,19 @@ export class SessionStore {
 
       for (const observation of observations) {
         const contentHash = computeObservationContentHash(memorySessionId, observation.title, observation.narrative);
+        const titleNormKey = computeTitleNormKey(project, observation.title);
+
+        // Tier-0 (#3038): cross-session normalized-title duplicate (incl. earlier items
+        // in THIS batch — already inserted and visible in-transaction) → bump + reuse.
+        if (dedup.enabled) {
+          const canonical = findTier0Canonical(this.db, project, titleNormKey);
+          if (canonical) {
+            this.db.prepare('UPDATE observations SET occurrence_count = occurrence_count + 1 WHERE id = ?').run(canonical.id);
+            observationIds.push(canonical.id);
+            continue;
+          }
+        }
+
         const inserted = obsStmt.get(
           memorySessionId,
           project,
@@ -2403,10 +2462,12 @@ export class SessionStore {
           contentHash,
           timestampIso,
           timestampEpoch,
-          generatedByModel || null
+          generatedByModel || null,
+          titleNormKey
         ) as { id: number } | null;
 
         if (inserted) {
+          if (dedup.enabled) this.maintainDedupOnInsert(project, inserted.id, observation.title, dedup);
           observationIds.push(inserted.id);
           continue;
         }
diff --git a/src/services/sqlite/dedup-store.ts b/src/services/sqlite/dedup-store.ts
index 2251e02978..dd77fa7e4f 100644
--- a/src/services/sqlite/dedup-store.ts
+++ b/src/services/sqlite/dedup-store.ts
@@ -12,7 +12,17 @@
 import { createHash } from 'crypto';
 import type { Database } from 'bun:sqlite';
 import { normalizeTitle, tokenizeWs } from '../dedup/normalize.js';
-import { buildIdfFn } from '../dedup/idf.js';
+import { buildIdfFn, idf } from '../dedup/idf.js';
+import { classifyPair, type ClassifyThresholds } from '../dedup/nearDuplicate.js';
+import { logger } from '../../utils/logger.js';
+
+/** Runtime dedup knobs resolved from settings. */
+export interface DedupRuntimeConfig {
+  cosineThreshold: number;
+  idfVetoDf: number;
+  minSharedTokens: number;
+  maxScan: number;
+}
 
 /**
  * Project-scoped exact-normalized-title key for O(1) Tier-0 lookup (#3038).
@@ -78,3 +88,50 @@ export function buildProjectIdf(db: Database, project: string): { idfFn: (t: str
 export function isFuzzyReady(db: Database, project: string, minDocs: number): boolean {
   return getProjectDocCount(db, project) >= minDocs;
 }
+
+/**
+ * Tier-1 (review-only) near-duplicate scan for a freshly-inserted observation.
+ * Compares its title against up to `maxScan` recent same-project titles and
+ * persists any 'candidate' verdicts into observation_dedup_candidates
+ * (INSERT OR IGNORE — UNIQUE(observation_id,duplicate_of_id) dedups). NEVER
+ * merges. The full-corpus sweep is the offline dedup-scan command. Returns the
+ * number of candidates persisted.
+ */
+export function recordTier1Candidates(
+  db: Database,
+  project: string,
+  newObsId: number,
+  title: string | null | undefined,
+  cfg: DedupRuntimeConfig
+): number {
+  if (!title) return 0;
+  const { idfFn, docCount } = buildProjectIdf(db, project);
+  const thresholds: ClassifyThresholds = {
+    cosineThreshold: cfg.cosineThreshold,
+    vetoThetaIdf: idf(cfg.idfVetoDf, docCount),
+    minSharedTokens: cfg.minSharedTokens,
+  };
+  const rows = db.prepare(
+    'SELECT id, title FROM observations WHERE project = ? AND id != ? AND title IS NOT NULL ' +
+    'ORDER BY created_at_epoch DESC, id DESC LIMIT ?'
+  ).all(project, newObsId, cfg.maxScan) as { id: number; title: string }[];
+  if (rows.length === cfg.maxScan) {
+    logger.debug('DEDUP', `Tier-1 scan hit MAX_SCAN=${cfg.maxScan} for project ${project}; older rows covered by dedup-scan`);
+  }
+  const ins = db.prepare(
+    'INSERT OR IGNORE INTO observation_dedup_candidates ' +
+    '(observation_id, duplicate_of_id, project, method, score, status, created_at, created_at_epoch) ' +
+    'VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
+  );
+  const nowIso = new Date().toISOString();
+  const nowEpoch = Date.now();
+  let count = 0;
+  for (const r of rows) {
+    const c = classifyPair(title, r.title, idfFn, thresholds);
+    if (c.tier === 'candidate') {
+      ins.run(newObsId, r.id, project, c.method, c.score, 'pending', nowIso, nowEpoch);
+      count++;
+    }
+  }
+  return count;
+}
diff --git a/tests/sqlite/session-store-dedup-integration.test.ts b/tests/sqlite/session-store-dedup-integration.test.ts
new file mode 100644
index 0000000000..61ebb42797
--- /dev/null
+++ b/tests/sqlite/session-store-dedup-integration.test.ts
@@ -0,0 +1,106 @@
+import { describe, it, expect, beforeEach, afterEach } from 'bun:test';
+import { SessionStore } from '../../src/services/sqlite/SessionStore.js';
+
+const ENV_KEYS = ['CLAUDE_MEM_DEDUP_ENABLED', 'CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS'] as const;
+const saved: Record<string, string | undefined> = {};
+
+function obs(title: string, narrative = 'n') {
+  return { type: 'discovery', title, subtitle: null as string | null, facts: [] as string[], narrative, concepts: [] as string[], files_read: [] as string[], files_modified: [] as string[] };
+}
+
+describe('storeObservation dedup integration (#3038)', () => {
+  let store: any;
+  beforeEach(() => {
+    for (const k of ENV_KEYS) saved[k] = process.env[k];
+    store = new SessionStore(':memory:');
+  });
+  afterEach(() => {
+    store.close();
+    for (const k of ENV_KEYS) { if (saved[k] === undefined) delete process.env[k]; else process.env[k] = saved[k]; }
+  });
+
+  function session(mem: string, project = 'p'): string {
+    const id = store.createSDKSession(`content-${mem}`, project, 'prompt');
+    store.updateMemorySessionId(id, mem);
+    return mem;
+  }
+  const rowCount = (project = 'p') => (store.db.prepare('SELECT COUNT(*) c FROM observations WHERE project = ?').get(project) as any).c;
+  const candCount = () => (store.db.prepare('SELECT COUNT(*) c FROM observation_dedup_candidates').get() as any).c;
+
+  it('Tier-0: collapses a normalized-equal title across sessions and bumps occurrence_count', () => {
+    process.env.CLAUDE_MEM_DEDUP_ENABLED = 'true';
+    const t = Date.now();
+    const a = store.storeObservation(session('s1'), 'p', obs('On-Demand Checkpoint.'), 1, 0, t);
+    const b = store.storeObservation(session('s2'), 'p', obs('on demand checkpoint'), 1, 0, t + 1000);
+    expect(b.id).toBe(a.id);
+    expect(rowCount()).toBe(1);
+    const occ = (store.db.prepare('SELECT occurrence_count FROM observations WHERE id = ?').get(a.id) as any).occurrence_count;
+    expect(occ).toBe(2);
+  });
+
+  it('Tier-0: does NOT collapse the same normalized title across DIFFERENT projects', () => {
+    process.env.CLAUDE_MEM_DEDUP_ENABLED = 'true';
+    const t = Date.now();
+    store.storeObservation(session('s1', 'p1'), 'p1', obs('Same Title'), 1, 0, t);
+    store.storeObservation(session('s2', 'p2'), 'p2', obs('Same Title'), 1, 0, t + 1000);
+    expect(rowCount('p1')).toBe(1);
+    expect(rowCount('p2')).toBe(1);
+  });
+
+  it('disabled (default): normalized-equal cross-session titles both insert (byte-identical legacy behavior)', () => {
+    process.env.CLAUDE_MEM_DEDUP_ENABLED = 'false';
+    const t = Date.now();
+    store.storeObservation(session('s1'), 'p', obs('On-Demand Checkpoint.'), 1, 0, t);
+    store.storeObservation(session('s2'), 'p', obs('on demand checkpoint'), 1, 0, t + 1000);
+    expect(rowCount()).toBe(2);
+    expect(candCount()).toBe(0);
+  });
+
+  it('Tier-1: persists a review-only candidate for a reorder near-dup once the corpus is warm', () => {
+    process.env.CLAUDE_MEM_DEDUP_ENABLED = 'true';
+    process.env.CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS = '2';
+    const mem = session('s1');
+    let t = Date.now();
+    store.storeObservation(mem, 'p', obs('alpha bravo charlie delta'), 1, 0, t++);
+    store.storeObservation(mem, 'p', obs('echo foxtrot golf hotel'), 1, 0, t++);
+    const r1 = store.storeObservation(mem, 'p', obs('build the worker service module'), 1, 0, t++);
+    const r2 = store.storeObservation(mem, 'p', obs('worker build the service module'), 1, 0, t++); // pure reorder
+    expect(r2.id).not.toBe(r1.id); // reorder is NOT Tier-0 exact
+    const cand = store.db.prepare(
+      'SELECT method, status FROM observation_dedup_candidates WHERE observation_id = ? AND duplicate_of_id = ?'
+    ).get(r2.id, r1.id) as any;
+    expect(cand?.method).toBe('idf_cosine');
+    expect(cand?.status).toBe('pending');
+  });
+
+  it('storeObservations batch: collapses an intra-batch normalized-equal pair to one row', () => {
+    process.env.CLAUDE_MEM_DEDUP_ENABLED = 'true';
+    const mem = session('s1');
+    const res = store.storeObservations(mem, 'p', [obs('Foo Bar!'), obs('foo bar'), obs('Distinct One')], null, 1, 0, Date.now());
+    expect(res.observationIds[0]).toBe(res.observationIds[1]); // the pair collapsed to one id
+    expect(rowCount()).toBe(2); // {foo bar} + {distinct one}
+    const occ = (store.db.prepare('SELECT occurrence_count FROM observations WHERE id = ?').get(res.observationIds[0]) as any).occurrence_count;
+    expect(occ).toBe(2);
+  });
+
+  it('storeObservations batch: disabled leaves legacy behavior (no collapse, no candidates)', () => {
+    process.env.CLAUDE_MEM_DEDUP_ENABLED = 'false';
+    const mem = session('s1');
+    store.storeObservations(mem, 'p', [obs('Foo Bar!'), obs('foo bar')], null, 1, 0, Date.now());
+    expect(rowCount()).toBe(2);
+    expect(candCount()).toBe(0);
+  });
+
+  it('cold-start: below MIN_PROJECT_DOCS, Tier-1 is skipped (no candidates) but Tier-0 still merges', () => {
+    process.env.CLAUDE_MEM_DEDUP_ENABLED = 'true';
+    process.env.CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS = '10';
+    const mem = session('s1');
+    let t = Date.now();
+    const r1 = store.storeObservation(mem, 'p', obs('build the worker service module'), 1, 0, t++);
+    store.storeObservation(mem, 'p', obs('worker build the service module'), 1, 0, t++); // reorder, but cold-start
+    expect(candCount()).toBe(0);
+    // Tier-0 exact still works even cold:
+    const dup = store.storeObservation(mem, 'p', obs('Build The Worker Service Module'), 1, 0, t++);
+    expect(dup.id).toBe(r1.id);
+  });
+});

From 62c4d75c5fa8bfdcc92656cab39ef4e95f15be30 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 01:41:31 +0200
Subject: [PATCH 14/19] feat(dedup): dedup-scan backfill + inverted-index
 corpus sweep (#3038)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Opt-in, idempotent (research Q-A/Q-C): backfillProjectDedup recomputes
title_norm_key for every row + DELETE/INSERT-rebuilds token_df + resets
dedup_meta in one transaction (this is how an EXISTING DB joins dedup and how
DF drift is reclaimed). sweepProjectCandidates finds existing near-dups via a
bounded inverted index (postings on df in 2..~4*sqrt(N) tokens, pairs sharing
>=minSharedTokens classified) — not O(N^2). runDedupScan covers all projects.
Persists review-only candidates (INSERT OR IGNORE, idempotent).
---
 src/services/sqlite/dedup-store.ts | 95 ++++++++++++++++++++++++++++++
 tests/sqlite/dedup-scan.test.ts    | 79 +++++++++++++++++++++++++
 2 files changed, 174 insertions(+)
 create mode 100644 tests/sqlite/dedup-scan.test.ts

diff --git a/src/services/sqlite/dedup-store.ts b/src/services/sqlite/dedup-store.ts
index dd77fa7e4f..c71ee950e3 100644
--- a/src/services/sqlite/dedup-store.ts
+++ b/src/services/sqlite/dedup-store.ts
@@ -89,6 +89,101 @@ export function isFuzzyReady(db: Database, project: string, minDocs: number): bo
   return getProjectDocCount(db, project) >= minDocs;
 }
 
+/**
+ * Idempotent backfill = the canonical "rebuild" for a project's IDF model (#3038,
+ * research Q-A/Q-C). Recomputes title_norm_key for every row (SQLite can't), then
+ * DELETE+INSERT rebuilds token_df and resets dedup_meta — all in one transaction,
+ * safe to re-run. This is how an EXISTING DB starts participating in dedup, and how
+ * post-deletion DF drift is reclaimed. Returns the project doc count.
+ */
+export function backfillProjectDedup(db: Database, project: string): number {
+  const rows = db.prepare('SELECT id, title FROM observations WHERE project = ?').all(project) as { id: number; title: string | null }[];
+  const updKey = db.prepare('UPDATE observations SET title_norm_key = ? WHERE id = ?');
+  const insDf = db.prepare('INSERT INTO token_df (project, token, df) VALUES (?, ?, ?)');
+  const tx = db.transaction(() => {
+    const dfMap = new Map<string, number>();
+    for (const r of rows) {
+      updKey.run(computeTitleNormKey(project, r.title), r.id);
+      for (const t of new Set(tokenizeWs(r.title))) dfMap.set(t, (dfMap.get(t) ?? 0) + 1);
+    }
+    db.prepare('DELETE FROM token_df WHERE project = ?').run(project);
+    for (const [t, df] of dfMap) insDf.run(project, t, df);
+    db.prepare(
+      'INSERT INTO dedup_meta (project, doc_count, last_rebuild_doc_count, deleted_since_rebuild) VALUES (?, ?, ?, 0) ' +
+      'ON CONFLICT(project) DO UPDATE SET doc_count = excluded.doc_count, last_rebuild_doc_count = excluded.last_rebuild_doc_count, deleted_since_rebuild = 0'
+    ).run(project, rows.length, rows.length);
+  });
+  tx();
+  return rows.length;
+}
+
+/**
+ * Full-corpus Tier-1 candidate sweep for an EXISTING project, via a bounded
+ * inverted index (research Q-C — NOT O(N^2)): only tokens appearing in 2..maxPostingDf
+ * rows form postings; pairs sharing >= minSharedTokens of them are classified.
+ * Persists review-only candidates (INSERT OR IGNORE). Returns the count persisted.
+ */
+export function sweepProjectCandidates(db: Database, project: string, cfg: DedupRuntimeConfig): number {
+  const rows = db.prepare(
+    'SELECT id, title FROM observations WHERE project = ? AND title IS NOT NULL ORDER BY id ASC'
+  ).all(project) as { id: number; title: string }[];
+  if (rows.length < 2) return 0;
+  const { idfFn, docCount } = buildProjectIdf(db, project);
+  const thresholds: ClassifyThresholds = {
+    cosineThreshold: cfg.cosineThreshold,
+    vetoThetaIdf: idf(cfg.idfVetoDf, docCount),
+    minSharedTokens: cfg.minSharedTokens,
+  };
+  const tokensPerRow = rows.map(r => new Set(tokenizeWs(r.title)));
+  const df = new Map<string, number>();
+  for (const set of tokensPerRow) for (const t of set) df.set(t, (df.get(t) ?? 0) + 1);
+  const maxPostingDf = Math.max(2, Math.ceil(Math.sqrt(rows.length)) * 4); // bound postings (skip ultra-common tokens)
+  const postings = new Map<string, number[]>();
+  tokensPerRow.forEach((set, i) => {
+    for (const t of set) {
+      const d = df.get(t)!;
+      if (d < 2 || d > maxPostingDf) continue;
+      let list = postings.get(t);
+      if (!list) { list = []; postings.set(t, list); }
+      list.push(i);
+    }
+  });
+  const shared = new Map<string, number>(); // "i:j" (i<j) -> shared discriminating-token count
+  for (const ids of postings.values()) {
+    for (let x = 0; x < ids.length; x++) for (let y = x + 1; y < ids.length; y++) {
+      const key = `${ids[x]}:${ids[y]}`;
+      shared.set(key, (shared.get(key) ?? 0) + 1);
+    }
+  }
+  const ins = db.prepare(
+    'INSERT OR IGNORE INTO observation_dedup_candidates ' +
+    '(observation_id, duplicate_of_id, project, method, score, status, created_at, created_at_epoch) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
+  );
+  const nowIso = new Date().toISOString();
+  const nowEpoch = Date.now();
+  let count = 0;
+  for (const [key, sh] of shared) {
+    if (sh < cfg.minSharedTokens) continue;
+    const [i, j] = key.split(':').map(Number);
+    const c = classifyPair(rows[i].title, rows[j].title, idfFn, thresholds);
+    if (c.tier === 'candidate') {
+      ins.run(rows[j].id, rows[i].id, project, c.method, c.score, 'pending', nowIso, nowEpoch);
+      count++;
+    }
+  }
+  return count;
+}
+
+/** Run a full dedup-scan (backfill + sweep) over every project. Idempotent. */
+export function runDedupScan(db: Database, cfg: DedupRuntimeConfig): { project: string; docs: number; candidates: number }[] {
+  const projects = (db.prepare('SELECT DISTINCT project FROM observations').all() as { project: string }[]).map(r => r.project);
+  return projects.map(project => {
+    const docs = backfillProjectDedup(db, project);
+    const candidates = sweepProjectCandidates(db, project, cfg);
+    return { project, docs, candidates };
+  });
+}
+
 /**
  * Tier-1 (review-only) near-duplicate scan for a freshly-inserted observation.
  * Compares its title against up to `maxScan` recent same-project titles and
diff --git a/tests/sqlite/dedup-scan.test.ts b/tests/sqlite/dedup-scan.test.ts
new file mode 100644
index 0000000000..ec285af587
--- /dev/null
+++ b/tests/sqlite/dedup-scan.test.ts
@@ -0,0 +1,79 @@
+import { describe, it, expect, beforeEach, afterEach } from 'bun:test';
+import { SessionStore } from '../../src/services/sqlite/SessionStore.js';
+import { backfillProjectDedup, sweepProjectCandidates, runDedupScan, computeTitleNormKey } from '../../src/services/sqlite/dedup-store.js';
+
+const CFG = { cosineThreshold: 0.8, idfVetoDf: 10, minSharedTokens: 2, maxScan: 2000 };
+
+describe('dedup-scan: backfill + sweep (#3038)', () => {
+  let store: any;
+  beforeEach(() => { store = new SessionStore(':memory:'); }); // dedup OFF by default -> no maintenance
+  afterEach(() => store.close());
+
+  function seed(titles: string[], project = 'p') {
+    const id = store.createSDKSession(`c-${project}`, project, 'prompt');
+    store.updateMemorySessionId(id, `m-${project}`);
+    let t = Date.now();
+    for (const title of titles) {
+      store.storeObservation(`m-${project}`, project, { type: 'discovery', title, subtitle: null, facts: [], narrative: `n-${title}`, concepts: [], files_read: [], files_modified: [] }, 1, 0, t++);
+    }
+  }
+  const dfCount = (project = 'p') => (store.db.prepare('SELECT COUNT(*) c FROM token_df WHERE project = ?').get(project) as any).c;
+  const docCount = (project = 'p') => (store.db.prepare('SELECT doc_count FROM dedup_meta WHERE project = ?').get(project) as any)?.doc_count ?? 0;
+  const candCount = () => (store.db.prepare('SELECT COUNT(*) c FROM observation_dedup_candidates').get() as any).c;
+
+  it('backfill (re)builds title_norm_key, token_df, and doc_count idempotently', () => {
+    seed(['Build The Worker', 'Ship The Release', 'Audit The Logs']);
+    // simulate a legacy/pre-dedup DB: blank the key and clear the IDF model
+    store.db.run("UPDATE observations SET title_norm_key = NULL");
+    expect(dfCount()).toBe(0);
+
+    const n = backfillProjectDedup(store.db, 'p');
+    expect(n).toBe(3);
+    expect(docCount()).toBe(3);
+    expect(dfCount()).toBeGreaterThan(0);
+    const keyed = (store.db.prepare("SELECT COUNT(*) c FROM observations WHERE title_norm_key IS NOT NULL").get() as any).c;
+    expect(keyed).toBe(3);
+    const buildKey = computeTitleNormKey('p', 'Build The Worker');
+    const row = store.db.prepare('SELECT title_norm_key FROM observations WHERE title = ?').get('Build The Worker') as any;
+    expect(row.title_norm_key).toBe(buildKey);
+
+    // idempotent: re-run -> same df rows + doc_count (not doubled)
+    const dfBefore = dfCount();
+    backfillProjectDedup(store.db, 'p');
+    expect(dfCount()).toBe(dfBefore);
+    expect(docCount()).toBe(3);
+  });
+
+  it('sweep finds an existing reorder near-dup and persists a review-only candidate', () => {
+    seed([
+      'alpha bravo charlie', 'delta echo foxtrot', 'golf hotel india',
+      'build the worker service module', 'worker build the service module', // reorder near-dup
+    ]);
+    backfillProjectDedup(store.db, 'p');
+    const n = sweepProjectCandidates(store.db, 'p', CFG);
+    expect(n).toBeGreaterThanOrEqual(1);
+
+    const ids = store.db.prepare("SELECT id FROM observations WHERE title LIKE '%worker%service%' OR title LIKE '%worker build%'").all() as any[];
+    expect(ids.length).toBe(2);
+    const cand = store.db.prepare('SELECT method, status FROM observation_dedup_candidates LIMIT 1').get() as any;
+    expect(cand.method).toBe('idf_cosine');
+    expect(cand.status).toBe('pending');
+  });
+
+  it('sweep is idempotent (re-run does not duplicate candidate rows)', () => {
+    seed(['build the worker service module', 'worker build the service module', 'totally distinct topic here']);
+    backfillProjectDedup(store.db, 'p');
+    sweepProjectCandidates(store.db, 'p', CFG);
+    const after1 = candCount();
+    sweepProjectCandidates(store.db, 'p', CFG);
+    expect(candCount()).toBe(after1); // UNIQUE(observation_id,duplicate_of_id) guards
+  });
+
+  it('runDedupScan covers every project', () => {
+    seed(['one alpha', 'two beta'], 'projA');
+    seed(['three gamma', 'four delta'], 'projB');
+    const report = runDedupScan(store.db, CFG);
+    expect(report.map(r => r.project).sort()).toEqual(['projA', 'projB']);
+    expect(report.every(r => r.docs === 2)).toBe(true);
+  });
+});

From e6db9a903b326eb2c7e88d9fea37921e01c7f6ee Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 01:45:39 +0200
Subject: [PATCH 15/19] feat(dedup): worker HTTP route + SessionStore surfaces
 for candidates & scan (#3038)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

GET /api/dedup/candidates (read-only, joined to both titles, project-scoped)
and POST /api/dedup/scan (opt-in backfill+sweep all projects) — thin glue over
tested SessionStore.listDedupCandidates / runDedupScan. Registered in
worker-service. Gives the Tier-1 candidates table a standalone consumer and the
dedup-scan its callable surface (CLI/viewer deferred).
---
 src/services/sqlite/SessionStore.ts           | 30 ++++++++++++++++
 src/services/worker-service.ts                |  2 ++
 .../worker/http/routes/DedupRoutes.ts         | 35 +++++++++++++++++++
 tests/sqlite/dedup-scan.test.ts               | 20 +++++++++++
 4 files changed, 87 insertions(+)
 create mode 100644 src/services/worker/http/routes/DedupRoutes.ts

diff --git a/src/services/sqlite/SessionStore.ts b/src/services/sqlite/SessionStore.ts
index 19f4258478..c78a1186b2 100644
--- a/src/services/sqlite/SessionStore.ts
+++ b/src/services/sqlite/SessionStore.ts
@@ -16,6 +16,7 @@ import { computeObservationContentHash } from './observations/store.js';
 import { SettingsDefaultsManager } from '../../shared/SettingsDefaultsManager.js';
 import {
   computeTitleNormKey, findTier0Canonical, bumpTokenDf, isFuzzyReady, recordTier1Candidates,
+  runDedupScan as runDedupScanAll,
   type DedupRuntimeConfig,
 } from './dedup-store.js';
 import { parseFileList } from './observations/files.js';
@@ -2233,6 +2234,35 @@ export class SessionStore {
     };
   }
 
+  // #3038 — read-only listing of Tier-1 near-duplicate candidates (joined to both titles).
+  listDedupCandidates(
+    project?: string,
+    limit = 100
+  ): Array<{
+    id: number; project: string; method: string; score: number; status: string; created_at_epoch: number;
+    observation_id: number; observation_title: string | null;
+    duplicate_of_id: number; duplicate_of_title: string | null;
+  }> {
+    const where = project ? 'WHERE c.project = ?' : '';
+    const params: SQLQueryBindings[] = project ? [project, limit] : [limit];
+    return this.db.prepare(`
+      SELECT c.id, c.project, c.method, c.score, c.status, c.created_at_epoch,
+             c.observation_id, o1.title AS observation_title,
+             c.duplicate_of_id, o2.title AS duplicate_of_title
+      FROM observation_dedup_candidates c
+      JOIN observations o1 ON o1.id = c.observation_id
+      JOIN observations o2 ON o2.id = c.duplicate_of_id
+      ${where}
+      ORDER BY c.score DESC, c.id DESC
+      LIMIT ?
+    `).all(...params) as any;
+  }
+
+  // #3038 — opt-in dedup-scan: backfill the IDF model + sweep all projects for candidates.
+  runDedupScan(): { project: string; docs: number; candidates: number }[] {
+    return runDedupScanAll(this.db, this.dedupConfig());
+  }
+
   // Forward IDF maintenance + Tier-1 candidate scan for a freshly-inserted observation.
   private maintainDedupOnInsert(
     project: string,
diff --git a/src/services/worker-service.ts b/src/services/worker-service.ts
index 5db8791290..74d68d20f6 100644
--- a/src/services/worker-service.ts
+++ b/src/services/worker-service.ts
@@ -102,6 +102,7 @@ import { SearchRoutes } from './worker/http/routes/SearchRoutes.js';
 import { SettingsRoutes } from './worker/http/routes/SettingsRoutes.js';
 import { LogsRoutes } from './worker/http/routes/LogsRoutes.js';
 import { MemoryRoutes } from './worker/http/routes/MemoryRoutes.js';
+import { DedupRoutes } from './worker/http/routes/DedupRoutes.js';
 import { CorpusRoutes } from './worker/http/routes/CorpusRoutes.js';
 import { ChromaRoutes } from './worker/http/routes/ChromaRoutes.js';
 
@@ -354,6 +355,7 @@ export class WorkerService implements WorkerRef {
     this.server.registerRoutes(new SettingsRoutes(this.settingsManager));
     this.server.registerRoutes(new LogsRoutes());
     this.server.registerRoutes(new MemoryRoutes(this.dbManager, 'claude-mem'));
+    this.server.registerRoutes(new DedupRoutes(this.dbManager));
     this.server.registerRoutes(new ServerV1Routes({
       getDatabase: () => this.dbManager.getConnection(),
     }));
diff --git a/src/services/worker/http/routes/DedupRoutes.ts b/src/services/worker/http/routes/DedupRoutes.ts
new file mode 100644
index 0000000000..e7f9fd10e6
--- /dev/null
+++ b/src/services/worker/http/routes/DedupRoutes.ts
@@ -0,0 +1,35 @@
+import express, { Request, Response } from 'express';
+import { BaseRouteHandler } from '../BaseRouteHandler.js';
+import type { DatabaseManager } from '../../DatabaseManager.js';
+
+/**
+ * #3038 near-duplicate dedup surfaces:
+ *  - GET  /api/dedup/candidates?project=&limit=  — read-only list of Tier-1
+ *    review-only near-dup candidates (joined to both observation titles).
+ *  - POST /api/dedup/scan  — opt-in idempotent backfill of the IDF model +
+ *    full-corpus candidate sweep across all projects.
+ * Both are thin wrappers over tested SessionStore methods.
+ */
+export class DedupRoutes extends BaseRouteHandler {
+  constructor(private dbManager: DatabaseManager) {
+    super();
+  }
+
+  setupRoutes(app: express.Application): void {
+    app.get('/api/dedup/candidates', this.handleListCandidates.bind(this));
+    app.post('/api/dedup/scan', this.handleScan.bind(this));
+  }
+
+  private handleListCandidates = this.wrapHandler(async (req: Request, res: Response): Promise<void> => {
+    const project = typeof req.query.project === 'string' && req.query.project.trim() ? req.query.project.trim() : undefined;
+    const requested = Number(req.query.limit);
+    const limit = Number.isFinite(requested) && requested > 0 ? Math.min(requested, 500) : 100;
+    const candidates = this.dbManager.getSessionStore().listDedupCandidates(project, limit);
+    res.json({ candidates });
+  });
+
+  private handleScan = this.wrapHandler(async (_req: Request, res: Response): Promise<void> => {
+    const scanned = this.dbManager.getSessionStore().runDedupScan();
+    res.json({ scanned });
+  });
+}
diff --git a/tests/sqlite/dedup-scan.test.ts b/tests/sqlite/dedup-scan.test.ts
index ec285af587..ca48fbcd11 100644
--- a/tests/sqlite/dedup-scan.test.ts
+++ b/tests/sqlite/dedup-scan.test.ts
@@ -69,6 +69,26 @@ describe('dedup-scan: backfill + sweep (#3038)', () => {
     expect(candCount()).toBe(after1); // UNIQUE(observation_id,duplicate_of_id) guards
   });
 
+  it('listDedupCandidates returns candidates joined to both titles, project-scoped', () => {
+    seed(['build the worker service module', 'worker build the service module', 'distinct subject matter']);
+    backfillProjectDedup(store.db, 'p');
+    sweepProjectCandidates(store.db, 'p', CFG);
+    const list = store.listDedupCandidates('p');
+    expect(list.length).toBeGreaterThanOrEqual(1);
+    expect(list[0].observation_title).toBeTruthy();
+    expect(list[0].duplicate_of_title).toBeTruthy();
+    expect(list[0].method).toBe('idf_cosine');
+    expect(store.listDedupCandidates('other-project')).toEqual([]);
+  });
+
+  it('store.runDedupScan() backfills + sweeps all projects via configured knobs', () => {
+    process.env.CLAUDE_MEM_DEDUP_COSINE_THRESHOLD = '0.80';
+    seed(['build the worker service module', 'worker build the service module'], 'pp');
+    const report = store.runDedupScan();
+    expect(report.find((r: any) => r.project === 'pp')?.docs).toBe(2);
+    delete process.env.CLAUDE_MEM_DEDUP_COSINE_THRESHOLD;
+  });
+
   it('runDedupScan covers every project', () => {
     seed(['one alpha', 'two beta'], 'projA');
     seed(['three gamma', 'four delta'], 'projB');

From fb854d8f890c37d48ada80c6f55e1eff23927f6e Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 01:58:36 +0200
Subject: [PATCH 16/19] fix(dedup): address security + code review findings
 (#3038)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Security:
- M1: single-flight guard on POST /api/dedup/scan (409 on overlap)
- M2: two explicit prepared statements in listDedupCandidates (drop ${where}
  interpolation + the as-any cast)
- L1: document the localhost-only trust model on DedupRoutes
- L2: CLAUDE_MEM_DEDUP_MAX_BACKFILL_ROWS cap (skip+warn oversized projects)
Code review:
- C1: occurrence_count retry-idempotency — a redelivered (session,content_hash)
  no longer bumps the count (single + batch paths)
- C2: regression test — Tier-0 merge AND content_hash retry leave token_df/
  doc_count flat (maintenance = real inserts only)
- C3: annotate dedup_meta drift columns as reserved (no delete hook yet)
- C4: POST /api/dedup/scan gated on CLAUDE_MEM_DEDUP_ENABLED (no row mutation
  when disabled)
- C5: buildProjectIdf loads project DF once into a Map (no per-token round-trip)
- C7: dedupConfig clamps NaN config values to safe defaults
- C8: shared candidateInsert() helper (DRY)
typecheck clean; 175 dedup/sqlite/settings tests green.
---
 src/services/sqlite/SessionStore.ts           | 61 +++++++++++++------
 src/services/sqlite/dedup-store.ts            | 52 ++++++++++------
 .../worker/http/routes/DedupRoutes.ts         | 29 ++++++++-
 src/shared/SettingsDefaultsManager.ts         |  2 +
 tests/dedup/config.test.ts                    |  2 +
 tests/sqlite/dedup-scan.test.ts               |  2 +-
 .../session-store-dedup-integration.test.ts   | 20 ++++++
 7 files changed, 128 insertions(+), 40 deletions(-)

diff --git a/src/services/sqlite/SessionStore.ts b/src/services/sqlite/SessionStore.ts
index c78a1186b2..9164f48734 100644
--- a/src/services/sqlite/SessionStore.ts
+++ b/src/services/sqlite/SessionStore.ts
@@ -13,7 +13,7 @@ import {
 } from '../../types/database.js';
 import type { ObservationSearchResult, SessionSummarySearchResult } from './types.js';
 import { computeObservationContentHash } from './observations/store.js';
-import { SettingsDefaultsManager } from '../../shared/SettingsDefaultsManager.js';
+import { SettingsDefaultsManager, type SettingsDefaults } from '../../shared/SettingsDefaultsManager.js';
 import {
   computeTitleNormKey, findTier0Canonical, bumpTokenDf, isFuzzyReady, recordTier1Candidates,
   runDedupScan as runDedupScanAll,
@@ -2223,14 +2223,20 @@ export class SessionStore {
   }
 
   // #3038 — resolve the dedup knobs from settings (cheap; off by default).
+  // Garbage/NaN values fall back to the safe defaults rather than disabling guards.
   private dedupConfig(): DedupRuntimeConfig & { enabled: boolean; minProjectDocs: number } {
+    const num = (key: keyof SettingsDefaults, fallback: number): number => {
+      const v = Number(SettingsDefaultsManager.get(key));
+      return Number.isFinite(v) ? v : fallback;
+    };
     return {
       enabled: SettingsDefaultsManager.getBool('CLAUDE_MEM_DEDUP_ENABLED'),
-      cosineThreshold: Number(SettingsDefaultsManager.get('CLAUDE_MEM_DEDUP_COSINE_THRESHOLD')),
-      idfVetoDf: SettingsDefaultsManager.getInt('CLAUDE_MEM_DEDUP_IDF_VETO_DF'),
-      minSharedTokens: SettingsDefaultsManager.getInt('CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS'),
-      maxScan: SettingsDefaultsManager.getInt('CLAUDE_MEM_DEDUP_MAX_SCAN'),
-      minProjectDocs: SettingsDefaultsManager.getInt('CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS'),
+      cosineThreshold: num('CLAUDE_MEM_DEDUP_COSINE_THRESHOLD', 0.8),
+      idfVetoDf: num('CLAUDE_MEM_DEDUP_IDF_VETO_DF', 10),
+      minSharedTokens: num('CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS', 2),
+      maxScan: num('CLAUDE_MEM_DEDUP_MAX_SCAN', 2000),
+      maxBackfillRows: num('CLAUDE_MEM_DEDUP_MAX_BACKFILL_ROWS', 50000),
+      minProjectDocs: num('CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS', 10),
     };
   }
 
@@ -2243,19 +2249,22 @@ export class SessionStore {
     observation_id: number; observation_title: string | null;
     duplicate_of_id: number; duplicate_of_title: string | null;
   }> {
-    const where = project ? 'WHERE c.project = ?' : '';
-    const params: SQLQueryBindings[] = project ? [project, limit] : [limit];
-    return this.db.prepare(`
-      SELECT c.id, c.project, c.method, c.score, c.status, c.created_at_epoch,
-             c.observation_id, o1.title AS observation_title,
-             c.duplicate_of_id, o2.title AS duplicate_of_title
-      FROM observation_dedup_candidates c
-      JOIN observations o1 ON o1.id = c.observation_id
-      JOIN observations o2 ON o2.id = c.duplicate_of_id
-      ${where}
-      ORDER BY c.score DESC, c.id DESC
-      LIMIT ?
-    `).all(...params) as any;
+    type Row = {
+      id: number; project: string; method: string; score: number; status: string; created_at_epoch: number;
+      observation_id: number; observation_title: string | null;
+      duplicate_of_id: number; duplicate_of_title: string | null;
+    };
+    // Two explicit prepared statements — no conditional SQL-fragment interpolation.
+    const select =
+      'SELECT c.id, c.project, c.method, c.score, c.status, c.created_at_epoch, ' +
+      'c.observation_id, o1.title AS observation_title, c.duplicate_of_id, o2.title AS duplicate_of_title ' +
+      'FROM observation_dedup_candidates c ' +
+      'JOIN observations o1 ON o1.id = c.observation_id ' +
+      'JOIN observations o2 ON o2.id = c.duplicate_of_id ';
+    const order = 'ORDER BY c.score DESC, c.id DESC LIMIT ?';
+    return project
+      ? this.db.prepare(`${select}WHERE c.project = ? ${order}`).all(project, limit) as Row[]
+      : this.db.prepare(`${select}${order}`).all(limit) as Row[];
   }
 
   // #3038 — opt-in dedup-scan: backfill the IDF model + sweep all projects for candidates.
@@ -2308,6 +2317,15 @@ export class SessionStore {
     // duplicate of any age — bump its occurrence_count and return it, no insert. Mirrors
     // the content_hash ON CONFLICT DO NOTHING semantics but cross-session + normalization-aware.
     if (dedup.enabled) {
+      // Retry idempotency: a redelivered identical (session, content_hash) must NOT
+      // bump occurrence_count — return the existing row, exactly as the legacy
+      // ON CONFLICT DO NOTHING path did. Only a genuinely-new observation that shares
+      // a normalized title (cross-session recurrence) counts as a Tier-0 occurrence.
+      const retry = this.db.prepare(
+        'SELECT id, created_at_epoch FROM observations WHERE memory_session_id = ? AND content_hash = ?'
+      ).get(memorySessionId, contentHash) as { id: number; created_at_epoch: number } | undefined;
+      if (retry) return { id: retry.id, createdAtEpoch: retry.created_at_epoch };
+
       const canonical = findTier0Canonical(this.db, project, titleNormKey);
       if (canonical) {
         this.db.prepare('UPDATE observations SET occurrence_count = occurrence_count + 1 WHERE id = ?').run(canonical.id);
@@ -2466,6 +2484,11 @@ export class SessionStore {
         // Tier-0 (#3038): cross-session normalized-title duplicate (incl. earlier items
         // in THIS batch — already inserted and visible in-transaction) → bump + reuse.
         if (dedup.enabled) {
+          // Retry idempotency (see storeObservation): identical (session, content_hash)
+          // redelivery returns the existing row without bumping occurrence_count.
+          const retry = lookupExistingStmt.get(memorySessionId, contentHash) as { id: number } | null;
+          if (retry) { observationIds.push(retry.id); continue; }
+
           const canonical = findTier0Canonical(this.db, project, titleNormKey);
           if (canonical) {
             this.db.prepare('UPDATE observations SET occurrence_count = occurrence_count + 1 WHERE id = ?').run(canonical.id);
diff --git a/src/services/sqlite/dedup-store.ts b/src/services/sqlite/dedup-store.ts
index c71ee950e3..e151b6749d 100644
--- a/src/services/sqlite/dedup-store.ts
+++ b/src/services/sqlite/dedup-store.ts
@@ -22,6 +22,7 @@ export interface DedupRuntimeConfig {
   idfVetoDf: number;
   minSharedTokens: number;
   maxScan: number;
+  maxBackfillRows: number;
 }
 
 /**
@@ -73,15 +74,25 @@ export function getProjectDocCount(db: Database, project: string): number {
   return row?.doc_count ?? 0;
 }
 
-/** Build a project-scoped `token -> idf` function plus the corpus size, from token_df. */
+/**
+ * Build a project-scoped `token -> idf` function plus the corpus size, from token_df.
+ * Loads the whole project DF table into a Map ONCE (not a per-token SQLite round-trip),
+ * so a scan classifying thousands of pairs does one query, not thousands.
+ */
 export function buildProjectIdf(db: Database, project: string): { idfFn: (t: string) => number; docCount: number } {
   const docCount = getProjectDocCount(db, project);
-  const dfStmt = db.prepare('SELECT df FROM token_df WHERE project = ? AND token = ?');
-  const dfLookup = (token: string): number => {
-    const row = dfStmt.get(project, token) as { df: number } | undefined;
-    return row?.df ?? 0;
-  };
-  return { idfFn: buildIdfFn(dfLookup, docCount), docCount };
+  const dfRows = db.prepare('SELECT token, df FROM token_df WHERE project = ?').all(project) as { token: string; df: number }[];
+  const dfMap = new Map<string, number>(dfRows.map(r => [r.token, r.df]));
+  return { idfFn: buildIdfFn((token: string) => dfMap.get(token) ?? 0, docCount), docCount };
+}
+
+/** Shared prepared-statement factory for persisting a review-only candidate (DRY). */
+function candidateInsert(db: Database) {
+  return db.prepare(
+    'INSERT OR IGNORE INTO observation_dedup_candidates ' +
+    '(observation_id, duplicate_of_id, project, method, score, status, created_at, created_at_epoch) ' +
+    'VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
+  );
 }
 
 /** Cold-start gate: fuzzy Tier-1 is only trustworthy once the corpus is large enough. */
@@ -96,7 +107,15 @@ export function isFuzzyReady(db: Database, project: string, minDocs: number): bo
  * safe to re-run. This is how an EXISTING DB starts participating in dedup, and how
  * post-deletion DF drift is reclaimed. Returns the project doc count.
  */
-export function backfillProjectDedup(db: Database, project: string): number {
+export function backfillProjectDedup(db: Database, project: string, maxRows: number = Number.POSITIVE_INFINITY): number {
+  const total = (db.prepare('SELECT COUNT(*) c FROM observations WHERE project = ?').get(project) as { c: number }).c;
+  if (total > maxRows) {
+    logger.warn('DEDUP', `Skipping dedup backfill for project ${project}: ${total} rows exceeds cap ${maxRows} (CLAUDE_MEM_DEDUP_MAX_BACKFILL_ROWS)`);
+    return 0;
+  }
+  // NOTE: dedup_meta.deleted_since_rebuild / last_rebuild_doc_count are reset here and are
+  // RESERVED for a future delete-hook-driven auto-rebuild; today the scan always does a full
+  // rebuild, so nothing increments deleted_since_rebuild (no runtime observation-delete path).
   const rows = db.prepare('SELECT id, title FROM observations WHERE project = ?').all(project) as { id: number; title: string | null }[];
   const updKey = db.prepare('UPDATE observations SET title_norm_key = ? WHERE id = ?');
   const insDf = db.prepare('INSERT INTO token_df (project, token, df) VALUES (?, ?, ?)');
@@ -128,6 +147,10 @@ export function sweepProjectCandidates(db: Database, project: string, cfg: Dedup
     'SELECT id, title FROM observations WHERE project = ? AND title IS NOT NULL ORDER BY id ASC'
   ).all(project) as { id: number; title: string }[];
   if (rows.length < 2) return 0;
+  if (rows.length > cfg.maxBackfillRows) {
+    logger.warn('DEDUP', `Skipping dedup sweep for project ${project}: ${rows.length} rows exceeds cap ${cfg.maxBackfillRows}`);
+    return 0;
+  }
   const { idfFn, docCount } = buildProjectIdf(db, project);
   const thresholds: ClassifyThresholds = {
     cosineThreshold: cfg.cosineThreshold,
@@ -155,10 +178,7 @@ export function sweepProjectCandidates(db: Database, project: string, cfg: Dedup
       shared.set(key, (shared.get(key) ?? 0) + 1);
     }
   }
-  const ins = db.prepare(
-    'INSERT OR IGNORE INTO observation_dedup_candidates ' +
-    '(observation_id, duplicate_of_id, project, method, score, status, created_at, created_at_epoch) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
-  );
+  const ins = candidateInsert(db);
   const nowIso = new Date().toISOString();
   const nowEpoch = Date.now();
   let count = 0;
@@ -178,7 +198,7 @@ export function sweepProjectCandidates(db: Database, project: string, cfg: Dedup
 export function runDedupScan(db: Database, cfg: DedupRuntimeConfig): { project: string; docs: number; candidates: number }[] {
   const projects = (db.prepare('SELECT DISTINCT project FROM observations').all() as { project: string }[]).map(r => r.project);
   return projects.map(project => {
-    const docs = backfillProjectDedup(db, project);
+    const docs = backfillProjectDedup(db, project, cfg.maxBackfillRows);
     const candidates = sweepProjectCandidates(db, project, cfg);
     return { project, docs, candidates };
   });
@@ -213,11 +233,7 @@ export function recordTier1Candidates(
   if (rows.length === cfg.maxScan) {
     logger.debug('DEDUP', `Tier-1 scan hit MAX_SCAN=${cfg.maxScan} for project ${project}; older rows covered by dedup-scan`);
   }
-  const ins = db.prepare(
-    'INSERT OR IGNORE INTO observation_dedup_candidates ' +
-    '(observation_id, duplicate_of_id, project, method, score, status, created_at, created_at_epoch) ' +
-    'VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
-  );
+  const ins = candidateInsert(db);
   const nowIso = new Date().toISOString();
   const nowEpoch = Date.now();
   let count = 0;
diff --git a/src/services/worker/http/routes/DedupRoutes.ts b/src/services/worker/http/routes/DedupRoutes.ts
index e7f9fd10e6..02900b6459 100644
--- a/src/services/worker/http/routes/DedupRoutes.ts
+++ b/src/services/worker/http/routes/DedupRoutes.ts
@@ -1,5 +1,6 @@
 import express, { Request, Response } from 'express';
 import { BaseRouteHandler } from '../BaseRouteHandler.js';
+import { SettingsDefaultsManager } from '../../../../shared/SettingsDefaultsManager.js';
 import type { DatabaseManager } from '../../DatabaseManager.js';
 
 /**
@@ -9,8 +10,17 @@ import type { DatabaseManager } from '../../DatabaseManager.js';
  *  - POST /api/dedup/scan  — opt-in idempotent backfill of the IDF model +
  *    full-corpus candidate sweep across all projects.
  * Both are thin wrappers over tested SessionStore methods.
+ *
+ * Trust model: like every other worker route, these are guarded only by the
+ * worker's 127.0.0.1 binding + localhost CORS — any local process can reach
+ * them. GET candidates exposes observation titles project-wide; no per-route
+ * auth, consistent with DataRoutes/SearchRoutes/MemoryRoutes.
  */
 export class DedupRoutes extends BaseRouteHandler {
+  // Single-flight guard: the scan is an expensive full-corpus, event-loop-blocking
+  // operation; reject overlapping runs instead of compounding CPU/memory pressure.
+  private static scanInProgress = false;
+
   constructor(private dbManager: DatabaseManager) {
     super();
   }
@@ -29,7 +39,22 @@ export class DedupRoutes extends BaseRouteHandler {
   });
 
   private handleScan = this.wrapHandler(async (_req: Request, res: Response): Promise<void> => {
-    const scanned = this.dbManager.getSessionStore().runDedupScan();
-    res.json({ scanned });
+    // Scan MUTATES observation rows (title_norm_key) — gate on the feature flag so a
+    // disabled install stays byte-identical to legacy behavior.
+    if (!SettingsDefaultsManager.getBool('CLAUDE_MEM_DEDUP_ENABLED')) {
+      res.status(409).json({ error: 'dedup_disabled', message: 'Set CLAUDE_MEM_DEDUP_ENABLED=true before running a dedup scan.' });
+      return;
+    }
+    if (DedupRoutes.scanInProgress) {
+      res.status(409).json({ error: 'scan_in_progress', message: 'A dedup scan is already running.' });
+      return;
+    }
+    DedupRoutes.scanInProgress = true;
+    try {
+      const scanned = this.dbManager.getSessionStore().runDedupScan();
+      res.json({ scanned });
+    } finally {
+      DedupRoutes.scanInProgress = false;
+    }
   });
 }
diff --git a/src/shared/SettingsDefaultsManager.ts b/src/shared/SettingsDefaultsManager.ts
index 6c09292b40..435ab0bddf 100644
--- a/src/shared/SettingsDefaultsManager.ts
+++ b/src/shared/SettingsDefaultsManager.ts
@@ -95,6 +95,7 @@ export interface SettingsDefaults {
   CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS: string;      // require >= N shared tokens before computing cosine (sparse-vector guard)
   CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS: string;       // cold-start: skip fuzzy Tier-1 below N docs/project (IDF unreliable)
   CLAUDE_MEM_DEDUP_MAX_SCAN: string;               // cap Tier-1 candidate scan per insert (logged when hit)
+  CLAUDE_MEM_DEDUP_MAX_BACKFILL_ROWS: string;      // dedup-scan safety valve: skip a project larger than this (avoids OOM)
 }
 
 export class SettingsDefaultsManager {
@@ -189,6 +190,7 @@ export class SettingsDefaultsManager {
     CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS: '2',             // >=2 shared tokens before cosine (kills sparse-vector noise)
     CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS: '10',            // cold-start gate: IDF unreliable below ~10 docs/project
     CLAUDE_MEM_DEDUP_MAX_SCAN: '2000',                  // per-insert candidate-scan cap (logged when exceeded)
+    CLAUDE_MEM_DEDUP_MAX_BACKFILL_ROWS: '50000',        // dedup-scan skips a project with more rows than this (memory safety)
   };
 
   static getAllDefaults(): SettingsDefaults {
diff --git a/tests/dedup/config.test.ts b/tests/dedup/config.test.ts
index 11721ef913..d316623734 100644
--- a/tests/dedup/config.test.ts
+++ b/tests/dedup/config.test.ts
@@ -8,6 +8,7 @@ const DEDUP_KEYS = [
   'CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS',
   'CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS',
   'CLAUDE_MEM_DEDUP_MAX_SCAN',
+  'CLAUDE_MEM_DEDUP_MAX_BACKFILL_ROWS',
 ] as const;
 
 // Env isolation (lesson from #3056/#3058): never leak DEDUP overrides across tests.
@@ -30,6 +31,7 @@ describe('dedup settings defaults', () => {
     expect(d.CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS).toBe('2');
     expect(d.CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS).toBe('10');
     expect(d.CLAUDE_MEM_DEDUP_MAX_SCAN).toBe('2000');
+    expect(d.CLAUDE_MEM_DEDUP_MAX_BACKFILL_ROWS).toBe('50000');
   });
 
   it('is disabled by default via getBool', () => {
diff --git a/tests/sqlite/dedup-scan.test.ts b/tests/sqlite/dedup-scan.test.ts
index ca48fbcd11..e99806c5c1 100644
--- a/tests/sqlite/dedup-scan.test.ts
+++ b/tests/sqlite/dedup-scan.test.ts
@@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'bun:test';
 import { SessionStore } from '../../src/services/sqlite/SessionStore.js';
 import { backfillProjectDedup, sweepProjectCandidates, runDedupScan, computeTitleNormKey } from '../../src/services/sqlite/dedup-store.js';
 
-const CFG = { cosineThreshold: 0.8, idfVetoDf: 10, minSharedTokens: 2, maxScan: 2000 };
+const CFG = { cosineThreshold: 0.8, idfVetoDf: 10, minSharedTokens: 2, maxScan: 2000, maxBackfillRows: 50000 };
 
 describe('dedup-scan: backfill + sweep (#3038)', () => {
   let store: any;
diff --git a/tests/sqlite/session-store-dedup-integration.test.ts b/tests/sqlite/session-store-dedup-integration.test.ts
index 61ebb42797..be96a5d00b 100644
--- a/tests/sqlite/session-store-dedup-integration.test.ts
+++ b/tests/sqlite/session-store-dedup-integration.test.ts
@@ -38,6 +38,26 @@ describe('storeObservation dedup integration (#3038)', () => {
     expect(occ).toBe(2);
   });
 
+  it('neither a Tier-0 merge NOR a content_hash retry grows token_df/doc_count (maintenance = real inserts only)', () => {
+    process.env.CLAUDE_MEM_DEDUP_ENABLED = 'true';
+    const t = Date.now();
+    store.storeObservation(session('s1'), 'p', obs('On-Demand Checkpoint.'), 1, 0, t);
+    const doc0 = (store.db.prepare("SELECT doc_count FROM dedup_meta WHERE project='p'").get() as any).doc_count;
+    const df0 = (store.db.prepare("SELECT COUNT(*) c FROM token_df WHERE project='p'").get() as any).c;
+    expect(doc0).toBe(1);
+    expect(df0).toBeGreaterThan(0);
+
+    store.storeObservation(session('s2'), 'p', obs('on demand checkpoint'), 1, 0, t + 1000); // Tier-0 merge
+    store.storeObservation('s1', 'p', obs('On-Demand Checkpoint.'), 1, 0, t);                  // content_hash retry
+
+    const doc1 = (store.db.prepare("SELECT doc_count FROM dedup_meta WHERE project='p'").get() as any).doc_count;
+    const df1 = (store.db.prepare("SELECT COUNT(*) c FROM token_df WHERE project='p'").get() as any).c;
+    expect(doc1).toBe(doc0); // merge + retry add no documents
+    expect(df1).toBe(df0);
+    const occ = (store.db.prepare('SELECT occurrence_count FROM observations LIMIT 1').get() as any).occurrence_count;
+    expect(occ).toBe(2); // exactly one merge bump (the retry did NOT bump)
+  });
+
   it('Tier-0: does NOT collapse the same normalized title across DIFFERENT projects', () => {
     process.env.CLAUDE_MEM_DEDUP_ENABLED = 'true';
     const t = Date.now();

From 7b89829c924cf95a567bf6dd032511a893cbfe98 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 02:02:15 +0200
Subject: [PATCH 17/19] docs(dedup): document CLAUDE_MEM_DEDUP_* settings +
 dedup-scan (#3038)

---
 docs/public/configuration.mdx | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/docs/public/configuration.mdx b/docs/public/configuration.mdx
index 9df3f3c106..79e3300f5a 100644
--- a/docs/public/configuration.mdx
+++ b/docs/public/configuration.mdx
@@ -66,6 +66,33 @@ Use [LiteLLM Gateway](configuration/litellm-gateway) when you want `CLAUDE_MEM_P
 | `CLAUDE_MEM_PYTHON_VERSION`   | `3.13`                          | Python version for chroma-mcp         |
 | `CLAUDE_CODE_PATH`            | _(auto-detect)_                 | Path to Claude Code CLI (for Windows) |
 
+### Near-Duplicate Deduplication
+
+Opt-in (off by default) dedup of observations whose titles describe the same recurring
+work but are worded slightly differently, so `content_hash` (byte-identical only) misses them.
+Two deterministic tiers:
+
+- **Tier 0 — exact-normalized-title** → safe silent auto-merge: a new observation whose
+  title matches an existing one after lowercasing / whitespace-collapse / punctuation-strip
+  is collapsed onto the existing row and its `occurrence_count` is bumped (cross-session).
+- **Tier 1 — IDF-weighted similarity** → *review-only* near-duplicate **candidates** are
+  recorded in `observation_dedup_candidates` (never auto-merged), so distinct work that
+  differs only in one rare token (`rdlp-api` vs `rdlp-plugin`) is never silently destroyed.
+
+| Setting                              | Default | Description                           |
+|--------------------------------------|---------|---------------------------------------|
+| `CLAUDE_MEM_DEDUP_ENABLED`           | `false` | Master switch. Off = byte-identical legacy behavior |
+| `CLAUDE_MEM_DEDUP_COSINE_THRESHOLD`  | `0.80`  | Min IDF-weighted cosine for a Tier-1 candidate |
+| `CLAUDE_MEM_DEDUP_IDF_VETO_DF`       | `10`    | A title token in ≤ N project rows is "discriminating" and vetoes a merge |
+| `CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS` | `2`     | Require ≥ N shared tokens before scoring (kills sparse-vector noise) |
+| `CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS`  | `10`    | Cold-start gate: skip Tier-1 below N docs/project (IDF unreliable) |
+| `CLAUDE_MEM_DEDUP_MAX_SCAN`          | `2000`  | Per-insert Tier-1 candidate-scan cap |
+| `CLAUDE_MEM_DEDUP_MAX_BACKFILL_ROWS` | `50000` | `dedup-scan` skips a project larger than this (memory safety) |
+
+After enabling on an existing database, run a one-time scan to backfill the IDF model and
+surface accumulated near-duplicates: `POST /api/dedup/scan` (worker, localhost-only).
+Review candidates via `GET /api/dedup/candidates?project=<name>`.
+
 ## Model Configuration
 
 Configure which Claude model compresses your observations (only applies when `CLAUDE_MEM_PROVIDER=claude`).

From 80b8a9de7a0a1688a9b2ad0fae1479bf7be5c74e Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 02:07:06 +0200
Subject: [PATCH 18/19] fix(dedup): truncate integer config knobs (review N1)
 for #3038
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

maxScan is bound as a SQL LIMIT — keep a fractional misconfig from reaching
the binding as a float.
---
 src/services/sqlite/SessionStore.ts | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/services/sqlite/SessionStore.ts b/src/services/sqlite/SessionStore.ts
index 9164f48734..f79674f0cf 100644
--- a/src/services/sqlite/SessionStore.ts
+++ b/src/services/sqlite/SessionStore.ts
@@ -2229,14 +2229,17 @@ export class SessionStore {
       const v = Number(SettingsDefaultsManager.get(key));
       return Number.isFinite(v) ? v : fallback;
     };
+    // Integer knobs are truncated — maxScan is bound as a SQL `LIMIT ?`, so a
+    // fractional misconfig must not reach the binding as a float (review N1).
+    const int = (key: keyof SettingsDefaults, fallback: number): number => Math.trunc(num(key, fallback));
     return {
       enabled: SettingsDefaultsManager.getBool('CLAUDE_MEM_DEDUP_ENABLED'),
       cosineThreshold: num('CLAUDE_MEM_DEDUP_COSINE_THRESHOLD', 0.8),
-      idfVetoDf: num('CLAUDE_MEM_DEDUP_IDF_VETO_DF', 10),
-      minSharedTokens: num('CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS', 2),
-      maxScan: num('CLAUDE_MEM_DEDUP_MAX_SCAN', 2000),
-      maxBackfillRows: num('CLAUDE_MEM_DEDUP_MAX_BACKFILL_ROWS', 50000),
-      minProjectDocs: num('CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS', 10),
+      idfVetoDf: int('CLAUDE_MEM_DEDUP_IDF_VETO_DF', 10),
+      minSharedTokens: int('CLAUDE_MEM_DEDUP_MIN_SHARED_TOKENS', 2),
+      maxScan: int('CLAUDE_MEM_DEDUP_MAX_SCAN', 2000),
+      maxBackfillRows: int('CLAUDE_MEM_DEDUP_MAX_BACKFILL_ROWS', 50000),
+      minProjectDocs: int('CLAUDE_MEM_DEDUP_MIN_PROJECT_DOCS', 10),
     };
   }
 

From 3234f3188da2938edffa3aa44ce118e93366d593 Mon Sep 17 00:00:00 2001
From: Mattias Carlsson <19933050+crippledgeek@users.noreply.github.com>
Date: Fri, 26 Jun 2026 02:16:06 +0200
Subject: [PATCH 19/19] fix(dedup): count only newly-persisted candidates, not
 ignored dups (#3038)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Greptile review: recordTier1Candidates/sweepProjectCandidates did count++ on
every INSERT OR IGNORE that classified as a candidate, so a re-run/redelivery
reported phantom counts for rows that were actually ignored by the
UNIQUE(observation_id,duplicate_of_id) guard. Count via .run().changes instead
— a second idempotent sweep now correctly returns 0. Tightened the idempotency
test to assert the return value, not just row count.
---
 src/services/sqlite/dedup-store.ts | 9 +++++----
 tests/sqlite/dedup-scan.test.ts    | 8 +++++---
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/services/sqlite/dedup-store.ts b/src/services/sqlite/dedup-store.ts
index e151b6749d..1f65258f05 100644
--- a/src/services/sqlite/dedup-store.ts
+++ b/src/services/sqlite/dedup-store.ts
@@ -187,8 +187,9 @@ export function sweepProjectCandidates(db: Database, project: string, cfg: Dedup
     const [i, j] = key.split(':').map(Number);
     const c = classifyPair(rows[i].title, rows[j].title, idfFn, thresholds);
     if (c.tier === 'candidate') {
-      ins.run(rows[j].id, rows[i].id, project, c.method, c.score, 'pending', nowIso, nowEpoch);
-      count++;
+      // Count only rows actually persisted — INSERT OR IGNORE returns changes=0 on a
+      // UNIQUE conflict (already-flagged pair), so re-runs report 0, not a phantom count.
+      count += ins.run(rows[j].id, rows[i].id, project, c.method, c.score, 'pending', nowIso, nowEpoch).changes;
     }
   }
   return count;
@@ -240,8 +241,8 @@ export function recordTier1Candidates(
   for (const r of rows) {
     const c = classifyPair(title, r.title, idfFn, thresholds);
     if (c.tier === 'candidate') {
-      ins.run(newObsId, r.id, project, c.method, c.score, 'pending', nowIso, nowEpoch);
-      count++;
+      // Count only newly-persisted rows (INSERT OR IGNORE → changes=0 on a UNIQUE conflict).
+      count += ins.run(newObsId, r.id, project, c.method, c.score, 'pending', nowIso, nowEpoch).changes;
     }
   }
   return count;
diff --git a/tests/sqlite/dedup-scan.test.ts b/tests/sqlite/dedup-scan.test.ts
index e99806c5c1..f2eb0076fa 100644
--- a/tests/sqlite/dedup-scan.test.ts
+++ b/tests/sqlite/dedup-scan.test.ts
@@ -63,10 +63,12 @@ describe('dedup-scan: backfill + sweep (#3038)', () => {
   it('sweep is idempotent (re-run does not duplicate candidate rows)', () => {
     seed(['build the worker service module', 'worker build the service module', 'totally distinct topic here']);
     backfillProjectDedup(store.db, 'p');
-    sweepProjectCandidates(store.db, 'p', CFG);
+    const firstReturned = sweepProjectCandidates(store.db, 'p', CFG);
     const after1 = candCount();
-    sweepProjectCandidates(store.db, 'p', CFG);
-    expect(candCount()).toBe(after1); // UNIQUE(observation_id,duplicate_of_id) guards
+    expect(firstReturned).toBe(after1); // count == rows actually persisted
+    const secondReturned = sweepProjectCandidates(store.db, 'p', CFG);
+    expect(candCount()).toBe(after1);   // UNIQUE(observation_id,duplicate_of_id) guards the data
+    expect(secondReturned).toBe(0);     // and the RETURN count reflects 0 newly-persisted (not ignored dups)
   });
 
   it('listDedupCandidates returns candidates joined to both titles, project-scoped', () => {