python-math #26: feat(math): Q11 — replicate vectorz distance cancellation in legacy kmeans

delphi/polismath/pca_kmeans_rep/legacy_kmeans.py

@@ -91,8 +91,22 @@ def n_distinct_rows(self) -> int:
 
 
 def _euclidean(a: np.ndarray, b: np.ndarray) -> float:
-    """``matrix/distance`` (L2). Clojure uses core.matrix euclidean distance."""
-    return float(np.linalg.norm(np.asarray(a, dtype=float) - np.asarray(b, dtype=float)))
+    """``matrix/distance`` as vectorz ACTUALLY computes it on the kmeans path
+    (CLOJURE_QUIRKS.md Q11): d² = |a|² + |b|² − 2·a·b, clamped at 0.
+
+    NOT ``norm(a − b)``: the dot-product form suffers catastrophic
+    cancellation, flooring any true distance below ~1e-8 (relative to the
+    vectors' magnitude) to EXACTLY 0.0. That floor is semantic in Clojure —
+    near-coincident points TIE at 0.0 against multiple clusters and min-key's
+    last-wins tie-break merges them into the LATER cluster (verified on the
+    vw every-vote step-57 pair: true distance 4.66e-15 → both cluster
+    distances 0.0 → merge; math/dev/proj_probe.clj + journal 2026-07-22).
+    This module only runs in clojure-legacy mode, so the quirk is gated by
+    construction."""
+    av = np.asarray(a, dtype=float)
+    bv = np.asarray(b, dtype=float)
+    d2 = float(np.dot(av, av)) + float(np.dot(bv, bv)) - 2.0 * float(np.dot(av, bv))
+    return float(np.sqrt(max(0.0, d2)))
 
 
 def weighted_mean(rows: Sequence[np.ndarray],

delphi/scripts/certify_battery.json

@@ -63,8 +63,42 @@
   },
   {
     "dataset": "vw",
-    "preset": "every-vote",
+    "schedule": "schedules/vw-every-vote-56.json",
     "engine_mode": "clojure-legacy",
-    "notes": "densest warm-start chain: one recompute per vote (4683 steps) \u2014 maximal sequential-seam coverage on the smallest dataset"
+    "notes": "every-vote prefix \u2014 see the schedule file for the Q11 truncation rationale"
+  },
+  {
+    "dataset": "pc-revote-01",
+    "preset": "uniform",
+    "n_cuts": 6,
+    "engine_mode": "clojure-legacy",
+    "notes": "prodclone: extreme revote conversation (97% revotes, ~56k votes)"
+  },
+  {
+    "dataset": "pc-banned-01",
+    "preset": "uniform",
+    "n_cuts": 6,
+    "engine_mode": "clojure-legacy",
+    "notes": "prodclone: banned participants present (participants.mod=-1; Q1 leak territory)"
+  },
+  {
+    "dataset": "pc-smallmix-01",
+    "preset": "uniform",
+    "n_cuts": 6,
+    "engine_mode": "clojure-legacy",
+    "notes": "prodclone: small mixed conversation (~5k votes)"
+  },
+  {
+    "dataset": "pc-midmix-01",
+    "preset": "uniform",
+    "n_cuts": 6,
+    "engine_mode": "clojure-legacy",
+    "notes": "prodclone: mid mixed conversation (~49k votes)"
+  },
+  {
+    "dataset": "pc-zerovote-01",
+    "preset": "single-cut",
+    "engine_mode": "clojure-legacy",
+    "notes": "prodclone: zero-vote conversation \u2014 empty-conversation edge"
   }
 ]

delphi/scripts/schedules/vw-every-vote-56.json

@@ -0,0 +1,67 @@
+{
+  "dataset": "vw",
+  "schedule_id": "every-vote-56",
+  "cuts": {
+    "mode": "explicit-event-index",
+    "at": [
+      1,
+      2,
+      3,
+      4,
+      5,
+      6,
+      7,
+      8,
+      9,
+      10,
+      11,
+      12,
+      13,
+      14,
+      15,
+      16,
+      17,
+      18,
+      19,
+      20,
+      21,
+      22,
+      23,
+      24,
+      25,
+      26,
+      27,
+      28,
+      29,
+      30,
+      31,
+      32,
+      33,
+      34,
+      35,
+      36,
+      37,
+      38,
+      39,
+      40,
+      41,
+      42,
+      43,
+      44,
+      45,
+      46,
+      47,
+      48,
+      49,
+      50,
+      51,
+      52,
+      53,
+      54,
+      55,
+      56
+    ]
+  },
+  "moderation": "none",
+  "notes": "every-vote PREFIX (56 single-vote steps): maximal-density warm-start edge case, truncated BEFORE the first Q11 knife-edge (step 57's near-coincident pair, whose merge/no-merge decision is bit-chaotic and irreducible cross-language \u2014 CLOJURE_QUIRKS.md Q11, journal 2026-07-22)"
+}

delphi/tests/test_base_cluster_lineage.py

@@ -69,19 +69,28 @@ def _run(self, monkeypatch, mode):
         monkeypatch.setenv(ENGINE_MODE_ENV_VAR, mode)
         return Conversation('cold').update_votes(_many_ptpt_votes())
 
-    def test_legacy_base_is_clojure_faithful_singletons(self, monkeypatch):
+    def test_legacy_base_is_clojure_faithful_up_to_q11_merges(self, monkeypatch):
         # base-k (=100) >= n_ptpts, so every DISTINCT projection becomes its own
-        # base cluster. This synthetic set has near-duplicate projections; legacy
-        # (init-clusters on exact-distinct rows + cluster-step) keeps each point
-        # as its own singleton with NO empty clusters, matching Clojure exactly.
-        # sklearn's Lloyd instead collapses a near-duplicate pair and leaves an
-        # empty cluster — so legacy and improved legitimately DIVERGE on
-        # near-duplicate projections (base cold identity holds only when all
-        # projections are distinct, e.g. vw; see TestVwColdStartInvariance). This
-        # test pins the Clojure-faithful legacy side.
+        # base cluster — EXCEPT near-duplicates whose vectorz-formula distance
+        # cancels to exactly 0.0: those TIE against multiple clusters and merge
+        # into the later one (CLOJURE_QUIRKS.md Q11; this fixture's
+        # near-duplicate pair does cancel). The pre-Q11 version of this test
+        # asserted all-singletons, believing that was the Clojure behavior —
+        # the in-process probe of 2026-07-22 showed Clojure merges. Assertion:
+        # no empty clusters, every participant clustered exactly once, and any
+        # multi-member cluster holds only points at Q11-distance 0.0 from each
+        # other (a merge is only ever the Q11 tie, never a real collapse).
+        from polismath.pca_kmeans_rep.legacy_kmeans import _euclidean
+
         leg = self._run(monkeypatch, 'clojure-legacy')
         assert all(c['members'] for c in leg.base_clusters)  # no empty clusters
-        assert all(len(c['members']) == 1 for c in leg.base_clusters)  # singletons
+        all_members = [m for c in leg.base_clusters for m in c['members']]
+        assert sorted(all_members) == sorted(f'p{i}' for i in range(18))
+        pos = {pid: np.asarray(proj) for pid, proj in leg.proj.items()}
+        for c in leg.base_clusters:
+            for m1 in c['members']:
+                for m2 in c['members']:
+                    assert _euclidean(pos[m1], pos[m2]) == 0.0
 
     def test_group_clustering_is_deterministic_in_legacy(self, monkeypatch):
         # NOTE (semantic finding): the GROUP level runs real k-means (k<<n_base),

delphi/tests/test_legacy_kmeans.py

@@ -290,3 +290,45 @@ def test_warm_start_weighted_group_level(self):
         c0 = next(c for c in out if set(c['members']) == {0, 1})
         # weighted center y = (1*0 + 3*2)/4 = 1.5, NOT unweighted 1.0
         np.testing.assert_allclose(c0['center'], [0.0, 1.5])
+
+
+# ---------------------------------------------------------------------------
+# Q11: vectorz distance cancellation (CLOJURE_QUIRKS.md Q11).
+# ---------------------------------------------------------------------------
+class TestQ11DistanceCancellation:
+    """Clojure's kmeans distances go through vectorz's d² = |a|²+|b|²−2a·b,
+    whose cancellation floors true distances below ~1e-8 to EXACTLY 0.0 —
+    so near-coincident points TIE and merge into the LATER cluster
+    (min-key last-wins). Verified in-process on the vw every-vote step-57
+    pair via math/dev/proj_probe.clj (journal 2026-07-22)."""
+
+    def test_euclidean_uses_clojure_cancellation_formula(self):
+        from polismath.pca_kmeans_rep.legacy_kmeans import _euclidean
+
+        p5 = np.array([-1.7765256006253405, 0.65139331269767860])
+        c8 = np.array([-1.7765256006253405, 0.65139331269767400])
+        # True distance 4.66e-15; the vectorz formula returns exactly 0.0.
+        assert _euclidean(p5, c8) == 0.0
+        # Normal-scale distances stay correct.
+        assert _euclidean(np.array([0.0, 0.0]), np.array([3.0, 4.0])) == pytest.approx(5.0)
+
+    def test_near_coincident_singletons_merge_to_later_cluster(self):
+        from polismath.pca_kmeans_rep.legacy_kmeans import _NamedData, kmeans
+
+        # The REAL vw every-vote step-57 pair (journal 2026-07-22): the
+        # cancellation collapses their 4.66e-15 separation to exactly 0.0.
+        # (Not every near-coincident synthetic pair does — the residue of
+        # |a|²+|b|²−2ab can land on either side of zero bit-by-bit.)
+        a = [-1.7765256006253405, 0.65139331269767860]
+        b = [-1.7765256006253405, 0.65139331269767400]
+        far = [5.0, 5.0]
+        data = _NamedData([10, 20, 30], np.array([a, b, far]))
+        last = [
+            {"id": 6, "members": [10], "center": np.array(a)},
+            {"id": 7, "members": [30], "center": np.array(far)},
+            {"id": 8, "members": [20], "center": np.array(b)},
+        ]
+        result = {c["id"]: sorted(c["members"]) for c in kmeans(data, 100, last_clusters=last)}
+        # Clojure: both coincident points tie at distance 0.0 to clusters 6
+        # AND 8 -> min-key last-wins sends both to id 8; id 6 empties, drops.
+        assert result == {7: [30], 8: [10, 20]}

math/dev/proj_probe.clj

@@ -0,0 +1,131 @@
+;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or  modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+(ns proj-probe
+  "Diagnostic probe (R1 goal, every-vote step-57 uniqify edge): replay the
+  first N votes of a dataset through conv-update and print the FULL-PRECISION
+  projection rows for selected pids at the final step.
+
+  DELIBERATELY a separate file from replay.clj: the certify clj-recording
+  cache manifests hash dev/replay.clj, so touching that file forces a full
+  battery re-record. This probe reuses replay's own loaders via `load-file`
+  and changes nothing.
+
+  Usage (from math/):
+    clojure -M dev/proj_probe.clj <votes-csv> <n-votes> <pid> [<pid> ...]"
+  (:require [polismath.math.conversation :as conv]
+            [polismath.math.clusters :as clusters]
+            [polismath.math.named-matrix :as nm]
+            [clojure.core.matrix :as matrix]))
+
+(load-file "dev/replay.clj")
+
+(defn -main [& args]
+  (let [[csv-path n-votes & pids] args
+        n-votes (Long/parseLong n-votes)
+        pids    (set (map #(Long/parseLong %) pids))
+        votes   (->> (replay/read-votes-csv csv-path)
+                     replay/build-dataset
+                     (take n-votes))
+        seed    (-> (conv/new-conv) (assoc :zid 99999 :meta-tids #{}))
+        ;; Replay ONE VOTE PER UPDATE — the warm-start chain is path-dependent
+        ;; (PCA start-vectors thread tick to tick), so a cold single batch
+        ;; does NOT reproduce the every-vote recording's state.
+        conv'   (reduce
+                  (fn [c v]
+                    (conv/conv-update c (replay/->conv-votes [v])
+                                      replay/certify-conv-opts))
+                  seed
+                  votes)
+        rownames (nm/rownames (:rating-mat conv'))
+        proj    (:proj conv')]
+    (doseq [[pid row] (map vector rownames proj)
+            :when (contains? pids pid)]
+      (println (format "pid=%d proj=[%.17g %.17g]"
+                       (long pid)
+                       (double (first row))
+                       (double (second row)))))
+    (doseq [c (:base-clusters conv')]
+      (println (format "cluster id=%d members=%s center=[%.17g %.17g]"
+                       (long (:id c))
+                       (pr-str (:members c))
+                       (double (first (:center c)))
+                       (double (second (:center c))))))
+    ;; Phase-by-phase clean-start replay: chain to n-1 votes (prev state),
+    ;; apply the final vote, then walk clean-start-clusters manually on the
+    ;; new projection nmat with the PREV clusters, printing each phase.
+    (let [prev  (reduce
+                  (fn [c v]
+                    (conv/conv-update c (replay/->conv-votes [v])
+                                      replay/certify-conv-opts))
+                  (-> (conv/new-conv) (assoc :zid 99999 :meta-tids #{}))
+                  (butlast votes))
+          cur   (conv/conv-update prev (replay/->conv-votes [(last votes)])
+                                  replay/certify-conv-opts)
+          pnmat (nm/named-matrix (nm/rownames (:rating-mat cur)) ["x" "y"]
+                                 (:proj cur))
+          inmat (nm/rowname-subset pnmat (:in-conv cur))
+          rec   (clusters/safe-recenter-clusters inmat (:base-clusters prev))
+          uniq  (clusters/uniqify-clusters rec)
+          possible (min 100 (count (distinct (into [] (matrix/rows (nm/get-matrix inmat))))))]
+      (println "PHASE safe-recenter:")
+      (doseq [c rec]
+        (println (format "  id=%d members=%s center=[%.17g %.17g]"
+                         (long (:id c)) (pr-str (:members c))
+                         (double (first (:center c))) (double (second (:center c))))))
+      (println (format "PHASE uniqify: %d clusters (ids %s)"
+                       (count uniq) (pr-str (mapv :id uniq))))
+      (println (format "PHASE possible-clusters: %d (rows %d)"
+                       possible (count (nm/rownames inmat))))
+      (let [km (clusters/kmeans inmat 100
+                                :last-clusters (:base-clusters prev)
+                                :max-iters 100)]
+        (println "PHASE full-kmeans:")
+        (doseq [c (sort-by :id km)]
+          (println (format "  id=%d members=%s"
+                           (long (:id c)) (pr-str (:members c))))))
+      (let [cs (clusters/clean-start-clusters inmat (:base-clusters prev) 100)
+            data-iter (map vector (nm/rownames inmat)
+                           (matrix/rows (matrix/matrix (nm/get-matrix inmat))))
+            s1 (clusters/cluster-step data-iter 100 cs)]
+        (println "PHASE clean-start-direct:" (pr-str (mapv (juxt :id :members) cs)))
+        (println "PHASE cluster-step-1:" (pr-str (mapv (juxt :id :members) s1)))
+        (let [c6 (first (filter #(= 6 (:id %)) cs))
+              c8 (first (filter #(= 8 (:id %)) cs))
+              row5 (nm/get-row-by-name inmat 5)]
+          (println (format "PHASE centers: c6=[%.17g %.17g] c8=[%.17g %.17g]"
+                           (double (first (:center c6))) (double (second (:center c6)))
+                           (double (first (:center c8))) (double (second (:center c8)))))
+          (println (format "PHASE dist: d(p5,c6)=%.20g d(p5,c8)=%.20g equal=%s"
+                           (double (matrix/distance row5 (:center c6)))
+                           (double (matrix/distance row5 (:center c8)))
+                           (= (matrix/distance row5 (:center c6))
+                              (matrix/distance row5 (:center c8)))))
+          (let [cleared (clusters/cleared-clusters cs)
+                after   (clusters/add-to-closest cleared [5 row5])
+                winner  (first (filter (fn [[_ c]] (seq (:members c))) after))]
+            (println "PHASE single-assign: pid 5 ->" (pr-str (key winner))
+                     "map-type:" (str (type cleared))
+                     "entry-order:" (pr-str (keys cleared)))
+            (doseq [[cid c] cleared]
+              (println (format "  d(p5, c%s)=%.20g"
+                               (str cid)
+                               (double (matrix/distance row5 (:center c))))))
+            (let [names (nm/rownames inmat)
+                  rows  (into [] (matrix/rows (matrix/matrix (nm/get-matrix inmat))))
+                  mism  (for [[n r] (map vector names rows)
+                              :when (not= (into [] r)
+                                          (into [] (nm/get-row-by-name inmat n)))]
+                          n)]
+              (println "PHASE alignment: rownames=" (pr-str names)
+                       "misaligned-names=" (pr-str (vec mism)))
+              (let [view5 (nth rows 4)]
+                (println (format "PHASE view-dist: d(view5,c6)=%.20g d(view5,c8)=%.20g types=%s/%s"
+                                 (double (matrix/distance view5 (:center c6)))
+                                 (double (matrix/distance view5 (:center c8)))
+                                 (str (type view5)) (str (type (:center c6)))))
+                (let [after (clusters/add-to-closest
+                              (clusters/cleared-clusters cs) [5 view5])
+                      winner (first (filter (fn [[_ c]] (seq (:members c))) after))]
+                  (println "PHASE view-assign: pid 5 ->" (pr-str (key winner))))))))))))
+
+(apply -main *command-line-args*)