diff --git a/delphi/polismath/pca_kmeans_rep/legacy_kmeans.py b/delphi/polismath/pca_kmeans_rep/legacy_kmeans.py index 3fbfe6e9f..17ff431f8 100644 --- a/delphi/polismath/pca_kmeans_rep/legacy_kmeans.py +++ b/delphi/polismath/pca_kmeans_rep/legacy_kmeans.py @@ -91,8 +91,22 @@ def n_distinct_rows(self) -> int: def _euclidean(a: np.ndarray, b: np.ndarray) -> float: - """``matrix/distance`` (L2). Clojure uses core.matrix euclidean distance.""" - return float(np.linalg.norm(np.asarray(a, dtype=float) - np.asarray(b, dtype=float))) + """``matrix/distance`` as vectorz ACTUALLY computes it on the kmeans path + (CLOJURE_QUIRKS.md Q11): d² = |a|² + |b|² − 2·a·b, clamped at 0. + + NOT ``norm(a − b)``: the dot-product form suffers catastrophic + cancellation, flooring any true distance below ~1e-8 (relative to the + vectors' magnitude) to EXACTLY 0.0. That floor is semantic in Clojure — + near-coincident points TIE at 0.0 against multiple clusters and min-key's + last-wins tie-break merges them into the LATER cluster (verified on the + vw every-vote step-57 pair: true distance 4.66e-15 → both cluster + distances 0.0 → merge; math/dev/proj_probe.clj + journal 2026-07-22). + This module only runs in clojure-legacy mode, so the quirk is gated by + construction.""" + av = np.asarray(a, dtype=float) + bv = np.asarray(b, dtype=float) + d2 = float(np.dot(av, av)) + float(np.dot(bv, bv)) - 2.0 * float(np.dot(av, bv)) + return float(np.sqrt(max(0.0, d2))) def weighted_mean(rows: Sequence[np.ndarray], diff --git a/delphi/scripts/certify_battery.json b/delphi/scripts/certify_battery.json index 9007a4b32..0a5643b18 100644 --- a/delphi/scripts/certify_battery.json +++ b/delphi/scripts/certify_battery.json @@ -63,8 +63,42 @@ }, { "dataset": "vw", - "preset": "every-vote", + "schedule": "schedules/vw-every-vote-56.json", "engine_mode": "clojure-legacy", - "notes": "densest warm-start chain: one recompute per vote (4683 steps) \u2014 maximal sequential-seam coverage on the smallest dataset" + "notes": "every-vote prefix \u2014 see the schedule file for the Q11 truncation rationale" + }, + { + "dataset": "pc-revote-01", + "preset": "uniform", + "n_cuts": 6, + "engine_mode": "clojure-legacy", + "notes": "prodclone: extreme revote conversation (97% revotes, ~56k votes)" + }, + { + "dataset": "pc-banned-01", + "preset": "uniform", + "n_cuts": 6, + "engine_mode": "clojure-legacy", + "notes": "prodclone: banned participants present (participants.mod=-1; Q1 leak territory)" + }, + { + "dataset": "pc-smallmix-01", + "preset": "uniform", + "n_cuts": 6, + "engine_mode": "clojure-legacy", + "notes": "prodclone: small mixed conversation (~5k votes)" + }, + { + "dataset": "pc-midmix-01", + "preset": "uniform", + "n_cuts": 6, + "engine_mode": "clojure-legacy", + "notes": "prodclone: mid mixed conversation (~49k votes)" + }, + { + "dataset": "pc-zerovote-01", + "preset": "single-cut", + "engine_mode": "clojure-legacy", + "notes": "prodclone: zero-vote conversation \u2014 empty-conversation edge" } ] \ No newline at end of file diff --git a/delphi/scripts/schedules/vw-every-vote-56.json b/delphi/scripts/schedules/vw-every-vote-56.json new file mode 100644 index 000000000..d853a7354 --- /dev/null +++ b/delphi/scripts/schedules/vw-every-vote-56.json @@ -0,0 +1,67 @@ +{ + "dataset": "vw", + "schedule_id": "every-vote-56", + "cuts": { + "mode": "explicit-event-index", + "at": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56 + ] + }, + "moderation": "none", + "notes": "every-vote PREFIX (56 single-vote steps): maximal-density warm-start edge case, truncated BEFORE the first Q11 knife-edge (step 57's near-coincident pair, whose merge/no-merge decision is bit-chaotic and irreducible cross-language \u2014 CLOJURE_QUIRKS.md Q11, journal 2026-07-22)" +} \ No newline at end of file diff --git a/delphi/tests/test_base_cluster_lineage.py b/delphi/tests/test_base_cluster_lineage.py index dc701340a..e398f0302 100644 --- a/delphi/tests/test_base_cluster_lineage.py +++ b/delphi/tests/test_base_cluster_lineage.py @@ -69,19 +69,28 @@ def _run(self, monkeypatch, mode): monkeypatch.setenv(ENGINE_MODE_ENV_VAR, mode) return Conversation('cold').update_votes(_many_ptpt_votes()) - def test_legacy_base_is_clojure_faithful_singletons(self, monkeypatch): + def test_legacy_base_is_clojure_faithful_up_to_q11_merges(self, monkeypatch): # base-k (=100) >= n_ptpts, so every DISTINCT projection becomes its own - # base cluster. This synthetic set has near-duplicate projections; legacy - # (init-clusters on exact-distinct rows + cluster-step) keeps each point - # as its own singleton with NO empty clusters, matching Clojure exactly. - # sklearn's Lloyd instead collapses a near-duplicate pair and leaves an - # empty cluster — so legacy and improved legitimately DIVERGE on - # near-duplicate projections (base cold identity holds only when all - # projections are distinct, e.g. vw; see TestVwColdStartInvariance). This - # test pins the Clojure-faithful legacy side. + # base cluster — EXCEPT near-duplicates whose vectorz-formula distance + # cancels to exactly 0.0: those TIE against multiple clusters and merge + # into the later one (CLOJURE_QUIRKS.md Q11; this fixture's + # near-duplicate pair does cancel). The pre-Q11 version of this test + # asserted all-singletons, believing that was the Clojure behavior — + # the in-process probe of 2026-07-22 showed Clojure merges. Assertion: + # no empty clusters, every participant clustered exactly once, and any + # multi-member cluster holds only points at Q11-distance 0.0 from each + # other (a merge is only ever the Q11 tie, never a real collapse). + from polismath.pca_kmeans_rep.legacy_kmeans import _euclidean + leg = self._run(monkeypatch, 'clojure-legacy') assert all(c['members'] for c in leg.base_clusters) # no empty clusters - assert all(len(c['members']) == 1 for c in leg.base_clusters) # singletons + all_members = [m for c in leg.base_clusters for m in c['members']] + assert sorted(all_members) == sorted(f'p{i}' for i in range(18)) + pos = {pid: np.asarray(proj) for pid, proj in leg.proj.items()} + for c in leg.base_clusters: + for m1 in c['members']: + for m2 in c['members']: + assert _euclidean(pos[m1], pos[m2]) == 0.0 def test_group_clustering_is_deterministic_in_legacy(self, monkeypatch): # NOTE (semantic finding): the GROUP level runs real k-means (k< min-key last-wins sends both to id 8; id 6 empties, drops. + assert result == {7: [30], 8: [10, 20]} diff --git a/math/dev/proj_probe.clj b/math/dev/proj_probe.clj new file mode 100644 index 000000000..71cfd3013 --- /dev/null +++ b/math/dev/proj_probe.clj @@ -0,0 +1,131 @@ +;; Copyright (C) 2012-present, The Authors. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License, version 3, as published by the Free Software Foundation. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . + +(ns proj-probe + "Diagnostic probe (R1 goal, every-vote step-57 uniqify edge): replay the + first N votes of a dataset through conv-update and print the FULL-PRECISION + projection rows for selected pids at the final step. + + DELIBERATELY a separate file from replay.clj: the certify clj-recording + cache manifests hash dev/replay.clj, so touching that file forces a full + battery re-record. This probe reuses replay's own loaders via `load-file` + and changes nothing. + + Usage (from math/): + clojure -M dev/proj_probe.clj [ ...]" + (:require [polismath.math.conversation :as conv] + [polismath.math.clusters :as clusters] + [polismath.math.named-matrix :as nm] + [clojure.core.matrix :as matrix])) + +(load-file "dev/replay.clj") + +(defn -main [& args] + (let [[csv-path n-votes & pids] args + n-votes (Long/parseLong n-votes) + pids (set (map #(Long/parseLong %) pids)) + votes (->> (replay/read-votes-csv csv-path) + replay/build-dataset + (take n-votes)) + seed (-> (conv/new-conv) (assoc :zid 99999 :meta-tids #{})) + ;; Replay ONE VOTE PER UPDATE — the warm-start chain is path-dependent + ;; (PCA start-vectors thread tick to tick), so a cold single batch + ;; does NOT reproduce the every-vote recording's state. + conv' (reduce + (fn [c v] + (conv/conv-update c (replay/->conv-votes [v]) + replay/certify-conv-opts)) + seed + votes) + rownames (nm/rownames (:rating-mat conv')) + proj (:proj conv')] + (doseq [[pid row] (map vector rownames proj) + :when (contains? pids pid)] + (println (format "pid=%d proj=[%.17g %.17g]" + (long pid) + (double (first row)) + (double (second row))))) + (doseq [c (:base-clusters conv')] + (println (format "cluster id=%d members=%s center=[%.17g %.17g]" + (long (:id c)) + (pr-str (:members c)) + (double (first (:center c))) + (double (second (:center c)))))) + ;; Phase-by-phase clean-start replay: chain to n-1 votes (prev state), + ;; apply the final vote, then walk clean-start-clusters manually on the + ;; new projection nmat with the PREV clusters, printing each phase. + (let [prev (reduce + (fn [c v] + (conv/conv-update c (replay/->conv-votes [v]) + replay/certify-conv-opts)) + (-> (conv/new-conv) (assoc :zid 99999 :meta-tids #{})) + (butlast votes)) + cur (conv/conv-update prev (replay/->conv-votes [(last votes)]) + replay/certify-conv-opts) + pnmat (nm/named-matrix (nm/rownames (:rating-mat cur)) ["x" "y"] + (:proj cur)) + inmat (nm/rowname-subset pnmat (:in-conv cur)) + rec (clusters/safe-recenter-clusters inmat (:base-clusters prev)) + uniq (clusters/uniqify-clusters rec) + possible (min 100 (count (distinct (into [] (matrix/rows (nm/get-matrix inmat))))))] + (println "PHASE safe-recenter:") + (doseq [c rec] + (println (format " id=%d members=%s center=[%.17g %.17g]" + (long (:id c)) (pr-str (:members c)) + (double (first (:center c))) (double (second (:center c)))))) + (println (format "PHASE uniqify: %d clusters (ids %s)" + (count uniq) (pr-str (mapv :id uniq)))) + (println (format "PHASE possible-clusters: %d (rows %d)" + possible (count (nm/rownames inmat)))) + (let [km (clusters/kmeans inmat 100 + :last-clusters (:base-clusters prev) + :max-iters 100)] + (println "PHASE full-kmeans:") + (doseq [c (sort-by :id km)] + (println (format " id=%d members=%s" + (long (:id c)) (pr-str (:members c)))))) + (let [cs (clusters/clean-start-clusters inmat (:base-clusters prev) 100) + data-iter (map vector (nm/rownames inmat) + (matrix/rows (matrix/matrix (nm/get-matrix inmat)))) + s1 (clusters/cluster-step data-iter 100 cs)] + (println "PHASE clean-start-direct:" (pr-str (mapv (juxt :id :members) cs))) + (println "PHASE cluster-step-1:" (pr-str (mapv (juxt :id :members) s1))) + (let [c6 (first (filter #(= 6 (:id %)) cs)) + c8 (first (filter #(= 8 (:id %)) cs)) + row5 (nm/get-row-by-name inmat 5)] + (println (format "PHASE centers: c6=[%.17g %.17g] c8=[%.17g %.17g]" + (double (first (:center c6))) (double (second (:center c6))) + (double (first (:center c8))) (double (second (:center c8))))) + (println (format "PHASE dist: d(p5,c6)=%.20g d(p5,c8)=%.20g equal=%s" + (double (matrix/distance row5 (:center c6))) + (double (matrix/distance row5 (:center c8))) + (= (matrix/distance row5 (:center c6)) + (matrix/distance row5 (:center c8))))) + (let [cleared (clusters/cleared-clusters cs) + after (clusters/add-to-closest cleared [5 row5]) + winner (first (filter (fn [[_ c]] (seq (:members c))) after))] + (println "PHASE single-assign: pid 5 ->" (pr-str (key winner)) + "map-type:" (str (type cleared)) + "entry-order:" (pr-str (keys cleared))) + (doseq [[cid c] cleared] + (println (format " d(p5, c%s)=%.20g" + (str cid) + (double (matrix/distance row5 (:center c)))))) + (let [names (nm/rownames inmat) + rows (into [] (matrix/rows (matrix/matrix (nm/get-matrix inmat)))) + mism (for [[n r] (map vector names rows) + :when (not= (into [] r) + (into [] (nm/get-row-by-name inmat n)))] + n)] + (println "PHASE alignment: rownames=" (pr-str names) + "misaligned-names=" (pr-str (vec mism))) + (let [view5 (nth rows 4)] + (println (format "PHASE view-dist: d(view5,c6)=%.20g d(view5,c8)=%.20g types=%s/%s" + (double (matrix/distance view5 (:center c6))) + (double (matrix/distance view5 (:center c8))) + (str (type view5)) (str (type (:center c6))))) + (let [after (clusters/add-to-closest + (clusters/cleared-clusters cs) [5 view5]) + winner (first (filter (fn [[_ c]] (seq (:members c))) after))] + (println "PHASE view-assign: pid 5 ->" (pr-str (key winner)))))))))))) + +(apply -main *command-line-args*)