From 736211a67feb852c63e57a90122abe0524ecc2e1 Mon Sep 17 00:00:00 2001 From: Julien Cornebise Date: Sat, 18 Jul 2026 02:12:29 +0100 Subject: [PATCH] python-math #14: feat(math): degenerate-tick group-clusterings overwrite + <2-participants edge (legacy mode) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Ports the approved 2026-07-21 Opus review verdict covering quirks Q4 and Q5 from `delphi/docs/CLOJURE_QUIRKS.md` into clojure-legacy mode (the engine setting that reproduces the old Clojure engine's behavior exactly): - Quirk Q4 (degenerate ticks still cluster): Clojure recomputes `:group-clusterings` EVERY tick unconditionally — `max-k-fn` is always >= 2 (conversation.clj:274-279) — so a tick with only 1 base cluster still runs k=2 kmeans on a single point. The clean-start initialization caps k at the number of distinct points, yielding 1 cluster carrying a lineage id; that degenerate value OVERWRITES `:group-clusterings`; and the recovery tick warm-starts from it, minting split ids via `(inc max-id)` (clusters.clj:267). - Quirk Q5 (<2-participants edge): Clojure also has no guard for fewer than 2 in-conv participants (the set of participants admitted to clustering) beyond the truly-empty short-circuit — a 1-participant tick runs the full pipeline. ## How it works clojure-legacy mode now skips both early returns and falls through to the normal legacy path, which reproduces all of the above exactly: the max_k arithmetic yields the k range [2]; `calculate_silhouette_sklearn` returns the 0.0 sentinel for singleton clusterings, matching Clojure's singleton rule (clusters.clj:350-353); and the advance of the group-k smoother (ported earlier in this stack as PR P6a) is preserved with the same `{2: 0.0}` input via the main path — the branch-local P6a advance block is removed as dead code. The 0-participant early return stays in both modes: it is unreachable past the empty short-circuit given the greedy in-conv floor from PR-E (#2623). improved mode (the engine setting that keeps Python's corrected behavior) keeps both guards byte-for-byte. ## Testing TDD: `tests/test_degenerate_tick_parity.py` was RED on the old code for the pinned divergences (stale 2-cluster group_clusterings after a collapse tick; empty structures on a single-participant tick), GREEN after. Existing P6a smoother tests are unchanged and green (24 passed with `test_group_k_smoother.py`; 37 passed with the lineage / greedy-carry / pca-warm-start / engine-mode neighbor suites). commit-id:bbb74f71 --- delphi/polismath/conversation/conversation.py | 52 +++-- delphi/tests/test_degenerate_tick_parity.py | 192 ++++++++++++++++++ 2 files changed, 223 insertions(+), 21 deletions(-) create mode 100644 delphi/tests/test_degenerate_tick_parity.py diff --git a/delphi/polismath/conversation/conversation.py b/delphi/polismath/conversation/conversation.py index f6e10a365..47a6ca150 100644 --- a/delphi/polismath/conversation/conversation.py +++ b/delphi/polismath/conversation/conversation.py @@ -826,7 +826,18 @@ def _compute_clusters(self, # Filter projections to only include in-conv participants in_conv_pids_list = [pid for pid in self.proj.keys() if pid in in_conv_pids] - if len(in_conv_pids_list) < 2: + legacy_mode = resolve_engine_mode() == ENGINE_MODE_LEGACY + + # Degenerate-tick port (journal 2026-07-21 verdict): Clojure has NO + # <2-participants guard past the truly-empty short-circuit — with one + # in-conv participant its graph still runs the full base->group chain + # (one base cluster, k=2 group clustering of one point). Legacy mode + # falls through and replicates that; improved mode keeps the guard. + # The 0-participant early return stays in BOTH modes: it is unreachable + # past the `not self.proj` short-circuit while the in-conv greedy floor + # guarantees >=1 participant (PR-E), and Clojure's kmeans on an empty + # matrix has nothing to warm-start either. + if len(in_conv_pids_list) == 0 or (not legacy_mode and len(in_conv_pids_list) < 2): logger.warning(f"Not enough participants meeting threshold ({len(in_conv_pids_list)})") self.base_clusters = [] self.group_clusters = [] @@ -838,11 +849,11 @@ def _compute_clusters(self, # Step 2: Base clustering (participants → ~100 base clusters) base_proj_values = np.array([self.proj[pid] for pid in in_conv_pids_list]) - # Adjust BASE_K if we have fewer participants + # Adjust BASE_K if we have fewer participants. (Clojure always passes + # :base-k=100, but its kmeans caps clusters at the distinct-row count, + # so min() here is outcome-equivalent.) actual_base_k = min(BASE_K, len(in_conv_pids_list)) - legacy_mode = resolve_engine_mode() == ENGINE_MODE_LEGACY - logger.info(f"Computing base clusters with k={actual_base_k}...") if legacy_mode: # PR-C: base-level warm start with lineage. Clojure threads the prior @@ -892,7 +903,22 @@ def _compute_clusters(self, logger.info(f"Created {len(base_clusters)} base clusters") # Step 3: Group clustering (base clusters → 2-5 groups) - if len(base_clusters) < 2: + # + # Degenerate-tick port (journal 2026-07-21 verdict, supersedes the P6a + # sentinel-only advance): Clojure has NO <2-base-cluster guard. Its + # max-k-fn is (min max-max-k (+ 2 (int (/ n 12)))) -> ALWAYS >= 2 + # (conversation.clj:274-279), so on a degenerate tick it still runs + # kmeans at k=2 on the single base-cluster center (clean-start caps + # clusters at the distinct-point count -> one cluster, lineage id + # preserved), stores the fresh 1-cluster :group-clusterings, and the + # next tick warm-starts from it — recovery splits mint ids via + # (inc (apply max ids)) (clusters.clj:267). Legacy mode falls through + # to the normal per-k loop below, which reproduces all of that + # (max_k arithmetic yields range [2]; silhouette of a singleton + # clustering is 0.0, matching Clojure's singleton rule + # clusters.clj:350-353, so the smoother advance is unchanged from + # P6a). Improved mode keeps the early return byte-for-byte. + if not legacy_mode and len(base_clusters) < 2: logger.warning(f"Not enough base clusters for group clustering ({len(base_clusters)})") self.base_clusters = base_clusters # Maintain consistent group-cluster schema: members are base-cluster IDs @@ -905,22 +931,6 @@ def _compute_clusters(self, else: self.group_clusters = [] self.subgroup_clusters = {} - # P6a: Clojure has NO <2-base-cluster guard. Its max-k-fn is - # (min max-max-k (+ 2 (int (/ n 12)))) -> ALWAYS >= 2 - # (conversation.clj:273-279), so on a degenerate tick with a NON-empty - # conv (we are past the `if not self.proj` empty short-circuit above) - # the Clojure graph still clusters at k=2 and feeds this_k=2 to the - # group-k smoother, ADVANCING its {last_k, last_k_count, smoothed_k} - # state. Mirror that in legacy mode (silhouette sentinel 0.0 -> this_k=2) - # instead of FREEZING the smoother memory — which self-corrected within - # <=4 ticks but diverged from Clojure meanwhile. Improved mode carries - # no smoother state, so it is unaffected. - if legacy_mode: - new_smoother_state, _ = group_k_smoother_update( - prev_group_k_smoother or {}, {2: 0.0}) - self.group_k_smoother = new_smoother_state - logger.info(f"Legacy degenerate-tick smoother advance: " - f"state={new_smoother_state}") return # Prepare base cluster centers and weights diff --git a/delphi/tests/test_degenerate_tick_parity.py b/delphi/tests/test_degenerate_tick_parity.py new file mode 100644 index 000000000..c2e210c75 --- /dev/null +++ b/delphi/tests/test_degenerate_tick_parity.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +""" +Degenerate-tick Clojure parity (approved port, journal 2026-07-21 verdict). + +Clojure's graph has NO guard for <2 base clusters or <2 in-conv participants +past the truly-empty short-circuit (conversation.clj:807-811). Its +:group-clusterings node recomputes EVERY tick, unconditionally +(conversation.clj:433-445): max-k-fn = min(max-k, 2 + n_base//12) >= 2 always +(conversation.clj:274-279), so a degenerate tick still runs kmeans at k=2 on +however many base-cluster centers exist (possibly one), stores the fresh +(possibly 1-cluster) value on the conv, and the NEXT tick warm-starts from it — +recovery splits mint ids via `(inc (apply max ids))` (clean-start-clusters, +clusters.clj:267). + +Python legacy mode used to early-return on both edges, keeping the last +NON-degenerate group_clusterings as the warm seed — different seeds, different +cluster ids across a degenerate episode. These tests pin the Clojure semantics +in 'clojure-legacy' mode and pin that 'improved' mode keeps its guards +byte-for-byte. +""" + +import os +import sys + +import pytest + +sys.path.append(os.path.abspath(os.path.dirname(__file__))) + +from polismath.pca_kmeans_rep.pca import PCA_IMPL_ENV_VAR +from polismath.utils.engine_mode import ENGINE_MODE_ENV_VAR +from polismath.conversation.conversation import Conversation + + +# --------------------------------------------------------------------------- +# Vote builders. Sign convention doesn't matter here — only bloc separation. +# --------------------------------------------------------------------------- + +N_CMTS = 8 +BLOC_A = [f'a{i}' for i in range(6)] +BLOC_B = [f'b{i}' for i in range(6)] + + +def _bloc_votes(pids, agree_first_half): + votes = [] + for pid in pids: + for t in range(N_CMTS): + first_half = t < N_CMTS // 2 + vote = 1.0 if (first_half == agree_first_half) else -1.0 + votes.append({'pid': pid, 'tid': f'c{t}', 'vote': vote}) + return votes + + +def _two_bloc_votes(): + """Two well-separated blocs -> >=2 base clusters, 2 group clusters.""" + return {'votes': _bloc_votes(BLOC_A, True) + _bloc_votes(BLOC_B, False)} + + +def _collapse_votes(): + """Bloc B revotes to match bloc A exactly -> every vote row identical -> + all projections identical -> a single base cluster (degenerate tick).""" + return {'votes': _bloc_votes(BLOC_B, True)} + + +def _recover_votes(): + """Bloc B revotes back to full opposition -> two blocs again.""" + return {'votes': _bloc_votes(BLOC_B, False)} + + +def _single_ptpt_votes(): + return {'votes': [{'pid': 'solo', 'tid': f'c{t}', 'vote': 1.0} + for t in range(3)]} + + +@pytest.fixture +def legacy_mode(monkeypatch): + monkeypatch.delenv(PCA_IMPL_ENV_VAR, raising=False) + monkeypatch.setenv(ENGINE_MODE_ENV_VAR, 'clojure-legacy') + + +@pytest.fixture +def improved_mode(monkeypatch): + monkeypatch.delenv(PCA_IMPL_ENV_VAR, raising=False) + monkeypatch.setenv(ENGINE_MODE_ENV_VAR, 'improved') + + +# --------------------------------------------------------------------------- +# Legacy mode: degenerate tick recomputes and threads group_clusterings +# --------------------------------------------------------------------------- + +class TestLegacyDegenerateTickOverwrite: + + def test_degenerate_tick_overwrites_group_clusterings(self, legacy_mode): + conv = Conversation('deg').update_votes(_two_bloc_votes()) + # Healthy tick sanity: a real multi-cluster clustering exists for k=2. + assert len(conv.group_clusterings.get(2, [])) == 2 + pre_collapse = conv.group_clusterings + + conv = conv.update_votes(_collapse_votes()) + assert len(conv.base_clusters) == 1, "scenario must be degenerate" + # Clojure recomputes :group-clusterings unconditionally: the stored map + # must be THIS tick's degenerate result (k range collapses to {2}, + # single cluster), not the stale pre-collapse map. + assert set(conv.group_clusterings.keys()) == {2} + assert len(conv.group_clusterings[2]) == 1 + assert conv.group_clusterings is not pre_collapse + + def test_degenerate_group_cluster_covers_the_base_cluster(self, legacy_mode): + conv = Conversation('deg').update_votes(_two_bloc_votes()) + conv = conv.update_votes(_collapse_votes()) + [base] = conv.base_clusters + [cluster] = conv.group_clusterings[2] + assert cluster['members'] == [base['id']] + # And the production selection is that same degenerate clustering. + assert [c['id'] for c in conv.group_clusters] == [cluster['id']] + + def test_recovery_tick_mints_inc_max_id(self, legacy_mode): + conv = Conversation('deg').update_votes(_two_bloc_votes()) + conv = conv.update_votes(_collapse_votes()) + [degenerate_cluster] = conv.group_clusterings[2] + x = degenerate_cluster['id'] + + conv = conv.update_votes(_recover_votes()) + assert len(conv.base_clusters) >= 2, "recovery must de-degenerate" + # Warm start from the DEGENERATE seed {2: [X]}: clean-start splits the + # most distal point into a NEW cluster with id (inc (apply max ids)) + # (clusters.clj:267) -> ids {X, X+1}. The old early-return would have + # warm-started from the stale pre-collapse clustering instead. + assert sorted(c['id'] for c in conv.group_clusterings[2]) == [x, x + 1] + + def test_degenerate_tick_still_advances_smoother(self, legacy_mode): + # P6a semantics preserved through the port: silhouette of the + # single-cluster clustering is 0.0 (Clojure singleton rule, + # clusters.clj:350-353), fed to the smoother as {2: 0.0}. + conv = Conversation('deg').update_votes(_two_bloc_votes()) + pre_state = dict(conv.group_k_smoother) + conv = conv.update_votes(_collapse_votes()) + assert conv.group_k_smoother.get('last_k') == 2 + expected_count = (pre_state.get('last_k_count', 0) + 1 + if pre_state.get('last_k') == 2 else 1) + assert conv.group_k_smoother.get('last_k_count') == expected_count + + +# --------------------------------------------------------------------------- +# Legacy mode: the <2-in-conv-participants edge runs the full chain +# --------------------------------------------------------------------------- + +class TestLegacySingleParticipant: + + def test_single_participant_runs_full_chain(self, legacy_mode): + conv = Conversation('solo').update_votes(_single_ptpt_votes()) + # Clojure has no <2-participants guard: one in-conv participant yields + # one base cluster, group-clusterings {2: [one cluster]}, and an + # advanced smoother — not empty structures. + assert len(conv.base_clusters) == 1 + assert conv.base_clusters[0]['members'] == ['solo'] + assert set(conv.group_clusterings.keys()) == {2} + assert len(conv.group_clusterings[2]) == 1 + assert len(conv.group_clusters) == 1 + assert conv.group_k_smoother.get('last_k') == 2 + assert conv.group_k_smoother.get('last_k_count') == 1 + + +# --------------------------------------------------------------------------- +# Improved mode: both guards keep their existing behavior byte-for-byte +# --------------------------------------------------------------------------- + +class TestImprovedGuardsUnchanged: + + def test_single_participant_early_return(self, improved_mode): + conv = Conversation('solo').update_votes(_single_ptpt_votes()) + assert conv.base_clusters == [] + assert conv.group_clusters == [] + assert conv.group_clusterings == {} + assert conv.group_k_smoother == {} + + def test_degenerate_tick_keeps_synthesized_cluster(self, improved_mode): + conv = Conversation('deg').update_votes(_two_bloc_votes()) + conv = conv.update_votes(_collapse_votes()) + assert len(conv.base_clusters) == 1 + [base] = conv.base_clusters + # Improved keeps the synthesized id-0 wrapper and stateless smoother. + assert conv.group_clusters == [{ + 'id': 0, + 'center': base['center'], + 'members': [base['id']], + }] + assert conv.group_clusterings == {} + assert conv.group_k_smoother == {} + + +if __name__ == '__main__': + pytest.main([__file__, '-v'])