diff --git a/delphi/polismath/conversation/conversation.py b/delphi/polismath/conversation/conversation.py index d06ec416c..9da945a2f 100644 --- a/delphi/polismath/conversation/conversation.py +++ b/delphi/polismath/conversation/conversation.py @@ -31,7 +31,6 @@ kmeans as legacy_kmeans, ) from polismath.utils.clj_hash import clojure_hash_map_key_order -from polismath.utils.engine_mode import resolve_engine_mode, ENGINE_MODE_LEGACY # Configure logging @@ -220,23 +219,20 @@ def __init__(self, self.group_clusters = [] self.subgroup_clusters = {} - # Warm-start state threaded across ticks in 'clojure-legacy' engine - # mode (see polismath.utils.engine_mode). Clojure carries these on the + # Warm-start state threaded across ticks. Clojure carries these on the # conv (conversation.clj:433-484): the per-k group clusterings and the # group-k-smoother state {last_k, last_k_count, smoothed_k}. Cold # default is empty (first tick); NOT persisted to/from dynamo — they # thread in-memory only, exactly as Clojure's math_main whitelist omits - # them (conv_man.clj:52-74). Unused in the default 'improved' mode. + # them (conv_man.clj:52-74). self.group_clusterings: Dict[Any, Any] = {} # k -> (labels, centers, member_lists, silhouette) self.group_k_smoother: Dict[str, Any] = {} # {last_k, last_k_count, smoothed_k} - # Persistent in-conv set for 'clojure-legacy' mode (PR-E). Clojure keeps - # in-conv on the conv and UNIONS into it every tick, so greedily-admitted - # participants never leave (conversation.clj:243-269). Empty on the first - # tick; threaded in-memory across update_votes (deepcopy in recompute), - # NOT persisted to dynamo — same lifetime as the other warm-start state. - # Unused/ignored in the default 'improved' mode (threshold-only - # selection) — it can hold carry state from an earlier clojure-legacy - # tick if the mode is switched mid-process. + # Persistent in-conv set (PR-E). Clojure keeps in-conv on the conv + # and UNIONS into it every tick, so greedily-admitted participants + # never leave (conversation.clj:243-269). Empty on the first tick; + # threaded in-memory across update_votes (deepcopy in recompute), + # NOT persisted to dynamo — same lifetime as the other warm-start + # state. self.in_conv: Set[Any] = set() self.proj = {} self.repness = None @@ -500,21 +496,15 @@ def _apply_moderation(self) -> None: structure so that tids, column indices, and dimensions match between Python and Clojure. - Participant bans (mod_out_ptpts) are a Python-only feature: the - Clojure worker's ingest path has no participants.mod filter, so - banned participants keep influencing every downstream node - (CLOJURE_QUIRKS Q1). 'clojure-legacy' mode replicates that leak — - the set is stored but NOT applied; 'improved' mode drops the rows. + Participant bans (mod_out_ptpts) are NOT a Polis feature (Julien + ruling 2026-07-27, POST_CUTOVER_IMPROVEMENTS.md item 1 — dropped): + no engine has ever honored them — the Clojure worker's ingest path + has no participants.mod filter (CLOJURE_QUIRKS Q1). The set is + ingested but never applied to the matrix. """ - # Filter out banned participants (remove rows) — improved mode only; - # legacy mode leaks like Clojure (Q1). # Preserve raw_rating_mat row order (vote encounter order) — see # update_votes() comment on why row order matters for Clojure parity. - if resolve_engine_mode() == ENGINE_MODE_LEGACY: - keep_ptpts = list(self.raw_rating_mat.index) - else: - keep_ptpts = [p for p in self.raw_rating_mat.index if p not in self.mod_out_ptpts] - self.rating_mat = self.raw_rating_mat.loc[keep_ptpts].copy() + self.rating_mat = self.raw_rating_mat.copy() # Zero out moderated-out comments (keep columns, set values to 0) # Clojure: (matrix/set-column m' i 0) — zeroes the column @@ -721,9 +711,8 @@ def _compute_pca(self, n_components: int = 2, Args: n_components: Number of principal components prev_pca: The previous tick's PCA result ({'center', 'comps'}) or - None. Consumed ONLY in 'clojure-legacy' engine mode as the - power-iteration warm start (Clojure :start-vectors, - conversation.clj:385). Ignored in the default 'improved' mode. + None. Consumed as the power-iteration warm start (Clojure + :start-vectors, conversation.clj:385). """ import time start_time = time.time() @@ -850,19 +839,16 @@ def _compute_clusters(self, Args: prev_base_clusters: The previous tick's base clusters - (list of {id, center, members}), or None. Consumed ONLY in - 'clojure-legacy' engine mode as the base-level k-means warm start - (Clojure :last-clusters (:base-clusters conv), conversation.clj:409). - Ignored in the default 'improved' mode. - prev_group_clusterings: The previous tick's per-k group clusterings, - or None. In 'clojure-legacy' mode this is {k: [id-carrying cluster - dicts]} — the warm start for per-k group k-means (Clojure - :last-clusters (last-clusterings k), conversation.clj:441). Ignored - in 'improved' mode (where it is never even written, so it stays {}). + (list of {id, center, members}), or None. Consumed as the + base-level k-means warm start (Clojure :last-clusters + (:base-clusters conv), conversation.clj:409). + prev_group_clusterings: The previous tick's per-k group + clusterings, or None: {k: [id-carrying cluster dicts]} — the + warm start for per-k group k-means (Clojure :last-clusters + (last-clusterings k), conversation.clj:441). prev_group_k_smoother: The previous tick's group-k-smoother state - {last_k, last_k_count, smoothed_k}, or None. Consumed ONLY in - 'clojure-legacy' mode (conversation.clj:457). Ignored in - 'improved' mode. + {last_k, last_k_count, smoothed_k}, or None + (conversation.clj:457). """ import time start_time = time.time() @@ -1087,14 +1073,10 @@ def _compute_repness(self) -> None: # Compute representativeness (needs participant IDs, not base-cluster IDs). # `mod_out=self.mod_out_tids` forwards moderated-out tids to the rep + consensus # selectors (Clojure parity per D11 / PR 9; matches repness.clj:222 and :296). - # In clojure-legacy mode, tid_order carries the first-vote arrival - # order so exact-score ties resolve like Clojure's stable sorts over - # named-matrix column order (FP-eaea8c1b7f / FP-0d73f006f4). - tid_order = ( - self.tid_arrival_order - if resolve_engine_mode() == ENGINE_MODE_LEGACY - else None - ) + # tid_order carries the first-vote arrival order so exact-score ties + # resolve like Clojure's stable sorts over named-matrix column order + # (FP-eaea8c1b7f / FP-0d73f006f4). + tid_order = self.tid_arrival_order self.repness = conv_repness(self.rating_mat, self._unfolded_group_clusters(), mod_out=self.mod_out_tids, @@ -1335,8 +1317,7 @@ def recompute(self) -> 'Conversation': # prior tick's values (deepcopied snapshots). This mirrors Clojure, # whose fnks read the incoming `conv` for :start-vectors # (conversation.clj:385) and :group-k-smoother (conversation.clj:457). - # In 'improved' mode (default) these are IGNORED and behavior is - # unchanged; only 'clojure-legacy' mode consumes them. + prev_pca = result.pca prev_base_clusters = getattr(result, 'base_clusters', []) prev_group_clusterings = getattr(result, 'group_clusterings', {}) @@ -1388,14 +1369,13 @@ def _compute_comment_priorities( `priority_metric(is_meta, A, P, S, E)` where E is the comment extremity computed from the CURRENT tick's PCA. - Which tick's group-votes feed A/D/S is mode-dependent (Q2): Clojure - shadows its current-tick group-votes input with `(:group-votes conv)` - — the PREVIOUS tick's stored value (conversation.clj:658) — so - 'clojure-legacy' mode uses `prev_group_votes` (empty on the first - tick, matching Clojure's nil). 'improved' mode uses the current - tick's (the sane behavior). Either way the CURRENT tick's group-votes - are stored on `self.group_votes` for the next tick's capture — the - in-memory analogue of Clojure persisting :group-votes in math_main. + The PREVIOUS tick's group-votes feed A/D/S (Q2): Clojure shadows + its current-tick group-votes input with `(:group-votes conv)` — + the previous tick's stored value (conversation.clj:658) — so + `prev_group_votes` is used (empty on the first tick, matching + Clojure's nil). The CURRENT tick's group-votes are stored on + `self.group_votes` for the next tick's capture — the in-memory + analogue of Clojure persisting :group-votes in math_main. Stores the result on `self.comment_priorities` and also returns it. TS server `nextComment.ts::getNextPrioritizedComment` consumes this @@ -1640,8 +1620,9 @@ def _compute_votes_base_buckets(self) -> Dict[str, Any]: bucket, matching `agg-bucket-votes-for-tid` over `bid-to-pid` (conversation.clj:593-608). Buckets are the base clusters SORTED BY :id; the aggregation domain is each bucket's member pids only — votes - from unclustered participants never appear (this is why the improved - int totals run up to +1 higher on some tids; FP-81fda13ef6). + from unclustered participants never appear (this is why the former + improved-mode int totals ran up to +1 higher on some tids; + FP-81fda13ef6). Values come from raw_rating_mat (D15 parity: the actual votes cast, not post-moderation zeros), same as `_compute_votes_base`. @@ -1824,13 +1805,9 @@ def _compute_group_votes(self) -> Dict[str, Any]: # RAW-rating-mat (conversation.clj:601-608): moderated-out comments # report the ACTUAL votes cast and true seen-counts, not the # post-zeroing pass-shaped columns (a zeroed column would tally - # A=0/D=0 with S = every member). Legacy mode mirrors that; improved - # mode keeps the zeroed-matrix tally it was snapshotted with (its - # S-inflation on moderated tids is a known later-fix). + # A=0/D=0 with S = every member). # tests/test_mod_update_parity.py TestGroupVotesTallyRawMatrix. - tally_mat = (self.raw_rating_mat - if resolve_engine_mode() == ENGINE_MODE_LEGACY - else self.rating_mat) + tally_mat = self.raw_rating_mat group_votes = {} @@ -1919,11 +1896,9 @@ def _compute_user_vote_counts(self) -> Dict[str, int]: import time start_time = time.time() # raw_rating_mat for the COLUMN view (preserves moderated-out comments — D15 - # parity), but filtered to rating_mat.index for the ROW view so banned - # participants (mod_out_ptpts, dropped by _apply_moderation in improved - # mode) don't leak into vote counts. In clojure-legacy mode - # rating_mat.index keeps banned rows (Q1 leak replication), so this - # matches Clojure's unfiltered user-vote-counts there. Both filters + # parity), row view via rating_mat.index. Since the ban-filter drop + # (bans are not a Polis feature), rating_mat.index keeps banned rows + # (Q1), matching Clojure's unfiltered user-vote-counts. Both filters # together give the moderation-applied state with un-zeroed values, # matching what Clojure produces. mat = self.raw_rating_mat.loc[self.rating_mat.index] @@ -1987,10 +1962,8 @@ def _get_in_conv_participants(self) -> Set[Any]: MUST be persisted to DynamoDB. See compdemocracy/polis#2358 and Clojure's approach in conv_man.clj:55, conversation.clj:244. - In the default 'improved' mode this is exactly the threshold set (no - carry, no greedy floor) — today's behavior, unchanged. In - 'clojure-legacy' mode it additionally ports the two Clojure steps the - Python pipeline was missing (conversation.clj:243-269): + Beyond the threshold set, this ports the two Clojure steps the + Python pipeline was originally missing (conversation.clj:243-269): 1. CARRY: union into the PERSISTENT in-conv set carried on the conv (`(or (:in-conv conv) #{})`, conversation.clj:247) so a participant, @@ -2012,16 +1985,10 @@ def _get_in_conv_participants(self) -> Set[Any]: # Participants meeting the vote threshold (Clojure conversation.clj:249-256). threshold_set = {pid for pid, count in vote_counts.items() if count >= threshold} - if resolve_engine_mode() != ENGINE_MODE_LEGACY: - # Improved (default): threshold set only — no carry, no greedy floor. - logger.info(f"Filtered {len(threshold_set)}/{len(vote_counts)} participants " - f"meeting vote threshold {threshold:.1f}") - return threshold_set - - # Legacy: carry forward the persisted in-conv set, then union the - # threshold set into it (Clojure `(into in-conv ...)`, conversation.clj:247-256). + # Carry forward the persisted in-conv set, then union the threshold + # set into it (Clojure `(into in-conv ...)`, conversation.clj:247-256). # The intersection with vote_counts is belt-and-braces: since the Q1 - # ban-leak replication, legacy-mode rating_mat keeps banned rows, so + # ban-leak replication, rating_mat keeps banned rows, so # vote_counts covers every carried pid and the intersection is inert # (append-only votes mean a counted pid can never vanish). It stays as # defense against any future row-view change re-opening the stale-carry @@ -2294,12 +2261,9 @@ def numpy_to_list(arr): # raw_rating_mat so that moderated-out columns report the actual votes cast, # not the post-D15 zeros (which would inflate every column's 'S' count). votes_base_start = time.time() - if resolve_engine_mode() == ENGINE_MODE_LEGACY: - # Clojure-exact per-base-cluster bucket vectors (agg-bucket-votes- - # for-tid parity); improved mode keeps the int totals. - result['votes-base'] = self._compute_votes_base_buckets() - else: - result['votes-base'] = self._compute_votes_base() + # Clojure-exact per-base-cluster bucket vectors (agg-bucket-votes- + # for-tid parity). + result['votes-base'] = self._compute_votes_base_buckets() logger.info(f"Votes base: {time.time() - votes_base_start:.4f}s") # Compute group votes with optimized approach @@ -2316,9 +2280,7 @@ def numpy_to_list(arr): # group-votes aggregates votes-base, which reads RAW-rating-mat # (conversation.clj:601-608) — moderated-out comments report the # actual votes cast, not the zeroed pass-shaped columns. - tally_mat = (self.raw_rating_mat - if resolve_engine_mode() == ENGINE_MODE_LEGACY - else self.rating_mat) + tally_mat = self.raw_rating_mat # Precompute indices for each participant for faster lookups ptpt_indices = {ptpt_id: i for i, ptpt_id in enumerate(tally_mat.index)} @@ -2388,7 +2350,6 @@ def numpy_to_list(arr): # Compute in one pass using existing structure if 'group-votes' in result: - gac_legacy = resolve_engine_mode() == ENGINE_MODE_LEGACY # Store consensus values per comment ID for tid in self.rating_mat.columns: # Try converting to integer for consistent keys @@ -2410,18 +2371,12 @@ def numpy_to_list(arr): agree_count = vote_stats.get('A', 0) total_count = vote_stats.get('S', 0) - if gac_legacy: - # Clojure parity (conversation.clj:639-641, - # FP-b3670cb052): every group's factor multiplies - # in, `:or {A 0 S 0}` — a zero-S group contributes - # (0+1)/(0+2) = 1/2, it is NOT skipped. - consensus_value *= (agree_count + 1.0) / (total_count + 2.0) - has_data = True - # Calculate probability with Laplace smoothing - elif total_count > 0: - prob = (agree_count + 1.0) / (total_count + 2.0) - consensus_value *= prob - has_data = True + # Clojure parity (conversation.clj:639-641, + # FP-b3670cb052): every group's factor multiplies + # in, `:or {A 0 S 0}` — a zero-S group contributes + # (0+1)/(0+2) = 1/2, it is NOT skipped. + consensus_value *= (agree_count + 1.0) / (total_count + 2.0) + has_data = True # Only store if we have actual data if has_data: @@ -2433,15 +2388,16 @@ def numpy_to_list(arr): # Calculate in-conv participants in_conv_start = time.time() - if resolve_engine_mode() == ENGINE_MODE_LEGACY and self.in_conv: - # Legacy (PR-E): serialize the PERSISTED carry+greedy set — exactly - # the participants that fed base clustering — so the blob's :in-conv + if self.in_conv: + # PR-E: serialize the PERSISTED carry+greedy set — exactly the + # participants that fed base clustering — so the blob's :in-conv # matches the clustered rows (Clojure serializes its carried # in-conv). Keyed off user-vote-counts (same source as self.in_conv) # to preserve pid types and row order. in_conv = [pid for pid in result['user-vote-counts'] if pid in self.in_conv] else: - # Improved (default): threshold set only — unchanged. + # Cold state (no clustering has persisted an in-conv set yet): + # threshold set only. in_conv = [] min_votes = min(7, self.comment_count) for pid, count in result['user-vote-counts'].items(): @@ -2497,8 +2453,7 @@ def numpy_to_list(arr): # Add math_tick value and return result['math_tick'] = math_tick_value - if resolve_engine_mode() == ENGINE_MODE_LEGACY: - self._apply_legacy_blob_shape(result) + self._apply_legacy_blob_shape(result) logger.info(f"Total to_dict time: {time.time() - overall_start_time:.4f}s") return result @@ -2743,7 +2698,6 @@ def from_dict(cls, data: Dict[str, Any]) -> 'Conversation': conv.meta_tids = set(moderation.get('meta_tids', [])) conv.mod_out_ptpts = set(moderation.get('mod_out_ptpts', [])) - legacy = resolve_engine_mode() == ENGINE_MODE_LEGACY # Best-effort inference (the blob carries no explicit flag): any # restored moderation set implies moderation was applied. A # moderated-then-emptied conversation restores as not-applied — the @@ -2752,37 +2706,33 @@ def from_dict(cls, data: Dict[str, Any]) -> 'Conversation': conv.mod_out_tids or conv.mod_in_tids or conv.meta_tids or conv.mod_out_ptpts ) - if legacy: - # Legacy blobs emit the real (possibly null) mod watermark; - # improved blobs reuse last_updated there, which is NOT a mod - # timestamp — leave the attribute at its None default for those. - conv.last_mod_timestamp = data.get('lastModTimestamp') - # Legacy blobs emit tids in Clojure column (arrival) order — - # restore the tracker so tie-breaking survives a warm restart. - conv.tid_arrival_order = list(data.get('tids', [])) + # Blobs emit the real (possibly null) mod watermark. + conv.last_mod_timestamp = data.get('lastModTimestamp') + # Blobs emit tids in Clojure column (arrival) order — restore the + # tracker so tie-breaking survives a warm restart. + conv.tid_arrival_order = list(data.get('tids', [])) # Restore PCA data pca_data = data.get('pca') if pca_data: center = np.array(pca_data['center']) comps = np.array(pca_data['comps']) - if legacy: - # Inverse of the legacy emission sign parity: blobs carry the - # Clojure-convention (negated) center; internal state stays in - # Delphi convention (see _apply_legacy_blob_shape). - center = -center - # Inverse of the legacy emission ORDER parity: blobs emit tids - # (and every tid-aligned pca array) in Clojure ARRIVAL order, - # while internal state aligns with the natsorted matrix - # columns. Without un-permuting, a warm restore would seed the - # next PCA with column-misaligned center/comps (#2649 review). - blob_tids = data.get('tids') or [] - if len(blob_tids) == center.shape[0]: - pos = {t: i for i, t in enumerate(blob_tids)} - perm = [pos[t] for t in natsorted(blob_tids)] - center = center[perm] - if comps.ndim == 2 and comps.shape[1] == len(perm): - comps = comps[:, perm] + # Inverse of the legacy emission sign parity: blobs carry the + # Clojure-convention (negated) center; internal state stays in + # Delphi convention (see _apply_legacy_blob_shape). + center = -center + # Inverse of the legacy emission ORDER parity: blobs emit tids + # (and every tid-aligned pca array) in Clojure ARRIVAL order, + # while internal state aligns with the natsorted matrix + # columns. Without un-permuting, a warm restore would seed the + # next PCA with column-misaligned center/comps (#2649 review). + blob_tids = data.get('tids') or [] + if len(blob_tids) == center.shape[0]: + pos = {t: i for i, t in enumerate(blob_tids)} + perm = [pos[t] for t in natsorted(blob_tids)] + center = center[perm] + if comps.ndim == 2 and comps.shape[1] == len(perm): + comps = comps[:, perm] conv.pca = { 'center': center, 'comps': comps @@ -2809,9 +2759,8 @@ def from_dict(cls, data: Dict[str, Any]) -> 'Conversation': folded_bc = data.get('base-clusters') if folded_bc: unfolded_bc = conv._unfold_base_clusters(folded_bc) - if legacy: - for c in unfolded_bc: - c['center'] = [-v for v in c['center']] + for c in unfolded_bc: + c['center'] = [-v for v in c['center']] conv.base_clusters = unfolded_bc # Restore group-votes — restructure-json-conv keeps :group-votes @@ -2992,9 +2941,7 @@ def float_to_decimal(obj): # RAW-rating-mat (conversation.clj:601-608) — moderated-out # comments report the actual votes cast, not the zeroed # pass-shaped columns. - tally_mat = (self.raw_rating_mat - if resolve_engine_mode() == ENGINE_MODE_LEGACY - else self.rating_mat) + tally_mat = self.raw_rating_mat # Precompute indices for each participant ptpt_indices = {} diff --git a/delphi/polismath/pca_kmeans_rep/legacy_kmeans.py b/delphi/polismath/pca_kmeans_rep/legacy_kmeans.py index b58831427..b336808cd 100644 --- a/delphi/polismath/pca_kmeans_rep/legacy_kmeans.py +++ b/delphi/polismath/pca_kmeans_rep/legacy_kmeans.py @@ -12,8 +12,8 @@ This is a DIFFERENT algorithm from the off-production ``clusters.py`` warm start (split-largest / merge-closest, clusters.py:302-364), which is NOT a port of the Clojure ``clean-start-clusters``. That module is intentionally left untouched; -this one is the faithful port and is wired only into the ``clojure-legacy`` -engine mode (see ``polismath.utils.engine_mode``). +this one is the faithful port wired into the engine (the only clustering path +since the mode collapse). Data model (mirrors Clojure's named-matrix + cluster maps): diff --git a/delphi/polismath/pca_kmeans_rep/repness.py b/delphi/polismath/pca_kmeans_rep/repness.py index 37ac140d8..e228793d1 100644 --- a/delphi/polismath/pca_kmeans_rep/repness.py +++ b/delphi/polismath/pca_kmeans_rep/repness.py @@ -9,7 +9,7 @@ import pandas as pd from typing import Any, Dict, Iterable, List, Optional, Tuple -from polismath.utils.engine_mode import ENGINE_MODE_LEGACY, resolve_engine_mode + from polismath.utils.general import AGREE, DISAGREE @@ -232,31 +232,23 @@ def compute_group_comment_stats_df(votes_long: pd.DataFrame, # Totals feed the "other" (rest) side of the comparison below. # - # clojure-legacy: Clojure's rest-stats sum per-group comment-stats over - # the OTHER GROUPS only (utils/mapv-rest, repness.clj:125-131), and group - # membership is unfolded through base clusters — so votes from - # participants in NO cluster never enter the comparison. Totals must - # therefore come from clustered voters only (FP-69c7a13580/FP-faac8c6125). - # - # improved: keeps the historical behavior where "other" included ALL - # participants not in the current group (even those not in any cluster). + # Clojure's rest-stats sum per-group comment-stats over the OTHER GROUPS + # only (utils/mapv-rest, repness.clj:125-131), and group membership is + # unfolded through base clusters — so votes from participants in NO + # cluster never enter the comparison. Totals must therefore come from + # clustered voters only (FP-69c7a13580/FP-faac8c6125). # # total_votes counts agree + disagree + PASS, matching Clojure's # `count-votes` (math/src/polismath/math/repness.clj:56-61, :70). # `count-votes` called with no `vote` arg uses `identity` as the filter # predicate; in Clojure 0 is truthy, so PASS (0) votes are kept. NaN # entries are already dropped above. Use size() to count non-NaN rows. - total_source = ( - votes_in_groups - if resolve_engine_mode() == ENGINE_MODE_LEGACY - else votes_only - ) - total_counts = total_source.groupby('comment').agg( + total_counts = votes_in_groups.groupby('comment').agg( total_agree=('vote', lambda x: (x == AGREE).sum()), total_disagree=('vote', lambda x: (x == DISAGREE).sum()), total_votes=('vote', 'size'), ) - # The comment universe stays votes_only-based in BOTH modes (Clojure + # The comment universe stays votes_only-based (Clojure # iterates every matrix column; a comment voted on only by unclustered # participants still gets an all-zero stats row). all_voted_comments = votes_only['comment'].unique() diff --git a/delphi/tests/test_clj_hash_order.py b/delphi/tests/test_clj_hash_order.py index cb8012a24..23f17494c 100644 --- a/delphi/tests/test_clj_hash_order.py +++ b/delphi/tests/test_clj_hash_order.py @@ -111,9 +111,3 @@ def test_legacy_greedy_tie_follows_clojure_hash_order_string_pids(monkeypatch): assert in_conv & {"14", "15", "16", "17"} == {"15", "17"} -def test_improved_greedy_unaffected(monkeypatch): - monkeypatch.setenv(ENGINE_MODE_ENV_VAR, "improved") - conv = _tie_conv() - in_conv = conv._get_in_conv_participants() - # Improved mode: threshold-only (min(7, n_cmts)=7 votes) — only pid 1. - assert in_conv == {1} diff --git a/delphi/tests/test_in_conv_greedy_carry.py b/delphi/tests/test_in_conv_greedy_carry.py index 25b6dd378..c066f12aa 100644 --- a/delphi/tests/test_in_conv_greedy_carry.py +++ b/delphi/tests/test_in_conv_greedy_carry.py @@ -64,13 +64,6 @@ def _mode(monkeypatch, mode): class TestGreedyFloor: - def test_improved_has_no_greedy_floor(self, monkeypatch): - # Current/improved behavior: only the 2 threshold-qualifiers cluster. - _mode(monkeypatch, 'improved') - conv = Conversation('g').update_votes(_votes(_TICK1_SPECS)) - assert _clustered_pids(conv) == {'H0', 'H1'} - assert conv.in_conv == set() # improved never populates the carry set - def test_legacy_greedy_fills_to_fifteen(self, monkeypatch): _mode(monkeypatch, 'clojure-legacy') conv = Conversation('g').update_votes(_votes(_TICK1_SPECS)) @@ -111,17 +104,6 @@ def test_legacy_greedy_admits_persist_after_growth(self, monkeypatch): # And the new qualifiers are in too. assert {f'Q{i}' for i in range(20)}.issubset(clustered) - def test_improved_drops_non_qualifiers_after_growth(self, monkeypatch): - _mode(monkeypatch, 'improved') - conv = Conversation('carry').update_votes(_votes(_TICK1_SPECS)) - conv = conv.update_votes(self._tick2_new_qualifiers()) - clustered = _clustered_pids(conv) - # No carry, no greedy: the below-threshold lows are NOT clustered. - assert not any(f'L{i}' in clustered for i in range(20)) - # Only the threshold-qualifiers (H0,H1 + Q0..Q19) cluster. - assert clustered == {'H0', 'H1'} | {f'Q{i}' for i in range(20)} - - class TestSerializedInConv: def test_legacy_blob_in_conv_includes_greedy_admits(self, monkeypatch): @@ -132,13 +114,6 @@ def test_legacy_blob_in_conv_includes_greedy_admits(self, monkeypatch): assert {'H0', 'H1'}.issubset(blob_in_conv) assert {f'L{i}' for i in range(13)}.issubset(blob_in_conv) # greedy admits - def test_improved_blob_in_conv_is_threshold_only(self, monkeypatch): - _mode(monkeypatch, 'improved') - conv = Conversation('blob').update_votes(_votes(_TICK1_SPECS)) - blob_in_conv = {str(p) for p in conv.to_dict()['in-conv']} - assert blob_in_conv == {'H0', 'H1'} # no greedy floor in improved - - class TestThresholdMonotonicity: @pytest.mark.parametrize('mode', ['improved', 'clojure-legacy']) diff --git a/delphi/tests/test_legacy_blob_shape.py b/delphi/tests/test_legacy_blob_shape.py index 7210fafda..3efc386ab 100644 --- a/delphi/tests/test_legacy_blob_shape.py +++ b/delphi/tests/test_legacy_blob_shape.py @@ -80,11 +80,6 @@ def legacy(monkeypatch): monkeypatch.setenv(ENGINE_MODE_ENV_VAR, "clojure-legacy") -@pytest.fixture() -def improved(monkeypatch): - monkeypatch.setenv(ENGINE_MODE_ENV_VAR, "improved") - - def _sorted_base_clusters(conv): return sorted(conv.base_clusters, key=lambda c: c["id"]) @@ -105,13 +100,6 @@ def test_legacy_group_clusters_members_are_bids(conv, legacy): assert sorted(all_members) == sorted(bc_ids) -def test_improved_group_clusters_members_stay_pids(conv, improved): - result = conv.to_dict() - pids = set(conv.rating_mat.index) - for gc in result["group-clusters"]: - assert set(gc["members"]) <= pids - - # --------------------------------------------------------------------------- # votes-base: per-base-cluster bucket lists in legacy mode. # --------------------------------------------------------------------------- @@ -154,14 +142,6 @@ def test_legacy_votes_base_excludes_unclustered_votes(conv, legacy): assert sum(entry["S"]) == 20 -def test_improved_votes_base_stays_int_totals(conv, improved): - result = conv.to_dict() - entry = next(iter(result["votes-base"].values())) - assert isinstance(entry["A"], int) - assert isinstance(entry["D"], int) - assert isinstance(entry["S"], int) - - # --------------------------------------------------------------------------- # pca: comment-projection / comment-extremity emitted + sign parity. # --------------------------------------------------------------------------- @@ -211,16 +191,6 @@ def test_legacy_sign_negation_of_center_and_projections(conv, legacy): ) -def test_improved_pca_emission_unchanged(conv, improved): - result = conv.to_dict() - np.testing.assert_allclose(result["pca"]["center"], np.asarray(conv.pca["center"])) - assert "comment-projection" not in result["pca"] - bc = result["base-clusters"] - by_id = {c["id"]: c for c in conv.base_clusters} - for i, bid in enumerate(bc["id"]): - assert bc["x"][i] == pytest.approx(by_id[bid]["center"][0]) - - # --------------------------------------------------------------------------- # repness: Clojure finalize-cmt-stats shape in legacy mode. # --------------------------------------------------------------------------- @@ -258,13 +228,6 @@ def test_legacy_repness_shape_and_direction_mapping(conv, legacy): assert "comment_id" not in got and "na" not in got and "rat" not in got -def test_improved_repness_stays_internal_shape(conv, improved): - result = conv.to_dict() - assert set(result["repness"].keys()) == { - "comment_ids", "group_repness", "comment_repness", "consensus_comments", - } - - # --------------------------------------------------------------------------- # repness rest-domain: "other" = the OTHER GROUPS only in legacy mode. # --------------------------------------------------------------------------- @@ -301,16 +264,6 @@ def test_legacy_repness_rest_domain_excludes_unclustered(legacy): assert row["ra"] == pytest.approx(3.0) -def test_improved_repness_rest_domain_includes_all_voters(improved): - from polismath.pca_kmeans_rep.repness import compute_group_comment_stats_df - - votes_long, groups = _rest_domain_fixture() - df = compute_group_comment_stats_df(votes_long, groups) - row = df.loc[(0, 0)] - # rest = group 1 + p99: na=1 ns=3 → other_pa = (1+1)/(3+2) = 0.4; ra = 1.875 - assert row["ra"] == pytest.approx(0.75 / 0.4) - - # --------------------------------------------------------------------------- # group-aware-consensus: zero-S groups contribute (A+1)/(S+2) = 1/2 in legacy. # --------------------------------------------------------------------------- @@ -337,16 +290,6 @@ def test_legacy_gac_multiplies_zero_s_groups(conv, legacy): assert result["group-aware-consensus"][10] == pytest.approx(expected) -def test_improved_gac_skips_zero_s_groups(conv, improved): - result = conv.to_dict() - stats = _gac_group_stats(result, 10) - expected = 1.0 - for a, s in stats.values(): - if s > 0: - expected *= (a + 1.0) / (s + 2.0) - assert result["group-aware-consensus"][10] == pytest.approx(expected) - - # --------------------------------------------------------------------------- # moderation-state semantics: None until moderation applied (legacy). # --------------------------------------------------------------------------- @@ -367,13 +310,6 @@ def test_legacy_mod_keys_populated_after_moderation(conv, legacy): assert result["lastModTimestamp"] is None -def test_improved_mod_keys_stay_lists(conv, improved): - result = conv.to_dict() - assert result["mod-in"] == [] - assert result["mod-out"] == [] - assert result["lastModTimestamp"] == conv.last_updated - - # --------------------------------------------------------------------------- # Arrival-order parity: Clojure's named-matrix column order is first-vote # arrival order (update-nmat appends unseen colnames in encounter order); @@ -422,11 +358,6 @@ def test_legacy_tids_emitted_in_arrival_order_with_aligned_pca(conv, legacy): assert result["pca"]["comment-extremity"][i] == pytest.approx(ext[tid]) -def test_improved_tids_stay_natsorted(conv, improved): - result = conv.to_dict() - assert result["tids"] == list(conv.rating_mat.columns) - - def test_legacy_from_dict_restores_arrival_order(conv, legacy): restored = Conversation.from_dict(conv.to_dict()) assert restored.tid_arrival_order == conv.tid_arrival_order @@ -458,12 +389,6 @@ def test_legacy_from_dict_restores_base_clusters_and_zid(conv, legacy): _assert_base_clusters_round_trip(conv, restored) -def test_improved_from_dict_restores_base_clusters_and_zid(conv, improved): - restored = Conversation.from_dict(conv.to_dict()) - assert restored.conversation_id == "legacy_blob_shape" - _assert_base_clusters_round_trip(conv, restored) - - def test_from_dict_preserves_falsy_conversation_id(): # #2656 review finding 2: `data.get('conversation_id') or data.get('zid')` # would discard a legitimately-falsy id (e.g. 0) — the key-presence check @@ -582,15 +507,6 @@ def test_legacy_single_vote_repness_and_consensus(legacy): assert d["consensus"]["disagree"] == [] -def test_improved_single_vote_guards_unchanged(improved): - d = _tiny_conv().to_dict() - assert d["pca"]["center"] == [0.0] - assert d["repness"]["group_repness"] == {0: []} or all( - not v for v in d["repness"]["group_repness"].values() - ) - assert d["consensus"] == {"agree": [], "disagree": []} - - # --------------------------------------------------------------------------- # from_dict inverse: legacy round-trip restores the internal convention. # --------------------------------------------------------------------------- @@ -628,13 +544,6 @@ def test_legacy_from_dict_round_trips_center_sign(conv, legacy): ) -def test_improved_from_dict_round_trips_center_sign(conv, improved): - restored = Conversation.from_dict(conv.to_dict()) - np.testing.assert_allclose( - np.asarray(restored.pca["center"]), np.asarray(conv.pca["center"]) - ) - - # --------------------------------------------------------------------------- # Tiny SHAPES beyond 1x1 (review finding on #2653): the relaxed small-dim # guards cover any `rows < 2 OR cols < 2` matrix. Expectations are REAL diff --git a/delphi/tests/test_mod_ptpt_leak_parity.py b/delphi/tests/test_mod_ptpt_leak_parity.py index 629bae8cc..094bddb31 100644 --- a/delphi/tests/test_mod_ptpt_leak_parity.py +++ b/delphi/tests/test_mod_ptpt_leak_parity.py @@ -51,12 +51,6 @@ def legacy_mode(monkeypatch): monkeypatch.setenv(ENGINE_MODE_ENV_VAR, 'clojure-legacy') -@pytest.fixture -def improved_mode(monkeypatch): - monkeypatch.delenv(PCA_IMPL_ENV_VAR, raising=False) - monkeypatch.setenv(ENGINE_MODE_ENV_VAR, 'improved') - - class TestLegacyBanLeak: def test_banned_participant_rows_kept(self, legacy_mode): @@ -89,15 +83,5 @@ def test_banned_participant_stays_in_conv(self, legacy_mode): assert 'a0' in conv.in_conv -class TestImprovedBanKept: - """Improved mode keeps the real ban feature byte-for-byte.""" - - def test_banned_participant_dropped_and_not_clustered(self, improved_mode): - conv = Conversation('leak').update_votes(_bloc_votes()) - conv = conv.update_moderation({'mod_out_ptpts': ['a0']}) - assert 'a0' not in conv.rating_mat.index - assert 'a0' not in _clustered_pids(conv) - - if __name__ == '__main__': pytest.main([__file__, '-v'])