diff --git a/delphi/scripts/generate_cold_start_clojure.py b/delphi/scripts/generate_cold_start_clojure.py index bc7444b19..a6c16eb8a 100755 --- a/delphi/scripts/generate_cold_start_clojure.py +++ b/delphi/scripts/generate_cold_start_clojure.py @@ -309,11 +309,29 @@ def copy_votes_with_fresh_timestamps(conn, source_zid: int, fake_zid: int) -> in """ Copy votes from source conversation to fake conversation with fresh timestamps. - Preserves vote ORDER by using sequential timestamps starting from now. - The poller finds votes by `created > last_poll_timestamp`, so fresh - timestamps ensure these votes are picked up. - - Uses a single INSERT ... SELECT for efficiency (no Python roundtrips). + Copies the FULL vote history, including revotes (multiple rows for the same + (pid, tid) pair). An earlier version deduplicated with + ``DISTINCT ON (pid, tid) ... ORDER BY created DESC`` ("keep the latest"), + which silently dropped superseded revote rows (vw: 128 of 4683). That made + the Clojure reference consume a DIFFERENT input than the Python side (which + feeds every CSV row and lets the engine's later-vote-wins merge resolve + revotes), and it erases the revote dynamics that sequential replay + specifically needs (see REPLAY_HARNESS_DESIGN.md §5: "Do NOT dedup + revotes"). Both engines implement later-vote-wins internally, so the dedup + was never necessary for correctness of the final matrix — only harmful for + input parity. + + Preserves vote ORDER by using sequential timestamps starting from now + (10 ms apart, strictly increasing, so Clojure's later-vote-wins resolves + revotes in source order). Source order is ``created ASC`` with ``ctid`` as + a tiebreak: for revotes of the same (pid, tid) sharing the same source + millisecond, physical row order approximates insertion order (the table is + append-only); the true relative order of same-ms revotes is ambiguous in + the source data itself. + + The poller finds votes by ``created > last_poll_timestamp``, so fresh + timestamps ensure these votes are picked up. Uses a single + INSERT ... SELECT for efficiency (no Python roundtrips). Returns the number of votes copied. """ @@ -324,7 +342,6 @@ def copy_votes_with_fresh_timestamps(conn, source_zid: int, fake_zid: int) -> in # Single INSERT ... SELECT with ROW_NUMBER() to generate sequential timestamps # This is much faster than executemany for large vote counts - # Use DISTINCT ON (pid, tid) to handle duplicate votes (keeps the latest) cursor.execute(""" INSERT INTO votes (zid, pid, tid, vote, weight_x_32767, created) SELECT @@ -333,14 +350,10 @@ def copy_votes_with_fresh_timestamps(conn, source_zid: int, fake_zid: int) -> in tid, vote, weight_x_32767, - %s + (ROW_NUMBER() OVER (ORDER BY created ASC) - 1) * 10 - FROM ( - SELECT DISTINCT ON (pid, tid) pid, tid, vote, weight_x_32767, created - FROM votes - WHERE zid = %s - ORDER BY pid, tid, created DESC - ) AS deduplicated - ORDER BY created ASC + %s + (ROW_NUMBER() OVER (ORDER BY created ASC, ctid ASC) - 1) * 10 + FROM votes + WHERE zid = %s + ORDER BY created ASC, ctid ASC """, (fake_zid, now_ms, source_zid)) copied_count = cursor.rowcount