Deflake ccov: Contain slot migration test failures with a recovery barrier

tests/unit/cluster/cluster-migrateslots.tcl

@@ -172,13 +172,128 @@ proc do_node_restart {idx} {
     }
 }
 
+proc has_active_slot_migration {node_idx} {
+    foreach migration [R $node_idx CLUSTER GETSLOTMIGRATIONS] {
+        if {[dict get $migration state] ni {success cancelled failed}} {
+            return 1
+        }
+    }
+    return 0
+}
+
+# Record which primary owns each slot, so a failed test that dies
+# mid-migration can have slot ownership restored by the recovery barrier.
+proc capture_canonical_slot_layout {} {
+    # Node IDs are how CLUSTER SLOTS reports owners, but tests address nodes
+    # by R index -- record the mapping so layouts can be compared later.
+    set ::canonical_node_ids {}
+    foreach n {0 1 2} {
+        dict set ::canonical_node_ids [R $n CLUSTER MYID] $n
+    }
+    # -1 marks slots with no recorded owner; the restore skips them.
+    for {set s 0} {$s < 16384} {incr s} {set ::canonical_slot_owner($s) -1}
+    # Each CLUSTER SLOTS entry is {start end {ip port id ...} ...} with the
+    # owning primary listed first; expand each range into a per-slot owner map.
+    foreach entry [R 0 CLUSTER SLOTS] {
+        set owner_id [lindex [lindex $entry 2] 2]
+        if {![dict exists $::canonical_node_ids $owner_id]} continue
+        set owner_idx [dict get $::canonical_node_ids $owner_id]
+        for {set s [lindex $entry 0]} {$s <= [lindex $entry 1]} {incr s} {
+            set ::canonical_slot_owner($s) $owner_idx
+        }
+    }
+}
+
+# Migrate any slots displaced from the canonical layout back to their
+# canonical owner. Contiguous displaced slots with the same source and
+# destination move as a single range.
+proc restore_canonical_slot_layout {} {
+    if {![info exists ::canonical_slot_owner]} return
+    # Build the current per-slot owner map, same shape as the canonical one.
+    set current {}
+    foreach entry [R 0 CLUSTER SLOTS] {
+        set owner_id [lindex [lindex $entry 2] 2]
+        if {![dict exists $::canonical_node_ids $owner_id]} continue
+        set owner_idx [dict get $::canonical_node_ids $owner_id]
+        for {set s [lindex $entry 0]} {$s <= [lindex $entry 1]} {incr s} {
+            dict set current $s $owner_idx
+        }
+    }
+    # Sweep all slots, run-length grouping consecutive displaced slots that
+    # share the same current and canonical owner, and migrate each group back
+    # as a single range. A run ends when the slot is no longer mismatched or
+    # its source/destination pair changes; the sweep deliberately goes one
+    # past the last slot (16384) so a run ending at slot 16383 is flushed.
+    # Slots with no canonical owner recorded (-1) or currently unserved
+    # (absent from CLUSTER SLOTS) are never treated as mismatched.
+    set run_start -1
+    for {set s 0} {$s <= 16384} {incr s} {
+        set mismatch 0
+        if {$s < 16384 && [dict exists $current $s]} {
+            set want $::canonical_slot_owner($s)
+            set have [dict get $current $s]
+            if {$want >= 0 && $have != $want} {set mismatch 1}
+        }
+        if {$mismatch && $run_start == -1} {
+            set run_start $s
+            set run_have $have
+            set run_want $want
+        } elseif {$run_start != -1 && (!$mismatch || $have != $run_have || $want != $run_want)} {
+            set want_id [R $run_want CLUSTER MYID]
+            # Best-effort: the barrier must never raise from block level.
+            catch {
+                assert_match "OK" [R $run_have CLUSTER MIGRATESLOTS SLOTSRANGE $run_start [expr {$s - 1}] NODE $want_id]
+                wait_for_migration $run_want $run_start
+            }
+            set run_start [expr {$mismatch ? $s : -1}]
+            if {$mismatch} {set run_have $have; set run_want $want}
+        }
+    }
+}
+
+# Recovery barrier for tests that start slot migrations. When such a test
+# fails an assertion mid-flight, it leaks state: an in-flight migration,
+# DEBUG SLOTMIGRATION PREVENT-* flags, and config tweaks. The next test to
+# call CLUSTER MIGRATESLOTS then hits a non-assertion error such as
+# "ERR I am already migrating slot", which aborts the whole test client and
+# discards the rest of the run. Calling this barrier after a
+# migration-starting test contains a failure to that single test. After
+# migrations drain, slot ownership displaced by the failed test is migrated
+# back to the canonical layout captured at cluster setup. All steps are
+# best-effort so the barrier itself cannot abort the
+# client from block level.
+proc recover_slot_migration_state {} {
+    catch {set_debug_prevent_pause 0}
+    catch {set_debug_prevent_failover 0}
+    for {set i 0} {$i < [llength $::servers]} {incr i} {
+        catch {R $i CONFIG SET rdb-key-save-delay 0}
+        catch {R $i CONFIG SET repl-timeout 60}
+    }
+    foreach n {0 1 2} {
+        catch {R $n CLUSTER CANCELSLOTMIGRATIONS}
+    }
+    catch {
+        foreach n {0 1 2} {
+            wait_for_condition 100 100 {
+                ![has_active_slot_migration $n]
+            } else {
+                fail "Node $n still has an active slot migration after recovery"
+            }
+        }
+        restore_canonical_slot_layout
+        wait_for_cluster_propagation
+    }
+}
+
+
 # Disable replica migration to prevent empty nodes from joining other shards.
 start_cluster 3 3 {tags {logreqres:skip external:skip cluster} overrides {cluster-allow-replica-migration no cluster-node-timeout 15000 cluster-databases 16}} {
 
     set node0_id [R 0 CLUSTER MYID]
     set node1_id [R 1 CLUSTER MYID]
     set node2_id [R 2 CLUSTER MYID]
     set fake_jobname "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+    capture_canonical_slot_layout
 
     test "General command interface" {
         assert_error "*wrong number of arguments*" {R 0 CLUSTER MIGRATESLOTS}
@@ -649,6 +764,7 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster} overrides {cluste
             wait_for_migration 2 16383
         }
     }
+    recover_slot_migration_state
 
     test "Import with hz set to 1" {
         assert_does_not_resync {
@@ -684,6 +800,7 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster} overrides {cluste
             R 2 CONFIG SET hz $old_hz
         }
     }
+    recover_slot_migration_state
 
     # Catch-all test for covering commands sent during incremental replication
     test "Single source import - Incremental Command Coverage" {
@@ -1153,6 +1270,7 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster} overrides {cluste
             assert_match "OK" [R 2 FLUSHDB SYNC]
         }
     }
+    recover_slot_migration_state
 
     test "Partial data removed on cancel" {
         assert_does_not_resync {
@@ -1544,6 +1662,7 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster} overrides {cluste
             set_debug_prevent_pause 0
         }
     }
+    recover_slot_migration_state
 
     test "FLUSH on target during import" {
         assert_does_not_resync {
@@ -1878,6 +1997,7 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster} overrides {cluste
             wait_for_migration 2 16383
         }
     }
+    recover_slot_migration_state
 
     test "Export client buffer excluded from maxmemory" {
         assert_does_not_resync {
@@ -1975,6 +2095,7 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster} overrides {cluste
             R 2 CONFIG SET rdb-key-save-delay 0
         }
     }
+    recover_slot_migration_state
 
     foreach testcase [list \
         [list 0 0 1] \
@@ -2135,6 +2256,7 @@ start_cluster 3 3 {tags {logreqres:skip external:skip cluster} overrides {cluste
             # Since we are restarting primaries, we need to ensure the cluster becomes stable
             wait_for_cluster_state ok
         }
+        recover_slot_migration_state
     }
 }