diff --git a/sdcm/nemesis/__init__.py b/sdcm/nemesis/__init__.py index d5bd7d7b5ad..976dc655162 100644 --- a/sdcm/nemesis/__init__.py +++ b/sdcm/nemesis/__init__.py @@ -2298,7 +2298,8 @@ def disrupt_truncate_large_partition(self): table = "test_table" ks_cf = f"{ks_name}.{table}" stress_cmd = ( - "scylla-bench -workload=sequential -mode=write -replication-factor=3 -partition-count=10 " + f"scylla-bench -workload=sequential -mode=write " + f"-replication-factor={self.tester.reliable_replication_factor} -partition-count=10 " + "-clustering-row-count=5555 -clustering-row-size=uniform:10..20 -concurrency=10 " + "-connection-count=10 -consistency-level=quorum -rows-per-request=10 -timeout=60s " + f"-keyspace {ks_name} -table {table}" diff --git a/sdcm/tester.py b/sdcm/tester.py index 1b17555386b..9652978464d 100644 --- a/sdcm/tester.py +++ b/sdcm/tester.py @@ -952,6 +952,10 @@ def run(self, result=None): def latency_results_file(self): return TestConfig.latency_results_file() + def _is_rf_rack_valid_keyspaces_enabled(self) -> bool: + """'rf_rack_valid_keyspaces' bounds a keyspace's RF by the number of racks in the DC.""" + return bool((self.params.get("append_scylla_yaml") or {}).get("rf_rack_valid_keyspaces", False)) + @property def reliable_replication_factor(self) -> int: """ @@ -965,11 +969,14 @@ def reliable_replication_factor(self) -> int: min_nodes_dc = min(n_db_nodes) # In case tablets are enabled, it's better to set RF smaller than dc-nodes-number, so decommission is allowed. - rf_candidate = ( - max([min_nodes_dc - 1, 1]) if is_tablets_feature_enabled(self.db_cluster.nodes[0]) else min_nodes_dc - ) + tablets_enabled = is_tablets_feature_enabled(self.db_cluster.nodes[0]) + rf_candidate = max([min_nodes_dc - 1, 1]) if tablets_enabled else min_nodes_dc # NOTE: use RF=3 at max to avoid problems on big setups - return min(rf_candidate, 3) + rf_candidate = min(rf_candidate, 3) + if tablets_enabled and self._is_rf_rack_valid_keyspaces_enabled(): + # 'rf_rack_valid_keyspaces' makes Scylla reject CREATE KEYSPACE with RF above the rack count + rf_candidate = min(rf_candidate, max(self.db_cluster.racks_count, 1)) + return rf_candidate @property def test_id(self): diff --git a/unit_tests/unit/test_tester.py b/unit_tests/unit/test_tester.py index 083c1cf7d5a..304eb056229 100644 --- a/unit_tests/unit/test_tester.py +++ b/unit_tests/unit/test_tester.py @@ -807,3 +807,61 @@ def test_get_cluster_docker_nonzero_monitor_nodes_builds_monitor_set_docker(tmp_ cluster_docker_mock.MonitorSetDocker.assert_called_once() assert tester.monitors is cluster_docker_mock.MonitorSetDocker.return_value + + +# --- Tests for ClusterTester.reliable_replication_factor RF-by-racks bounding --- + + +def _rf_tester(n_db_nodes, racks_count, append_scylla_yaml=None, tablets_enabled=True): + tester = MagicMock(spec=ClusterTester) + tester.params = {"n_db_nodes": n_db_nodes, "append_scylla_yaml": append_scylla_yaml} + tester.db_cluster = MagicMock() + tester.db_cluster.nodes = [MagicMock()] + tester.db_cluster.racks_count = racks_count + tester._is_rf_rack_valid_keyspaces_enabled = types.MethodType( + ClusterTester._is_rf_rack_valid_keyspaces_enabled, tester + ) + with patch("sdcm.tester.is_tablets_feature_enabled", return_value=tablets_enabled): + return ClusterTester.reliable_replication_factor.fget(tester) + + +@pytest.mark.parametrize( + "tablets_enabled, rf_rack_valid, n_db_nodes, racks_count, expected_rf", + [ + pytest.param(True, True, [6], 1, 1, id="tablets-rfrack-one-rack-bites"), + pytest.param(True, True, [6], 3, 3, id="tablets-rfrack-multi-rack-unaffected"), + pytest.param(True, True, [6], 4, 3, id="tablets-rfrack-cap-at-3-still-wins"), + pytest.param(True, False, [6], 1, 3, id="tablets-rfrack-disabled-no-change"), + pytest.param(False, True, [6], 1, 3, id="no-tablets-rfrack-enabled-no-change"), + pytest.param(False, True, [2, 3], 1, 2, id="no-tablets-path-untouched"), + pytest.param(True, True, [1], 0, 1, id="never-degenerates-to-zero"), + ], +) +def test_reliable_replication_factor(tablets_enabled, rf_rack_valid, n_db_nodes, racks_count, expected_rf): + append_scylla_yaml = {"rf_rack_valid_keyspaces": rf_rack_valid} if rf_rack_valid is not None else {} + rf = _rf_tester( + n_db_nodes=n_db_nodes, + racks_count=racks_count, + append_scylla_yaml=append_scylla_yaml, + tablets_enabled=tablets_enabled, + ) + assert rf == expected_rf + + +@pytest.mark.parametrize( + "append_scylla_yaml, expected", + [ + pytest.param(None, False, id="none"), + pytest.param({}, False, id="empty-dict"), + pytest.param({"enable_tablets": True}, False, id="unrelated-key"), + pytest.param({"rf_rack_valid_keyspaces": False}, False, id="explicitly-disabled"), + pytest.param({"rf_rack_valid_keyspaces": True}, True, id="explicitly-enabled"), + ], +) +def test_is_rf_rack_valid_keyspaces_enabled(append_scylla_yaml, expected): + tester = MagicMock(spec=ClusterTester) + tester.params = {"append_scylla_yaml": append_scylla_yaml} + tester._is_rf_rack_valid_keyspaces_enabled = types.MethodType( + ClusterTester._is_rf_rack_valid_keyspaces_enabled, tester + ) + assert tester._is_rf_rack_valid_keyspaces_enabled() is expected