Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 59 additions & 40 deletions docs/plans/nemesis/nemesis-precheck.md

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion docs/plans/progress.json
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@
"status": "in_progress",
"created": "2026-06-11",
"phases_total": 5,
"phases_done": 3
"phases_done": 4
},
{
"id": "k8s-multitenancy-dict-config",
Expand Down
10 changes: 5 additions & 5 deletions sdcm/nemesis/monkey/add_remove_dc.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,11 @@ def __init__(self, *args, **kwargs):
self.new_ks_name: str = "keyspace_new_dc"
self.new_ks_rf: int = len(self.runner.cluster.racks)

def precheck(self, node) -> str | None:
if self.runner.cluster.test_config.MULTI_REGION:
return "Skipped for multi-dc scenario (https://github.com/scylladb/scylla-cluster-tests/issues/5369)"
return None

@property
def num_nodes_in_new_dc(self) -> int:
"""Return new datacenter size based on supported RF change size."""
Expand Down Expand Up @@ -270,11 +275,6 @@ def finalizer(self, exc_type, *_):

def disrupt(self) -> None:
"""Execute the add/remove datacenter nemesis workflow."""
if self.runner.cluster.test_config.MULTI_REGION:
raise UnsupportedNemesis(
"Skipped for multi-dc scenario (https://github.com/scylladb/scylla-cluster-tests/issues/5369)"
)

if self.runner.tester.prepare_phase_active.is_set():
raise UnsupportedNemesis(
"Skipped during prepare phase, due to stress commands potentially writing to the new DC."
Expand Down
37 changes: 25 additions & 12 deletions sdcm/nemesis/monkey/network.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,13 @@
# ---------------------------------------------------------------------------


def extra_network_interface_precheck(runner) -> str | None:
"""Skip reason when the cluster has no secondary network interface configured."""
if not runner.cluster.extra_network_interface:
return "for this nemesis to work, you need to set `extra_network_interface: True`"
return None


def install_iptables(target_node) -> None:
"""Install iptables on Ubuntu nodes where it is missing by default."""
if target_node.distro.is_ubuntu: # iptables is missing in a minimized Ubuntu installation
Expand Down Expand Up @@ -209,16 +216,18 @@ class RandomInterruptionNetworkMonkey(NemesisBaseClass):
# Test communication address (ip_ssh_connections) is defined as "public" for the relevant pipelines in "two_interfaces.yaml"
additional_params = {"ip_ssh_connections": "public"}

def precheck(self, node) -> str | None:
if self.runner._is_it_on_kubernetes(): # k8s uses Chaos Mesh, no secondary interface needed
return None
return extra_network_interface_precheck(self.runner)

def disrupt(self):
"""Apply a random network interruption (loss, corruption, delay, or bandwidth cap) for a random duration."""
list_of_timeout_options = [10, 60, 120, 300, 500]
if self.runner._is_it_on_kubernetes():
self._disrupt_k8s(list_of_timeout_options)
return

if not self.runner.cluster.extra_network_interface:
raise UnsupportedNemesis("for this nemesis to work, you need to set `extra_network_interface: True`")

if not self.runner.target_node.install_traffic_control():
raise UnsupportedNemesis("Traffic control package not installed on system")

Expand Down Expand Up @@ -367,6 +376,11 @@ class BlockNetworkMonkey(NemesisBaseClass):
# Test communication address (ip_ssh_connections) is defined as "public" for the relevant pipelines in "two_interfaces.yaml"
additional_params = {"ip_ssh_connections": "public"}

def precheck(self, node) -> str | None:
if self.runner._is_it_on_kubernetes(): # k8s uses Chaos Mesh, no secondary interface needed
return None
return extra_network_interface_precheck(self.runner)

def disrupt(self):
"""Block all network traffic on the target node using 100% packet loss for a random duration."""
list_of_timeout_options = [10, 60, 120, 300, 500]
Expand All @@ -375,9 +389,6 @@ def disrupt(self):
self._disrupt_k8s(list_of_timeout_options)
return

if not self.runner.cluster.extra_network_interface:
raise UnsupportedNemesis("for this nemesis to work, you need to set `extra_network_interface: True`")

if not self.runner.target_node.install_traffic_control():
raise UnsupportedNemesis("Traffic control package not installed on system")

Expand Down Expand Up @@ -438,12 +449,14 @@ class RejectInterNodeNetworkMonkey(NemesisBaseClass):
networking = True
free_tier_set = True

def disrupt(self):
"""Drop or reject inter-node gossip/streaming traffic (ports 7000/7001) with a random iptables rule."""
def precheck(self, node) -> str | None:
# Temporary disable due to https://github.com/scylladb/scylla/issues/6522
if SkipPerIssues("https://github.com/scylladb/scylladb/issues/6522", self.runner.tester.params):
raise UnsupportedNemesis("https://github.com/scylladb/scylladb/issues/6522")
return "https://github.com/scylladb/scylladb/issues/6522"
return None

def disrupt(self):
"""Drop or reject inter-node gossip/streaming traffic (ports 7000/7001) with a random iptables rule."""
install_iptables(self.runner.target_node)

textual_matching_rule, matching_rule = iptables_randomly_get_random_matching_rule(rnd=self.runner.random)
Expand Down Expand Up @@ -547,11 +560,11 @@ class StopStartInterfacesNetworkMonkey(NemesisBaseClass):
# Test communication address (ip_ssh_connections) is defined as "public" for the relevant pipelines in "two_interfaces.yaml"
additional_params = {"ip_ssh_connections": "public"}

def precheck(self, node) -> str | None:
return extra_network_interface_precheck(self.runner)

def disrupt(self):
"""Take the secondary network interface down for a random duration, then bring it back up."""
if not self.runner.cluster.extra_network_interface:
raise UnsupportedNemesis("for this nemesis to work, you need to set `extra_network_interface: True`")

list_of_timeout_options = [10, 60, 120, 300, 500]
wait_time = self.runner.random.choice(list_of_timeout_options)
self.runner.log.debug("Taking down eth1 for %dsec", wait_time)
Expand Down
36 changes: 21 additions & 15 deletions sdcm/nemesis/monkey/sla.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,21 +14,30 @@ class SlaMonkeyBase(NemesisBaseClass):

sla = True

# Whether the nemesis needs the cassandra-stress default table prefilled by
# the prepare phase. Checked in precheck() via the prepare_write_cmd param.
requires_prefilled_cs_data: bool = True

additional_configs = [
"configurations/nemesis/additional_configs/sla_config.yaml",
"configurations/auth_cassandra.yaml",
]

def validate_sla_preconditions(self):
"""Common validation for all SLA nemeses."""
def precheck(self, node) -> str | None:
"""Common static preconditions for all SLA nemeses."""
if not self.runner.cluster.params.get("sla"):
raise UnsupportedNemesis("SLA nemesis can be run during SLA test only")
return "SLA nemesis can be run during SLA test only"

if not self.runner.cluster.nodes[0].is_enterprise:
raise UnsupportedNemesis("SLA feature is only supported by Scylla Enterprise")
if not node.is_enterprise:
return "SLA feature is only supported by Scylla Enterprise"

if not self.runner.cluster.params.get("authenticator"):
raise UnsupportedNemesis("SLA feature can't work without authenticator")
return "SLA feature can't work without authenticator"

if self.requires_prefilled_cs_data and not self.get_cassandra_stress_write_cmds():
return "SLA nemesis needs cassandra-stress default table 'keyspace1.standard1' is created and prefilled"

return None

def get_cassandra_stress_write_cmds(self):
write_cmds = self.runner.tester.params.get("prepare_write_cmd")
Expand Down Expand Up @@ -57,7 +66,11 @@ def get_cassandra_stress_definition(self, stress_cmds, default_data_set_size=500
return column_definition, data_set_size

def get_stress_params(self):
"""Return (column_definition, dataset_size) for SLA stress tests."""
"""Return (column_definition, dataset_size) for SLA stress tests.

The missing-command case is already pruned by precheck(); the raise below
stays as an in-method safety net.
"""
stress_cmds = self.get_cassandra_stress_write_cmds()
if not stress_cmds:
raise UnsupportedNemesis(
Expand All @@ -78,10 +91,9 @@ class RemoveServiceLevelMonkey(SlaMonkeyBase):
"""Remove service level while load is running, then re-create it."""

disruptive = True
requires_prefilled_cs_data = False # works on pre-defined roles, not on cassandra-stress data

def disrupt(self):
self.validate_sla_preconditions()

if not getattr(self.runner.tester, "roles", None):
raise UnsupportedNemesis("This nemesis is supported with Service Level and role are pre-defined")

Expand Down Expand Up @@ -115,7 +127,6 @@ class SlaIncreaseSharesDuringLoad(SlaMonkeyBase):
disruptive = False

def disrupt(self):
self.validate_sla_preconditions()
column_definition, dataset_size = self.get_stress_params()

prometheus_stats = PrometheusDBStats(host=self.runner.monitoring_set.nodes[0].external_address)
Expand All @@ -134,7 +145,6 @@ class SlaDecreaseSharesDuringLoad(SlaMonkeyBase):
disruptive = False

def disrupt(self):
self.validate_sla_preconditions()
column_definition, dataset_size = self.get_stress_params()

prometheus_stats = PrometheusDBStats(host=self.runner.monitoring_set.nodes[0].external_address)
Expand All @@ -154,7 +164,6 @@ class SlaReplaceUsingDetachDuringLoad(SlaMonkeyBase):
disruptive = True

def disrupt(self):
self.validate_sla_preconditions()
column_definition, dataset_size = self.get_stress_params()

prometheus_stats = PrometheusDBStats(host=self.runner.monitoring_set.nodes[0].external_address)
Expand All @@ -174,7 +183,6 @@ class SlaReplaceUsingDropDuringLoad(SlaMonkeyBase):
disruptive = True

def disrupt(self):
self.validate_sla_preconditions()
column_definition, dataset_size = self.get_stress_params()

prometheus_stats = PrometheusDBStats(host=self.runner.monitoring_set.nodes[0].external_address)
Expand All @@ -194,7 +202,6 @@ class SlaIncreaseSharesByAttachAnotherSlDuringLoad(SlaMonkeyBase):
disruptive = True

def disrupt(self):
self.validate_sla_preconditions()
column_definition, dataset_size = self.get_stress_params()

prometheus_stats = PrometheusDBStats(host=self.runner.monitoring_set.nodes[0].external_address)
Expand All @@ -213,7 +220,6 @@ class SlaMaximumAllowedSlsWithMaxSharesDuringLoad(SlaMonkeyBase):
disruptive = False

def disrupt(self):
self.validate_sla_preconditions()
column_definition, dataset_size = self.get_stress_params()

prometheus_stats = PrometheusDBStats(host=self.runner.monitoring_set.nodes[0].external_address)
Expand Down
18 changes: 9 additions & 9 deletions unit_tests/unit/nemesis/monkey/test_add_remove_dc.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,18 +108,18 @@ def runner(base_runner):
return base_runner


def test_disrupt_raises_unsupported_for_multi_region(runner):
"""MULTI_REGION runs are rejected before any topology changes start."""
def test_precheck_skips_for_multi_region(runner):
"""MULTI_REGION runs are pruned before the nemesis is ever scheduled."""
runner.cluster.test_config.MULTI_REGION = True

with pytest.raises(UnsupportedNemesis, match="multi-dc scenario"):
AddRemoveDcNemesis(runner).disrupt()
assert AddRemoveDcNemesis(runner).precheck(node=runner.cluster.nodes[0]) == (
"Skipped for multi-dc scenario (https://github.com/scylladb/scylla-cluster-tests/issues/5369)"
)

runner.tester.create_keyspace.assert_not_called()
runner.cluster.add_nodes.assert_not_called()
runner.run_repair.assert_not_called()
runner.decommission_nodes.assert_not_called()
assert not runner.executed

def test_precheck_keeps_for_single_region(runner):
"""Single-region runs keep the nemesis in the rotation."""
assert AddRemoveDcNemesis(runner).precheck(node=runner.cluster.nodes[0]) is None


@pytest.mark.parametrize(
Expand Down
36 changes: 30 additions & 6 deletions unit_tests/unit/nemesis/monkey/test_network.py
Original file line number Diff line number Diff line change
Expand Up @@ -240,23 +240,47 @@ def test_rate_limit_returns_string_with_suffix(mock_prom_class, runner):


# ---------------------------------------------------------------------------
# UnsupportedNemesis guards
# precheck guards
# ---------------------------------------------------------------------------

EXTRA_INTERFACE_MONKEYS = [
pytest.param(RandomInterruptionNetworkMonkey, id="random-interruption"),
pytest.param(BlockNetworkMonkey, id="block"),
pytest.param(StopStartInterfacesNetworkMonkey, id="stop-start"),
]


@pytest.mark.parametrize("monkey_class", EXTRA_INTERFACE_MONKEYS)
def test_precheck_skips_when_no_extra_interface(runner, monkey_class):
"""precheck() skips monkeys requiring extra_network_interface when it's absent."""
runner.cluster.extra_network_interface = False

assert monkey_class(runner).precheck(node=runner.cluster.data_nodes[0]) == (
"for this nemesis to work, you need to set `extra_network_interface: True`"
)


@pytest.mark.parametrize("monkey_class", EXTRA_INTERFACE_MONKEYS)
def test_precheck_keeps_when_extra_interface_set(runner, monkey_class):
"""precheck() keeps the nemesis when extra_network_interface is configured."""
runner.cluster.extra_network_interface = True

assert monkey_class(runner).precheck(node=runner.cluster.data_nodes[0]) is None


@pytest.mark.parametrize(
"monkey_class",
[
pytest.param(RandomInterruptionNetworkMonkey, id="random-interruption"),
pytest.param(BlockNetworkMonkey, id="block"),
pytest.param(StopStartInterfacesNetworkMonkey, id="stop-start"),
],
)
def test_raises_when_no_extra_interface(runner, monkey_class):
"""Verify monkeys requiring extra_network_interface raise UnsupportedNemesis when it's absent."""
def test_precheck_keeps_on_kubernetes_without_extra_interface(runner, monkey_class):
"""On k8s the Chaos Mesh path is used, so no secondary interface is required."""
runner._is_it_on_kubernetes.return_value = True
runner.cluster.extra_network_interface = False
with pytest.raises(UnsupportedNemesis, match="extra_network_interface"):
monkey_class(runner).disrupt()

assert monkey_class(runner).precheck(node=runner.cluster.data_nodes[0]) is None


@pytest.mark.parametrize(
Expand Down
Loading