From 3a9dd4495bdb040c22659371fb8ca3ce5f89c83c Mon Sep 17 00:00:00 2001 From: Jakub Smolar Date: Mon, 18 May 2026 12:05:18 +0200 Subject: [PATCH] feature(s3_object_storage): configure rolling upgrade test to use S3-backed keyspaces - Pre-create all keyspaces used by rolling-upgrade.yaml with S3 storage via pre_create_keyspace config param - Call _pre_create_keyspace() in test_rolling_upgrade before prepare_keyspaces_and_tables() so S3 keyspaces exist before any hardcoded CREATE KEYSPACE IF NOT EXISTS runs - Add S3 storage clause to hardcoded CREATE KEYSPACE statements in fill_db_data.prepare_keyspaces_and_tables for scylla_bench and keyspace_fill_db_data --- configurations/object_storage_s3.yaml | 3 +++ sdcm/fill_db_data.py | 8 ++++++-- test-cases/upgrades/rolling-upgrade.yaml | 8 ++++++++ upgrade_test.py | 9 ++++++--- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/configurations/object_storage_s3.yaml b/configurations/object_storage_s3.yaml index 6baf8b1c1dd..4c6e3c10221 100644 --- a/configurations/object_storage_s3.yaml +++ b/configurations/object_storage_s3.yaml @@ -1,3 +1,6 @@ +experimental_features: + - keyspace-storage-options + append_scylla_yaml: auto_snapshot: false object_storage_endpoints: diff --git a/sdcm/fill_db_data.py b/sdcm/fill_db_data.py index 2e6979b7e85..b53343a3bf8 100644 --- a/sdcm/fill_db_data.py +++ b/sdcm/fill_db_data.py @@ -3772,11 +3772,15 @@ def prepare_keyspaces_and_tables(self): session.default_consistency_level = ConsistencyLevel.QUORUM session.execute(""" CREATE KEYSPACE IF NOT EXISTS scylla_bench - WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND durable_writes = true; + WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} + AND durable_writes = true + AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}; """) session.execute(f""" CREATE KEYSPACE IF NOT EXISTS {self.base_ks} - WITH replication = {{'class': 'NetworkTopologyStrategy', 'replication_factor': '3'}} AND durable_writes = true; + WITH replication = {{'class': 'NetworkTopologyStrategy', 'replication_factor': '3'}} + AND durable_writes = true + AND storage = {{'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}}; """) session.set_keyspace(self.base_ks) diff --git a/test-cases/upgrades/rolling-upgrade.yaml b/test-cases/upgrades/rolling-upgrade.yaml index f3c78fc8be4..7cd444219e1 100644 --- a/test-cases/upgrades/rolling-upgrade.yaml +++ b/test-cases/upgrades/rolling-upgrade.yaml @@ -1,5 +1,13 @@ test_duration: 360 +pre_create_keyspace: + - "CREATE KEYSPACE IF NOT EXISTS keyspace1 WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}" + - "CREATE KEYSPACE IF NOT EXISTS keyspace_entire_test WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}" + - "CREATE KEYSPACE IF NOT EXISTS scylla_bench WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}" + - "CREATE KEYSPACE IF NOT EXISTS scylla_bench_during_upgrade WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}" + - "CREATE KEYSPACE IF NOT EXISTS keyspace_complex WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}" + - "CREATE KEYSPACE IF NOT EXISTS keyspace_fill_db_data WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}" + # workloads write_stress_during_entire_test: cassandra-stress write no-warmup cl=QUORUM n=20100200 -schema 'keyspace=keyspace_entire_test replication(strategy=NetworkTopologyStrategy,replication_factor=3) compression=LZ4Compressor compaction(strategy=SizeTieredCompactionStrategy)' -mode cql3 native compression=lz4 -rate threads=100 -pop seq=1..20100200 -log interval=5 verify_stress_after_cluster_upgrade: cassandra-stress read no-warmup cl=QUORUM n=20100200 -schema 'keyspace=keyspace_entire_test replication(strategy=NetworkTopologyStrategy,replication_factor=3) compression=LZ4Compressor' -mode cql3 native compression=lz4 -rate threads=1000 -pop seq=1..20100200 -log interval=5 diff --git a/upgrade_test.py b/upgrade_test.py index aa109f4ea49..f1a92988014 100644 --- a/upgrade_test.py +++ b/upgrade_test.py @@ -287,7 +287,7 @@ def _upgrade_node(self, node, upgrade_sstables=True, new_scylla_repo=None, new_v # flush all memtables to SSTables with self.actions_log.action_scope("stopping node"): node.run_nodetool("drain", timeout=15 * 60, coredump_on_timeout=True, long_running=True, retry=0) - node.run_nodetool("snapshot") + # node.run_nodetool("snapshot") node.stop_scylla_server() with self.actions_log.action_scope("upgrading packages"): node.remoter.run("sudo rpm -UvhR --oldpackage /tmp/scylla/*development*", ignore_status=True) @@ -306,7 +306,7 @@ def _upgrade_node(self, node, upgrade_sstables=True, new_scylla_repo=None, new_v # flush all memtables to SSTables with self.actions_log.action_scope("stop node"): node.run_nodetool("drain", timeout=15 * 60, coredump_on_timeout=True, long_running=True, retry=0) - node.run_nodetool("snapshot") + # node.run_nodetool("snapshot") node.stop_scylla_server(verify_down=False) orig_is_enterprise = node.is_product_enterprise @@ -464,7 +464,7 @@ def _rollback_node(self, node, upgrade_sstables=True): with self.actions_log.action_scope("stop node"): node.run_nodetool("drain", timeout=15 * 60, coredump_on_timeout=True, long_running=True, retry=0) # backup the data - node.run_nodetool("snapshot") + # node.run_nodetool("snapshot") node.stop_scylla_server(verify_down=False) if node.distro.is_rhel_like: @@ -705,6 +705,9 @@ def test_rolling_upgrade(self): # noqa: PLR0914, PLR0915 self.upgrade_os(self.db_cluster.nodes) self.actions_log.info("Preparing test keyspaces and tables") + # Pre-create keyspaces with custom storage (e.g. S3) before prepare_keyspaces_and_tables, + # so the hardcoded CREATE KEYSPACE IF NOT EXISTS calls inside it reuse the pre-created ones. + self._pre_create_keyspace() # prepare test keyspaces and tables before upgrade to avoid schema change during mixed cluster. self.prepare_keyspaces_and_tables() self.actions_log.info("Running s-b to create schemas to avoid #11459")