feature(s3_object_storage): configure rolling upgrade test to use S3-backed keyspaces

configurations/object_storage_s3.yaml

@@ -1,3 +1,6 @@
+experimental_features:
+  - keyspace-storage-options
+
 append_scylla_yaml:
   auto_snapshot: false
   object_storage_endpoints:

sdcm/fill_db_data.py

@@ -3772,11 +3772,15 @@ def prepare_keyspaces_and_tables(self):
             session.default_consistency_level = ConsistencyLevel.QUORUM
             session.execute("""
                 CREATE KEYSPACE IF NOT EXISTS scylla_bench
-                WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND durable_writes = true;
+                WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'}
+                AND durable_writes = true
+                AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'};
                 """)
             session.execute(f"""
                 CREATE KEYSPACE IF NOT EXISTS {self.base_ks}
-                WITH replication = {{'class': 'NetworkTopologyStrategy', 'replication_factor': '3'}} AND durable_writes = true;
+                WITH replication = {{'class': 'NetworkTopologyStrategy', 'replication_factor': '3'}}
+                AND durable_writes = true
+                AND storage = {{'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}};
                 """)
             session.set_keyspace(self.base_ks)
 

test-cases/upgrades/rolling-upgrade.yaml

@@ -1,5 +1,13 @@
 test_duration: 360
 
+pre_create_keyspace:
+  - "CREATE KEYSPACE IF NOT EXISTS keyspace1 WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}"
+  - "CREATE KEYSPACE IF NOT EXISTS keyspace_entire_test WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}"
+  - "CREATE KEYSPACE IF NOT EXISTS scylla_bench WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}"
+  - "CREATE KEYSPACE IF NOT EXISTS scylla_bench_during_upgrade WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}"
+  - "CREATE KEYSPACE IF NOT EXISTS keyspace_complex WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}"
+  - "CREATE KEYSPACE IF NOT EXISTS keyspace_fill_db_data WITH replication = {'class': 'NetworkTopologyStrategy', 'replication_factor': '3'} AND storage = {'type': 'S3', 'endpoint': 's3.us-east-1.amazonaws.com', 'bucket': 'manager-backup-tests-us-east-1'}"
+
 # workloads
 write_stress_during_entire_test: cassandra-stress write no-warmup cl=QUORUM n=20100200 -schema 'keyspace=keyspace_entire_test replication(strategy=NetworkTopologyStrategy,replication_factor=3) compression=LZ4Compressor compaction(strategy=SizeTieredCompactionStrategy)' -mode cql3 native compression=lz4 -rate threads=100 -pop seq=1..20100200 -log interval=5
 verify_stress_after_cluster_upgrade: cassandra-stress read no-warmup cl=QUORUM n=20100200 -schema 'keyspace=keyspace_entire_test replication(strategy=NetworkTopologyStrategy,replication_factor=3) compression=LZ4Compressor' -mode cql3 native compression=lz4 -rate threads=1000 -pop seq=1..20100200 -log interval=5

upgrade_test.py

@@ -287,7 +287,7 @@ def _upgrade_node(self, node, upgrade_sstables=True, new_scylla_repo=None, new_v
             # flush all memtables to SSTables
             with self.actions_log.action_scope("stopping node"):
                 node.run_nodetool("drain", timeout=15 * 60, coredump_on_timeout=True, long_running=True, retry=0)
-                node.run_nodetool("snapshot")
+                # node.run_nodetool("snapshot")
                 node.stop_scylla_server()
             with self.actions_log.action_scope("upgrading packages"):
                 node.remoter.run("sudo rpm -UvhR --oldpackage /tmp/scylla/*development*", ignore_status=True)
@@ -306,7 +306,7 @@ def _upgrade_node(self, node, upgrade_sstables=True, new_scylla_repo=None, new_v
             # flush all memtables to SSTables
             with self.actions_log.action_scope("stop node"):
                 node.run_nodetool("drain", timeout=15 * 60, coredump_on_timeout=True, long_running=True, retry=0)
-                node.run_nodetool("snapshot")
+                # node.run_nodetool("snapshot")
                 node.stop_scylla_server(verify_down=False)
 
             orig_is_enterprise = node.is_product_enterprise
@@ -464,7 +464,7 @@ def _rollback_node(self, node, upgrade_sstables=True):
         with self.actions_log.action_scope("stop node"):
             node.run_nodetool("drain", timeout=15 * 60, coredump_on_timeout=True, long_running=True, retry=0)
             # backup the data
-            node.run_nodetool("snapshot")
+            # node.run_nodetool("snapshot")
             node.stop_scylla_server(verify_down=False)
 
         if node.distro.is_rhel_like:
@@ -705,6 +705,9 @@ def test_rolling_upgrade(self):  # noqa: PLR0914, PLR0915
         self.upgrade_os(self.db_cluster.nodes)
 
         self.actions_log.info("Preparing test keyspaces and tables")
+        # Pre-create keyspaces with custom storage (e.g. S3) before prepare_keyspaces_and_tables,
+        # so the hardcoded CREATE KEYSPACE IF NOT EXISTS calls inside it reuse the pre-created ones.
+        self._pre_create_keyspace()
         # prepare test keyspaces and tables before upgrade to avoid schema change during mixed cluster.
         self.prepare_keyspaces_and_tables()
         self.actions_log.info("Running s-b to create schemas to avoid #11459")