fix(integration-tests): run the integration suite in parallel

Jenkinsfile

@@ -183,7 +183,7 @@ pipeline {
                 }
             }
             options {
-                timeout(time: 40, unit: 'MINUTES')
+                timeout(time: 45, unit: 'MINUTES')
             }
             steps {
                 catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {

docker/env/version

@@ -1 +1 @@
-1.90-2025.1-scylla-driver-v3.29.9
+1.91-2025.1-pytest-xdist_2

requirements.in

@@ -23,6 +23,7 @@ pyzmq==22.3.0
 tenacity==8.0.1
 pytest==7.2.0
 pytest-random-order==1.0.4
+pytest-xdist==3.8.0  # parallel integration tests (sct.py integration-tests -n)
 moto==5.2.2  # AWS mocking for keystore unit tests
 tcconfig==0.29.1
 mysql-connector-python==9.1.0

requirements.txt

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile requirements.in --generate-hashes
+#    uv pip compile requirements.in --generate-hashes --python-version 3.10 --output-file requirements.txt
 aenum==3.1.17 \
     --hash=sha256:0dad0421b2fbe30e3fb623b2a0a23eff823407df53829d6a72595e7f76f3d872 \
     --hash=sha256:8b883a37a04e74cc838ac442bdd28c266eae5bbf13e1342c7ef123ed25230139 \
@@ -296,6 +296,7 @@ boto3==1.28.4 \
     # via
     #   -r requirements.in
     #   gimme-aws-creds
+    #   moto
 boto3-stubs==1.28.4 \
     --hash=sha256:7a68d2a456d995318a6237eb5c6039f63b6a9bf902f216ff2261f87015fbb1c3 \
     --hash=sha256:a5b1cb7cdfd83f34b573c9af4eda80a62638f2ac78fae9c972721d3837e1b918
@@ -306,6 +307,7 @@ botocore==1.31.4 \
     # via
     #   awscli
     #   boto3
+    #   moto
     #   s3transfer
 botocore-stubs==1.42.41 \
     --hash=sha256:9423110fb0e391834bd2ed44ae5f879d8cb370a444703d966d30842ce2bcb5f0 \
@@ -657,6 +659,7 @@ cryptography==46.0.7 \
     #   fido2
     #   google-auth
     #   jwcrypto
+    #   moto
     #   msal
     #   paramiko
     #   pyjwt
@@ -706,6 +709,10 @@ exceptiongroup==1.3.1 \
     --hash=sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219 \
     --hash=sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598
     # via pytest
+execnet==2.1.2 \
+    --hash=sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd \
+    --hash=sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec
+    # via pytest-xdist
 fabric==2.6.0 \
     --hash=sha256:47f184b070272796fd2f9f0436799e18f2ccba4ee8ee587796fca192acd46cd2 \
     --hash=sha256:7a71714b8b8f28cf828eceb155196f43ebac1bd4c849b7161ed5993d1cbcaa40
@@ -1252,7 +1259,9 @@ markupsafe==3.0.3 \
     --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \
     --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \
     --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50
-    # via jinja2
+    # via
+    #   jinja2
+    #   werkzeug
 mbstrdecoder==1.1.4 \
     --hash=sha256:03dae4ec50ec0d2ff4743e63fdbd5e0022815857494d35224b60775d3d934a8c \
     --hash=sha256:8105ef9cf6b7d7d69fe7fd6b68a2d8f281ca9b365d7a9b670be376b2e6c81b21
@@ -2000,10 +2009,15 @@ pytest==7.2.0 \
     # via
     #   -r requirements.in
     #   pytest-random-order
+    #   pytest-xdist
 pytest-random-order==1.0.4 \
     --hash=sha256:6b2159342a4c8c10855bc4fc6d65ee890fc614cb2b4ff688979b008a82a0ff52 \
     --hash=sha256:72279a7f823969e18b10e438950f58330d17e0fcffb57cbd7929770cd687ecb2
     # via -r requirements.in
+pytest-xdist==3.8.0 \
+    --hash=sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88 \
+    --hash=sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1
+    # via -r requirements.in
 python-dateutil==2.9.0.post0 \
     --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \
     --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
@@ -2078,6 +2092,7 @@ pyyaml==6.0.1 \
     #   kubernetes
     #   okta
     #   pre-commit
+    #   responses
     #   scylla-driver
 pyzmq==22.3.0 \
     --hash=sha256:08c4e315a76ef26eb833511ebf3fa87d182152adf43dedee8d79f998a2162a0b \
@@ -2144,12 +2159,14 @@ requests==2.32.2 \
     #   google-cloud-storage
     #   jira
     #   kubernetes
+    #   moto
     #   msal
     #   msrest
     #   pygithub
     #   python-jenkins
     #   requests-oauthlib
     #   requests-toolbelt
+    #   responses
 requests-oauthlib==2.0.0 \
     --hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \
     --hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9
@@ -2413,6 +2430,7 @@ urllib3==1.26.20 \
     #   kubernetes
     #   pygithub
     #   requests
+    #   responses
 uv==0.2.24 \
     --hash=sha256:06b95969d73c804bbee63fb1bde54d1a7570826b446a2b5b9136534ebf55a168 \
     --hash=sha256:0ea783d55d10bd3cf619fc1a7cb3f14e3157ffaa578a6afa068d0257923a31a4 \
@@ -2556,7 +2574,9 @@ wrapt==2.1.2 \
 xmltodict==1.0.4 \
     --hash=sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61 \
     --hash=sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a
-    # via okta
+    # via
+    #   moto
+    #   okta
 yarl==1.23.0 \
     --hash=sha256:03214408cfa590df47728b84c679ae4ef00be2428e11630277be0727eba2d7cc \
     --hash=sha256:041b1a4cefacf65840b4e295c6985f334ba83c30607441ae3cf206a0eed1a2e4 \

sct.py

@@ -1318,9 +1318,12 @@ def unit_tests(test, junit_xml):
 
 
 @cli.command("integration-tests", help="Run all the SCT internal integration-tests")
-@click.option("-t", "--test", required=False, default="", help="Run specific test file from unit-tests directory")
+@click.option(
+    "-t", "--test", required=False, default=[""], multiple=True, help="Run specific test file from unit-tests directory"
+)
+@click.option("-n", required=False, default=4, help="Sets number of parallel tests to run, default is 4")
 @click.option("--junit-xml", required=False, default="", help="Path to write JUnit XML report")
-def integration_tests(test, junit_xml):
+def integration_tests(test, n, junit_xml):
     get_test_config().logdir()
     add_file_logger()
 
@@ -1336,7 +1339,17 @@ def integration_tests(test, junit_xml):
         )
         local_cluster.setup_prerequisites()
 
-    args = ["-v", "-p", "no:warnings", "-m", "integration", "unit_tests/{}".format(test)]
+    args = [
+        "-v",
+        "-p",
+        "no:warnings",
+        "-m",
+        "integration",
+        "--dist",
+        "loadgroup",
+        f"-n{n}",
+        *(f"unit_tests/{t}" for t in test),
+    ]
     if junit_xml:
         args.append(f"--junit-xml={junit_xml}")
     sys.exit(pytest.main(args))

sdcm/utils/version_utils.py

@@ -612,13 +612,26 @@ def resolve_latest_repo_symlink(url: str) -> str:
     return resolved_url
 
 
-def get_specific_tag_of_docker_image(docker_repo: str):
+def get_specific_tag_of_docker_image(docker_repo: str, architecture: Literal["x86_64", "aarch64"] = "x86_64") -> str:
+    """
+    Get the latest docker image tag from ScyllaDB S3 storage for given nightly docker repo.
+
+    :param docker_repo: docker repository name, e.g. 'scylladb/scylla-nightly'
+    :param architecture: architecture of the docker image, either 'x86_64' or 'aarch64' (default: 'x86_64')
+    :return: docker image tag string if found
+    :raises ValueError: if docker repo is not supported or tag info cannot be found
+    """
+
     if docker_repo == "scylladb/scylla-nightly":
         product = "scylla"
         branch = "master"
     elif docker_repo == "scylladb/scylla-enterprise-nightly":
         product = "scylla-enterprise"
-        branch = "enterprise"
+        # The `enterprise` rolling branch stopped producing builds when
+        # enterprise development folded into the unified releases, and its
+        # relocatables are gone from downloads.scylladb.com. `enterprise-2024.1`
+        # is the only branch still publishing scylla-enterprise-nightly images.
+        branch = "enterprise-2024.1"
     else:
         raise ValueError(f"SCT doesn't support getting latest from {docker_repo}")
 
@@ -632,9 +645,11 @@ def get_specific_tag_of_docker_image(docker_repo: str):
     #    url-id: 2022-08-29T08:05:34Z
     #    docker-image-name: scylla-nightly:5.2.0-dev-0.20220829.67c91e8bcd61
     build_info = yaml.safe_load(res.content)
-    tag = build_info["docker-image-name"].split(":", maxsplit=1)[1]
-    LOGGER.debug("found %s for %s repo", tag, docker_repo)
-    return tag
+    if tag := build_info.get(f"docker-image-name-{architecture}", build_info.get("docker-image-name")):
+        tag = tag.split(":", maxsplit=1)[1]
+        LOGGER.debug("found %s for %s repo", tag, docker_repo)
+        return tag
+    raise ValueError(f"Cannot find docker image tag info in {build_url} for architecture={architecture}")
 
 
 def transform_non_semver_scylla_version_to_semver(scylla_version: str):

unit_tests/integration/test_config_get_version_based_on_conf.py

@@ -153,158 +153,6 @@ def test_baremetal(monkeypatch):
     assert not _is_enterprise
 
 
-def test_unified_package(monkeypatch):
-    monkeypatch.setenv("SCT_CLUSTER_BACKEND", "gce")
-    monkeypatch.setenv(
-        "SCT_UNIFIED_PACKAGE",
-        (
-            "https://downloads.scylladb.com/unstable/scylla/master/relocatable/2023-11-13T03:04:27Z/"
-            "scylla-unified-5.5.0~dev-0.20231113.7b08886e8dd8.x86_64.tar.gz"
-        ),
-    )
-    monkeypatch.setenv(
-        "SCT_GCE_IMAGE_DB", "https://www.googleapis.com/compute/v1/projects/centos-cloud/global/images/family/centos-7"
-    )
-
-    conf = sct_config.SCTConfiguration()
-    conf.verify_configuration()
-
-    # unified_package should auto-set use_preinstalled_scylla to False
-    assert conf.get("use_preinstalled_scylla") is False
-
-    _version, _is_enterprise = conf.get_version_based_on_conf()
-
-    assert "5.5.0" in _version
-    assert not _is_enterprise
-
-
-def test_unified_package_aws_sets_ubuntu_user(monkeypatch):
-    """Test that unified_package on AWS forces ami_db_scylla_user='ubuntu'."""
-    monkeypatch.setenv("SCT_CLUSTER_BACKEND", "aws")
-    monkeypatch.setenv(
-        "SCT_UNIFIED_PACKAGE",
-        (
-            "https://downloads.scylladb.com/unstable/scylla/master/relocatable/2023-11-13T03:04:27Z/"
-            "scylla-unified-5.5.0~dev-0.20231113.7b08886e8dd8.x86_64.tar.gz"
-        ),
-    )
-    monkeypatch.setenv("SCT_AMI_ID_DB_SCYLLA", "ami-test123")
-    monkeypatch.setenv("SCT_CONFIG_FILES", "internal_test_data/minimal_test_case.yaml")
-
-    conf = sct_config.SCTConfiguration()
-    conf.verify_configuration()
-
-    assert conf.get("use_preinstalled_scylla") is False
-    assert conf.get("ami_db_scylla_user") == "ubuntu"
-
-
-@pytest.mark.parametrize(
-    "scylla_version, backend, expected_ami_ssm",
-    [
-        pytest.param(
-            "relocatable:latest",
-            "aws",
-            "resolve:ssm:/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id",
-            id="relocatable-latest-aws-x86_64",
-        ),
-        pytest.param(
-            "relocatable:master:x86_64",
-            "aws",
-            "resolve:ssm:/aws/service/canonical/ubuntu/server/24.04/stable/current/amd64/hvm/ebs-gp3/ami-id",
-            id="relocatable-master-x86_64-aws",
-        ),
-        pytest.param(
-            "relocatable:master:aarch64",
-            "aws",
-            "resolve:ssm:/aws/service/canonical/ubuntu/server/24.04/stable/current/arm64/hvm/ebs-gp3/ami-id",
-            id="relocatable-master-aarch64-aws",
-        ),
-        pytest.param("relocatable:latest", "gce", None, id="relocatable-latest-gce"),
-    ],
-)
-def test_relocatable_version_resolves_unified_package(scylla_version, backend, expected_ami_ssm, monkeypatch):
-    """Test that relocatable:<branch> scylla_version resolves to unified_package URL without crashing."""
-    monkeypatch.setenv("SCT_CLUSTER_BACKEND", backend)
-    monkeypatch.setenv("SCT_SCYLLA_VERSION", scylla_version)
-    if backend == "gce":
-        monkeypatch.setenv(
-            "SCT_GCE_IMAGE_DB",
-            "https://www.googleapis.com/compute/v1/projects/centos-cloud/global/images/family/centos-7",
-        )
-
-    fake_url = (
-        "https://downloads.scylladb.com/unstable/scylla/master/relocatable/latest/"
-        "scylla-unified-6.3.0~dev-0.20260101.abcdef123456.x86_64.tar.gz"
-    )
-    with (
-        unittest.mock.patch("sdcm.sct_config.latest_unified_package", return_value=fake_url),
-        unittest.mock.patch(
-            "sdcm.sct_config.convert_name_to_ami_if_needed",
-            side_effect=lambda param, region_names: param,
-        ),
-    ):
-        conf = sct_config.SCTConfiguration()
-        conf.verify_configuration()
-
-    assert conf.get("unified_package") == fake_url
-    assert conf.get("scylla_version") == ""
-    assert conf.get("use_preinstalled_scylla") is False
-    if expected_ami_ssm:
-        # On AWS, ami_id_db_scylla should be auto-set to the Ubuntu 24.04 SSM resolve pattern
-        assert conf.get("ami_id_db_scylla") == expected_ami_ssm
-
-
-def test_unified_package_aws_auto_resolves_ami(monkeypatch):
-    """Test that unified_package on AWS auto-resolves ami_id_db_scylla to Ubuntu 24.04 base AMI when not set."""
-    monkeypatch.setenv("SCT_CLUSTER_BACKEND", "aws")
-    monkeypatch.setenv(
-        "SCT_UNIFIED_PACKAGE",
-        (
-            "https://downloads.scylladb.com/unstable/scylla/master/relocatable/2023-11-13T03:04:27Z/"
-            "scylla-unified-5.5.0~dev-0.20231113.7b08886e8dd8.x86_64.tar.gz"
-        ),
-    )
-
-    with unittest.mock.patch(
-        "sdcm.sct_config.convert_name_to_ami_if_needed",
-        side_effect=lambda param, region_names: param,
-    ):
-        conf = sct_config.SCTConfiguration()
-        conf.verify_configuration()
-
-    assert conf.get("use_preinstalled_scylla") is False
-    assert conf.get("ami_db_scylla_user") == "ubuntu"
-    # ami_id_db_scylla should be auto-set to the Ubuntu 24.04 SSM resolve pattern
-    assert "resolve:ssm:" in conf.get("ami_id_db_scylla")
-    assert "ubuntu" in conf.get("ami_id_db_scylla")
-
-
-def test_unified_package_aws_verify_passes_without_ami(monkeypatch):
-    """Test that verify_configuration passes with unified_package on AWS even if ami_id_db_scylla is not explicitly set."""
-    monkeypatch.setenv("SCT_CLUSTER_BACKEND", "aws")
-    monkeypatch.setenv(
-        "SCT_UNIFIED_PACKAGE",
-        (
-            "https://downloads.scylladb.com/unstable/scylla/master/relocatable/2023-11-13T03:04:27Z/"
-            "scylla-unified-5.5.0~dev-0.20231113.7b08886e8dd8.x86_64.tar.gz"
-        ),
-    )
-
-    # Mock convert_name_to_ami_if_needed to pass through the SSM pattern (simulates AMI resolution)
-    with unittest.mock.patch(
-        "sdcm.sct_config.convert_name_to_ami_if_needed",
-        side_effect=lambda param, region_names: param,
-    ):
-        conf = sct_config.SCTConfiguration()
-        # This should NOT raise AssertionError about ami_id_db_scylla,
-        # because unified_package is set and _check_version_supplied should not
-        # require backend-specific images when unified_package is provided
-        conf.verify_configuration()
-
-    assert conf.get("use_preinstalled_scylla") is False
-    assert conf.get("unified_package") is not None
-
-
 def test_aws_ami_missing_scylla_version_tag(monkeypatch):
     """Test that missing scylla_version tag in AWS AMI raises clear ValueError."""
     monkeypatch.setenv("SCT_CLUSTER_BACKEND", "aws")

unit_tests/integration/test_sstable_load_utils.py

@@ -33,7 +33,7 @@ def cleanup_thread():
 
     cs_thread.run()
 
-    output, _ = cs_thread.parse_results()
+    output, _ = cs_thread.verify_results()
 
     column_num = SstableLoadUtils.calculate_columns_count_in_table(
         docker_scylla, keyspace_name="keyspace_refresh", table_name="standard1"

unit_tests/test_alternator_streams_kcl.py

@@ -23,6 +23,7 @@
 pytestmark = [
     pytest.mark.usefixtures("events"),
     pytest.mark.integration,
+    pytest.mark.xdist_group("docker_heavy"),
 ]
 
 

unit_tests/test_cassandra_harry.py

@@ -19,6 +19,7 @@
 pytestmark = [
     pytest.mark.usefixtures("events"),
     pytest.mark.integration,
+    pytest.mark.xdist_group("docker_heavy"),
 ]