diff --git a/databricks/sdk/mixins/compute.py b/databricks/sdk/mixins/compute.py index 164887fb3..45524ca67 100644 --- a/databricks/sdk/mixins/compute.py +++ b/databricks/sdk/mixins/compute.py @@ -81,7 +81,7 @@ def select_spark_version( ml: bool = False, genomics: bool = False, gpu: bool = False, - scala: str = "2.12", + scala: str = "", spark_version: str = None, photon: bool = False, graviton: bool = False, @@ -95,17 +95,19 @@ def select_spark_version( :param genomics: bool :param gpu: bool :param scala: str + Scala version to filter on, e.g. "2.12" or "2.13". Leave empty (the default) to + consider every Scala version, matching the Go SDK's zero-value behavior. :param spark_version: str :param photon: bool :param graviton: bool :returns: `spark_version` compatible string """ - # Logic ported from https://github.com/databricks/databricks-sdk-go/blob/main/service/compute/spark_version.go + # Logic ported from https://github.com/databricks/databricks-sdk-go/blob/main/service/compute/ext_spark_version.go versions = [] sv = self.spark_versions() for version in sv.versions: - if "-scala" + scala not in version.key: + if scala and "-scala" + scala not in version.key: continue matches = ( ("apache-spark-" not in version.key) diff --git a/tests/test_compute_mixins.py b/tests/test_compute_mixins.py index ec895b022..8eb20f805 100644 --- a/tests/test_compute_mixins.py +++ b/tests/test_compute_mixins.py @@ -1,7 +1,16 @@ +import json + import pytest from databricks.sdk.mixins.compute import SemVer +SPARK_VERSIONS_RESPONSE = { + "versions": [ + {"key": "16.4.x-scala2.12", "name": "16.4"}, + {"key": "18.2.x-scala2.13", "name": "18.2"}, + ] +} + @pytest.mark.parametrize( "given,expected", @@ -40,3 +49,25 @@ def test_sorting_semver(): SemVer(1, 0, 0), SemVer(12, 0, 0), ] + + +def test_select_spark_version_latest_ignores_scala_by_default(w, requests_mock): + # Regression test for https://github.com/databricks/databricks-sdk-py/issues/1487: + # select_spark_version(latest=True) implicitly filtered to the "2.12" default + # scala version before picking the latest, instead of considering every scala + # version like the Go SDK it's ported from does. + requests_mock.get( + "http://localhost/api/2.1/clusters/spark-versions", + text=json.dumps(SPARK_VERSIONS_RESPONSE), + ) + + assert w.clusters.select_spark_version(latest=True) == "18.2.x-scala2.13" + + +def test_select_spark_version_latest_still_honors_an_explicit_scala(w, requests_mock): + requests_mock.get( + "http://localhost/api/2.1/clusters/spark-versions", + text=json.dumps(SPARK_VERSIONS_RESPONSE), + ) + + assert w.clusters.select_spark_version(latest=True, scala="2.12") == "16.4.x-scala2.12"