From d1b18659104d8582377c9679d36f49d26c525cf8 Mon Sep 17 00:00:00 2001 From: Imran Hasan Date: Wed, 22 Jul 2026 14:25:58 -0700 Subject: [PATCH 1/5] Add serverless Scala JAR job example --- CODEOWNERS | 1 + serverless-jar-job-example/.gitignore | 7 + serverless-jar-job-example/README.md | 96 +++++++++++++ serverless-jar-job-example/build.sbt | 26 ++++ .../project/build.properties | 1 + .../project/plugins.sbt | 2 + .../example/ServerlessJarJobExample.scala | 134 ++++++++++++++++++ 7 files changed, 267 insertions(+) create mode 100644 serverless-jar-job-example/.gitignore create mode 100644 serverless-jar-job-example/README.md create mode 100644 serverless-jar-job-example/build.sbt create mode 100644 serverless-jar-job-example/project/build.properties create mode 100644 serverless-jar-job-example/project/plugins.sbt create mode 100644 serverless-jar-job-example/src/main/scala/com/databricks/labs/example/ServerlessJarJobExample.scala diff --git a/CODEOWNERS b/CODEOWNERS index 174adc20..3efdd748 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -18,6 +18,7 @@ ka-chat-bot @taiga-db marketplace-provisioning @michellejm metascan @nfx @alexott runtime-packages @nfx @alexott +serverless-jar-job-example @imran-a-hasan sql_migration_copilot @robertwhiffin tacklebox @Jonathan-Choi uc-catalog-cloning @esiol-db @vasco-lopes diff --git a/serverless-jar-job-example/.gitignore b/serverless-jar-job-example/.gitignore new file mode 100644 index 00000000..85f0489f --- /dev/null +++ b/serverless-jar-job-example/.gitignore @@ -0,0 +1,7 @@ +target/ +project/target/ +project/project/ +.bsp/ +.idea/ +.metals/ +.bloop/ diff --git a/serverless-jar-job-example/README.md b/serverless-jar-job-example/README.md new file mode 100644 index 00000000..8c39d28f --- /dev/null +++ b/serverless-jar-job-example/README.md @@ -0,0 +1,96 @@ +--- +title: "Serverless Scala JAR job example" +language: scala +author: "Imran Hasan" +date: 2026-07-22 + +tags: +- scala +- spark-connect +- serverless +- jar +- jobs +- dbutils +--- + +# Serverless Scala JAR job example + +A minimal, self-contained Scala JAR job that runs on **Databricks serverless +compute**. It is meant as a copy-and-go starting point: it builds into a single +fat JAR with `sbt assembly` and demonstrates the three building blocks you are +most likely to need. + +1. **Spark basics** — get the session, build a DataFrame, read a Unity Catalog table. +2. **UDF basics** — scalar, map-returning, SQL-registered, and a UDF closing over a locally-defined class. +3. **dbutils basics** — the job/notebook context and a `dbutils.fs` read/write round-trip against a UC volume. + +## Serverless specifics + +Serverless JAR jobs run under **Spark Connect**, which differs from a classic +cluster job in a few ways this example follows: + +- Get the session with `SparkSession.builder().getOrCreate()`. There is **no** + `SparkContext` / RDD API — those throw under Spark Connect. +- Build against the **Spark Connect** client (`databricks-connect`) and the + **`databricks-dbutils-scala`** SDK (package `com.databricks.sdk.scala.dbutils`). + Both are provided by the runtime, so they are marked `% provided` and are not + shipped inside your JAR. +- Target **Scala 2.13** and **JDK 17**. +- Obtain `dbutils` on the driver and pass it into your code. Do not hold it in an + `object`-level `val`: the initializer can otherwise run inside a UDF closure on + an executor, where it fails. + +## Layout + +``` +serverless-jar-job-example/ +├── build.sbt # Scala 2.13.16, JDK 17, provided deps, assembly config +├── project/ +│ ├── plugins.sbt # sbt-assembly plugin +│ └── build.properties # sbt version +└── src/main/scala/com/databricks/labs/example/ + └── ServerlessJarJobExample.scala +``` + +## Build + +You need [sbt](https://www.scala-sbt.org/) and JDK 17. + +```bash +sbt clean assembly +``` + +This produces the fat JAR at: + +``` +target/scala-2.13/ServerlessJarJobExample-2.13-assembly.jar +``` + +## Deploy and run + +1. Upload the assembled JAR to a Unity Catalog volume, e.g. with the Databricks CLI: + + ```bash + databricks fs cp \ + target/scala-2.13/ServerlessJarJobExample-2.13-assembly.jar \ + dbfs:/Volumes////ServerlessJarJobExample.jar + ``` + +2. Create a job with a **JAR task** on **serverless** compute: + - Main class: `com.databricks.labs.example.ServerlessJarJobExample` + - Dependent library: the uploaded JAR + - Parameter (`args(0)`): a writable UC volume base path, + e.g. `/Volumes///` + +3. Run the job. On success the driver log ends with: + + ``` + ServerlessJarJobExample: all examples completed successfully + ``` + +The single argument is the base path of a UC volume the job can write to; the +`dbutils.fs` example writes a temporary file there and cleans it up afterwards. + +## References + +- [Use JARs in Databricks jobs](https://docs.databricks.com/aws/en/jobs/how-to/use-jars-in-workflows) diff --git a/serverless-jar-job-example/build.sbt b/serverless-jar-job-example/build.sbt new file mode 100644 index 00000000..a5b13ec9 --- /dev/null +++ b/serverless-jar-job-example/build.sbt @@ -0,0 +1,26 @@ +name := "ServerlessJarJobExample" + +// Serverless JAR jobs require Scala 2.13 and JDK 17. +scalaVersion := "2.13.16" +javacOptions ++= Seq("--release", "17") +scalacOptions ++= Seq("-release", "17") + +// Spark Connect client. Provided by the serverless runtime, so it is not +// shipped inside the assembled JAR. +libraryDependencies += "com.databricks" %% "databricks-connect" % "17.3.2" % "provided" + +// dbutils for Scala. Use the databricks-dbutils-scala SDK (API package +// com.databricks.sdk.scala.dbutils); it is in the serverless provided-library +// list, so it is marked provided as well. Add any of your own dependencies +// here -- use %% for Scala libraries so sbt resolves the _2.13 artifact. +libraryDependencies += "com.databricks" %% "databricks-dbutils-scala" % "0.1.4" % "provided" + +// Fork a JVM on `sbt run` so the javaOptions below are applied. +fork := true +javaOptions += "--add-opens=java.base/java.nio=ALL-UNNAMED" + +// Build a deployable fat JAR with `sbt assembly`. No custom assemblyMergeStrategy +// is needed: every heavy dependency is provided by the runtime and marked +// provided, so the assembly contains only this project's own classes. +Compile / mainClass := Some("com.databricks.labs.example.ServerlessJarJobExample") +assembly / assemblyJarName := s"ServerlessJarJobExample-${scalaBinaryVersion.value}-assembly.jar" diff --git a/serverless-jar-job-example/project/build.properties b/serverless-jar-job-example/project/build.properties new file mode 100644 index 00000000..bc739060 --- /dev/null +++ b/serverless-jar-job-example/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.10.3 diff --git a/serverless-jar-job-example/project/plugins.sbt b/serverless-jar-job-example/project/plugins.sbt new file mode 100644 index 00000000..9443a4ff --- /dev/null +++ b/serverless-jar-job-example/project/plugins.sbt @@ -0,0 +1,2 @@ +// sbt-assembly builds the uber JAR (`sbt assembly`) that the job deploys. +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.1") diff --git a/serverless-jar-job-example/src/main/scala/com/databricks/labs/example/ServerlessJarJobExample.scala b/serverless-jar-job-example/src/main/scala/com/databricks/labs/example/ServerlessJarJobExample.scala new file mode 100644 index 00000000..e011f764 --- /dev/null +++ b/serverless-jar-job-example/src/main/scala/com/databricks/labs/example/ServerlessJarJobExample.scala @@ -0,0 +1,134 @@ +package com.databricks.labs.example + +import com.databricks.sdk.scala.dbutils.DBUtils +import org.apache.spark.sql.connect.SparkSession +import org.apache.spark.sql.functions.{col, udf} + +import java.util.UUID + +/** + * A minimal, self-contained example of a Scala JAR job that runs on Databricks + * serverless compute. It demonstrates the three building blocks you are most + * likely to need: + * + * 1. Spark basics -- get the session, build a DataFrame, read a UC table. + * 2. UDF basics -- scalar, map-returning, SQL-registered, and closing + * over a locally-defined class. + * 3. dbutils basics -- the notebook/job context and a `dbutils.fs` + * read/write round-trip against a Unity Catalog volume. + * + * Serverless JAR jobs run under Spark Connect, so a few things differ from a + * classic cluster job: + * + * - Get the session with `SparkSession.builder().getOrCreate()`. There is no + * `SparkContext` / RDD API (those throw under Spark Connect). + * - Build against `databricks-connect` (Spark Connect) and the + * `databricks-dbutils-scala` SDK. Both are provided by the runtime, so they + * are marked `% provided` in build.sbt and are not shipped in the JAR. + * - Target Scala 2.13 and JDK 17 (see build.sbt). + * + * Run it with a single argument: the base path of a UC volume you can write to, + * e.g. `/Volumes///`. + */ +object ServerlessJarJobExample { + + def main(args: Array[String]): Unit = { + require( + args.nonEmpty, + "ServerlessJarJobExample requires a writable UC volume base path as args(0), " + + "e.g. /Volumes///" + ) + val volumeBasePath = args(0) + + // Under Spark Connect, getOrCreate() returns the session wired to serverless. + val spark = SparkSession.builder().getOrCreate() + + // dbutils comes from the databricks-dbutils-scala SDK. Obtain it on the + // driver and pass it into helpers -- do NOT store it in an object-level val, + // or its initializer can run inside a UDF closure on an executor and fail. + val dbutils = DBUtils.getDBUtils() + + sparkBasics(spark) + udfBasics(spark) + dbutilsBasics(spark, dbutils, volumeBasePath) + + println("ServerlessJarJobExample: all examples completed successfully") + } + + // --------------------------------------------------------------------------- + // 1. Spark basics + // --------------------------------------------------------------------------- + + def sparkBasics(spark: SparkSession): Unit = { + println(s"Spark version: ${spark.version}") + + // Build a DataFrame and collect a few rows. + val ids = spark.range(10).limit(3).collect().toSeq + println(s"spark.range(10).limit(3) = ${ids.mkString(", ")}") + + // Read a Unity Catalog table. `samples` is a built-in public catalog + // available on Databricks workspaces, so this needs no setup. + val rows = spark.read.table("samples.nyctaxi.trips").limit(10).count() + println(s"Read $rows rows from samples.nyctaxi.trips") + } + + // --------------------------------------------------------------------------- + // 2. UDF basics + // --------------------------------------------------------------------------- + + def udfBasics(spark: SparkSession): Unit = { + import spark.implicits._ + + // A scalar UDF returning a String. + val greet = udf((v: Int) => s"hello world $v") + val greeted = spark.range(3).select(greet(col("id")).as("v")).as[String].collect().toSeq + println(s"scalar UDF: ${greeted.mkString(", ")}") + + // A UDF returning a Map. + val toMap = udf((v: Int) => Map(v -> "some value")) + val mapped = + spark.range(1).select(toMap(col("id")).as("m")).as[Map[Int, String]].collect().toSeq + println(s"map UDF: ${mapped.mkString(", ")}") + + // Register a UDF for use from SQL. + spark.udf.register("greet", greet) + val fromSql = spark.sql("SELECT greet(1) AS v").as[String].collect().head + println(s"SQL-registered UDF: $fromSql") + + // A UDF closing over a locally-defined class. Spark ships the closure (and + // the class) to the executors, so this exercises closure serialization. + val withClass = udf((x: Int) => new Multiplier(x).result) + val computed = spark.range(5).select(withClass(col("id"))).as[Int].collect().sorted.toSeq + println(s"UDF with local class: ${computed.mkString(", ")}") + } + + /** A small class captured by a UDF closure and shipped to the executors. */ + private class Multiplier(x: Int) { + def result: Int = x * 42 + 5 + } + + // --------------------------------------------------------------------------- + // 3. dbutils basics + // --------------------------------------------------------------------------- + + def dbutilsBasics(spark: SparkSession, dbutils: DBUtils, volumeBasePath: String): Unit = { + import spark.implicits._ + + // The job/notebook context carries tags such as the job and run identifiers. + println(s"notebook context tags: ${dbutils.notebook.getContext().tags}") + + // A Spark read/write round-trip against a UC volume, cleaned up with + // dbutils.fs.rm afterwards. + val path = s"$volumeBasePath/example_${UUID.randomUUID().toString}" + try { + Seq("hello world").toDF("value").write.mode("overwrite").text(path) + val readBack = spark.read.text(path).as[String].collect().toSeq + require(readBack == Seq("hello world"), s"volume round-trip mismatch, got $readBack") + println(s"dbutils.fs volume round-trip OK at $path") + } finally { + // Best-effort cleanup. + try dbutils.fs.rm(path, true) + catch { case _: Throwable => () } + } + } +} From 417173d5ece51bc598c0f12cb37dba80ef492a4f Mon Sep 17 00:00:00 2001 From: Imran Hasan Date: Wed, 22 Jul 2026 14:39:06 -0700 Subject: [PATCH 2/5] Fix comments --- serverless-jar-job-example/README.md | 22 +++++-------------- .../example/ServerlessJarJobExample.scala | 7 ++---- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/serverless-jar-job-example/README.md b/serverless-jar-job-example/README.md index 8c39d28f..043da101 100644 --- a/serverless-jar-job-example/README.md +++ b/serverless-jar-job-example/README.md @@ -17,8 +17,7 @@ tags: A minimal, self-contained Scala JAR job that runs on **Databricks serverless compute**. It is meant as a copy-and-go starting point: it builds into a single -fat JAR with `sbt assembly` and demonstrates the three building blocks you are -most likely to need. +fat JAR with `sbt assembly` and demonstrates three basic building blocks: 1. **Spark basics** — get the session, build a DataFrame, read a Unity Catalog table. 2. **UDF basics** — scalar, map-returning, SQL-registered, and a UDF closing over a locally-defined class. @@ -36,9 +35,6 @@ cluster job in a few ways this example follows: Both are provided by the runtime, so they are marked `% provided` and are not shipped inside your JAR. - Target **Scala 2.13** and **JDK 17**. -- Obtain `dbutils` on the driver and pass it into your code. Do not hold it in an - `object`-level `val`: the initializer can otherwise run inside a UDF closure on - an executor, where it fails. ## Layout @@ -68,21 +64,13 @@ target/scala-2.13/ServerlessJarJobExample-2.13-assembly.jar ## Deploy and run -1. Upload the assembled JAR to a Unity Catalog volume, e.g. with the Databricks CLI: - - ```bash - databricks fs cp \ - target/scala-2.13/ServerlessJarJobExample-2.13-assembly.jar \ - dbfs:/Volumes////ServerlessJarJobExample.jar - ``` - -2. Create a job with a **JAR task** on **serverless** compute: +1. Create a job with a **JAR task** on **serverless** compute: - Main class: `com.databricks.labs.example.ServerlessJarJobExample` - - Dependent library: the uploaded JAR + - Environment JAR dependency: the uploaded JAR (drag and drop it into the file selector, or browse to select it from a Unity Catalog volume or workspace location) - Parameter (`args(0)`): a writable UC volume base path, e.g. `/Volumes///` -3. Run the job. On success the driver log ends with: +2. Run the job. On success the driver log ends with: ``` ServerlessJarJobExample: all examples completed successfully @@ -93,4 +81,4 @@ The single argument is the base path of a UC volume the job can write to; the ## References -- [Use JARs in Databricks jobs](https://docs.databricks.com/aws/en/jobs/how-to/use-jars-in-workflows) +- [Create and run JARs on serverless compute](https://docs.databricks.com/aws/en/jobs/how-to/use-jars-in-workflows) diff --git a/serverless-jar-job-example/src/main/scala/com/databricks/labs/example/ServerlessJarJobExample.scala b/serverless-jar-job-example/src/main/scala/com/databricks/labs/example/ServerlessJarJobExample.scala index e011f764..4ea9e326 100644 --- a/serverless-jar-job-example/src/main/scala/com/databricks/labs/example/ServerlessJarJobExample.scala +++ b/serverless-jar-job-example/src/main/scala/com/databricks/labs/example/ServerlessJarJobExample.scala @@ -8,8 +8,7 @@ import java.util.UUID /** * A minimal, self-contained example of a Scala JAR job that runs on Databricks - * serverless compute. It demonstrates the three building blocks you are most - * likely to need: + * serverless compute. It demonstrates three basic building blocks: * * 1. Spark basics -- get the session, build a DataFrame, read a UC table. * 2. UDF basics -- scalar, map-returning, SQL-registered, and closing @@ -43,9 +42,7 @@ object ServerlessJarJobExample { // Under Spark Connect, getOrCreate() returns the session wired to serverless. val spark = SparkSession.builder().getOrCreate() - // dbutils comes from the databricks-dbutils-scala SDK. Obtain it on the - // driver and pass it into helpers -- do NOT store it in an object-level val, - // or its initializer can run inside a UDF closure on an executor and fail. + // dbutils comes from the databricks-dbutils-scala SDK. val dbutils = DBUtils.getDBUtils() sparkBasics(spark) From 42ac59c35ca2d1a9b1e9b8e98332d0df9923d764 Mon Sep 17 00:00:00 2001 From: Imran Hasan Date: Thu, 23 Jul 2026 12:19:04 -0700 Subject: [PATCH 3/5] Use DAB --- serverless-jar-job-example/README.md | 56 +++++++++++-------- serverless-jar-job-example/databricks.yml | 40 +++++++++++++ .../serverless-jar-job-example.job.yml | 22 ++++++++ 3 files changed, 96 insertions(+), 22 deletions(-) create mode 100644 serverless-jar-job-example/databricks.yml create mode 100644 serverless-jar-job-example/resources/serverless-jar-job-example.job.yml diff --git a/serverless-jar-job-example/README.md b/serverless-jar-job-example/README.md index 043da101..aab9a7b8 100644 --- a/serverless-jar-job-example/README.md +++ b/serverless-jar-job-example/README.md @@ -16,8 +16,10 @@ tags: # Serverless Scala JAR job example A minimal, self-contained Scala JAR job that runs on **Databricks serverless -compute**. It is meant as a copy-and-go starting point: it builds into a single -fat JAR with `sbt assembly` and demonstrates three basic building blocks: +compute**, packaged as a **Databricks Asset Bundle**. It is meant as a +copy-and-go starting point: the bundle builds the JAR with `sbt assembly`, +uploads it, and defines a serverless JAR job. The code demonstrates three basic +building blocks: 1. **Spark basics** — get the session, build a DataFrame, read a Unity Catalog table. 2. **UDF basics** — scalar, map-returning, SQL-registered, and a UDF closing over a locally-defined class. @@ -40,6 +42,9 @@ cluster job in a few ways this example follows: ``` serverless-jar-job-example/ +├── databricks.yml # bundle: build the JAR, define variables and target +├── resources/ +│ └── serverless-jar-job-example.job.yml # the serverless JAR job ├── build.sbt # Scala 2.13.16, JDK 17, provided deps, assembly config ├── project/ │ ├── plugins.sbt # sbt-assembly plugin @@ -48,37 +53,44 @@ serverless-jar-job-example/ └── ServerlessJarJobExample.scala ``` -## Build +## Prerequisites -You need [sbt](https://www.scala-sbt.org/) and JDK 17. +- The [Databricks CLI](https://docs.databricks.com/dev-tools/cli/install.html), + authenticated to your workspace. +- [sbt](https://www.scala-sbt.org/) and JDK 17 (the bundle invokes `sbt assembly`). -```bash -sbt clean assembly -``` +## Configure -This produces the fat JAR at: +Edit `databricks.yml`: -``` -target/scala-2.13/ServerlessJarJobExample-2.13-assembly.jar -``` +- Set the `dev` target's `workspace.host` to your workspace URL. +- Set the `volume_base` variable default to a Unity Catalog volume you can write + to, e.g. `/Volumes///` (or pass it at deploy time with + `--var volume_base=...`). ## Deploy and run -1. Create a job with a **JAR task** on **serverless** compute: - - Main class: `com.databricks.labs.example.ServerlessJarJobExample` - - Environment JAR dependency: the uploaded JAR (drag and drop it into the file selector, or browse to select it from a Unity Catalog volume or workspace location) - - Parameter (`args(0)`): a writable UC volume base path, - e.g. `/Volumes///` +The bundle builds the JAR, uploads it, and creates the job in one step: -2. Run the job. On success the driver log ends with: +```bash +databricks bundle deploy -t dev +databricks bundle run -t dev serverless-jar-job-example +``` + +`bundle deploy` runs `sbt assembly` to produce the fat JAR at +`target/scala-2.13/ServerlessJarJobExample-2.13-assembly.jar`, uploads it to the +workspace artifact path, and registers the serverless JAR job. On success the +run's driver log ends with: - ``` - ServerlessJarJobExample: all examples completed successfully - ``` +``` +ServerlessJarJobExample: all examples completed successfully +``` -The single argument is the base path of a UC volume the job can write to; the -`dbutils.fs` example writes a temporary file there and cleans it up afterwards. +The job's single argument (`args(0)`) is the `volume_base` variable — a UC volume +the job can write to; the `dbutils.fs` example writes a temporary file there and +cleans it up afterwards. ## References - [Create and run JARs on serverless compute](https://docs.databricks.com/aws/en/jobs/how-to/use-jars-in-workflows) +- [Databricks Asset Bundles](https://docs.databricks.com/dev-tools/bundles/index.html) diff --git a/serverless-jar-job-example/databricks.yml b/serverless-jar-job-example/databricks.yml new file mode 100644 index 00000000..46eb8d47 --- /dev/null +++ b/serverless-jar-job-example/databricks.yml @@ -0,0 +1,40 @@ +# Databricks Asset Bundle for the serverless Scala JAR job example. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +# +# The bundle builds the fat JAR with `sbt assembly`, uploads it, and defines a +# job with a single spark_jar_task on serverless compute. Deploy and run with: +# +# databricks bundle deploy -t dev +# databricks bundle run -t dev serverless-jar-job-example +bundle: + name: serverless-jar-job-example + uuid: bbd857bc-f581-4885-a744-0d951cf4440a + +include: + - resources/*.yml + +variables: + volume_base: + description: >- + Writable Unity Catalog volume base path passed to the job as args(0), + e.g. /Volumes///. The dbutils.fs example writes a + temporary file under this path and cleans it up. + default: /Volumes/main/default/my-volume + +# Build the deployable fat JAR with sbt assembly. The `files` source path is the +# artifact sbt produces (see build.sbt: assembly / assemblyJarName). +artifacts: + jar: + type: jar + path: . + build: sbt ";clean; assembly" + files: + - source: target/scala-2.13/ServerlessJarJobExample-2.13-assembly.jar + +targets: + dev: + mode: development + default: true + workspace: + # Replace with your workspace URL. + host: https://.cloud.databricks.com diff --git a/serverless-jar-job-example/resources/serverless-jar-job-example.job.yml b/serverless-jar-job-example/resources/serverless-jar-job-example.job.yml new file mode 100644 index 00000000..71936daf --- /dev/null +++ b/serverless-jar-job-example/resources/serverless-jar-job-example.job.yml @@ -0,0 +1,22 @@ +# Job definition for the serverless Scala JAR job example. +resources: + jobs: + serverless-jar-job-example: + name: serverless-jar-job-example + tasks: + - task_key: main + spark_jar_task: + main_class_name: com.databricks.labs.example.ServerlessJarJobExample + # args(0): the writable UC volume base path. + parameters: + - ${var.volume_base} + environment_key: default + # Serverless JAR tasks run under a serverless environment. `client` selects + # the serverless environment version; the assembled JAR is listed as a + # dependency so the job can load it. + environments: + - environment_key: default + spec: + client: "4" + java_dependencies: + - ${workspace.artifact_path}/.internal/ServerlessJarJobExample-2.13-assembly.jar From c4980602346cc0c81a699eebe61985f5969fdb90 Mon Sep 17 00:00:00 2001 From: Imran Hasan Date: Thu, 23 Jul 2026 12:20:04 -0700 Subject: [PATCH 4/5] Client 5 --- .../resources/serverless-jar-job-example.job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/serverless-jar-job-example/resources/serverless-jar-job-example.job.yml b/serverless-jar-job-example/resources/serverless-jar-job-example.job.yml index 71936daf..84bff578 100644 --- a/serverless-jar-job-example/resources/serverless-jar-job-example.job.yml +++ b/serverless-jar-job-example/resources/serverless-jar-job-example.job.yml @@ -17,6 +17,6 @@ resources: environments: - environment_key: default spec: - client: "4" + client: "5" java_dependencies: - ${workspace.artifact_path}/.internal/ServerlessJarJobExample-2.13-assembly.jar From 708296ddea197a911583fa3b2790435c22492c09 Mon Sep 17 00:00:00 2001 From: Imran Hasan Date: Thu, 23 Jul 2026 14:15:33 -0700 Subject: [PATCH 5/5] Update deps --- serverless-jar-job-example/build.sbt | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/serverless-jar-job-example/build.sbt b/serverless-jar-job-example/build.sbt index a5b13ec9..80606bb2 100644 --- a/serverless-jar-job-example/build.sbt +++ b/serverless-jar-job-example/build.sbt @@ -5,15 +5,17 @@ scalaVersion := "2.13.16" javacOptions ++= Seq("--release", "17") scalacOptions ++= Seq("-release", "17") -// Spark Connect client. Provided by the serverless runtime, so it is not -// shipped inside the assembled JAR. -libraryDependencies += "com.databricks" %% "databricks-connect" % "17.3.2" % "provided" +// Spark Connect client version. This tracks the serverless environment version: +// bump the major version only in step with a major environment_version release, +// and the minor/patch version continuously as new client releases ship. +val databricksConnectVersion = "17.3.2" -// dbutils for Scala. Use the databricks-dbutils-scala SDK (API package -// com.databricks.sdk.scala.dbutils); it is in the serverless provided-library -// list, so it is marked provided as well. Add any of your own dependencies -// here -- use %% for Scala libraries so sbt resolves the _2.13 artifact. -libraryDependencies += "com.databricks" %% "databricks-dbutils-scala" % "0.1.4" % "provided" +// Spark Connect client. Provided by the serverless runtime, so it is not +// shipped inside the assembled JAR. The databricks-dbutils-scala SDK (API +// package com.databricks.sdk.scala.dbutils) comes in transitively, so there is +// no need to declare it separately. Add any of your own dependencies here -- +// use %% for Scala libraries so sbt resolves the _2.13 artifact. +libraryDependencies += "com.databricks" %% "databricks-connect" % databricksConnectVersion % "provided" // Fork a JVM on `sbt run` so the javaOptions below are applied. fork := true @@ -24,3 +26,7 @@ javaOptions += "--add-opens=java.base/java.nio=ALL-UNNAMED" // provided, so the assembly contains only this project's own classes. Compile / mainClass := Some("com.databricks.labs.example.ServerlessJarJobExample") assembly / assemblyJarName := s"ServerlessJarJobExample-${scalaBinaryVersion.value}-assembly.jar" + +// The Scala library is already on the serverless runtime classpath, so there is +// no need to bundle it -- excluding it keeps the assembled JAR small. +assembly / assemblyOption := (assembly / assemblyOption).value.withIncludeScala(false)