diff --git a/CODEOWNERS b/CODEOWNERS index 174adc20..3efdd748 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -18,6 +18,7 @@ ka-chat-bot @taiga-db marketplace-provisioning @michellejm metascan @nfx @alexott runtime-packages @nfx @alexott +serverless-jar-job-example @imran-a-hasan sql_migration_copilot @robertwhiffin tacklebox @Jonathan-Choi uc-catalog-cloning @esiol-db @vasco-lopes diff --git a/serverless-jar-job-example/.gitignore b/serverless-jar-job-example/.gitignore new file mode 100644 index 00000000..85f0489f --- /dev/null +++ b/serverless-jar-job-example/.gitignore @@ -0,0 +1,7 @@ +target/ +project/target/ +project/project/ +.bsp/ +.idea/ +.metals/ +.bloop/ diff --git a/serverless-jar-job-example/README.md b/serverless-jar-job-example/README.md new file mode 100644 index 00000000..aab9a7b8 --- /dev/null +++ b/serverless-jar-job-example/README.md @@ -0,0 +1,96 @@ +--- +title: "Serverless Scala JAR job example" +language: scala +author: "Imran Hasan" +date: 2026-07-22 + +tags: +- scala +- spark-connect +- serverless +- jar +- jobs +- dbutils +--- + +# Serverless Scala JAR job example + +A minimal, self-contained Scala JAR job that runs on **Databricks serverless +compute**, packaged as a **Databricks Asset Bundle**. It is meant as a +copy-and-go starting point: the bundle builds the JAR with `sbt assembly`, +uploads it, and defines a serverless JAR job. The code demonstrates three basic +building blocks: + +1. **Spark basics** — get the session, build a DataFrame, read a Unity Catalog table. +2. **UDF basics** — scalar, map-returning, SQL-registered, and a UDF closing over a locally-defined class. +3. **dbutils basics** — the job/notebook context and a `dbutils.fs` read/write round-trip against a UC volume. + +## Serverless specifics + +Serverless JAR jobs run under **Spark Connect**, which differs from a classic +cluster job in a few ways this example follows: + +- Get the session with `SparkSession.builder().getOrCreate()`. There is **no** + `SparkContext` / RDD API — those throw under Spark Connect. +- Build against the **Spark Connect** client (`databricks-connect`) and the + **`databricks-dbutils-scala`** SDK (package `com.databricks.sdk.scala.dbutils`). + Both are provided by the runtime, so they are marked `% provided` and are not + shipped inside your JAR. +- Target **Scala 2.13** and **JDK 17**. + +## Layout + +``` +serverless-jar-job-example/ +├── databricks.yml # bundle: build the JAR, define variables and target +├── resources/ +│ └── serverless-jar-job-example.job.yml # the serverless JAR job +├── build.sbt # Scala 2.13.16, JDK 17, provided deps, assembly config +├── project/ +│ ├── plugins.sbt # sbt-assembly plugin +│ └── build.properties # sbt version +└── src/main/scala/com/databricks/labs/example/ + └── ServerlessJarJobExample.scala +``` + +## Prerequisites + +- The [Databricks CLI](https://docs.databricks.com/dev-tools/cli/install.html), + authenticated to your workspace. +- [sbt](https://www.scala-sbt.org/) and JDK 17 (the bundle invokes `sbt assembly`). + +## Configure + +Edit `databricks.yml`: + +- Set the `dev` target's `workspace.host` to your workspace URL. +- Set the `volume_base` variable default to a Unity Catalog volume you can write + to, e.g. `/Volumes///` (or pass it at deploy time with + `--var volume_base=...`). + +## Deploy and run + +The bundle builds the JAR, uploads it, and creates the job in one step: + +```bash +databricks bundle deploy -t dev +databricks bundle run -t dev serverless-jar-job-example +``` + +`bundle deploy` runs `sbt assembly` to produce the fat JAR at +`target/scala-2.13/ServerlessJarJobExample-2.13-assembly.jar`, uploads it to the +workspace artifact path, and registers the serverless JAR job. On success the +run's driver log ends with: + +``` +ServerlessJarJobExample: all examples completed successfully +``` + +The job's single argument (`args(0)`) is the `volume_base` variable — a UC volume +the job can write to; the `dbutils.fs` example writes a temporary file there and +cleans it up afterwards. + +## References + +- [Create and run JARs on serverless compute](https://docs.databricks.com/aws/en/jobs/how-to/use-jars-in-workflows) +- [Databricks Asset Bundles](https://docs.databricks.com/dev-tools/bundles/index.html) diff --git a/serverless-jar-job-example/build.sbt b/serverless-jar-job-example/build.sbt new file mode 100644 index 00000000..80606bb2 --- /dev/null +++ b/serverless-jar-job-example/build.sbt @@ -0,0 +1,32 @@ +name := "ServerlessJarJobExample" + +// Serverless JAR jobs require Scala 2.13 and JDK 17. +scalaVersion := "2.13.16" +javacOptions ++= Seq("--release", "17") +scalacOptions ++= Seq("-release", "17") + +// Spark Connect client version. This tracks the serverless environment version: +// bump the major version only in step with a major environment_version release, +// and the minor/patch version continuously as new client releases ship. +val databricksConnectVersion = "17.3.2" + +// Spark Connect client. Provided by the serverless runtime, so it is not +// shipped inside the assembled JAR. The databricks-dbutils-scala SDK (API +// package com.databricks.sdk.scala.dbutils) comes in transitively, so there is +// no need to declare it separately. Add any of your own dependencies here -- +// use %% for Scala libraries so sbt resolves the _2.13 artifact. +libraryDependencies += "com.databricks" %% "databricks-connect" % databricksConnectVersion % "provided" + +// Fork a JVM on `sbt run` so the javaOptions below are applied. +fork := true +javaOptions += "--add-opens=java.base/java.nio=ALL-UNNAMED" + +// Build a deployable fat JAR with `sbt assembly`. No custom assemblyMergeStrategy +// is needed: every heavy dependency is provided by the runtime and marked +// provided, so the assembly contains only this project's own classes. +Compile / mainClass := Some("com.databricks.labs.example.ServerlessJarJobExample") +assembly / assemblyJarName := s"ServerlessJarJobExample-${scalaBinaryVersion.value}-assembly.jar" + +// The Scala library is already on the serverless runtime classpath, so there is +// no need to bundle it -- excluding it keeps the assembled JAR small. +assembly / assemblyOption := (assembly / assemblyOption).value.withIncludeScala(false) diff --git a/serverless-jar-job-example/databricks.yml b/serverless-jar-job-example/databricks.yml new file mode 100644 index 00000000..46eb8d47 --- /dev/null +++ b/serverless-jar-job-example/databricks.yml @@ -0,0 +1,40 @@ +# Databricks Asset Bundle for the serverless Scala JAR job example. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +# +# The bundle builds the fat JAR with `sbt assembly`, uploads it, and defines a +# job with a single spark_jar_task on serverless compute. Deploy and run with: +# +# databricks bundle deploy -t dev +# databricks bundle run -t dev serverless-jar-job-example +bundle: + name: serverless-jar-job-example + uuid: bbd857bc-f581-4885-a744-0d951cf4440a + +include: + - resources/*.yml + +variables: + volume_base: + description: >- + Writable Unity Catalog volume base path passed to the job as args(0), + e.g. /Volumes///. The dbutils.fs example writes a + temporary file under this path and cleans it up. + default: /Volumes/main/default/my-volume + +# Build the deployable fat JAR with sbt assembly. The `files` source path is the +# artifact sbt produces (see build.sbt: assembly / assemblyJarName). +artifacts: + jar: + type: jar + path: . + build: sbt ";clean; assembly" + files: + - source: target/scala-2.13/ServerlessJarJobExample-2.13-assembly.jar + +targets: + dev: + mode: development + default: true + workspace: + # Replace with your workspace URL. + host: https://.cloud.databricks.com diff --git a/serverless-jar-job-example/project/build.properties b/serverless-jar-job-example/project/build.properties new file mode 100644 index 00000000..bc739060 --- /dev/null +++ b/serverless-jar-job-example/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.10.3 diff --git a/serverless-jar-job-example/project/plugins.sbt b/serverless-jar-job-example/project/plugins.sbt new file mode 100644 index 00000000..9443a4ff --- /dev/null +++ b/serverless-jar-job-example/project/plugins.sbt @@ -0,0 +1,2 @@ +// sbt-assembly builds the uber JAR (`sbt assembly`) that the job deploys. +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.3.1") diff --git a/serverless-jar-job-example/resources/serverless-jar-job-example.job.yml b/serverless-jar-job-example/resources/serverless-jar-job-example.job.yml new file mode 100644 index 00000000..84bff578 --- /dev/null +++ b/serverless-jar-job-example/resources/serverless-jar-job-example.job.yml @@ -0,0 +1,22 @@ +# Job definition for the serverless Scala JAR job example. +resources: + jobs: + serverless-jar-job-example: + name: serverless-jar-job-example + tasks: + - task_key: main + spark_jar_task: + main_class_name: com.databricks.labs.example.ServerlessJarJobExample + # args(0): the writable UC volume base path. + parameters: + - ${var.volume_base} + environment_key: default + # Serverless JAR tasks run under a serverless environment. `client` selects + # the serverless environment version; the assembled JAR is listed as a + # dependency so the job can load it. + environments: + - environment_key: default + spec: + client: "5" + java_dependencies: + - ${workspace.artifact_path}/.internal/ServerlessJarJobExample-2.13-assembly.jar diff --git a/serverless-jar-job-example/src/main/scala/com/databricks/labs/example/ServerlessJarJobExample.scala b/serverless-jar-job-example/src/main/scala/com/databricks/labs/example/ServerlessJarJobExample.scala new file mode 100644 index 00000000..4ea9e326 --- /dev/null +++ b/serverless-jar-job-example/src/main/scala/com/databricks/labs/example/ServerlessJarJobExample.scala @@ -0,0 +1,131 @@ +package com.databricks.labs.example + +import com.databricks.sdk.scala.dbutils.DBUtils +import org.apache.spark.sql.connect.SparkSession +import org.apache.spark.sql.functions.{col, udf} + +import java.util.UUID + +/** + * A minimal, self-contained example of a Scala JAR job that runs on Databricks + * serverless compute. It demonstrates three basic building blocks: + * + * 1. Spark basics -- get the session, build a DataFrame, read a UC table. + * 2. UDF basics -- scalar, map-returning, SQL-registered, and closing + * over a locally-defined class. + * 3. dbutils basics -- the notebook/job context and a `dbutils.fs` + * read/write round-trip against a Unity Catalog volume. + * + * Serverless JAR jobs run under Spark Connect, so a few things differ from a + * classic cluster job: + * + * - Get the session with `SparkSession.builder().getOrCreate()`. There is no + * `SparkContext` / RDD API (those throw under Spark Connect). + * - Build against `databricks-connect` (Spark Connect) and the + * `databricks-dbutils-scala` SDK. Both are provided by the runtime, so they + * are marked `% provided` in build.sbt and are not shipped in the JAR. + * - Target Scala 2.13 and JDK 17 (see build.sbt). + * + * Run it with a single argument: the base path of a UC volume you can write to, + * e.g. `/Volumes///`. + */ +object ServerlessJarJobExample { + + def main(args: Array[String]): Unit = { + require( + args.nonEmpty, + "ServerlessJarJobExample requires a writable UC volume base path as args(0), " + + "e.g. /Volumes///" + ) + val volumeBasePath = args(0) + + // Under Spark Connect, getOrCreate() returns the session wired to serverless. + val spark = SparkSession.builder().getOrCreate() + + // dbutils comes from the databricks-dbutils-scala SDK. + val dbutils = DBUtils.getDBUtils() + + sparkBasics(spark) + udfBasics(spark) + dbutilsBasics(spark, dbutils, volumeBasePath) + + println("ServerlessJarJobExample: all examples completed successfully") + } + + // --------------------------------------------------------------------------- + // 1. Spark basics + // --------------------------------------------------------------------------- + + def sparkBasics(spark: SparkSession): Unit = { + println(s"Spark version: ${spark.version}") + + // Build a DataFrame and collect a few rows. + val ids = spark.range(10).limit(3).collect().toSeq + println(s"spark.range(10).limit(3) = ${ids.mkString(", ")}") + + // Read a Unity Catalog table. `samples` is a built-in public catalog + // available on Databricks workspaces, so this needs no setup. + val rows = spark.read.table("samples.nyctaxi.trips").limit(10).count() + println(s"Read $rows rows from samples.nyctaxi.trips") + } + + // --------------------------------------------------------------------------- + // 2. UDF basics + // --------------------------------------------------------------------------- + + def udfBasics(spark: SparkSession): Unit = { + import spark.implicits._ + + // A scalar UDF returning a String. + val greet = udf((v: Int) => s"hello world $v") + val greeted = spark.range(3).select(greet(col("id")).as("v")).as[String].collect().toSeq + println(s"scalar UDF: ${greeted.mkString(", ")}") + + // A UDF returning a Map. + val toMap = udf((v: Int) => Map(v -> "some value")) + val mapped = + spark.range(1).select(toMap(col("id")).as("m")).as[Map[Int, String]].collect().toSeq + println(s"map UDF: ${mapped.mkString(", ")}") + + // Register a UDF for use from SQL. + spark.udf.register("greet", greet) + val fromSql = spark.sql("SELECT greet(1) AS v").as[String].collect().head + println(s"SQL-registered UDF: $fromSql") + + // A UDF closing over a locally-defined class. Spark ships the closure (and + // the class) to the executors, so this exercises closure serialization. + val withClass = udf((x: Int) => new Multiplier(x).result) + val computed = spark.range(5).select(withClass(col("id"))).as[Int].collect().sorted.toSeq + println(s"UDF with local class: ${computed.mkString(", ")}") + } + + /** A small class captured by a UDF closure and shipped to the executors. */ + private class Multiplier(x: Int) { + def result: Int = x * 42 + 5 + } + + // --------------------------------------------------------------------------- + // 3. dbutils basics + // --------------------------------------------------------------------------- + + def dbutilsBasics(spark: SparkSession, dbutils: DBUtils, volumeBasePath: String): Unit = { + import spark.implicits._ + + // The job/notebook context carries tags such as the job and run identifiers. + println(s"notebook context tags: ${dbutils.notebook.getContext().tags}") + + // A Spark read/write round-trip against a UC volume, cleaned up with + // dbutils.fs.rm afterwards. + val path = s"$volumeBasePath/example_${UUID.randomUUID().toString}" + try { + Seq("hello world").toDF("value").write.mode("overwrite").text(path) + val readBack = spark.read.text(path).as[String].collect().toSeq + require(readBack == Seq("hello world"), s"volume round-trip mismatch, got $readBack") + println(s"dbutils.fs volume round-trip OK at $path") + } finally { + // Best-effort cleanup. + try dbutils.fs.rm(path, true) + catch { case _: Throwable => () } + } + } +}