From 2e3f0103afaddb22ecf3f27305c68aaf789d1917 Mon Sep 17 00:00:00 2001 From: Grigory Panov Date: Tue, 8 Sep 2026 11:46:16 +0200 Subject: [PATCH] Derive default Scala cluster runtime from DBR version --- .agents/rules/template-versions.md | 6 ++- .../default-scala-classic/input.json | 7 +++ .../default-scala-classic/out.test.toml | 4 ++ .../default-scala-classic/output.txt | 42 +++++++++++++++ .../templates/default-scala-classic/script | 7 +++ .../templates/default-scala-classic/test.toml | 3 ++ libs/template/renderer_test.go | 51 +++++++++++++++++++ .../library/template_variables.tmpl | 8 +++ .../resources/{{.project_name}}.job.yml.tmpl | 2 +- 9 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 acceptance/bundle/templates/default-scala-classic/input.json create mode 100644 acceptance/bundle/templates/default-scala-classic/out.test.toml create mode 100644 acceptance/bundle/templates/default-scala-classic/output.txt create mode 100644 acceptance/bundle/templates/default-scala-classic/script create mode 100644 acceptance/bundle/templates/default-scala-classic/test.toml diff --git a/.agents/rules/template-versions.md b/.agents/rules/template-versions.md index cd0c4b2b64f..2d8980af0ef 100644 --- a/.agents/rules/template-versions.md +++ b/.agents/rules/template-versions.md @@ -3,24 +3,26 @@ description: Version pins in bundle templates (DBR, DB Connect, serverless env, globs: - "libs/template/templates/**/library/versions.tmpl" - "libs/template/templates/**/library/template_variables.tmpl" + - "libs/template/templates/**/build.sbt.tmpl" - "libs/template/templates/**/resources/*.job.yml.tmpl" - "libs/template/templates/**/pyproject.toml.tmpl" paths: - "libs/template/templates/**/library/versions.tmpl" - "libs/template/templates/**/library/template_variables.tmpl" + - "libs/template/templates/**/build.sbt.tmpl" - "libs/template/templates/**/resources/*.job.yml.tmpl" - "libs/template/templates/**/pyproject.toml.tmpl" --- # Bundle template version pins -Bundle templates pin the runtime versions they render. Most pins are macros in `libs/template/templates/default/library/versions.tmpl` (DBR, DB Connect, serverless environment, Python, and the `databricks-bundles` package), and that single file serves five shipped templates: `default-minimal`, `default-python`, `lakeflow-pipelines` and `pydabs` all alias it through `"template_dir": "../default"` in their `databricks_template_schema.json`. `dbt-sql` defines its own subset, `default-scala` defines its own in `library/template_variables.tmpl`, and two templates hardcode a version in the files they render instead of going through a macro. The `.tmpl` files are the source of truth for the current values; the rules below describe how the pins relate, not what they are. +Bundle templates pin the runtime versions they render. Most pins are macros in `libs/template/templates/default/library/versions.tmpl` (DBR, DB Connect, serverless environment, Python, and the `databricks-bundles` package), and that single file serves five shipped templates: `default-minimal`, `default-python`, `lakeflow-pipelines` and `pydabs` alias it through `"template_dir": "../default"` in their `databricks_template_schema.json`, while the deprecated `cli-pipelines` is registered in `libs/template/template.go` with `lakeflow-pipelines`'s reader. The `default` template itself is not shipped. `dbt-sql` defines its own subset, `default-scala` defines its own in `library/template_variables.tmpl`, and two templates hardcode a version in the files they render instead of going through a macro. The `.tmpl` files are the source of truth for the current values; the rules below describe how the pins relate, not what they are. **RULE: Keep the serverless environment version, the Python pins, the DB Connect pin, and `latest_lts_dbr_version` mutually compatible. This is a hard constraint.** A serverless environment version dictates a runtime Python version, and the DB Connect pin must support that Python. The DBR LTS pin belongs in the same check rather than being treated as independent — the two have drifted before: `latest_lts_dbr_version` reached 16.4 in #3671 while the DB Connect pin stayed on 15.4 until #6378. Cross-check the [serverless environment version release notes](https://docs.databricks.com/aws/en/release-notes/serverless/environment-version/) and the [DB Connect requirements](https://docs.databricks.com/dev-tools/databricks-connect/python/index.html#requirements) whenever you change any of the four. **RULE: Within that constraint, keep `default`'s `conservative_db_connect_version_spec` as low as compatibility allows.** The DB Connect client is only forward-compatible (it reaches compute of its own version and higher), so the lowest compatible pin maximizes the range of DBR versions a customer can connect to; a higher pin rules out customers on older DBR, and they can move to a newer version themselves after initializing the template. The floor is the *oldest* release that passes two tests: it is still supported, *and* the DBR version it targets runs `python_version_spec`'s Python on the cluster — not merely that the client supports that Python. Support status only rules out the short-lived non-LTS releases; an LTS stays supported for years, so older LTS releases are ruled out by the runtime test instead. DB Connect 15.4 is the worked example: it is still supported and its client supports Python 3.12, but DBR 15 clusters run Python 3.11, so it fails the second test and 16.4 is the floor. Never read this as "pin the current DBR LTS" — that is the newest release, the opposite of a floor, and #6378 rejected exactly that bump. Keep the upper bound at the next minor release, so the spec stays inside one minor: widening it lets a freshly initialized project resolve a client far newer than the floor advertises, which is the opposite of conservative. Do not raise the pin merely to match a newer serverless environment version — raise it only when the constraint above forces it, or when the pinned release falls out of support (see the [DBR release notes](https://docs.databricks.com/aws/en/release-notes/runtime/) for supported versions). See PR #3897 and PR #6378 for prior history. -This rule governs `default`'s pin. `default-scala` deliberately does the opposite: it tracks the newest LTS in `dbr_version` and derives its client from that macro in `template/{{.project_name}}/build.sbt.tmpl` as an open-ended `.+` dependency. Do not lower `dbr_version`, or pin that dependency, to satisfy the rule above. +This rule governs `default`'s pin. `default-scala` deliberately does the opposite: it tracks the newest LTS in `dbr_version`. Its client in `template/{{.project_name}}/build.sbt.tmpl` uses that macro as an open-ended `.+` dependency, while its classic-cluster `spark_version` combines `dbr_version` with `scala_major_minor_version`. Keep both macros compatible with the selected DBR release. Do not lower `dbr_version`, or pin the client dependency, to satisfy the rule above. **RULE: When bumping the serverless environment version or the Python version, update every occurrence, not just the macros.** More than one template defines `serverless_environment_version`, and those definitions must stay in sync. Two templates hardcode the value in the files they render instead — `default-scala` in one job and `lakeflow-integrations` in two — and `lakeflow-integrations` also hardcodes `requires-python`. A Python bump additionally needs `default_python_version`, which supplies the notebook kernel version in `src/sample_notebook.ipynb.tmpl`. Grep `environment_version`, `requires-python` and `default_python_version` under `libs/template/templates/` and move every hit together. diff --git a/acceptance/bundle/templates/default-scala-classic/input.json b/acceptance/bundle/templates/default-scala-classic/input.json new file mode 100644 index 00000000000..c728d78f632 --- /dev/null +++ b/acceptance/bundle/templates/default-scala-classic/input.json @@ -0,0 +1,7 @@ +{ + "project_name": "my_default_scala", + "compute_type": "standard cluster", + "artifacts_dest_path": "/Volumes/test-folder", + "default_catalog": "main", + "personal_schemas": "yes" +} diff --git a/acceptance/bundle/templates/default-scala-classic/out.test.toml b/acceptance/bundle/templates/default-scala-classic/out.test.toml new file mode 100644 index 00000000000..86234a2fd03 --- /dev/null +++ b/acceptance/bundle/templates/default-scala-classic/out.test.toml @@ -0,0 +1,4 @@ +Cloud = false +Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/templates/default-scala-classic/output.txt b/acceptance/bundle/templates/default-scala-classic/output.txt new file mode 100644 index 00000000000..4dd1b6125a7 --- /dev/null +++ b/acceptance/bundle/templates/default-scala-classic/output.txt @@ -0,0 +1,42 @@ + +>>> [CLI] bundle init default-scala --config-file ./input.json --output-dir output + +Welcome to the default-scala template for Declarative Automation Bundles! + +A workspace was selected based on your current profile. For information about how to change this, see https://docs.databricks.com/dev-tools/cli/profiles.html. +workspace_host: [DATABRICKS_URL] + +✨ Your new project has been created in the 'my_default_scala' directory! + +Please refer to the README.md file for "getting started" instructions. +See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> diff.py [TESTROOT]/bundle/templates/default-scala-classic/../default-scala/output/my_default_scala/resources/my_default_scala.job.yml output/my_default_scala/resources/my_default_scala.job.yml +--- [TESTROOT]/bundle/templates/default-scala-classic/../default-scala/output/my_default_scala/resources/my_default_scala.job.yml ++++ output/my_default_scala/resources/my_default_scala.job.yml +@@ -12,4 +12,5 @@ + tasks: + - task_key: main_task ++ job_cluster_key: job_cluster + spark_jar_task: + main_class_name: com.examples.Main +@@ -19,9 +20,12 @@ + - "--schema" + - "{{job.parameters.schema}}" +- environment_key: default +- environments: +- - environment_key: default +- spec: +- environment_version: "5" +- java_dependencies: +- - ${workspace.artifact_path}/.internal/my_default_scala-assembly-0.1.jar ++ libraries: ++ - jar: ${workspace.artifact_path}/.internal/my_default_scala-assembly-0.1.jar ++ job_clusters: ++ - job_cluster_key: job_cluster ++ new_cluster: ++ spark_version: 17.3.x-scala2.13 ++ node_type_id: [NODE_TYPE_ID] ++ autoscale: ++ min_workers: 1 ++ max_workers: 4 diff --git a/acceptance/bundle/templates/default-scala-classic/script b/acceptance/bundle/templates/default-scala-classic/script new file mode 100644 index 00000000000..b206e044bec --- /dev/null +++ b/acceptance/bundle/templates/default-scala-classic/script @@ -0,0 +1,7 @@ +# With standard cluster compute, the default-scala job uses a classic job cluster +# instead of a serverless environment. Diff only the job resource to lock in the +# spark_version derived from dbr_version and scala_major_minor_version. +trap 'rm -rf output' EXIT + +trace $CLI bundle init default-scala --config-file ./input.json --output-dir output +trace diff.py $TESTDIR/../default-scala/output/my_default_scala/resources/my_default_scala.job.yml output/my_default_scala/resources/my_default_scala.job.yml diff --git a/acceptance/bundle/templates/default-scala-classic/test.toml b/acceptance/bundle/templates/default-scala-classic/test.toml new file mode 100644 index 00000000000..fb24ad07098 --- /dev/null +++ b/acceptance/bundle/templates/default-scala-classic/test.toml @@ -0,0 +1,3 @@ +# This test reads the sibling default-scala test's committed golden via $TESTDIR, so it +# must run in phase 1 (after phase 0 regenerates that golden in update mode). +Phase = 1 diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index 837584c452a..64ffdb6dbdc 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -1,6 +1,7 @@ package template import ( + "bytes" "context" "io/fs" "os" @@ -134,6 +135,56 @@ func TestBuiltinDbtTemplateValid(t *testing.T) { } } +func TestDefaultScalaSparkVersionCombinesVersionMacros(t *testing.T) { + tmpl, err := template.New("root").ParseFS(builtinTemplates, "templates/default-scala/library/*") + require.NoError(t, err) + + var dbrVersion bytes.Buffer + require.NoError(t, tmpl.ExecuteTemplate(&dbrVersion, "dbr_version", nil)) + var scalaVersion bytes.Buffer + require.NoError(t, tmpl.ExecuteTemplate(&scalaVersion, "scala_major_minor_version", nil)) + var sparkVersion bytes.Buffer + require.NoError(t, tmpl.ExecuteTemplate(&sparkVersion, "spark_version", nil)) + + assert.Equal(t, strings.TrimSpace(dbrVersion.String()), dbrVersion.String()) + assert.Equal(t, strings.TrimSpace(scalaVersion.String()), scalaVersion.String()) + assert.Equal(t, strings.TrimSpace(sparkVersion.String()), sparkVersion.String()) + assert.Equal(t, + dbrVersion.String()+".x-scala"+scalaVersion.String(), + sparkVersion.String(), + ) +} + +func TestDefaultScalaRenderedVersionsUseMacros(t *testing.T) { + // Define every macro referenced by the two template files parsed below. + tmpl := template.Must(template.New("root").Parse(` +{{define "dbr_version"}}99.9{{end}} +{{define "scala_version"}}9.9.9{{end}} +{{define "spark_version"}}99.9.x-scala9.9{{end}} +{{define "organization"}}com.example{{end}} +{{define "version"}}0.1{{end}} +{{define "main_class_name"}}com.example.Main{{end}} +`)) + tmpl, err := tmpl.ParseFS( + builtinTemplates, + "templates/default-scala/template/{{.project_name}}/build.sbt.tmpl", + "templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl", + ) + require.NoError(t, err) + + config := map[string]any{ + "compute_type": "standard cluster", + "project_name": "my_project", + } + var buildFile bytes.Buffer + require.NoError(t, tmpl.ExecuteTemplate(&buildFile, "build.sbt.tmpl", config)) + assert.Contains(t, buildFile.String(), `"databricks-connect" % "99.9.+"`) + + var jobFile bytes.Buffer + require.NoError(t, tmpl.ExecuteTemplate(&jobFile, "{{.project_name}}.job.yml.tmpl", config)) + assert.Contains(t, jobFile.String(), "spark_version: 99.9.x-scala9.9") +} + func TestRendererWithAssociatedTemplateInLibrary(t *testing.T) { tmpDir := t.TempDir() diff --git a/libs/template/templates/default-scala/library/template_variables.tmpl b/libs/template/templates/default-scala/library/template_variables.tmpl index c595cd13f69..ff9749a676a 100644 --- a/libs/template/templates/default-scala/library/template_variables.tmpl +++ b/libs/template/templates/default-scala/library/template_variables.tmpl @@ -7,6 +7,10 @@ 0.241.0 {{- end }} +{{/* dbr_version selects the Databricks Connect client, and together with + * scala_major_minor_version identifies the DBR image used by spark_version. + * Keep both compatible with the same DBR release. + */}} {{ define `dbr_version` -}} 17.3 {{- end }} @@ -15,6 +19,10 @@ 2.13 {{- end }} +{{ define `spark_version` -}} + {{template `dbr_version` .}}.x-scala{{template `scala_major_minor_version` .}} +{{- end }} + {{ define `scala_version` -}} {{template `scala_major_minor_version` .}}.16 {{- end}} diff --git a/libs/template/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index f700ff1811a..ed10b6f28f0 100644 --- a/libs/template/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ b/libs/template/templates/default-scala/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -38,7 +38,7 @@ resources: job_clusters: - job_cluster_key: job_cluster new_cluster: - spark_version: 17.3.x-scala2.13 + spark_version: {{template `spark_version` .}} node_type_id: i3.xlarge autoscale: min_workers: 1