From 897f89def5c0999b0636871b79d705bbfbeee241 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 11 Sep 2026 11:43:52 +0000 Subject: [PATCH 1/2] test: reproduce config-sync dropping remote cluster additions --- .../databricks.yml.tmpl | 66 +++++++++++++++++++ .../cluster_policy_remote_addition/hello.py | 3 + .../out.test.toml | 4 ++ .../cluster_policy_remote_addition/output.txt | 49 ++++++++++++++ .../cluster_policy_remote_addition/script | 29 ++++++++ .../cluster_policy_remote_addition/test.toml | 4 ++ 6 files changed, 155 insertions(+) create mode 100644 acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl create mode 100644 acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/hello.py create mode 100644 acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/out.test.toml create mode 100644 acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt create mode 100644 acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script create mode 100644 acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl new file mode 100644 index 00000000000..b0f61018ee4 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl @@ -0,0 +1,66 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + cluster_policies: + policy: + name: test-policy-$UNIQUE_NAME + definition: + # This rule controls only PolicyTag, not the Spark setting edited below. + custom_tags.PolicyTag: + type: fixed + value: from-policy + jobs: + test_job: + name: test-job-$UNIQUE_NAME + max_concurrent_runs: 1 + job_clusters: + - job_cluster_key: shared + new_cluster: + policy_id: ${resources.cluster_policies.policy.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + custom_tags: + PolicyTag: from-policy + - job_cluster_key: ungated + new_cluster: + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + tasks: + - task_key: shared_task + job_cluster_key: shared + spark_python_task: + python_file: ./hello.py + - task_key: ungated_task + job_cluster_key: ungated + spark_python_task: + python_file: ./hello.py + - task_key: own_cluster + new_cluster: + policy_id: ${resources.cluster_policies.policy.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + custom_tags: + PolicyTag: from-policy + spark_python_task: + python_file: ./hello.py + - task_key: loop + for_each_task: + inputs: "[1]" + task: + task_key: nested_cluster + new_cluster: + policy_id: ${resources.cluster_policies.policy.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + custom_tags: + PolicyTag: from-policy + spark_python_task: + python_file: ./hello.py diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/hello.py b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/hello.py new file mode 100644 index 00000000000..8cb8f7a2d91 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/hello.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python3 + +print("hello") diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/out.test.toml b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/out.test.toml new file mode 100644 index 00000000000..0db72c1b195 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt new file mode 100644 index 00000000000..24d5f4a3184 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt @@ -0,0 +1,49 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created cluster_policies.policy +Created jobs.test_job +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle config-remote-sync +No changes detected. + + +=== Sync remote user edits unrelated to the attached policy + +>>> [CLI] bundle config-remote-sync --save +Detected changes in 1 resource(s): + +Resource: resources.jobs.test_job + job_clusters[job_cluster_key='shared'].new_cluster.num_workers: replace + job_clusters[job_cluster_key='shared'].new_cluster.spark_conf: add + job_clusters[job_cluster_key='ungated'].new_cluster.num_workers: replace + job_clusters[job_cluster_key='ungated'].new_cluster.spark_conf: add + max_concurrent_runs: replace + tasks[task_key='loop'].for_each_task.task.new_cluster.num_workers: replace + tasks[task_key='loop'].for_each_task.task.new_cluster.spark_conf: add + tasks[task_key='own_cluster'].new_cluster.num_workers: replace + tasks[task_key='own_cluster'].new_cluster.spark_conf: add + + + +>>> [CLI] bundle validate -o json +json.resources.jobs.test_job.job_clusters[0].new_cluster.num_workers = 2; +json.resources.jobs.test_job.job_clusters[0].new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; +json.resources.jobs.test_job.job_clusters[1].new_cluster.num_workers = 2; +json.resources.jobs.test_job.job_clusters[1].new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; +json.resources.jobs.test_job.max_concurrent_runs = 5; +json.resources.jobs.test_job.tasks[0].for_each_task.task.new_cluster.num_workers = 2; +json.resources.jobs.test_job.tasks[0].for_each_task.task.new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; +json.resources.jobs.test_job.tasks[1].new_cluster.num_workers = 2; +json.resources.jobs.test_job.tasks[1].new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.policy + delete resources.jobs.test_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script new file mode 100644 index 00000000000..70f04d2e8f1 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script @@ -0,0 +1,29 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +job_id=$(read_id.py test_job) + +trace $CLI bundle config-remote-sync | contains.py "No changes detected" + +title "Sync remote user edits unrelated to the attached policy\n" +edit_resource.py jobs "$job_id" <<'EOF' +r["max_concurrent_runs"] = 5 +clusters = [jc["new_cluster"] for jc in r["job_clusters"]] +for task in r["tasks"]: + if "new_cluster" in task: + clusters.append(task["new_cluster"]) + if "for_each_task" in task: + clusters.append(task["for_each_task"]["task"]["new_cluster"]) +for cluster in clusters: + cluster["spark_conf"] = {"spark.sql.shuffle.partitions": "7"} + cluster["num_workers"] = 2 +EOF + +trace $CLI bundle config-remote-sync --save +# The snapshot must include the user addition for every cluster, including those with a policy. +trace $CLI bundle validate -o json | gron.py | grep -E 'max_concurrent_runs|num_workers|spark.sql.shuffle.partitions' diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml new file mode 100644 index 00000000000..119189805fd --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml @@ -0,0 +1,4 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = false +Ignore = [".databricks", "databricks.yml"] From 86da9c2601c31cc9e7f7e4a1e976c23b15587e84 Mon Sep 17 00:00:00 2001 From: Ilya Kuznetsov Date: Fri, 11 Sep 2026 12:03:27 +0000 Subject: [PATCH 2/2] test: simplify policy sync case and show YAML diff --- .../databricks.yml.tmpl | 41 +----------------- .../cluster_policy_remote_addition/output.txt | 42 +++++++------------ .../cluster_policy_remote_addition/script | 19 ++------- .../cluster_policy_remote_addition/test.toml | 2 +- 4 files changed, 21 insertions(+), 83 deletions(-) diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl index b0f61018ee4..346c8366609 100644 --- a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl @@ -9,38 +9,15 @@ resources: policy: name: test-policy-$UNIQUE_NAME definition: - # This rule controls only PolicyTag, not the Spark setting edited below. + # The policy does not constrain spark_conf. custom_tags.PolicyTag: type: fixed value: from-policy jobs: test_job: name: test-job-$UNIQUE_NAME - max_concurrent_runs: 1 - job_clusters: - - job_cluster_key: shared - new_cluster: - policy_id: ${resources.cluster_policies.policy.id} - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 - custom_tags: - PolicyTag: from-policy - - job_cluster_key: ungated - new_cluster: - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 tasks: - - task_key: shared_task - job_cluster_key: shared - spark_python_task: - python_file: ./hello.py - - task_key: ungated_task - job_cluster_key: ungated - spark_python_task: - python_file: ./hello.py - - task_key: own_cluster + - task_key: test new_cluster: policy_id: ${resources.cluster_policies.policy.id} spark_version: $DEFAULT_SPARK_VERSION @@ -50,17 +27,3 @@ resources: PolicyTag: from-policy spark_python_task: python_file: ./hello.py - - task_key: loop - for_each_task: - inputs: "[1]" - task: - task_key: nested_cluster - new_cluster: - policy_id: ${resources.cluster_policies.policy.id} - spark_version: $DEFAULT_SPARK_VERSION - node_type_id: $NODE_TYPE_ID - num_workers: 1 - custom_tags: - PolicyTag: from-policy - spark_python_task: - python_file: ./hello.py diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt index 24d5f4a3184..f2756c75248 100644 --- a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt @@ -6,38 +6,24 @@ Created jobs.test_job Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> [CLI] bundle config-remote-sync -No changes detected. - - -=== Sync remote user edits unrelated to the attached policy - >>> [CLI] bundle config-remote-sync --save Detected changes in 1 resource(s): Resource: resources.jobs.test_job - job_clusters[job_cluster_key='shared'].new_cluster.num_workers: replace - job_clusters[job_cluster_key='shared'].new_cluster.spark_conf: add - job_clusters[job_cluster_key='ungated'].new_cluster.num_workers: replace - job_clusters[job_cluster_key='ungated'].new_cluster.spark_conf: add - max_concurrent_runs: replace - tasks[task_key='loop'].for_each_task.task.new_cluster.num_workers: replace - tasks[task_key='loop'].for_each_task.task.new_cluster.spark_conf: add - tasks[task_key='own_cluster'].new_cluster.num_workers: replace - tasks[task_key='own_cluster'].new_cluster.spark_conf: add - - - ->>> [CLI] bundle validate -o json -json.resources.jobs.test_job.job_clusters[0].new_cluster.num_workers = 2; -json.resources.jobs.test_job.job_clusters[0].new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; -json.resources.jobs.test_job.job_clusters[1].new_cluster.num_workers = 2; -json.resources.jobs.test_job.job_clusters[1].new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; -json.resources.jobs.test_job.max_concurrent_runs = 5; -json.resources.jobs.test_job.tasks[0].for_each_task.task.new_cluster.num_workers = 2; -json.resources.jobs.test_job.tasks[0].for_each_task.task.new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; -json.resources.jobs.test_job.tasks[1].new_cluster.num_workers = 2; -json.resources.jobs.test_job.tasks[1].new_cluster.spark_conf.spark.sql.shuffle.partitions = "7"; + tasks[task_key='test'].new_cluster.spark_conf: add + + + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -26,4 +26,6 @@ + custom_tags: + PolicyTag: from-policy ++ spark_conf: ++ spark.sql.shuffle.partitions: "7" + spark_python_task: + python_file: ./hello.py >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script index 70f04d2e8f1..afe89d9a834 100644 --- a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script @@ -8,22 +8,11 @@ trap cleanup EXIT trace $CLI bundle deploy job_id=$(read_id.py test_job) -trace $CLI bundle config-remote-sync | contains.py "No changes detected" - -title "Sync remote user edits unrelated to the attached policy\n" edit_resource.py jobs "$job_id" <<'EOF' -r["max_concurrent_runs"] = 5 -clusters = [jc["new_cluster"] for jc in r["job_clusters"]] -for task in r["tasks"]: - if "new_cluster" in task: - clusters.append(task["new_cluster"]) - if "for_each_task" in task: - clusters.append(task["for_each_task"]["task"]["new_cluster"]) -for cluster in clusters: - cluster["spark_conf"] = {"spark.sql.shuffle.partitions": "7"} - cluster["num_workers"] = 2 +r["tasks"][0]["new_cluster"]["spark_conf"] = {"spark.sql.shuffle.partitions": "7"} EOF +cp databricks.yml databricks.yml.backup trace $CLI bundle config-remote-sync --save -# The snapshot must include the user addition for every cluster, including those with a policy. -trace $CLI bundle validate -o json | gron.py | grep -E 'max_concurrent_runs|num_workers|spark.sql.shuffle.partitions' +trace diff.py databricks.yml.backup databricks.yml +rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml index 119189805fd..b13ee63c9a2 100644 --- a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml @@ -1,4 +1,4 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = false -Ignore = [".databricks", "databricks.yml"] +Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]