Do not report cluster fields supplied by a cluster policy as drift

.nextchanges/bundles/cluster-policy-no-drift.md

@@ -0,0 +1 @@
+* direct: `bundle plan` no longer reports a permanent update on a cluster that uses a cluster policy: when the cluster spec sets `policy_id`, a field present in the remote but absent from the bundle config is not treated as drift. ([#6531](https://github.com/databricks/cli/pull/6531))

acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl

@@ -0,0 +1,27 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  cluster_policies:
+    policy:
+      name: test-policy-$UNIQUE_NAME
+      definition:
+        # The policy does not constrain spark_conf.
+        custom_tags.PolicyTag:
+          type: fixed
+          value: from-policy
+  jobs:
+    test_job:
+      name: test-job-$UNIQUE_NAME
+      tasks:
+        - task_key: test
+          new_cluster:
+            policy_id: ${resources.cluster_policies.policy.id}
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+          spark_python_task:
+            python_file: ./hello.py

acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/hello.py

@@ -0,0 +1,3 @@
+#!/usr/bin/env python3
+
+print("hello")

acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt

@@ -0,0 +1,35 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Created cluster_policies.policy
+Created jobs.test_job
+Files: 5 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> [CLI] bundle config-remote-sync --save
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.test_job
+  tasks[task_key='test'].new_cluster.spark_conf: add
+
+
+
+>>> diff.py databricks.yml.backup databricks.yml
+--- databricks.yml.backup
++++ databricks.yml
+@@ -24,4 +24,6 @@
+             node_type_id: [NODE_TYPE_ID]
+             num_workers: 1
++            spark_conf:
++              spark.sql.shuffle.partitions: "7"
+           spark_python_task:
+             python_file: ./hello.py
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.policy
+  delete resources.jobs.test_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 2 deleted

acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script

@@ -0,0 +1,18 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+job_id=$(read_id.py test_job)
+
+edit_resource.py jobs "$job_id" <<'EOF'
+r["tasks"][0]["new_cluster"]["spark_conf"] = {"spark.sql.shuffle.partitions": "7"}
+EOF
+
+cp databricks.yml databricks.yml.backup
+trace $CLI bundle config-remote-sync --save
+trace diff.py databricks.yml.backup databricks.yml
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml

@@ -0,0 +1,4 @@
+Cloud = true
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+RecordRequests = false
+Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]

acceptance/bundle/resources/clusters/cluster_policy/databricks.yml.tmpl

@@ -0,0 +1,27 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  cluster_policies:
+    my_policy:
+      name: test-policy-$UNIQUE_NAME
+      definition:
+        # Only a map key: the clusters resource force-sends spark_version, so a scalar the
+        # config omits arrives as "" and a fixed policy rejects it. The scalar-supply case is
+        # covered by fixed_addition, whose job-cluster new_cluster fields are omitempty.
+        custom_tags.CostCenter:
+          type: fixed
+          value: from-policy
+
+  clusters:
+    my_cluster:
+      cluster_name: test-cluster-$UNIQUE_NAME
+      spark_version: $DEFAULT_SPARK_VERSION
+      node_type_id: $NODE_TYPE_ID
+      # Shared pool avoids a cold boot; the policy tag under test is unaffected.
+      instance_pool_id: $TEST_INSTANCE_POOL_ID
+      num_workers: 1
+      policy_id: ${resources.cluster_policies.my_policy.id}

acceptance/bundle/resources/clusters/cluster_policy/output.txt

@@ -0,0 +1,34 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Created cluster_policies.my_policy
+Created clusters.my_cluster
+Files: 4 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== A cluster policy tag on a standalone cluster is not drift
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+=== ...classified remote_addition
+
+>>> [CLI] bundle plan -o json
+{
+  "custom_tags": {
+    "action": "skip",
+    "reason": "remote_addition",
+    "remote": {
+      "CostCenter": "from-policy"
+    }
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.my_policy
+  delete resources.clusters.my_cluster
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 2 deleted

acceptance/bundle/resources/clusters/cluster_policy/script

@@ -0,0 +1,15 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+
+title "A cluster policy tag on a standalone cluster is not drift\n"
+trace $CLI bundle plan
+
+title "...classified remote_addition\n"
+trace $CLI bundle plan -o json | jq -S '.plan["resources.clusters.my_cluster"].changes
+  | with_entries(select(.key | test("custom_tags")))'

acceptance/bundle/resources/clusters/cluster_policy/test.toml

@@ -0,0 +1,12 @@
+# The standalone clusters resource boots a real cluster at deploy, which is slow and has hit
+# capacity limits on cloud; CloudSlow keeps it out of the per-PR run (it still runs locally and
+# in the full cloud sweep). Output is projected to the policy tag so it is cloud-portable.
+Cloud = true
+CloudSlow = true
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+RecordRequests = false
+
+Ignore = [
+    "databricks.yml",
+]

acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/databricks.yml.tmpl

@@ -0,0 +1,33 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  cluster_policies:
+    my_policy:
+      name: test-policy-$UNIQUE_NAME
+      definition:
+        custom_tags.CostCenter:
+          type: fixed
+          value: from-policy
+
+  jobs:
+    my_job:
+      name: test-job-$UNIQUE_NAME
+      job_clusters:
+        # The config sets the fixed attribute to a value the policy forbids.
+        - job_cluster_key: main
+          new_cluster:
+            policy_id: ${resources.cluster_policies.my_policy.id}
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+            custom_tags:
+              CostCenter: not-what-the-policy-says
+      tasks:
+        - task_key: main
+          job_cluster_key: main
+          spark_python_task:
+            python_file: ./hello_world.py

acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/hello_world.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/output.txt

@@ -0,0 +1,21 @@
+
+=== A cluster tag contradicting a fixed policy value is rejected at create
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Error: cannot create resources.jobs.my_job: Cluster validation error: Validation failed for custom_tags, CostCenter must be from-policy (is "not-what-the-policy-says") (400 INVALID_PARAMETER_VALUE)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create
+HTTP Status: 400 Bad Request
+API error_code: INVALID_PARAMETER_VALUE
+API message: Cluster validation error: Validation failed for custom_tags, CostCenter must be from-policy (is "not-what-the-policy-says")
+
+Files: 4 uploaded, 0 deleted
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.my_policy
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 1 deleted

acceptance/bundle/resources/jobs/cluster_policy/config_conflicts_policy/script

@@ -0,0 +1,9 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+title "A cluster tag contradicting a fixed policy value is rejected at create\n"
+musterr trace $CLI bundle deploy

acceptance/bundle/resources/jobs/cluster_policy/default_flag/databricks.yml.tmpl

@@ -0,0 +1,40 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  cluster_policies:
+    my_policy:
+      name: test-policy-$UNIQUE_NAME
+      definition:
+        # A defaultValue element: the backend fills it into an omitted attribute only when
+        # the request sets apply_policy_default_values (unlike a fixed element).
+        custom_tags.DefaultTag:
+          type: unlimited
+          defaultValue: from-default
+          isOptional: true
+
+  jobs:
+    my_job:
+      name: test-job-$UNIQUE_NAME
+      job_clusters:
+        - job_cluster_key: flag_off
+          new_cluster:
+            policy_id: ${resources.cluster_policies.my_policy.id}
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+        - job_cluster_key: flag_on
+          new_cluster:
+            policy_id: ${resources.cluster_policies.my_policy.id}
+            apply_policy_default_values: true
+            spark_version: $DEFAULT_SPARK_VERSION
+            node_type_id: $NODE_TYPE_ID
+            num_workers: 1
+      tasks:
+        - task_key: main
+          job_cluster_key: flag_off
+          spark_python_task:
+            python_file: ./hello_world.py

acceptance/bundle/resources/jobs/cluster_policy/default_flag/hello_world.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/resources/jobs/cluster_policy/default_flag/output.txt

@@ -0,0 +1,35 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Created cluster_policies.my_policy
+Created jobs.my_job
+Files: 4 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+=== defaultValue is supplied only with apply_policy_default_values (flag_on), not flag_off
+[
+  {
+    "custom_tags": null,
+    "job_cluster_key": "flag_off"
+  },
+  {
+    "custom_tags": {
+      "DefaultTag": "from-default"
+    },
+    "job_cluster_key": "flag_on"
+  }
+]
+
+=== Neither is drift on a second plan
+
+>>> [CLI] bundle plan
+Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.cluster_policies.my_policy
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Destroy: 2 deleted

acceptance/bundle/resources/jobs/cluster_policy/default_flag/script

@@ -0,0 +1,16 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+job_id=$(read_id.py my_job)
+
+title "defaultValue is supplied only with apply_policy_default_values (flag_on), not flag_off\n"
+$CLI jobs get "$job_id" | jq -S '[.settings.job_clusters[]
+  | {job_cluster_key, custom_tags: .new_cluster.custom_tags}]'
+
+title "Neither is drift on a second plan\n"
+trace $CLI bundle plan