diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl new file mode 100644 index 00000000000..346c8366609 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/databricks.yml.tmpl @@ -0,0 +1,29 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + cluster_policies: + policy: + name: test-policy-$UNIQUE_NAME + definition: + # The policy does not constrain spark_conf. + custom_tags.PolicyTag: + type: fixed + value: from-policy + jobs: + test_job: + name: test-job-$UNIQUE_NAME + tasks: + - task_key: test + new_cluster: + policy_id: ${resources.cluster_policies.policy.id} + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 1 + custom_tags: + PolicyTag: from-policy + spark_python_task: + python_file: ./hello.py diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/hello.py b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/hello.py new file mode 100644 index 00000000000..8cb8f7a2d91 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/hello.py @@ -0,0 +1,3 @@ +#!/usr/bin/env python3 + +print("hello") diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/out.test.toml b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/out.test.toml new file mode 100644 index 00000000000..0db72c1b195 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/out.test.toml @@ -0,0 +1,4 @@ +Cloud = true +GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt new file mode 100644 index 00000000000..f2756c75248 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Created cluster_policies.policy +Created jobs.test_job +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle config-remote-sync --save +Detected changes in 1 resource(s): + +Resource: resources.jobs.test_job + tasks[task_key='test'].new_cluster.spark_conf: add + + + +>>> diff.py databricks.yml.backup databricks.yml +--- databricks.yml.backup ++++ databricks.yml +@@ -26,4 +26,6 @@ + custom_tags: + PolicyTag: from-policy ++ spark_conf: ++ spark.sql.shuffle.partitions: "7" + spark_python_task: + python_file: ./hello.py + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.cluster_policies.policy + delete resources.jobs.test_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Destroy: 2 deleted diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script new file mode 100644 index 00000000000..afe89d9a834 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +job_id=$(read_id.py test_job) + +edit_resource.py jobs "$job_id" <<'EOF' +r["tasks"][0]["new_cluster"]["spark_conf"] = {"spark.sql.shuffle.partitions": "7"} +EOF + +cp databricks.yml databricks.yml.backup +trace $CLI bundle config-remote-sync --save +trace diff.py databricks.yml.backup databricks.yml +rm databricks.yml.backup diff --git a/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml new file mode 100644 index 00000000000..b13ee63c9a2 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/cluster_policy_remote_addition/test.toml @@ -0,0 +1,4 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = false +Ignore = [".databricks", "databricks.yml", "databricks.yml.backup"]