diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md new file mode 100644 index 00000000000..2357fefcb61 --- /dev/null +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -0,0 +1 @@ +* direct: resources.job\_runs: new lifecycle.triggers.on\_file\_change setting to restart the run when monitored files change. Can be set to a series of paths or globs. ([#6309](https://github.com/databricks/cli/pull/6309)) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 158fcc5d1c0..411d7b29e49 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -925,11 +925,14 @@ resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState STATE resources.job_runs.*.lifecycle *resources.JobRunLifecycle INPUT resources.job_runs.*.lifecycle resources.Lifecycle INPUT resources.job_runs.*.lifecycle.prevent_destroy bool INPUT -resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT -resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT +resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT +resources.job_runs.*.lifecycle.triggers_state *resources.JobRunTriggersState INPUT STATE +resources.job_runs.*.lifecycle.triggers_state.on_bundle_deploy string INPUT STATE +resources.job_runs.*.lifecycle.triggers_state.on_file_change map[string]string INPUT STATE +resources.job_runs.*.lifecycle.triggers_state.on_file_change.* string INPUT STATE resources.job_runs.*.modified_status string INPUT resources.job_runs.*.notebook_params map[string]string ALL resources.job_runs.*.notebook_params.* string ALL diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt index 39123f3370f..f69c5934d83 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -30,21 +30,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { - "action": "recreate", - "reason": "immutable", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - }, - "new": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } - }, - "lifecycle.triggers.on_bundle_deploy": { + "lifecycle.triggers_state.on_bundle_deploy": { "action": "recreate", "reason": "immutable", "old": "[UUID]", @@ -108,15 +94,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { - "action": "skip", - "reason": "trigger removed", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } - }, "result_state": { "action": "skip", "reason": "remote_already_set", diff --git a/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml new file mode 100644 index 00000000000..0a6df75047a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: seed.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.plan.create.json b/acceptance/bundle/resources/job_runs/on_file_change/out.plan.create.json new file mode 100644 index 00000000000..8fb3fcdfd59 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.plan.create.json @@ -0,0 +1,57 @@ +{ + "plan_version": [PLAN_VERSION], + "cli_version": "[CLI_VERSION]", + "plan": { + "resources.job_runs.my_run": { + "depends_on": [ + { + "node": "resources.jobs.my_job", + "label": "${resources.jobs.my_job.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "job_id": 0, + "lifecycle": { + "triggers_state": { + "on_file_change": { + "seed.txt": "[FILE_HASH]" + } + } + }, + "result_state": "SUCCESS" + }, + "vars": { + "job_id": "${resources.jobs.my_job.id}" + } + } + }, + "resources.jobs.my_job": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "my-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/test" + }, + "task_key": "main" + } + ] + } + } + } + } +} diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml new file mode 100644 index 00000000000..dd03baaabb6 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -0,0 +1,4 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt new file mode 100644 index 00000000000..37e4b63a2bc --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -0,0 +1,264 @@ + +=== first deploy triggers a run +>>> [CLI] bundle plan -o json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 8 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== redeploy with unchanged file plans nothing +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== touch without content change plans nothing +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== editing the file re-fires +>>> jq .plan["resources.job_runs.my_run"] tmp.plan1.json +{ + "depends_on": [ + { + "node": "resources.jobs.my_job", + "label": "${resources.jobs.my_job.id}" + } + ], + "action": "recreate", + "new_state": { + "value": { + "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers_state": { + "on_file_change": { + "seed.txt": "[FILE_HASH][0]" + } + } + }, + "result_state": "SUCCESS" + } + }, + "remote_state": { + "job_id": [MY_JOB_ID], + "result_state": "SUCCESS", + "run_id": [MY_RUN_ID], + "run_name": "my-job", + "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", + "run_type": "JOB_RUN", + "state": { + "life_cycle_state": "TERMINATED", + "result_state": "SUCCESS" + } + }, + "changes": { + "lifecycle.triggers_state.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "[FILE_HASH][0]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } + } +} + +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== deleting the file warns and re-fires +>>> [CLI] bundle plan -o json +Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" + in databricks.yml:18:29 + + +>>> jq .plan["resources.job_runs.my_run"] tmp.plan2.json +{ + "depends_on": [ + { + "node": "resources.jobs.my_job", + "label": "${resources.jobs.my_job.id}" + } + ], + "action": "recreate", + "new_state": { + "value": { + "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers_state": { + "on_file_change": { + "seed.txt": "[FILE_HASH][2]" + } + } + }, + "result_state": "SUCCESS" + } + }, + "remote_state": { + "job_id": [MY_JOB_ID], + "result_state": "SUCCESS", + "run_id": [MY_RUN_ID_2], + "run_name": "my-job", + "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID]", + "run_type": "JOB_RUN", + "state": { + "life_cycle_state": "TERMINATED", + "result_state": "SUCCESS" + } + }, + "changes": { + "lifecycle.triggers_state.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][2]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } + } +} +Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" + in databricks.yml:18:29 + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +>>> [CLI] bundle plan +Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" + in databricks.yml:18:29 + +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== restoring the file re-fires again +>>> [CLI] bundle plan -o json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== removing on_file_change is unchanged and does not run +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 4 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== re-adding on_file_change does not re-fire an unchanged file +>>> print_state.py +{ + "triggers_state": { + "on_file_change": { + "seed.txt": "[FILE_HASH][0]" + } + } +} + +>>> [CLI] bundle plan -o json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 4 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script new file mode 100644 index 00000000000..53758e64a30 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -0,0 +1,83 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +plan_no_changes() { + trace $CLI bundle plan | contains.py "0 to add, 0 to change" +} + +title "first deploy triggers a run" +# Deploy reads the raw plan so it keeps the DMS version_id it needs; the committed +# golden is that plan with the DMS stamp stripped, so it matches across DMS variants. +trace $CLI bundle plan -o json > tmp.plan.create.json +nostamp < tmp.plan.create.json > out.plan.create.json +$CLI bundle deploy $(readplanarg tmp.plan.create.json) +trace read_id.py my_job +# Name the first run so the recreated one becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "redeploy with unchanged file plans nothing" +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "touch without content change plans nothing" +python -c "import os; os.utime('seed.txt', None)" +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "editing the file re-fires" +update_file.py seed.txt "v1" "v2" +$CLI bundle plan -o json > tmp.plan1.json +trace jq '.plan["resources.job_runs.my_run"]' tmp.plan1.json +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan1.json) +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now +plan_no_changes + +title "deleting the file warns and re-fires" +mv seed.txt tmp.seed.txt +trace $CLI bundle plan -o json > tmp.plan2.json +trace jq '.plan["resources.job_runs.my_run"]' tmp.plan2.json +$CLI bundle deploy $(readplanarg tmp.plan2.json) +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now +plan_no_changes + +title "restoring the file re-fires again" +mv tmp.seed.txt seed.txt +trace $CLI bundle plan -o json > tmp.plan3.json +$CLI bundle deploy $(readplanarg tmp.plan3.json) +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now +plan_no_changes + +title "removing on_file_change is unchanged and does not run" +update_file.py databricks.yml " lifecycle: + triggers: + - on_file_change: seed.txt +" "" +trace $CLI bundle plan +$CLI bundle plan -o json > tmp.plan4.json +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan4.json) +trace print_requests.py //jobs/run-now +plan_no_changes + +title "re-adding on_file_change does not re-fire an unchanged file" +# Removing the trigger left the last hash in state, so re-adding it compares +# against that and only re-fires when the file changed while disarmed. +trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle' +update_file.py databricks.yml ' job_id: ${resources.jobs.my_job.id} +' ' job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: seed.txt +' +trace $CLI bundle plan -o json > tmp.plan5.json +$CLI bundle deploy $(readplanarg tmp.plan5.json) +plan_no_changes diff --git a/acceptance/bundle/resources/job_runs/on_file_change/seed.txt b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt @@ -0,0 +1 @@ +v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change/test.toml b/acceptance/bundle/resources/job_runs/on_file_change/test.toml new file mode 100644 index 00000000000..f5010a202d7 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/test.toml @@ -0,0 +1,14 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.*.*"] + +# Content hashes are stable but unreadable, and the parent's `\d{8,}` rule +# mangles any digit run inside them. Order=1 runs before it; Distinct keeps +# different hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_build/databricks.yml new file mode 100644 index 00000000000..ddc7a4d8896 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: job-runs-on-file-change-build + +experimental: + scripts: + prebuild: cp source.txt watched.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_build/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt new file mode 100644 index 00000000000..b674e4063a7 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt @@ -0,0 +1,53 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Executing 'prebuild' script +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-build/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== a prebuild file change re-fires in the same deploy +>>> [CLI] bundle deploy +Executing 'prebuild' script +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-build/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-build/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/script b/acceptance/bundle/resources/job_runs/on_file_change_build/script new file mode 100644 index 00000000000..e350d2ba6e6 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/script @@ -0,0 +1,18 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +# Name the first run so the recreated one becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "a prebuild file change re-fires in the same deploy" +update_file.py source.txt "v1" "v2" +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/source.txt b/acceptance/bundle/resources/job_runs/on_file_change_build/source.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/source.txt @@ -0,0 +1 @@ +v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_build/watched.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/watched.txt @@ -0,0 +1 @@ +v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml new file mode 100644 index 00000000000..e0d709d3fac --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-errors + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: /tmp/seed.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt new file mode 100644 index 00000000000..78981922613 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt @@ -0,0 +1 @@ +a diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml new file mode 100644 index 00000000000..0ce9819887c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml @@ -0,0 +1,24 @@ +bundle: + name: job-runs-on-file-change-errors + +sync: + exclude: + - mixed/excluded.txt + # Literal path: the script rewrites the trigger by substring, and allexcluded/* would hit this line. + - allexcluded/a.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: ../shared/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt new file mode 100644 index 00000000000..79c53955ef8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt @@ -0,0 +1 @@ +nested diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt new file mode 100644 index 00000000000..bbde3dc9de9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt @@ -0,0 +1 @@ +excluded diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt new file mode 100644 index 00000000000..2fa992c0b8b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt @@ -0,0 +1 @@ +keep diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt new file mode 100644 index 00000000000..73f170fe8e9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -0,0 +1,91 @@ + +=== a pattern that escapes the sync root is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob that only matches directories warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: no synced files match "dirs/*" + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning + +=== a glob matching a directory and an excluded file warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: no synced files match "mixed/*" + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning + +=== a glob matching only excluded files warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: no synced files match "allexcluded/*" + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning + +=== a pattern that matches nothing warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: no synced files match "missing/*.txt" + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning + +=== ** is not supported +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: ** in "src/**" is not supported; use * for a single directory level + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== an absolute pattern is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "/tmp/seed.txt" must be relative to the defining YAML file + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script new file mode 100644 index 00000000000..faaa5de8d3c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -0,0 +1,31 @@ +title "a pattern that escapes the sync root is rejected" +trace musterr $CLI bundle validate + +title "a glob that only matches directories warns" +# dirs/* matches dirs/nested, a directory, so nothing is left to hash. +update_file.py databricks.yml "../shared/*.txt" "dirs/*" +trace $CLI bundle validate + +title "a glob matching a directory and an excluded file warns" +# mixed/excluded.txt is dropped by sync.exclude; mixed/nested is a directory. +update_file.py databricks.yml "dirs/*" "mixed/*" +trace $CLI bundle validate + +title "a glob matching only excluded files warns" +# Every match is dropped by sync.exclude. +update_file.py databricks.yml "mixed/*" "allexcluded/*" +trace $CLI bundle validate + +title "a pattern that matches nothing warns" +update_file.py databricks.yml "allexcluded/*" "missing/*.txt" +trace $CLI bundle validate + +title "** is not supported" +update_file.py databricks.yml "missing/*.txt" "src/**" +trace musterr $CLI bundle validate + +title "an absolute pattern is rejected" +# Copied from a committed file rather than passed to update_file.py: Git Bash on +# Windows rewrites a leading-slash argument into a Windows path. +cp absolute_pattern.yml databricks.yml +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml new file mode 100644 index 00000000000..7b88d613d4c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml @@ -0,0 +1,2 @@ +# The bundle is never deployed here, so there are no requests worth recording. +RecordRequests = false diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml new file mode 100644 index 00000000000..d47fcf44049 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: job-runs-on-file-change-glob + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: migrations/* + - on_file_change: mysubdir/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore new file mode 100644 index 00000000000..57433d43f47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore @@ -0,0 +1,2 @@ +# Matches migrations/* but sync skips it, so a content edit must not re-fire. +migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt new file mode 100644 index 00000000000..5626abf0f72 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt @@ -0,0 +1 @@ +one diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt new file mode 100644 index 00000000000..f719efd430d --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt @@ -0,0 +1 @@ +two diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt new file mode 100644 index 00000000000..2bdf67abb16 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt @@ -0,0 +1 @@ +three diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt new file mode 100644 index 00000000000..ea10ec85c10 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt @@ -0,0 +1 @@ +ignored diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt new file mode 100644 index 00000000000..62e0af52c19 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt @@ -0,0 +1 @@ +sub diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml new file mode 100644 index 00000000000..dd03baaabb6 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -0,0 +1,4 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt new file mode 100644 index 00000000000..5349b686287 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -0,0 +1,158 @@ + +=== deploy hashes the matched files +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 10 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== editing one matched file re-fires + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle.triggers_state.on_file_change['migrations/*']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== editing a file in a subdirectory re-fires + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle.triggers_state.on_file_change['mysubdir/*.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][2]", + "new": "[FILE_HASH][3]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle.triggers_state.on_file_change['migrations/*']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "[FILE_HASH][4]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== removing a matched file re-fires + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle.triggers_state.on_file_change['migrations/*']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "[FILE_HASH][5]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a file below the glob does not re-fire + +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a gitignored file does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== removing one pattern does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Files: 7 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script new file mode 100644 index 00000000000..324efdb49a9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -0,0 +1,67 @@ +# Fixture is named dot_gitignore so a real .gitignore does not affect git while developing. +mv dot_gitignore .gitignore + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. +changes() { + $CLI bundle plan -o json > tmp.plan.json + trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +} + +title "deploy hashes the matched files" +trace $CLI bundle deploy +trace read_id.py my_job +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "editing one matched file re-fires\n" +update_file.py migrations/001.txt "one" "one-edited" +changes +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "editing a file in a subdirectory re-fires\n" +# printf keeps LF on Windows, where update_file.py's text-mode rewrite would +# restore CRLF and leave the file hashing differently than the deploy above. +printf 'sub-edited\n' > mysubdir/a.txt +changes +printf 'sub\n' > mysubdir/a.txt + +title "a new matching file re-fires\n" +printf 'three\n' > migrations/003.txt +changes +rm migrations/003.txt + +title "removing a matched file re-fires\n" +mv migrations/002.txt tmp.002.txt +changes +mv tmp.002.txt migrations/002.txt + +title "editing a file below the glob does not re-fire\n" +# migrations/* matches the archive directory itself, which is skipped: the glob +# does not descend into it. +update_file.py migrations/archive/003.txt "three" "three-edited" +changes +update_file.py migrations/archive/003.txt "three-edited" "three" + +title "editing a gitignored file does not re-fire" +update_file.py migrations/ignored.txt "ignored" "ignored-edited" +trace $CLI bundle plan +update_file.py migrations/ignored.txt "ignored-edited" "ignored" + +title "removing one pattern does not re-fire" +cp databricks.yml tmp.databricks.yml +update_file.py databricks.yml " - on_file_change: mysubdir/*.txt" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now +mv tmp.databricks.yml databricks.yml diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml new file mode 100644 index 00000000000..45b345e15a7 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -0,0 +1,13 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.002.txt", "tmp.databricks.yml", ".gitignore"] + +# See on_file_change/test.toml: mask content hashes before the parent's +# `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml new file mode 100644 index 00000000000..2bdd1c06d13 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: job-runs-on-file-change-include-sync-root + +include: + - resources/job_run.yml + +sync: + paths: + - . + - ../shared.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml new file mode 100644 index 00000000000..f6761be7e15 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml @@ -0,0 +1,9 @@ +resources: + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched-resources.txt + - on_file_change: ../watched-bundle.txt + - on_file_change: ../../shared.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched-resources.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched-resources.txt new file mode 100644 index 00000000000..8b4e79d3c47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched-resources.txt @@ -0,0 +1 @@ +include-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched-bundle.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched-bundle.txt new file mode 100644 index 00000000000..6f22941eacf --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched-bundle.txt @@ -0,0 +1 @@ +bundle-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt new file mode 100644 index 00000000000..015ce22241c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt @@ -0,0 +1,77 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include-sync-root/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> file_triggers +{ + "bundle/resources/watched-resources.txt": "[FILE_HASH][0]", + "bundle/watched-bundle.txt": "[FILE_HASH][1]", + "shared.txt": "[FILE_HASH][2]" +} + +=== editing the file next to the included YAML re-fires +>>> changes +{ + "lifecycle.triggers_state.on_file_change['bundle/resources/watched-resources.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][3]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing the file at the bundle root re-fires +>>> changes +{ + "lifecycle.triggers_state.on_file_change['bundle/watched-bundle.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "[FILE_HASH][4]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing the file at the sync root re-fires +>>> changes +{ + "lifecycle.triggers_state.on_file_change['shared.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][2]", + "new": "[FILE_HASH][5]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include-sync-root/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script new file mode 100644 index 00000000000..7f7934d00cf --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script @@ -0,0 +1,32 @@ +cleanup() { + withdir bundle trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +changes() { + withdir bundle $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + +file_triggers() { + withdir bundle print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle.triggers_state.on_file_change' +} + +title "first deploy" +withdir bundle trace $CLI bundle deploy +trace file_triggers + +title "editing the file next to the included YAML re-fires" +# printf keeps LF on Windows, where update_file.py's text-mode rewrite uses CRLF. +printf 'include-v2\n' > bundle/resources/watched-resources.txt +trace changes +printf 'include-v1\n' > bundle/resources/watched-resources.txt + +title "editing the file at the bundle root re-fires" +printf 'bundle-v2\n' > bundle/watched-bundle.txt +trace changes +printf 'bundle-v1\n' > bundle/watched-bundle.txt + +title "editing the file at the sync root re-fires" +printf 'parent-v2\n' > shared.txt +trace changes +printf 'parent-v1\n' > shared.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt new file mode 100644 index 00000000000..4ddcc71c374 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt @@ -0,0 +1 @@ +parent-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml new file mode 100644 index 00000000000..b0362c07bc0 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml @@ -0,0 +1,9 @@ +RecordRequests = false +Ignore = [".databricks"] + +# Mask hashes before the parent's `\d{8,}` rule; Distinct keeps them distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/databricks.yml new file mode 100644 index 00000000000..ca80131494b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: job-runs-on-file-change-predeploy + +experimental: + scripts: + predeploy: cp source.txt watched.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt new file mode 100644 index 00000000000..4778b7e19d7 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt @@ -0,0 +1,53 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Executing 'predeploy' script +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-predeploy/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== a predeploy file change re-fires in the same deploy +>>> [CLI] bundle deploy +Executing 'predeploy' script +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-predeploy/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-predeploy/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/script b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/script new file mode 100644 index 00000000000..80661900eab --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/script @@ -0,0 +1,18 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +# Name the first run so the recreated one becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "a predeploy file change re-fires in the same deploy" +update_file.py source.txt "v1" "v2" +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/source.txt b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/source.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/source.txt @@ -0,0 +1 @@ +v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/watched.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/watched.txt @@ -0,0 +1 @@ +v1 diff --git a/bundle/config/mutator/normalize_paths_test.go b/bundle/config/mutator/normalize_paths_test.go index 8bee6c5064b..9ad1dcad128 100644 --- a/bundle/config/mutator/normalize_paths_test.go +++ b/bundle/config/mutator/normalize_paths_test.go @@ -53,6 +53,45 @@ func TestNormalizePaths(t *testing.T) { require.Equal(t, "src/notebook.py", newValue.MustString()) } +func TestNormalizePaths_jobRunOnFileChange(t *testing.T) { + tmpDir := t.TempDir() + pattern := "../data/*.txt" + m := NormalizePaths() + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "run1": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + TriggersState: nil, + }, + }, + }, + }, + }, + BundleRootPath: tmpDir, + } + + location := dyn.Location{File: filepath.Join(tmpDir, "resources", "run.yml")} + path := dyn.MustPathFromString("resources.job_runs.run1.lifecycle.triggers[0].on_file_change") + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPath(v, path, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return dyn.NewValue(value.MustString(), []dyn.Location{location}), nil + }) + }) + require.NoError(t, err) + + diags := bundle.Apply(t.Context(), b, m) + require.NoError(t, diags.Error()) + + newValue, err := dyn.GetByPath(b.Config.Value(), path) + require.NoError(t, err) + require.Equal(t, "data/*.txt", newValue.MustString()) +} + func TestNormalizePath_absolutePath(t *testing.T) { value, err := normalizePath("/notebook.py", dyn.Location{}, "/tmp") assert.NoError(t, err) diff --git a/bundle/config/mutator/paths/job_run_paths_visitor.go b/bundle/config/mutator/paths/job_run_paths_visitor.go new file mode 100644 index 00000000000..3bc56bacf72 --- /dev/null +++ b/bundle/config/mutator/paths/job_run_paths_visitor.go @@ -0,0 +1,24 @@ +package paths + +import ( + "github.com/databricks/cli/libs/dyn" +) + +// VisitJobRunPaths visits local paths on job_runs so NormalizePaths can rewrite +// them relative to the bundle root. Not used by TranslatePaths: hashing still +// needs a local glob, not a workspace path. +func VisitJobRunPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("job_runs"), + dyn.AnyKey(), + dyn.Key("lifecycle"), + dyn.Key("triggers"), + dyn.AnyIndex(), + dyn.Key("on_file_change"), + ) + + return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return fn(path, TranslateModeLocalRelative, value) + }) +} diff --git a/bundle/config/mutator/paths/job_run_paths_visitor_test.go b/bundle/config/mutator/paths/job_run_paths_visitor_test.go new file mode 100644 index 00000000000..93df73d3c9e --- /dev/null +++ b/bundle/config/mutator/paths/job_run_paths_visitor_test.go @@ -0,0 +1,35 @@ +package paths + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" +) + +func TestVisitJobRunPaths(t *testing.T) { + watched := "watched.txt" + root := config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "run0": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &watched}, + }, + TriggersState: nil, + }, + }, + }, + }, + } + + actual := collectVisitedPaths(t, root, VisitJobRunPaths) + expected := []dyn.Path{ + dyn.MustPathFromString("resources.job_runs.run0.lifecycle.triggers[0].on_file_change"), + } + + assert.ElementsMatch(t, expected, actual) +} diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go index bdf42188fde..cadc52b75fe 100644 --- a/bundle/config/mutator/paths/visitor.go +++ b/bundle/config/mutator/paths/visitor.go @@ -10,6 +10,7 @@ type VisitFunc func(path dyn.Path, mode TranslateMode, value dyn.Value) (dyn.Val func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { visitors := []func(dyn.Value, VisitFunc) (dyn.Value, error){ VisitJobPaths, + VisitJobRunPaths, VisitJobLibrariesPaths, VisitAppPaths, VisitArtifactPaths, diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go new file mode 100644 index 00000000000..35cffd0dcbe --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -0,0 +1,212 @@ +package mutator + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + pathlib "path" + "path/filepath" + "slices" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + libsync "github.com/databricks/cli/libs/sync" +) + +// Every diagnostic below is reported against an on_file_change entry. +const fileTriggerPrefix = "lifecycle.triggers.on_file_change: " + +type resolveJobRunFileTriggers struct{} + +// ResolveJobRunFileTriggers expands on_file_change globs into trigger state. +func ResolveJobRunFileTriggers() bundle.Mutator { + return &resolveJobRunFileTriggers{} +} + +func (*resolveJobRunFileTriggers) Name() string { + return "ResolveJobRunFileTriggers" +} + +func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + // Sorted so diagnostics from several job_runs come out in a stable order. + names := make([]string, 0, len(b.Config.Resources.JobRuns)) + for name, jr := range b.Config.Resources.JobRuns { + // A job_run declared with an empty YAML body is a nil entry here. + if jr != nil && jr.HasOnFileChange() { + names = append(names, name) + } + } + if len(names) == 0 { + return diags + } + slices.Sort(names) + + // Listing the sync files walks the tree, so only do it once the loop above + // found a pattern that needs matching against it. + syncable, err := listSyncableRelPaths(ctx, b) + if err != nil { + return diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("list sync files: %s", err), + }) + } + + for _, name := range names { + jr := b.Config.Resources.JobRuns[name] + out := make(map[string]string) + for i, t := range jr.Lifecycle.Triggers { + if t.OnFileChange == nil { + continue + } + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) + pattern, fingerprint, d := resolveFileTrigger(b, path, *t.OnFileChange, syncable) + diags = diags.Extend(d) + if !d.HasError() { + out[pattern] = fingerprint + } + } + jr.Lifecycle.TriggersState = &resources.JobRunTriggersState{OnFileChange: out} + } + return diags +} + +func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, error) { + // Match sync's effective include set, not just Sync.Include, so a pattern can + // hash the internal dir sync force-includes. + // + // This is the candidate set, not sync's post-snapshot upload set: NewSnapshotState + // additionally drops files whose notebook type it cannot determine, such as a + // malformed .ipynb. Those stay watched on purpose - editing one is exactly when the + // run should re-fire - so a pattern can hash a file that this deploy will not upload. + includes, err := b.GetSyncIncludePatterns(ctx) + if err != nil { + return nil, err + } + fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, includes, b.Config.Sync.Exclude) + if err != nil { + return nil, err + } + files, err := fl.Files(ctx) + if err != nil { + return nil, err + } + out := make([]string, 0, len(files)) + for _, f := range files { + out = append(out, filepath.ToSlash(f.Relative)) + } + slices.Sort(out) + return out, nil +} + +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string) (string, string, diag.Diagnostics) { + relPattern, diags := validateFileTriggerPattern(b, loc, pattern) + if diags.HasError() { + return "", "", diags + } + + h := sha256.New() + matches := 0 + for _, rel := range syncable { + matched, err := pathlib.Match(relPattern, rel) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(loc), + }) + continue + } + if !matched { + continue + } + hash, err := hashFile(b.SyncRoot, rel) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("hash %q: %s", rel, err), + Locations: b.Config.GetLocations(loc), + }) + continue + } + h.Write([]byte(rel)) + h.Write([]byte{0}) + h.Write([]byte(hash)) + h.Write([]byte{0}) + matches++ + } + if matches == 0 && !diags.HasError() { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fileTriggerPrefix + fmt.Sprintf("no synced files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) + } + return relPattern, hex.EncodeToString(h.Sum(nil)), diags +} + +func validateFileTriggerPattern(b *bundle.Bundle, loc, pattern string) (string, diag.Diagnostics) { + var diags diag.Diagnostics + // A double star looks recursive but path.Match treats it as two ordinary stars. + if strings.Contains(pattern, "**") { + return "", diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("** in %q is not supported; use * for a single directory level", pattern), + Locations: b.Config.GetLocations(loc), + }) + } + // Reject a genuinely absolute path; Join would otherwise silently reinterpret it + // as relative to the bundle root. filepath.IsAbs is OS-aware and pathlib.IsAbs + // covers a POSIX path on Windows. A string that is rooted only on the other OS + // (e.g. "c:foo" or "\foo" on POSIX) is a valid relative name here and falls + // through to the sync-root containment check below. + if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) { + return "", diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("pattern %q must be relative to the defining YAML file", pattern), + Locations: b.Config.GetLocations(loc), + }) + } + // NormalizePaths has already rewritten YAML-relative globs to be bundle-root + // relative. Join that onto the bundle root, then require the result stay + // under the sync root (an ancestor of the bundle when sync.paths uses ..). + joined := filepath.Join(b.BundleRootPath, filepath.FromSlash(pattern)) + relPattern, err := filepath.Rel(b.SyncRootPath, joined) + if err != nil || !filepath.IsLocal(relPattern) { + return "", diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(loc), + }) + } + relPattern = filepath.ToSlash(relPattern) + _, err = pathlib.Match(relPattern, "") + if err != nil { + return "", diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(loc), + }) + } + return relPattern, diags +} + +func hashFile(root fs.FS, path string) (string, error) { + f, err := root.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go new file mode 100644 index 00000000000..2af318b0178 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -0,0 +1,101 @@ +package mutator_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveJobRunFileTriggersHashesThroughSyncRoot(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "watched.txt"), []byte("watched"), 0o644)) + root := vfs.MustNew(dir) + pattern := "watched.txt" + b := &bundle.Bundle{ + BundleRootPath: dir, + SyncRootPath: dir, + SyncRoot: root, + WorktreeRoot: root, + Config: config.Root{ + Bundle: config.Bundle{Target: "default"}, + Sync: config.Sync{Paths: []string{"."}}, + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, + contentHash("watched.txt\x00"+contentHash("watched")+"\x00"), + b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange["watched.txt"], + ) +} + +// Rooted patterns must be rejected the same way on every host: filepath.IsAbs +// alone only knows the local flavour, so a Windows-rooted pattern used to pass +// validation on POSIX and fail on Windows. +func TestResolveJobRunFileTriggersRejectsRootedPatterns(t *testing.T) { + // Only genuinely absolute paths on this OS are rejected. A string that is + // rooted only on Windows (e.g. "c:foo" or "\foo") is a valid relative name on + // POSIX and is left to the sync-root containment check, so it is not listed + // here; on Windows filepath.IsAbs classifies those forms itself. + for _, pattern := range []string{ + "/abs/watched.txt", + } { + t.Run(pattern, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "watched.txt"), []byte("native"), 0o644)) + root := vfs.MustNew(dir) + b := &bundle.Bundle{ + BundleRootPath: dir, + SyncRootPath: dir, + SyncRoot: root, + WorktreeRoot: root, + Config: config.Root{ + Bundle: config.Bundle{Target: "default"}, + Sync: config.Sync{Paths: []string{"."}}, + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError(), "expected %q to be rejected", pattern) + assert.Contains(t, diags[0].Summary, "must be relative to the defining YAML file") + }) + } +} + +func contentHash(content string) string { + sum := sha256.Sum256([]byte(content)) + return hex.EncodeToString(sum[:]) +} diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index a7c5ada925e..d33a7263bde 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -3,6 +3,7 @@ package mutator import ( "context" "fmt" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -25,31 +26,44 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D if jr == nil || jr.Lifecycle == nil { continue } - // Recreate-every-deploy cannot coexist with prevent_destroy. - if jr.HasOnBundleDeploy() && jr.Lifecycle.PreventDestroy { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", - Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), - }) - } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) - if t.OnBundleDeploy == nil { + if t.OnBundleDeploy == nil && t.OnFileChange == nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + Locations: b.Config.GetLocations(path), + }) + continue + } + if t.OnBundleDeploy != nil && t.OnFileChange != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", Locations: b.Config.GetLocations(path), }) continue } - if !*t.OnBundleDeploy { + if t.OnBundleDeploy != nil && !*t.OnBundleDeploy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "lifecycle.triggers.on_bundle_deploy must be true when set", Locations: b.Config.GetLocations(path + ".on_bundle_deploy"), }) } + if t.OnFileChange != nil { + if strings.TrimSpace(*t.OnFileChange) == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_file_change must be non-empty when set", + Locations: b.Config.GetLocations(path + ".on_file_change"), + }) + continue + } + // Report bad patterns at validate time; hashing only runs on deploy. + _, patternDiags := validateFileTriggerPattern(b, path+".on_file_change", *t.OnFileChange) + diags = diags.Extend(patternDiags) + } } } return diags diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 9c225457471..cb5d13e8f2f 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -14,6 +14,10 @@ func TestValidateJobRunTriggers(t *testing.T) { trueVal := true falseVal := false + fileChange := "seed.txt" + emptyFile := "" + whitespaceFile := " \t" + tests := []struct { name string triggers []resources.JobRunTrigger @@ -26,12 +30,32 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnBundleDeploy: &trueVal}, }, }, + { + name: "on_file_change set", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + }, + { + name: "both triggers as separate entries", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + {OnBundleDeploy: &trueVal}, + }, + }, { name: "empty entry", triggers: []resources.JobRunTrigger{ {}, }, - summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + }, + { + name: "both keys on one entry", + triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &trueVal, OnFileChange: &fileChange}, + }, + summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", }, { name: "on_bundle_deploy false", @@ -41,12 +65,27 @@ func TestValidateJobRunTriggers(t *testing.T) { summary: "lifecycle.triggers.on_bundle_deploy must be true when set", }, { - name: "on_bundle_deploy with prevent_destroy", + name: "on_file_change empty", triggers: []resources.JobRunTrigger{ - {OnBundleDeploy: &trueVal}, + {OnFileChange: &emptyFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, + { + name: "on_file_change whitespace", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &whitespaceFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, + { + // A trigger with prevent_destroy is valid at this stage; the recreate a + // fired trigger plans is rejected generically by checkForPreventDestroy. + name: "trigger with prevent_destroy is allowed here", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", }, { name: "prevent_destroy alone", @@ -56,14 +95,18 @@ func TestValidateJobRunTriggers(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() b := &bundle.Bundle{ + BundleRootPath: root, + SyncRootPath: root, Config: config.Root{ Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ "my_run": { Lifecycle: &resources.JobRunLifecycle{ - Lifecycle: resources.Lifecycle{PreventDestroy: tt.preventDestroy}, - Triggers: tt.triggers, + Lifecycle: resources.Lifecycle{PreventDestroy: tt.preventDestroy}, + Triggers: tt.triggers, + TriggersState: nil, }, }, }, diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 167530465e8..8b7428e8fb8 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -29,6 +29,36 @@ type JobRun struct { ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` } +// JobRunLifecycle extends Lifecycle with run-fire triggers. +type JobRunLifecycle struct { + Lifecycle + + // Triggers that cause the run to re-fire (in addition to config changes). + Triggers []JobRunTrigger `json:"triggers,omitempty"` + + // Resolved fingerprint for the planner; not user config. + TriggersState *JobRunTriggersState `json:"triggers_state,omitempty" bundle:"internal"` +} + +// JobRunTrigger is one lifecycle.triggers entry. +type JobRunTrigger struct { + OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob relative to the defining YAML file; must resolve under the sync root +} + +// JobRunTriggersState is the resolved fingerprint of lifecycle.triggers. +type JobRunTriggersState struct { + OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` + OnFileChange map[string]string `json:"on_file_change,omitempty"` +} + +// IsEmpty reports whether no trigger is armed. An empty state is left off the +// job run entirely, so this has to cover every field above: a new fingerprint +// added without extending it would be dropped instead of persisted. +func (s JobRunTriggersState) IsEmpty() bool { + return s.OnBundleDeploy == "" && len(s.OnFileChange) == 0 +} + // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. func (r *JobRun) HasOnBundleDeploy() bool { if r.Lifecycle == nil { @@ -42,6 +72,19 @@ func (r *JobRun) HasOnBundleDeploy() bool { return false } +// HasOnFileChange reports whether any trigger re-fires when matched files change. +func (r *JobRun) HasOnFileChange() bool { + if r.Lifecycle == nil { + return false + } + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + return true + } + } + return false +} + func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 88d130239ee..db2b130d313 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -25,16 +25,3 @@ type LifecycleWithStarted struct { // Supported only for apps, clusters, and sql_warehouses. Started *bool `json:"started,omitempty"` } - -// JobRunLifecycle extends Lifecycle with run-fire triggers. -type JobRunLifecycle struct { - Lifecycle - - // Triggers that cause the run to re-fire (in addition to config changes). - Triggers []JobRunTrigger `json:"triggers,omitempty"` -} - -// JobRunTrigger is one lifecycle.triggers entry. -type JobRunTrigger struct { - OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` -} diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9eb671f430c..9174c6255b6 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,15 +26,9 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour -// JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. -type JobRunTriggersState struct { - // Fresh UUID each plan while armed so Old!=New forces recreate. - OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` -} - -// JobRunLifecycleState holds local-only lifecycle fields persisted in state. +// JobRunLifecycleState is the local-only trigger fingerprint. type JobRunLifecycleState struct { - Triggers *JobRunTriggersState `json:"triggers,omitempty"` + TriggersState *resources.JobRunTriggersState `json:"triggers_state,omitempty"` } // JobRunState is the RunNow request plus the outcome required for planning. @@ -44,8 +38,8 @@ type JobRunState struct { // Always SUCCESS during planning and cleared before persistence. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Local-only; listed in knownMissingInRemoteType. Nested under lifecycle to - // mirror config and avoid colliding with a future Jobs API field. + // Local-only fingerprint. Nested under lifecycle to avoid colliding with a + // future Jobs API field; nil when no trigger is armed so the key is omitted. Lifecycle *JobRunLifecycleState `json:"lifecycle,omitempty"` } @@ -57,8 +51,8 @@ func (s JobRunState) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } -// JobRunRemote embeds RunNow so every StateType path is a valid RemoteType path -// (see TestRemoteSuperset), plus the run's output-only fields for a faithful view. +// JobRunRemote is the RunNow request plus the run's output-only fields. It has no +// lifecycle: GetRun never returns the fingerprints (see knownMissingInRemoteType). type JobRunRemote struct { jobs.RunNow @@ -99,10 +93,15 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { ResultState: jobs.RunResultStateSuccess, Lifecycle: nil, } + var ts resources.JobRunTriggersState + if input.Lifecycle != nil && input.Lifecycle.TriggersState != nil { + ts = *input.Lifecycle.TriggersState + } if input.HasOnBundleDeploy() { - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{OnBundleDeploy: uuid.NewString()}, - } + ts.OnBundleDeploy = uuid.NewString() + } + if !ts.IsEmpty() { + state.Lifecycle = &JobRunLifecycleState{TriggersState: &ts} } return state } @@ -178,13 +177,12 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, } // RemapState extracts the fields used for diffing: the RunNow request and the -// outcome the run reached. +// outcome the run reached. Lifecycle has no remote counterpart. func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - // Local-only lifecycle fingerprints stay unset on the remapped remote. - Lifecycle: nil, + Lifecycle: nil, } } @@ -369,17 +367,28 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // still going, so a run that may yet succeed is not recreated. A run that // stopped without succeeding keeps its recreate. A SKIPPED run reports no // result_state either, so the lifecycle state is what tells the two apart. -// Clearing a trigger skips its local-only fingerprint without re-firing the run. +// Removing a trigger drops the change and leaves the last fingerprint in state, +// so re-adding the trigger only re-fires when the watched files changed +// meanwhile. All other trigger changes recreate the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { switch path.String() { - case "lifecycle", "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy": - // A cleared trigger sets New empty; structdiff may report it at lifecycle, - // lifecycle.triggers, or the leaf. + case "lifecycle": + // Dropped when the trigger is removed (New nil) and when both sides are + // present, where the trigger fields below classify the change instead. + // Arming from no lifecycle at all keeps its recreate. + if change.New == nil || change.Old != nil { + change.Reason = deployplan.ReasonDrop + } + case "lifecycle.triggers_state.on_bundle_deploy": if change.New == nil || change.New == "" { - change.Action = deployplan.Skip - change.Reason = "trigger removed" + change.Reason = deployplan.ReasonDrop + } + case "lifecycle.triggers_state.on_file_change": + // As above: an emptied map is a removal, and pattern entries classify a map + // that still has both sides. + if isEmptyFileTriggerMap(change.New) || change.Old != nil { + change.Reason = deployplan.ReasonDrop } - return nil case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { @@ -387,10 +396,17 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa } change.Action = deployplan.Skip change.Reason = "run in progress" - return nil default: - return nil + // A single on_file_change pattern entry, e.g. + // lifecycle.triggers_state.on_file_change['seed.txt']. Removing one pattern + // drops its change and leaves the last fingerprint in state. + if parent := path.Parent(); parent != nil && parent.String() == "lifecycle.triggers_state.on_file_change" { + if change.New == nil { + change.Reason = deployplan.ReasonDrop + } + } } + return nil } // DoDelete deletes the run via jobs/runs/delete, on both destroy and the @@ -428,6 +444,14 @@ func (r *ResourceJobRun) cancelRun(ctx context.Context, runID int64) error { return nil } +func isEmptyFileTriggerMap(v any) bool { + if v == nil { + return true + } + m, ok := v.(map[string]string) + return ok && len(m) == 0 +} + func parseRunID(id string) (int64, error) { result, err := strconv.ParseInt(id, 10, 64) if err != nil { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index a2efdc0e6d3..ecda3243f50 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -9,6 +9,8 @@ import ( "sync/atomic" "testing" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -255,6 +257,39 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } +func TestJobRunStateOmitsEmptyLifecycle(t *testing.T) { + var state JobRunState + + require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) + + assert.Nil(t, state.Lifecycle) + serialized, err := json.Marshal(state) + require.NoError(t, err) + assert.NotContains(t, string(serialized), `"lifecycle"`) +} + +func TestJobRunPrepareStateCopiesResolvedTriggers(t *testing.T) { + enabled := true + triggers := &resources.JobRunTriggersState{ + OnFileChange: map[string]string{"*.txt": "hash"}, + } + input := &resources.JobRun{ + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &enabled}}, + TriggersState: triggers, + }, + } + + state := (&ResourceJobRun{}).PrepareState(input) + + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.TriggersState) + assert.NotSame(t, triggers, state.Lifecycle.TriggersState) + assert.Equal(t, triggers.OnFileChange, state.Lifecycle.TriggersState.OnFileChange) + assert.NotEmpty(t, state.Lifecycle.TriggersState.OnBundleDeploy) + assert.Empty(t, triggers.OnBundleDeploy) +} + // The planner diffs RemapState(remote) against PrepareState(config), so a run // that did not end in SUCCESS has to surface as a difference on result_state. func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { @@ -270,6 +305,7 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) + assert.Nil(t, state.Lifecycle) }) } } @@ -368,3 +404,128 @@ func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { assert.False(t, cancelled.Load(), "a run that already finished has nothing to cancel") } + +func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { + r := &ResourceJobRun{} + var lifecycle JobRunLifecycleState + for _, tt := range []struct { + name string + path string + old any + new any + action deployplan.ActionType + reason string + }{ + { + name: "cleared lifecycle", + path: "lifecycle", + old: lifecycle, + new: nil, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "added lifecycle", + path: "lifecycle", + old: nil, + new: lifecycle, + action: deployplan.Recreate, + reason: "", + }, + { + name: "changed lifecycle", + path: "lifecycle", + old: lifecycle, + new: lifecycle, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "cleared on_bundle_deploy string", + path: "lifecycle.triggers_state.on_bundle_deploy", + old: "uuid", + new: "", + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "nil on_bundle_deploy", + path: "lifecycle.triggers_state.on_bundle_deploy", + old: "uuid", + new: nil, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "rotated on_bundle_deploy", + path: "lifecycle.triggers_state.on_bundle_deploy", + old: "old-uuid", + new: "new-uuid", + action: deployplan.Recreate, + reason: "", + }, + { + name: "cleared on_file_change", + path: "lifecycle.triggers_state.on_file_change", + old: map[string]string{"*.txt": "hash"}, + new: nil, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "empty on_file_change maps", + path: "lifecycle.triggers_state.on_file_change", + old: map[string]string{}, + new: map[string]string{}, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "added on_file_change map", + path: "lifecycle.triggers_state.on_file_change", + old: nil, + new: map[string]string{"*.txt": "hash"}, + action: deployplan.Recreate, + reason: "", + }, + { + name: "changed on_file_change map", + path: "lifecycle.triggers_state.on_file_change", + old: map[string]string{"*.txt": "old"}, + new: map[string]string{"*.txt": "new"}, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "cleared on_file_change pattern", + path: "lifecycle.triggers_state.on_file_change['*.txt']", + old: "hash", + new: nil, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "changed on_file_change pattern", + path: "lifecycle.triggers_state.on_file_change['*.txt']", + old: "old", + new: "new", + action: deployplan.Recreate, + reason: "", + }, + { + name: "result_state with unreadable remote", + path: "result_state", + old: jobs.RunResultStateSuccess, + new: nil, + action: deployplan.Recreate, + reason: "", + }, + } { + t.Run(tt.name, func(t *testing.T) { + change := &ChangeDesc{Action: deployplan.Recreate, Old: tt.old, New: tt.new} + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath(tt.path), change, nil)) + assert.Equal(t, tt.action, change.Action) + assert.Equal(t, tt.reason, change.Reason) + }) + } +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 1cba568d103..9d7f6770180 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -94,7 +94,7 @@ var knownMissingInStateType = map[string][]string{ "job_runs": { // State stores trigger fingerprints, not the config trigger list / prevent_destroy. "lifecycle.prevent_destroy", - "lifecycle.triggers[*]", + "lifecycle.triggers", }, "dashboards": { "file_path", diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index f50d3f5bc98..215a156b0f8 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1035,7 +1035,10 @@ resources: "$fields": "on_bundle_deploy": "description": |- - If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + If true, re-fire the run on every bundle deploy. + "on_file_change": + "description": |- + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 4025c55c40b..659537a54f9 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -183,13 +183,9 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand log.Info(ctx, "Phase: deploy") // Core mutators that CRUD resources and modify deployment state. These - // mutators need informed consent if they are potentially destructive. - bundle.ApplySeqContext( - ctx, b, - scripts.Execute(config.ScriptPreDeploy), - lock.Acquire(lock.GoalDeploy), - ) - + // mutators need informed consent if they are potentially destructive. The + // predeploy script ran in ProcessBundleRet, ahead of this phase. + bundle.ApplyContext(ctx, b, lock.Acquire(lock.GoalDeploy)) if logdiag.HasError(ctx) { // lock is not acquired here return diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 6db8c5cc7e2..6d499819f61 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1533,8 +1533,12 @@ "type": "object", "properties": { "on_bundle_deploy": { - "description": "If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy.", + "description": "If true, re-fire the run on every bundle deploy.", "$ref": "#/$defs/bool" + }, + "on_file_change": { + "description": "Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported.", + "$ref": "#/$defs/string" } }, "additionalProperties": false diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index f273e576814..2dcd549eba2 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -10,6 +10,7 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" @@ -19,6 +20,7 @@ import ( "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/scripts" "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/build" @@ -431,6 +433,28 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } } + // The predeploy script can generate or rewrite files that on_file_change + // watches, so it has to run before those files are fingerprinted below. It + // stays ahead of the deployment lock, as it was when phases.Deploy ran it. + if opts.Deploy { + bundle.ApplyContext(ctx, b, scripts.Execute(config.ScriptPreDeploy)) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } + + // Fingerprint on_file_change triggers once, after every step that can produce + // a watched file: build and the predeploy script. Reads the sync root that + // phases.Initialize resolves, so it is skipped along with it; `bundle deploy + // --plan` recomputes fingerprints that the loaded plan then overrides with the + // ones it recorded. + if !opts.SkipInitialize { + bundle.ApplyContext(ctx, b, mutator.ResolveJobRunFileTriggers()) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } + if opts.Deploy { var outputHandler sync.OutputHandler if opts.Verbose { diff --git a/python/databricks/bundles/job_runs/_models/job_run_trigger.py b/python/databricks/bundles/job_runs/_models/job_run_trigger.py index 50e7c0fe12c..75bdf970a4e 100644 --- a/python/databricks/bundles/job_runs/_models/job_run_trigger.py +++ b/python/databricks/bundles/job_runs/_models/job_run_trigger.py @@ -17,7 +17,12 @@ class JobRunTrigger: on_bundle_deploy: VariableOrOptional[bool] = None """ - If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + If true, re-fire the run on every bundle deploy. + """ + + on_file_change: VariableOrOptional[str] = None + """ + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. """ @classmethod @@ -33,7 +38,12 @@ class JobRunTriggerDict(TypedDict, total=False): on_bundle_deploy: VariableOrOptional[bool] """ - If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + If true, re-fire the run on every bundle deploy. + """ + + on_file_change: VariableOrOptional[str] + """ + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. """