job_runs: wait for run completion, expose output, dedupe deploy retries

.nextchanges/bundles/job-runs-idempotency-wait.md

@@ -0,0 +1 @@
+direct: the experimental `job_runs` resource now waits for the run to finish (printing the run page URL while it waits), exposes its output fields (e.g. `${resources.job_runs.<name>.state.result_state}`), and dedupes runs on deploy retries.

acceptance/bundle/resources/job_runs/basic/output.txt

@@ -34,6 +34,7 @@ Resources:
 >>> [CLI] bundle deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-basic/default/files...
 Deploying resources...
+Waiting for job run to complete: [DATABRICKS_URL]/?o=[NUMID]#job/[MY_JOB_ID]/run/[MY_RUN_ID]
 Updating deployment state...
 Deployment complete!
 
@@ -68,6 +69,7 @@ Resources:
   "method": "POST",
   "path": "/api/2.2/jobs/run-now",
   "body": {
+    "idempotency_token": "[IDEMPOTENCY_TOKEN]",
     "job_id": [MY_JOB_ID]
   }
 }
@@ -88,12 +90,3 @@ All files and directories at the following location will be deleted: /Workspace/
 
 Deleting files...
 Destroy complete!
-
->>> print_requests.py //jobs/runs/delete
-{
-  "method": "POST",
-  "path": "/api/2.2/jobs/runs/delete",
-  "body": {
-    "run_id": [MY_RUN_ID]
-  }
-}

acceptance/bundle/resources/job_runs/basic/script

@@ -1,7 +1,8 @@
 cleanup() {
+    # destroy leaves the run in place (immutable historical record), so it makes
+    # no jobs/runs/delete call.
     trace $CLI bundle destroy --auto-approve
-    # destroy deletes the triggered run via jobs/runs/delete (also unlinks out.requests.txt)
-    trace print_requests.py //jobs/runs/delete
+    rm -f out.requests.txt
 }
 trap cleanup EXIT
 

acceptance/bundle/resources/job_runs/destroy_redeploy/databricks.yml

@@ -0,0 +1,15 @@
+bundle:
+  name: job-runs-destroy-redeploy
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+  job_runs:
+    my_run:
+      job_id: ${resources.jobs.my_job.id}

acceptance/bundle/resources/job_runs/destroy_redeploy/output.txt

@@ -0,0 +1,57 @@
+
+=== initial deploy triggers the run
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-destroy-redeploy/default/files...
+Deploying resources...
+Waiting for job run to complete: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID]
+Updating deployment state...
+Deployment complete!
+
+=== destroy removes the resource from state but leaves the run in place (no jobs/runs/delete)
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.job_runs.my_run
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-destroy-redeploy/default
+
+Deleting files...
+Destroy complete!
+
+>>> print_requests.py --keep //jobs/runs/delete
+
+=== redeploy after destroy succeeds and triggers a fresh run
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-destroy-redeploy/default/files...
+Deploying resources...
+Waiting for job run to complete: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID]
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //jobs/run-now
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "idempotency_token": "[IDEMPOTENCY_TOKEN]",
+    "job_id": [NUMID]
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/run-now",
+  "body": {
+    "idempotency_token": "[IDEMPOTENCY_TOKEN]",
+    "job_id": [NUMID]
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.job_runs.my_run
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-destroy-redeploy/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/job_runs/destroy_redeploy/script

@@ -0,0 +1,20 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+title "initial deploy triggers the run"
+trace $CLI bundle deploy
+
+title "destroy removes the resource from state but leaves the run in place (no jobs/runs/delete)"
+trace $CLI bundle destroy --auto-approve
+trace print_requests.py --keep //jobs/runs/delete
+
+title "redeploy after destroy succeeds and triggers a fresh run"
+# Destroy never deleted the run, so its idempotency token is not tombstoned and
+# run-now cannot error on reuse. The bundle's job is recreated with a new id, so
+# the run config differs and a new run is triggered (rather than deduping to the
+# old one).
+trace $CLI bundle deploy
+trace print_requests.py //jobs/run-now

acceptance/bundle/resources/job_runs/failed_run/databricks.yml

@@ -0,0 +1,26 @@
+bundle:
+  name: job-runs-failed-run
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+    # Reads the run's result_state. The run finishes FAILED (see test.toml stub),
+    # but the deploy still completes and this job is created with the resolved tag.
+    downstream_job:
+      name: downstream-job
+      tags:
+        run_result: ${resources.job_runs.my_run.state.result_state}
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+  job_runs:
+    my_run:
+      job_id: ${resources.jobs.my_job.id}

acceptance/bundle/resources/job_runs/failed_run/output.txt

@@ -0,0 +1,74 @@
+
+=== a run that finishes FAILED is surfaced, not fatal: the deploy still completes
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== the downstream job was created with the run's FAILED result_state (deploy did not abort)
+>>> print_requests.py //jobs/create
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/create",
+  "body": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "my-job",
+    "queue": {
+      "enabled": true
+    },
+    "tasks": [
+      {
+        "notebook_task": {
+          "notebook_path": "/Workspace/test"
+        },
+        "task_key": "main"
+      }
+    ]
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/create",
+  "body": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "max_concurrent_runs": 1,
+    "name": "downstream-job",
+    "queue": {
+      "enabled": true
+    },
+    "tags": {
+      "run_result": "FAILED"
+    },
+    "tasks": [
+      {
+        "notebook_task": {
+          "notebook_path": "/Workspace/test"
+        },
+        "task_key": "main"
+      }
+    ]
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.job_runs.my_run
+  delete resources.jobs.downstream_job
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-failed-run/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/job_runs/failed_run/script

@@ -0,0 +1,11 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+title "a run that finishes FAILED is surfaced, not fatal: the deploy still completes"
+trace $CLI bundle deploy
+
+title "the downstream job was created with the run's FAILED result_state (deploy did not abort)"
+trace print_requests.py //jobs/create

acceptance/bundle/resources/job_runs/failed_run/test.toml

@@ -0,0 +1,13 @@
+# Force my_run's GetRun to report a terminal FAILED result so we can assert a
+# failed run is surfaced (readable downstream), not turned into a deploy failure.
+# The first handler registered for a pattern wins, so this overrides the default
+# runs/get handler.
+[[Server]]
+Pattern = "GET /api/2.2/jobs/runs/get"
+Response.Body = '''{
+  "run_id": 1,
+  "state": {
+    "life_cycle_state": "TERMINATED",
+    "result_state": "FAILED"
+  }
+}'''

acceptance/bundle/resources/job_runs/flip_back/databricks.yml

@@ -0,0 +1,17 @@
+bundle:
+  name: job-runs-flip-back
+
+resources:
+  jobs:
+    my_job:
+      name: my-job
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Workspace/test
+
+  job_runs:
+    my_run:
+      job_id: ${resources.jobs.my_job.id}
+      job_parameters:
+        env: dev

acceptance/bundle/resources/job_runs/flip_back/output.txt

@@ -0,0 +1,42 @@
+
+=== deploy with env=dev triggers the first run
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-flip-back/default/files...
+Deploying resources...
+Waiting for job run to complete: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID]
+Updating deployment state...
+Deployment complete!
+
+=== flip env dev -> prod: the config changed, so a fresh, different run is triggered
+>>> update_file.py databricks.yml env: dev env: prod
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-flip-back/default/files...
+Deploying resources...
+Waiting for job run to complete: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID]
+Updating deployment state...
+Deployment complete!
+
+=== flip env prod -> dev: dedupes back to the original dev run, does not start a new one
+>>> update_file.py databricks.yml env: prod env: dev
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-flip-back/default/files...
+Deploying resources...
+Waiting for job run to complete: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID]
+Updating deployment state...
+Deployment complete!
+
+=== no run was ever deleted (destroy/recreate leave runs in place), and the dev run id was reused
+>>> print_requests.py --keep //jobs/runs/delete
+flip-back reused the original dev run ([NUMID]); prod had its own run ([NUMID])
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.job_runs.my_run
+  delete resources.jobs.my_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-flip-back/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/job_runs/flip_back/script

@@ -0,0 +1,36 @@
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+STATE=.databricks/bundle/default/resources.json
+run_id() {
+    jq -r '.state["resources.job_runs.my_run"].__id__' "$STATE"
+}
+
+title "deploy with env=dev triggers the first run"
+trace $CLI bundle deploy
+dev1=$(run_id)
+
+title "flip env dev -> prod: the config changed, so a fresh, different run is triggered"
+trace update_file.py databricks.yml "env: dev" "env: prod"
+trace $CLI bundle deploy
+prod=$(run_id)
+
+# The dev->prod recreate left the original dev run in place (delete is a noop),
+# so its idempotency token is still live. Flipping back to dev re-issues run-now
+# with that same token, which dedupes to the original dev run instead of starting
+# a new one. This is a direct consequence of the noop delete + content-hash token.
+title "flip env prod -> dev: dedupes back to the original dev run, does not start a new one"
+trace update_file.py databricks.yml "env: prod" "env: dev"
+trace $CLI bundle deploy
+dev2=$(run_id)
+
+title "no run was ever deleted (destroy/recreate leave runs in place), and the dev run id was reused"
+trace print_requests.py --keep //jobs/runs/delete
+if [ "$dev1" = "$dev2" ] && [ "$dev1" != "$prod" ]; then
+    echo "flip-back reused the original dev run ($dev1); prod had its own run ($prod)"
+else
+    echo "unexpected run ids: dev1=$dev1 prod=$prod dev2=$dev2"
+fi