Integrate deployment metadata service for locking and state

acceptance/bundle/dms/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: metadata-service-test
+
+resources:
+  jobs:
+    test_job:
+      name: test-job

acceptance/bundle/dms/deploy-error/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: metadata-service-error-test
+
+resources:
+  jobs:
+    test_job:
+      name: test-job

acceptance/bundle/dms/deploy-error/output.txt

@@ -0,0 +1,61 @@
+
+>>> musterr [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/metadata-service-error-test/default/files...
+Deploying resources...
+Error: cannot create resources.jobs.test_job: Invalid job configuration. (400 INVALID_PARAMETER_VALUE)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create
+HTTP Status: 400 Bad Request
+API error_code: INVALID_PARAMETER_VALUE
+API message: Invalid job configuration.
+
+Updating deployment state...
+
+>>> print_requests.py --get //bundle
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "q": {
+    "deployment_id": "[UUID]"
+  },
+  "body": {
+    "target_name": "default"
+  }
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "version_type": "VERSION_TYPE_DEPLOY",
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations",
+  "q": {
+    "resource_key": "resources.jobs.test_job"
+  },
+  "body": {
+    "resource_key": "resources.jobs.test_job",
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "status": "OPERATION_STATUS_FAILED",
+    "error_message": "Invalid job configuration."
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "name": "deployments/[UUID]/versions/1",
+    "completion_reason": "VERSION_COMPLETE_FAILURE"
+  }
+}

acceptance/bundle/dms/deploy-error/script

@@ -0,0 +1,5 @@
+# Deploy with the metadata service enabled, expecting a resource creation failure.
+trace musterr $CLI bundle deploy
+
+# Print the metadata service requests to verify the failed operation is reported.
+trace print_requests.py --get //bundle

acceptance/bundle/dms/deploy-error/test.toml

@@ -0,0 +1,8 @@
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+EnvMatrix.DATABRICKS_BUNDLE_MANAGED_STATE = ["true"]
+RecordRequests = true
+
+[[Server]]
+Pattern = "POST /api/2.2/jobs/create"
+Response.StatusCode = 400
+Response.Body = '{"error_code": "INVALID_PARAMETER_VALUE", "message": "Invalid job configuration."}'

acceptance/bundle/dms/initial-register/databricks.yml

@@ -0,0 +1,7 @@
+bundle:
+  name: initial-register-test
+
+resources:
+  jobs:
+    test_job:
+      name: test-job

acceptance/bundle/dms/initial-register/output.txt

@@ -0,0 +1,61 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/initial-register-test/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/initial-register-test/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py --get //bundle
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "q": {
+    "deployment_id": "[UUID]"
+  },
+  "body": {
+    "target_name": "default"
+  }
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "version_type": "VERSION_TYPE_DEPLOY",
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations",
+  "q": {
+    "resource_key": "resources.jobs.test_job"
+  },
+  "body": {
+    "resource_key": "resources.jobs.test_job",
+    "action_type": "OPERATION_ACTION_TYPE_INITIAL_REGISTER",
+    "resource_id": "[NUMID]",
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "name": "deployments/[UUID]/versions/1",
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}

acceptance/bundle/dms/initial-register/script

@@ -0,0 +1,11 @@
+# First deploy without metadata service - creates the job using filesystem lock.
+trace $CLI bundle deploy
+
+# Enable metadata service and deploy again.
+# Since this is the first DMS deployment (version 1) and the job hasn't changed
+# (Skip action), it should be reported as INITIAL_REGISTER.
+export DATABRICKS_BUNDLE_MANAGED_STATE=true
+trace $CLI bundle deploy
+
+# Print metadata service requests from the second deploy.
+trace print_requests.py --get //bundle

acceptance/bundle/dms/initial-register/test.toml

@@ -0,0 +1,5 @@
+# Override parent: start without managed state so the first deploy uses filesystem lock.
+# The script enables managed state before the second deploy.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]
+EnvMatrix.DATABRICKS_BUNDLE_MANAGED_STATE = ["false"]
+RecordRequests = true

acceptance/bundle/dms/output.txt

@@ -0,0 +1,117 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/metadata-service-test/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py --get //bundle
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "q": {
+    "deployment_id": "[UUID]"
+  },
+  "body": {
+    "target_name": "default"
+  }
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "version_type": "VERSION_TYPE_DEPLOY",
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations",
+  "q": {
+    "resource_key": "resources.jobs.test_job"
+  },
+  "body": {
+    "resource_key": "resources.jobs.test_job",
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "resource_id": "[NUMID]",
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "name": "deployments/[UUID]/versions/1",
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.jobs.test_job
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/metadata-service-test/default
+
+Deleting files...
+Destroy complete!
+
+>>> print_requests.py --get //bundle
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "q": {
+    "deployment_id": "[UUID]"
+  },
+  "body": {
+    "target_name": "default"
+  }
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "2"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "version_type": "VERSION_TYPE_DESTROY",
+    "target_name": "default"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/operations",
+  "q": {
+    "resource_key": "resources.jobs.test_job"
+  },
+  "body": {
+    "resource_key": "resources.jobs.test_job",
+    "action_type": "OPERATION_ACTION_TYPE_DELETE",
+    "resource_id": "[NUMID]",
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete",
+  "body": {
+    "name": "deployments/[UUID]/versions/2",
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}
+{
+  "method": "DELETE",
+  "path": "/api/2.0/bundle/deployments/[UUID]"
+}

acceptance/bundle/dms/release-lock-error/databricks.yml

@@ -0,0 +1,11 @@
+bundle:
+  name: dms-release-lock-error
+
+targets:
+  fail-complete:
+    default: true
+
+resources:
+  jobs:
+    test_job:
+      name: test-job

acceptance/bundle/dms/release-lock-error/output.txt

@@ -0,0 +1,56 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-release-lock-error/fail-complete/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+Warn: Failed to release deployment lock: complete version: simulated complete version failure
+
+>>> print_requests.py --get //bundle
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments",
+  "q": {
+    "deployment_id": "[UUID]"
+  },
+  "body": {
+    "target_name": "fail-complete"
+  }
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/bundle/deployments/[UUID]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions",
+  "q": {
+    "version_id": "1"
+  },
+  "body": {
+    "cli_version": "[DEV_VERSION]",
+    "version_type": "VERSION_TYPE_DEPLOY",
+    "target_name": "fail-complete"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations",
+  "q": {
+    "resource_key": "resources.jobs.test_job"
+  },
+  "body": {
+    "resource_key": "resources.jobs.test_job",
+    "action_type": "OPERATION_ACTION_TYPE_CREATE",
+    "resource_id": "[NUMID]",
+    "status": "OPERATION_STATUS_SUCCEEDED"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete",
+  "body": {
+    "name": "deployments/[UUID]/versions/1",
+    "completion_reason": "VERSION_COMPLETE_SUCCESS"
+  }
+}