diff --git a/.nextchanges/bundles/cascade-on-destroy.md b/.nextchanges/bundles/cascade-on-destroy.md new file mode 100644 index 00000000000..161f49fd1ae --- /dev/null +++ b/.nextchanges/bundles/cascade-on-destroy.md @@ -0,0 +1 @@ +Added a `cascade_on_destroy` field to the pipeline resource to control whether destroying a pipeline also deletes its datasets (MVs, STs, Views). When unset, the server default applies; set `cascade_on_destroy: false` to retain the datasets on destroy. Supported with the direct deployment engine ([#5846](https://github.com/databricks/cli/pull/5846)). diff --git a/acceptance/bundle/config-remote-sync/cli_defaults/output.txt b/acceptance/bundle/config-remote-sync/cli_defaults/output.txt index 307a945ea93..f3e2372b81a 100644 --- a/acceptance/bundle/config-remote-sync/cli_defaults/output.txt +++ b/acceptance/bundle/config-remote-sync/cli_defaults/output.txt @@ -57,7 +57,7 @@ The following resources will be deleted: delete resources.pipelines.pipeline1 This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.pipeline1 All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt b/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt index 54a1e342788..52e37685043 100644 --- a/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt +++ b/acceptance/bundle/config-remote-sync/job_pipeline_task/output.txt @@ -41,7 +41,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt b/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt index e14b94f76f5..19b20f8befa 100644 --- a/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt @@ -60,7 +60,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/resolve_variables/output.txt b/acceptance/bundle/config-remote-sync/resolve_variables/output.txt index 388f7ab3bbf..8cb8849da4d 100644 --- a/acceptance/bundle/config-remote-sync/resolve_variables/output.txt +++ b/acceptance/bundle/config-remote-sync/resolve_variables/output.txt @@ -143,7 +143,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/config-remote-sync/validation_errors/output.txt b/acceptance/bundle/config-remote-sync/validation_errors/output.txt index c4ce29efa0a..372e4fa1ae8 100644 --- a/acceptance/bundle/config-remote-sync/validation_errors/output.txt +++ b/acceptance/bundle/config-remote-sync/validation_errors/output.txt @@ -51,7 +51,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/deploy/pipeline-config-dots/output.txt b/acceptance/bundle/deploy/pipeline-config-dots/output.txt index 48b6576f221..8b338fcfbb3 100644 --- a/acceptance/bundle/deploy/pipeline-config-dots/output.txt +++ b/acceptance/bundle/deploy/pipeline-config-dots/output.txt @@ -23,7 +23,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.my_schema This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/deploy/snapshot-comparison/output.txt b/acceptance/bundle/deploy/snapshot-comparison/output.txt index b5bb597e264..c908a5fa61a 100644 --- a/acceptance/bundle/deploy/snapshot-comparison/output.txt +++ b/acceptance/bundle/deploy/snapshot-comparison/output.txt @@ -43,7 +43,7 @@ The following resources will be deleted: delete resources.pipelines.test_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.test_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/snapshot-test-1-[UNIQUE_NAME]/default @@ -58,7 +58,7 @@ The following resources will be deleted: delete resources.pipelines.test_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.test_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/snapshot-test-2-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/destroy/all-resources/output.txt b/acceptance/bundle/destroy/all-resources/output.txt index 5fd8d432a82..49bed70e906 100644 --- a/acceptance/bundle/destroy/all-resources/output.txt +++ b/acceptance/bundle/destroy/all-resources/output.txt @@ -14,7 +14,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.my_schema This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt index ce1530c469a..c5f958ff3cf 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/output.txt +++ b/acceptance/bundle/destroy/jobs-and-pipeline/output.txt @@ -53,7 +53,7 @@ The following resources will be deleted: delete resources.pipelines.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.bar All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/migrate/runas/output.txt b/acceptance/bundle/migrate/runas/output.txt index f8a9b475bdc..f58deae5c7c 100644 --- a/acceptance/bundle/migrate/runas/output.txt +++ b/acceptance/bundle/migrate/runas/output.txt @@ -138,7 +138,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dabs_revenue-[UNIQUE_NAME]/production diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index d35bfa350f0..6c785d4ca2a 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2405,6 +2405,7 @@ resources.models.*.permissions[*].service_principal_name string ALL resources.models.*.permissions[*].user_name string ALL resources.pipelines.*.allow_duplicate_names bool ALL resources.pipelines.*.budget_policy_id string ALL +resources.pipelines.*.cascade_on_destroy *bool INPUT STATE resources.pipelines.*.catalog string ALL resources.pipelines.*.cause string REMOTE resources.pipelines.*.channel string ALL diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt index b7eec30702b..0d338041e65 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt +++ b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt @@ -108,7 +108,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 4c51088f686..7f0ed6a6975 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -90,7 +90,7 @@ The following resources will be deleted: delete resources.pipelines.mypipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.mypipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt index 2e15e6e4108..f5b15b7345e 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/output.txt @@ -30,7 +30,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt index 89f7921475b..6508649dd4c 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/output.txt @@ -18,7 +18,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt b/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt index 2f3ce99457f..cedb5f75f0d 100644 --- a/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/empty_list/output.txt @@ -13,7 +13,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt index 2e15e6e4108..f5b15b7345e 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/output.txt @@ -30,7 +30,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt index 3b82b04aa94..e512edbbf53 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/output.txt @@ -22,7 +22,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/resources/permissions/pipelines/update/output.txt b/acceptance/bundle/resources/permissions/pipelines/update/output.txt index e5395eeceea..4c576ab4d1c 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/update/output.txt @@ -123,7 +123,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/permissions-test/default diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt index 8cea9634565..2ff2983a465 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt @@ -47,7 +47,7 @@ The following resources will be deleted: delete resources.pipelines.pipeline_one This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.pipeline_one All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/auto-approve/output.txt b/acceptance/bundle/resources/pipelines/auto-approve/output.txt index 3016a4c248d..3f793420787 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/output.txt +++ b/acceptance/bundle/resources/pipelines/auto-approve/output.txt @@ -63,7 +63,7 @@ The following resources will be deleted: delete resources.pipelines.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.bar All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt index 04129e78ce1..be82456d7c1 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/output.txt @@ -43,7 +43,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-lakeflow-pipeline-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt index 77bfdc09d32..cb839853340 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/output.txt @@ -150,7 +150,7 @@ The following resources will be deleted: delete resources.pipelines.my This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt index 74943571027..43fc3727d26 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/output.txt @@ -134,7 +134,7 @@ The following resources will be deleted: delete resources.pipelines.my This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/pipelines/recreate/output.txt b/acceptance/bundle/resources/pipelines/recreate/output.txt index b6a280d38d5..bd1304b9fdf 100644 --- a/acceptance/bundle/resources/pipelines/recreate/output.txt +++ b/acceptance/bundle/resources/pipelines/recreate/output.txt @@ -65,7 +65,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/pipelines/update/output.txt b/acceptance/bundle/resources/pipelines/update/output.txt index 213e8a0cc64..3cd4c3a9736 100644 --- a/acceptance/bundle/resources/pipelines/update/output.txt +++ b/acceptance/bundle/resources/pipelines/update/output.txt @@ -72,7 +72,7 @@ The following resources will be deleted: delete resources.pipelines.my This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default diff --git a/acceptance/bundle/resources/schemas/auto-approve/output.txt b/acceptance/bundle/resources/schemas/auto-approve/output.txt index 375c94fe652..d27e9a6a6c7 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/output.txt +++ b/acceptance/bundle/resources/schemas/auto-approve/output.txt @@ -72,7 +72,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.bar This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/summary/modified_status/output.txt b/acceptance/bundle/summary/modified_status/output.txt index 92338e101c9..9cb3e7b0517 100644 --- a/acceptance/bundle/summary/modified_status/output.txt +++ b/acceptance/bundle/summary/modified_status/output.txt @@ -189,7 +189,7 @@ This action will result in the deletion of the following UC schemas. Any underly delete resources.schemas.my_schema This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt index 2206b10cd25..cc0c54574dc 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/output.txt +++ b/acceptance/bundle/templates/default-python/integration_classic/output.txt @@ -93,7 +93,7 @@ The following resources will be deleted: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev @@ -491,7 +491,7 @@ The following resources will be deleted: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.project_name_[UNIQUE_NAME]_etl All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod diff --git a/acceptance/pipelines/destroy/auto-approve/output.txt b/acceptance/pipelines/destroy/auto-approve/output.txt index 76bf97b4e2f..8ab0e6e1eb9 100644 --- a/acceptance/pipelines/destroy/auto-approve/output.txt +++ b/acceptance/pipelines/destroy/auto-approve/output.txt @@ -19,7 +19,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy/default diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/databricks.yml b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/databricks.yml new file mode 100644 index 00000000000..0c4f487853b --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-cascade-on-destroy-tf-error +resources: + pipelines: + my_pipeline: + name: test-pipeline-cascade-tf + cascade_on_destroy: false diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml new file mode 100644 index 00000000000..65156e0457c --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/output.txt b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/output.txt new file mode 100644 index 00000000000..e1636043a2e --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/output.txt @@ -0,0 +1,11 @@ + +=== pipelines deploy fails with cascade_on_destroy on terraform engine +>>> errcode [CLI] pipelines deploy +Error: cascade_on_destroy is only supported in direct deployment mode + in databricks.yml:7:27 + +cascade_on_destroy is not yet supported by the terraform provider. Set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use it. +Learn more at https://docs.databricks.com/dev-tools/bundles/direct + + +Exit code: 1 diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/script b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/script new file mode 100644 index 00000000000..ba05370f088 --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/script @@ -0,0 +1,2 @@ +title "pipelines deploy fails with cascade_on_destroy on terraform engine" +trace errcode $CLI pipelines deploy diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml new file mode 100644 index 00000000000..ec557241780 --- /dev/null +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml @@ -0,0 +1,8 @@ +Local = true +RecordRequests = false + +# Static validation only; no need to run on cloud. +Cloud = false + +# cascade_on_destroy is direct-only; assert the terraform engine rejects it. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/databricks.yml b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/databricks.yml new file mode 100644 index 00000000000..2f735bad878 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-pipeline-destroy-cascade-divergence +resources: + pipelines: + my_pipeline: + name: test-pipeline-cascade-divergence + cascade_on_destroy: true diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/out.test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/output.txt b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/output.txt new file mode 100644 index 00000000000..0446e46b40f --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/output.txt @@ -0,0 +1,47 @@ + +=== Deploy a pipeline with cascade_on_destroy: true (state records true) +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade-divergence/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade-divergence/default/state/metadata.json" + }, + "edition": "ADVANCED", + "name": "test-pipeline-cascade-divergence" + } +} + +=== Edit cascade_on_destroy true -> false in YAML WITHOUT deploying (config diverges from state) +=== Destroy: banner and DELETE both reflect STATE (cascade=true), not the edited YAML +>>> [CLI] pipelines destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.my_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.my_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade-divergence/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //api/2.0/pipelines +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[UUID]", + "q": { + "cascade": "true" + } +} diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/script b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/script new file mode 100644 index 00000000000..feb1cf52f4a --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/script @@ -0,0 +1,13 @@ +title "Deploy a pipeline with cascade_on_destroy: true (state records true)" +trace $CLI pipelines deploy +trace print_requests.py //api/2.0/pipelines + +title "Edit cascade_on_destroy true -> false in YAML WITHOUT deploying (config diverges from state)" +update_file.py databricks.yml "cascade_on_destroy: true" "cascade_on_destroy: false" + +# The approval banner and the DELETE request must both reflect the persisted state (cascade=true), +# not the edited-but-undeployed YAML. Otherwise the prompt would promise dataset retention while +# the execution cascades and deletes them. +title "Destroy: banner and DELETE both reflect STATE (cascade=true), not the edited YAML" +trace $CLI pipelines destroy --auto-approve +trace print_requests.py //api/2.0/pipelines diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/test.toml new file mode 100644 index 00000000000..18d7f81b515 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/test.toml @@ -0,0 +1,4 @@ +# cascade_on_destroy only affects the direct engine's delete request; the terraform engine +# does not support the attribute yet and rejects it. Restrict this test to direct +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml new file mode 100644 index 00000000000..af159a9e9c1 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: test-pipeline-destroy-cascade +resources: + pipelines: + my_pipeline: + name: test-pipeline-cascade + cascade_on_destroy: false diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt new file mode 100644 index 00000000000..7d405350746 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/output.txt @@ -0,0 +1,73 @@ + +=== Deploy a pipeline configured with cascade_on_destroy: false +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/state/metadata.json" + }, + "edition": "ADVANCED", + "name": "test-pipeline-cascade" + } +} + +=== Change only cascade_on_destroy (false -> true): expect a state-only update, no pipeline update API call +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines + +=== Set cascade_on_destroy back to false and deploy so state reflects it before destroy +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines + +=== Unset cascade_on_destroy entirely: expect a state-only update that persists nil to state +>>> [CLI] pipelines deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?w=[NUMID] + +>>> print_requests.py //api/2.0/pipelines + +=== Destroy: cascade_on_destroy is unset in state, so the delete uses the backend default (no cascade param) +>>> [CLI] pipelines destroy --auto-approve +The following resources will be deleted: + delete resources.pipelines.my_pipeline + +This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: + delete resources.pipelines.my_pipeline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy-cascade/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //api/2.0/pipelines +{ + "method": "DELETE", + "path": "/api/2.0/pipelines/[UUID]" +} diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/script b/acceptance/pipelines/destroy/destroy-pipeline-cascade/script new file mode 100644 index 00000000000..017780d8412 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/script @@ -0,0 +1,27 @@ +title "Deploy a pipeline configured with cascade_on_destroy: false" +trace $CLI pipelines deploy + +# Flush the deploy requests: cascade_on_destroy is delete-time-only, so the create request must not carry it. +trace print_requests.py //api/2.0/pipelines + +title "Change only cascade_on_destroy (false -> true): expect a state-only update, no pipeline update API call" +update_file.py databricks.yml "cascade_on_destroy: false" "cascade_on_destroy: true" +trace $CLI pipelines deploy +# No POST/PATCH to /pipelines is expected here; the cascade-only change is persisted to state. +trace print_requests.py //api/2.0/pipelines + +title "Set cascade_on_destroy back to false and deploy so state reflects it before destroy" +update_file.py databricks.yml "cascade_on_destroy: true" "cascade_on_destroy: false" +trace $CLI pipelines deploy +trace print_requests.py //api/2.0/pipelines + +title "Unset cascade_on_destroy entirely: expect a state-only update that persists nil to state" +update_file.py databricks.yml " cascade_on_destroy: false +" "" +trace $CLI pipelines deploy +# No pipeline API call is expected; the cascade-only change is persisted to state. +trace print_requests.py //api/2.0/pipelines + +title "Destroy: cascade_on_destroy is unset in state, so the delete uses the backend default (no cascade param)" +trace $CLI pipelines destroy --auto-approve +trace print_requests.py //api/2.0/pipelines diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml new file mode 100644 index 00000000000..18d7f81b515 --- /dev/null +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/test.toml @@ -0,0 +1,4 @@ +# cascade_on_destroy only affects the direct engine's delete request; the terraform engine +# does not support the attribute yet and rejects it. Restrict this test to direct +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +RecordRequests = true diff --git a/acceptance/pipelines/destroy/destroy-pipeline/output.txt b/acceptance/pipelines/destroy/destroy-pipeline/output.txt index 4e5ab020852..70cf60cff61 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline/output.txt +++ b/acceptance/pipelines/destroy/destroy-pipeline/output.txt @@ -12,7 +12,7 @@ The following resources will be deleted: delete resources.pipelines.my_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.my_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-destroy/default diff --git a/acceptance/pipelines/destroy/force-lock/output.txt b/acceptance/pipelines/destroy/force-lock/output.txt index f253d542a91..f7cf597f67b 100644 --- a/acceptance/pipelines/destroy/force-lock/output.txt +++ b/acceptance/pipelines/destroy/force-lock/output.txt @@ -27,7 +27,7 @@ The following resources will be deleted: delete resources.pipelines.foo This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-pipeline-force-lock/default diff --git a/acceptance/pipelines/e2e/output.txt b/acceptance/pipelines/e2e/output.txt index 8e029977b23..8148765cd12 100644 --- a/acceptance/pipelines/e2e/output.txt +++ b/acceptance/pipelines/e2e/output.txt @@ -106,7 +106,7 @@ The following resources will be deleted: delete resources.pipelines.lakeflow_project_etl_2 This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.lakeflow_project_etl delete resources.pipelines.lakeflow_project_etl_2 diff --git a/bundle/config/mutator/validate_cascade_on_destroy.go b/bundle/config/mutator/validate_cascade_on_destroy.go new file mode 100644 index 00000000000..023b72b991a --- /dev/null +++ b/bundle/config/mutator/validate_cascade_on_destroy.go @@ -0,0 +1,48 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/diag" +) + +type validateCascadeOnDestroy struct { + engine engine.EngineType +} + +// ValidateCascadeOnDestroy returns a mutator that errors when a pipeline sets +// cascade_on_destroy while using the terraform deployment engine. The terraform +// provider does not support the attribute yet, so it is only honored in direct +// deployment mode. +func ValidateCascadeOnDestroy(e engine.EngineType) bundle.Mutator { + return &validateCascadeOnDestroy{engine: e} +} + +func (m *validateCascadeOnDestroy) Name() string { + return "ValidateCascadeOnDestroy" +} + +func (m *validateCascadeOnDestroy) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + if m.engine.IsDirect() { + return nil + } + + var diags diag.Diagnostics + for key, pipeline := range b.Config.Resources.Pipelines { + if pipeline.CascadeOnDestroy == nil { + continue + } + path := "resources.pipelines." + key + ".cascade_on_destroy" + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "cascade_on_destroy is only supported in direct deployment mode", + Detail: "cascade_on_destroy is not yet supported by the terraform provider. " + + "Set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use it.\n" + + "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", + Locations: b.Config.GetLocations(path), + }) + } + return diags +} diff --git a/bundle/config/mutator/validate_cascade_on_destroy_test.go b/bundle/config/mutator/validate_cascade_on_destroy_test.go new file mode 100644 index 00000000000..2af79a69f13 --- /dev/null +++ b/bundle/config/mutator/validate_cascade_on_destroy_test.go @@ -0,0 +1,69 @@ +package mutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" +) + +func pipelineWithCascade(cascade *bool) *resources.Pipeline { + return &resources.Pipeline{ + CreatePipeline: pipelines.CreatePipeline{Name: "my_pipeline"}, + CascadeOnDestroy: cascade, + } +} + +func TestValidateCascadeOnDestroyDirectEngineReturnsNil(t *testing.T) { + no := false + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "my_pipeline": pipelineWithCascade(&no), + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateCascadeOnDestroy(engine.EngineDirect)) + assert.Empty(t, diags) +} + +func TestValidateCascadeOnDestroyTerraformEngineUnsetReturnsNil(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "my_pipeline": pipelineWithCascade(nil), + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateCascadeOnDestroy(engine.EngineTerraform)) + assert.Empty(t, diags) +} + +func TestValidateCascadeOnDestroyTerraformEngineSetEmitsError(t *testing.T) { + no := false + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Pipelines: map[string]*resources.Pipeline{ + "my_pipeline": pipelineWithCascade(&no), + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateCascadeOnDestroy(engine.EngineTerraform)) + assert.Len(t, diags, 1) + assert.Equal(t, "cascade_on_destroy is only supported in direct deployment mode", diags[0].Summary) + assert.Contains(t, diags[0].Detail, "https://docs.databricks.com/dev-tools/bundles/direct") +} diff --git a/bundle/config/resources/pipeline.go b/bundle/config/resources/pipeline.go index c442a01a641..7c205f7dc14 100644 --- a/bundle/config/resources/pipeline.go +++ b/bundle/config/resources/pipeline.go @@ -16,6 +16,11 @@ type Pipeline struct { pipelines.CreatePipeline //nolint CreatePipeline also defines Id field with the same json tag "id" Permissions []PipelinePermission `json:"permissions,omitempty"` + + // CascadeOnDestroy controls whether destroying the pipeline also deletes its datasets + // Nil (unset) uses the server default, which cascades. Set to false to retain datasets + // when the pipeline is deleted. Delete-time only: never sent on create/update. + CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` } func (p *Pipeline) UnmarshalJSON(b []byte) error { diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline.go b/bundle/deploy/terraform/tfdyn/convert_pipeline.go index 944ce0858c7..2eacbf714af 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline.go @@ -21,7 +21,9 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err return dyn.InvalidValue, err } - vout, err = dyn.DropKeys(vout, []string{"dry_run"}) + // Current Terraform provider does not support the cascade_on_destroy attribute yet. + // ValidateCascadeOnDestroy rejects it here and provides an explicit error message. + vout, err = dyn.DropKeys(vout, []string{"dry_run", "cascade_on_destroy"}) if err != nil { return dyn.InvalidValue, err } diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index 20d2eb08245..a02d11abf8c 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -4,14 +4,38 @@ import ( "context" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structdiff" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/pipelines" ) +// PipelineState is the state type for Pipeline resources. It extends CreatePipeline with +// the CascadeOnDestroy field, a delete-time setting that is not part of the pipeline spec +type PipelineState struct { + pipelines.CreatePipeline + + // CascadeOnDestroy controls whether deleting the pipeline also deletes its datasets (MVs, + // STs, Views). Nil means the server default (cascade) applies. Read from persisted state at + // delete time; never sent on create/update. + CascadeOnDestroy *bool `json:"cascade_on_destroy,omitempty"` +} + +func (s *PipelineState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s PipelineState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + // PipelineRemote is the return type for DoRead. It embeds CreatePipeline so that all // paths in StateType are valid paths in RemoteType. +// Note that cascade_on_destroy is intentionally absent: it is a delete-time-only setting +// that the GET response never returns, so the engine suppresses its drift automatically type PipelineRemote struct { pipelines.CreatePipeline @@ -49,12 +73,19 @@ func (*ResourcePipeline) New(client *databricks.WorkspaceClient) *ResourcePipeli } } -func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *pipelines.CreatePipeline { - return &input.CreatePipeline +func (*ResourcePipeline) PrepareState(input *resources.Pipeline) *PipelineState { + return &PipelineState{ + CreatePipeline: input.CreatePipeline, + CascadeOnDestroy: input.CascadeOnDestroy, + } } -func (*ResourcePipeline) RemapState(remote *PipelineRemote) *pipelines.CreatePipeline { - return &remote.CreatePipeline +func (*ResourcePipeline) RemapState(remote *PipelineRemote) *PipelineState { + return &PipelineState{ + CreatePipeline: remote.CreatePipeline, + // cascade_on_destroy is input-only and absent from PipelineRemote, so it stays nil here. + CascadeOnDestroy: nil, + } } func (r *ResourcePipeline) DoRead(ctx context.Context, id string) (*PipelineRemote, error) { @@ -124,15 +155,21 @@ func makePipelineRemote(p *pipelines.GetPipelineResponse) *PipelineRemote { } } -func (r *ResourcePipeline) DoCreate(ctx context.Context, config *pipelines.CreatePipeline) (string, *PipelineRemote, error) { - response, err := r.client.Pipelines.Create(ctx, *config) +func (r *ResourcePipeline) DoCreate(ctx context.Context, config *PipelineState) (string, *PipelineRemote, error) { + response, err := r.client.Pipelines.Create(ctx, config.CreatePipeline) if err != nil { return "", nil, err } return response.PipelineId, nil, nil } -func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *pipelines.CreatePipeline, _ *PlanEntry) (*PipelineRemote, error) { +func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *PipelineState, entry *PlanEntry) (*PipelineRemote, error) { + // cascade_on_destroy is a delete-time-only setting with no update API, so a change to it + // alone must persist to state without a pipeline Update call. + if !entry.Changes.HasChangeExcept("cascade_on_destroy") { + return nil, nil + } + request := pipelines.EditPipeline{ AllowDuplicateNames: config.AllowDuplicateNames, BudgetPolicyId: config.BudgetPolicyId, @@ -174,8 +211,29 @@ func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *pipe return nil, r.client.Pipelines.Update(ctx, request) } -func (r *ResourcePipeline) DoDelete(ctx context.Context, id string, _ *pipelines.CreatePipeline) error { - return r.client.Pipelines.DeleteByPipelineId(ctx, id) +func (r *ResourcePipeline) DoDelete(ctx context.Context, id string, state *PipelineState) error { + if state.CascadeOnDestroy == nil { + // No explicit cascade_on_destroy in config: preserve the backend default (cascade). + return r.client.Pipelines.DeleteByPipelineId(ctx, id) + } + return r.client.Pipelines.Delete(ctx, pipelines.DeletePipelineRequest{ + PipelineId: id, + Cascade: *state.CascadeOnDestroy, + Force: false, + // Cascade is marshaled as `url:"cascade,omitempty"`, so a false value would be dropped from + // the query string. We specify `cascade` in ForceSendFields so the SDK send cascade=false explicitly + ForceSendFields: []string{"Cascade"}, + }) +} + +// OverrideChangeDesc forces a state-only Update when cascade_on_destroy changes locally. +// The field is a delete-time-only, client-side setting absent from PipelineRemote, so its +// Remote value is always nil. We add this override to force a state-only update. +func (*ResourcePipeline) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *PipelineRemote) error { + if path.String() == "cascade_on_destroy" && !structdiff.IsEqual(change.Old, change.New) { + change.Action = deployplan.Update + } + return nil } // Note, terraform provider either diff --git a/bundle/direct/dresources/pipeline_test.go b/bundle/direct/dresources/pipeline_test.go index cd5c9e26208..3629255a600 100644 --- a/bundle/direct/dresources/pipeline_test.go +++ b/bundle/direct/dresources/pipeline_test.go @@ -4,7 +4,11 @@ import ( "reflect" "testing" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) // TestPipelineRemote verifies that all fields from pipelines.GetPipelineResponse @@ -18,3 +22,34 @@ func TestPipelineRemote(t *testing.T) { "RunAs": true, // Available through embedded CreatePipeline }) } + +func TestPipelineOverrideChangeDescCascadeOnDestroy(t *testing.T) { + r := &ResourcePipeline{} + cascadePath := structpath.MustParsePath("cascade_on_destroy") + ptr := func(b bool) *bool { return &b } + + t.Run("unset forces state-only update", func(t *testing.T) { + change := &ChangeDesc{Action: deployplan.Skip, Old: ptr(false), New: nil} + require.NoError(t, r.OverrideChangeDesc(t.Context(), cascadePath, change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + }) + + t.Run("changed value forces state-only update", func(t *testing.T) { + change := &ChangeDesc{Action: deployplan.Skip, Old: ptr(true), New: ptr(false)} + require.NoError(t, r.OverrideChangeDesc(t.Context(), cascadePath, change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + }) + + t.Run("unchanged value is left as skip", func(t *testing.T) { + change := &ChangeDesc{Action: deployplan.Skip, Old: ptr(false), New: ptr(false)} + require.NoError(t, r.OverrideChangeDesc(t.Context(), cascadePath, change, nil)) + assert.Equal(t, deployplan.Skip, change.Action) + }) + + t.Run("other paths are untouched", func(t *testing.T) { + namePath := structpath.MustParsePath("name") + change := &ChangeDesc{Action: deployplan.Skip, Old: "a", New: nil} + require.NoError(t, r.OverrideChangeDesc(t.Context(), namePath, change, nil)) + assert.Equal(t, deployplan.Skip, change.Action) + }) +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index ac04be010b9..128e3b8e778 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -25,6 +25,11 @@ var knownMissingInRemoteType = map[string][]string{ "model_serving_endpoints": { "rate_limits", }, + "pipelines": { + // Note that this is a deliberate omission: cascade_on_destroy is a client-side-only field. + // It has no remote counterpart, so it is never expected in RemoteType. + "cascade_on_destroy", + }, "quality_monitors": { "skip_builtin_dashboard", "warehouse_id", diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 10832fe04a6..412cfb0ae16 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1287,6 +1287,9 @@ resources: path: ./pipeline.py ``` "$fields": + "cascade_on_destroy": + "description": |- + Whether destroying the pipeline also deletes its datasets (MVs, STs, Views). Defaults to true (the server default). Set to false to retain the datasets when the pipeline is deleted. Only affects the delete operation. "ingestion_definition": "$fields": "objects": diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2496c7033ad..d8170ffa76d 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -34,7 +34,8 @@ func assertRootPathExists(ctx context.Context, b *bundle.Bundle) (bool, error) { var destroyApprovalGroups = []approvalGroup{ {group: "schemas", message: deleteSchemaMessage}, - {group: "pipelines", message: deletePipelineMessage}, + // Pipelines are handled separately in approvalForDestroy so the message reflects each + // pipeline's cascade_on_destroy setting; see logPipelineDeleteApproval. {group: "volumes", message: deleteVolumeMessage}, {group: "database_instances", message: deleteDatabaseInstanceMessage}, {group: "synced_database_tables", message: deleteSyncedDatabaseTableMessage}, @@ -45,7 +46,44 @@ var destroyApprovalGroups = []approvalGroup{ {group: "genie_spaces", message: deleteGenieSpaceMessage}, } -func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (bool, error) { +// logPipelineDeleteApproval prints the pipeline deletions. If cascade_on_destroy is true, we will include +// a note that datasets will be deleted as well. +func logPipelineDeleteApproval(ctx context.Context, b *bundle.Bundle, actions []deployplan.Action, engine engine.EngineType) error { + pipelineDeletes := filterGroup(actions, "pipelines", deployplan.Delete) + + var cascading, retaining []deployplan.Action + for _, a := range pipelineDeletes { + cascade, err := pipelineDeletionCascades(b, a, engine) + if err != nil { + return err + } + if cascade { + cascading = append(cascading, a) + } else { + retaining = append(retaining, a) + } + } + + for _, grp := range []struct { + message string + actions []deployplan.Action + }{ + {deletePipelineWithCascadeMessage, cascading}, + {deletePipelineNoCascadeMessage, retaining}, + } { + if len(grp.actions) == 0 { + continue + } + cmdio.LogString(ctx, grp.message) + for _, a := range grp.actions { + cmdio.Log(ctx, a) + } + cmdio.LogString(ctx, "") + } + return nil +} + +func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) (bool, error) { deleteActions := plan.GetActions() // Deletes of resources that are already gone remotely only clean up the state, @@ -69,6 +107,9 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. } logApprovalGroups(ctx, deleteActions, destroyApprovalGroups, true, deployplan.Delete) + if err := logPipelineDeleteApproval(ctx, b, deleteActions, engine); err != nil { + return false, err + } cmdio.LogString(ctx, "All files and directories at the following location will be deleted: "+b.Config.Workspace.RootPath) cmdio.LogString(ctx, "") @@ -174,7 +215,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { } } - hasApproval, err := approvalForDestroy(ctx, b, plan) + hasApproval, err := approvalForDestroy(ctx, b, plan, engine) if err != nil { logdiag.LogError(ctx, err) return diff --git a/bundle/phases/messages.go b/bundle/phases/messages.go index 8f07b1e2429..388c0faa067 100644 --- a/bundle/phases/messages.go +++ b/bundle/phases/messages.go @@ -59,8 +59,11 @@ const DataLossWarning = "Deleting data assets such as schemas, pipelines, or vol const ( deleteSchemaMessage = `This action will result in the deletion of the following UC schemas. Any underlying data may be lost:` - deletePipelineMessage = `This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them:` + deletePipelineWithCascadeMessage = `This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion:` + + deletePipelineNoCascadeMessage = `This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines. +The Streaming Tables (STs) and Materialized Views (MVs) managed by them are retained, as 'cascade_on_destroy' is set to false:` deleteVolumeMessage = `This action will result in the deletion of the following volumes. For managed volumes, the files stored in the volume are also deleted from your diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 0af9394f243..aee481b2b14 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -2,6 +2,7 @@ package phases import ( "context" + "encoding/json" "errors" "fmt" @@ -12,6 +13,7 @@ import ( "github.com/databricks/cli/bundle/deploy" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/dyn" ) @@ -26,10 +28,38 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), mutator.ValidateLifecycleStarted(engine), + mutator.ValidateCascadeOnDestroy(engine), statemgmt.CheckRunningResource(engine), ) } +// pipelineDeletionCascades reports whether deleting the pipeline referenced by a delete action +// also deletes its datasets (MVs, STs, Views). This is the server default (cascade) unless +// cascade_on_destroy is explicitly set to false. +// +// Currently, this feature is only supported by the direct engine. We will read from the persisted +// state to determine the value. For the Terraform engine, this parameter cannot be configured, so +// there is no state to read from and we return the default of true. +func pipelineDeletionCascades(b *bundle.Bundle, action deployplan.Action, engine engine.EngineType) (bool, error) { + if !engine.IsDirect() { + return true, nil + } + + entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry(action.ResourceKey) + if !ok || len(entry.State) == 0 { + return true, nil + } + + var state dresources.PipelineState + if err := json.Unmarshal(entry.State, &state); err != nil { + return false, fmt.Errorf("parsing persisted state for %s: %w", action.ResourceKey, err) + } + if state.CascadeOnDestroy == nil { + return true, nil + } + return *state.CascadeOnDestroy, nil +} + // checkForPreventDestroy checks if the resource has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. // If it does, it returns an error. func checkForPreventDestroy(b *bundle.Bundle, actions []deployplan.Action) error { diff --git a/bundle/phases/plan_test.go b/bundle/phases/plan_test.go index 73249b4e13c..dd95de77542 100644 --- a/bundle/phases/plan_test.go +++ b/bundle/phases/plan_test.go @@ -1,15 +1,98 @@ package phases import ( + "encoding/json" + "path/filepath" "testing" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestPipelineDeletionCascades(t *testing.T) { + // The value must be read from persisted state (what the delete uses), not config. + b := &bundle.Bundle{} + data := dstate.NewDatabase("lineage", 1) + data.State["resources.pipelines.cascade_true"] = dstate.ResourceEntry{ID: "1", State: json.RawMessage(`{"cascade_on_destroy": true}`)} + data.State["resources.pipelines.cascade_false"] = dstate.ResourceEntry{ID: "2", State: json.RawMessage(`{"cascade_on_destroy": false}`)} + data.State["resources.pipelines.cascade_unset"] = dstate.ResourceEntry{ID: "3", State: json.RawMessage(`{}`)} + data.State["resources.pipelines.cascade_empty"] = dstate.ResourceEntry{ID: "4"} + b.DeploymentBundle.StateDB.OpenWithData(filepath.Join(t.TempDir(), "resources.json"), data) + + // A bundle whose direct state DB was never opened, as happens under the terraform + // engine. pipelineDeletionCascades must not touch the state DB there. + bTerraform := &bundle.Bundle{} + + tests := []struct { + name string + bundle *bundle.Bundle + engine engine.EngineType + resourceKey string + want bool + }{ + { + name: "cascade_on_destroy explicitly true", + bundle: b, + engine: engine.EngineDirect, + resourceKey: "resources.pipelines.cascade_true", + want: true, + }, + { + name: "cascade_on_destroy explicitly false", + bundle: b, + engine: engine.EngineDirect, + resourceKey: "resources.pipelines.cascade_false", + want: false, + }, + { + name: "cascade_on_destroy unset defaults to cascade", + bundle: b, + engine: engine.EngineDirect, + resourceKey: "resources.pipelines.cascade_unset", + want: true, + }, + { + name: "empty persisted state defaults to cascade", + bundle: b, + engine: engine.EngineDirect, + resourceKey: "resources.pipelines.cascade_empty", + want: true, + }, + { + name: "missing state entry defaults to cascade", + bundle: b, + engine: engine.EngineDirect, + resourceKey: "resources.pipelines.does_not_exist", + want: true, + }, + { + name: "terraform engine defaults to cascade without reading state", + bundle: bTerraform, + engine: engine.EngineTerraform, + resourceKey: "resources.pipelines.cascade_false", + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + action := deployplan.Action{ + ResourceKey: tt.resourceKey, + ActionType: deployplan.Delete, + } + cascade, err := pipelineDeletionCascades(tt.bundle, action, tt.engine) + require.NoError(t, err) + assert.Equal(t, tt.want, cascade) + }) + } +} + func TestCheckPreventDestroyForAllResources(t *testing.T) { for resourceType := range config.SupportedResources() { t.Run(resourceType, func(t *testing.T) { diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 4c78bd7c384..6ece80651a9 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1597,6 +1597,10 @@ "description": "[Public Preview] Budget policy of this pipeline.", "$ref": "#/$defs/string" }, + "cascade_on_destroy": { + "description": "Whether destroying the pipeline also deletes its datasets (MVs, STs, Views). Defaults to true (the server default). Set to false to retain the datasets when the pipeline is deleted. Only affects the delete operation.", + "$ref": "#/$defs/bool" + }, "catalog": { "description": "A catalog in Unity Catalog to publish data from this pipeline to. If `target` is specified, tables in this pipeline are published to a `target` schema inside `catalog` (for example, `catalog`.`target`.`table`). If `target` is not specified, no data is published to Unity Catalog.", "$ref": "#/$defs/string"