diff --git a/.nextchanges/bundles/uc-secrets.md b/.nextchanges/bundles/uc-secrets.md new file mode 100644 index 00000000000..423d709bf2b --- /dev/null +++ b/.nextchanges/bundles/uc-secrets.md @@ -0,0 +1 @@ +Add support for UC secrets resource ([#5861](https://github.com/databricks/cli/pull/5861)) diff --git a/.nextchanges/version b/.nextchanges/version index 81c871de46b..1cac385c6cb 100644 --- a/.nextchanges/version +++ b/.nextchanges/version @@ -1 +1 @@ -1.10.0 +1.11.0 diff --git a/acceptance/bundle/invariant/configs/secret.yml.tmpl b/acceptance/bundle/invariant/configs/secret.yml.tmpl new file mode 100644 index 00000000000..011f26726d1 --- /dev/null +++ b/acceptance/bundle/invariant/configs/secret.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: "Test secret value" + +resources: + secrets: + foo: + catalog_name: main + schema_name: default + name: test-secret-$UNIQUE_NAME + value: ${var.secret_value} + comment: "Test secret" + grants: + - principal: account users + privileges: + - USE_SCHEMA diff --git a/acceptance/bundle/invariant/configs/secret.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/secret.yml.tmpl-init.sh new file mode 100644 index 00000000000..f28895c1bdc --- /dev/null +++ b/acceptance/bundle/invariant/configs/secret.yml.tmpl-init.sh @@ -0,0 +1 @@ +export BUNDLE_VAR_secret_value="my-test-secret-value" diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 407bac1bf19..db094ab2b2e 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -35,6 +35,7 @@ EnvMatrix.INPUT_CONFIG = [ "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_uppercase_name.yml.tmpl", + "secret.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", "sql_warehouse.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index a91f990df3b..46d9e0f5565 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -36,3 +36,6 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml # migrate deploys via Terraform first, and the TF provider rejects an uppercase # volume schema_name ("inconsistent final plan"). Covered by no_drift on direct. EnvMatrixExclude.no_volume_uppercase = ["INPUT_CONFIG=volume_uppercase_name.yml.tmpl"] + +# Secrets are not supported in terraform mode. +EnvMatrixExclude.no_secret = ["INPUT_CONFIG=secret.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 1b0b5d6585a..7890a8975c2 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -44,6 +44,7 @@ EnvMatrix.INPUT_CONFIG = [ "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_uppercase_name.yml.tmpl", + "secret.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", "sql_warehouse.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 55e77c64640..d621c35c6df 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -62,6 +62,7 @@ EnvMatrix.INPUT_CONFIG = [ "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_uppercase_name.yml.tmpl", + "secret.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", "sql_warehouse.yml.tmpl", diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index d35bfa350f0..90bf70b196e 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3473,6 +3473,33 @@ resources.secret_scopes.*.permissions.acls[*] workspace.AclItem ALL resources.secret_scopes.*.permissions.acls[*].permission workspace.AclPermission ALL resources.secret_scopes.*.permissions.acls[*].principal string ALL resources.secret_scopes.*.permissions.scope_name string ALL +resources.secrets.*.catalog_name string ALL +resources.secrets.*.comment string ALL +resources.secrets.*.create_time *time.Time REMOTE STATE +resources.secrets.*.created_by string REMOTE STATE +resources.secrets.*.effective_owner string REMOTE STATE +resources.secrets.*.effective_value string REMOTE STATE +resources.secrets.*.expire_time *time.Time ALL +resources.secrets.*.fingerprint string STATE +resources.secrets.*.full_name string REMOTE STATE +resources.secrets.*.id string INPUT +resources.secrets.*.lifecycle resources.Lifecycle INPUT +resources.secrets.*.lifecycle.prevent_destroy bool INPUT +resources.secrets.*.metastore_id string REMOTE STATE +resources.secrets.*.modified_status string INPUT +resources.secrets.*.name string ALL +resources.secrets.*.owner string REMOTE STATE +resources.secrets.*.schema_name string ALL +resources.secrets.*.update_time *time.Time REMOTE STATE +resources.secrets.*.updated_by string REMOTE STATE +resources.secrets.*.url string INPUT +resources.secrets.*.value string ALL +resources.secrets.*.grants.full_name string ALL +resources.secrets.*.grants.securable_type string ALL +resources.secrets.*.grants[*] catalog.PrivilegeAssignment ALL +resources.secrets.*.grants[*].principal string ALL +resources.secrets.*.grants[*].privileges []catalog.Privilege ALL +resources.secrets.*.grants[*].privileges[*] catalog.Privilege ALL resources.sql_warehouses.*.auto_stop_mins int ALL resources.sql_warehouses.*.channel *sql.Channel ALL resources.sql_warehouses.*.channel.dbsql_version string ALL diff --git a/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl new file mode 100644 index 00000000000..16d033b6042 --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: "Test secret value" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret_${UNIQUE_NAME} + value: ${var.secret_value} + comment: "Test secret for acceptance testing" diff --git a/acceptance/bundle/resources/secrets/basic/out.test.toml b/acceptance/bundle/resources/secrets/basic/out.test.toml new file mode 100644 index 00000000000..fe4076cdf9b --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/basic/output.txt b/acceptance/bundle/resources/secrets/basic/output.txt new file mode 100644 index 00000000000..1e3d1b9f46b --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/output.txt @@ -0,0 +1,102 @@ + +>>> [CLI] bundle validate --var secret_value=initial-secret-value +Name: test-bundle-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle summary --var secret_value=initial-secret-value +Name: test-bundle-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default +Resources: + Secrets: + secret1: + Name: main.default.test_secret_[UNIQUE_NAME] + URL: (not deployed) + +>>> [CLI] bundle deploy --var secret_value=initial-secret-value +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "main", + "comment": "Test secret for acceptance testing", + "name": "test_secret_[UNIQUE_NAME]", + "schema_name": "default", + "value": "initial-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret_[UNIQUE_NAME]' comment='Test secret for acceptance testing' value='' effective_value=None + +=== Update comment and re-deploy +>>> update_file.py databricks.yml Test secret for acceptance testing Updated comment + +>>> [CLI] bundle deploy --var secret_value=initial-secret-value +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/main.default.test_secret_[UNIQUE_NAME]", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "main", + "comment": "Updated comment", + "name": "test_secret_[UNIQUE_NAME]", + "schema_name": "default", + "value": "initial-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret_[UNIQUE_NAME]' comment='Updated comment' value='' effective_value=None + +=== Restore comment and re-deploy +>>> update_file.py databricks.yml Updated comment Test secret for acceptance testing + +>>> [CLI] bundle deploy --var secret_value=initial-secret-value +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/main.default.test_secret_[UNIQUE_NAME]", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "main", + "comment": "Test secret for acceptance testing", + "name": "test_secret_[UNIQUE_NAME]", + "schema_name": "default", + "value": "initial-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret_[UNIQUE_NAME]' comment='Test secret for acceptance testing' value='' effective_value=None + +>>> [CLI] bundle destroy --auto-approve --var secret_value=initial-secret-value +The following resources will be deleted: + delete resources.secrets.secret1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/secrets/basic/script b/acceptance/bundle/resources/secrets/basic/script new file mode 100755 index 00000000000..d7dd7b53f6f --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve --var secret_value=initial-secret-value + rm out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle validate --var secret_value=initial-secret-value +trace $CLI bundle summary --var secret_value=initial-secret-value +trace $CLI bundle deploy --var secret_value=initial-secret-value + +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment value effective_value + +title "Update comment and re-deploy" +trace update_file.py databricks.yml "Test secret for acceptance testing" "Updated comment" +trace $CLI bundle deploy --var secret_value=initial-secret-value +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment value effective_value + +title "Restore comment and re-deploy" +trace update_file.py databricks.yml "Updated comment" "Test secret for acceptance testing" +trace $CLI bundle deploy --var secret_value=initial-secret-value +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment value effective_value + +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/secrets/basic/test.toml b/acceptance/bundle/resources/secrets/basic/test.toml new file mode 100644 index 00000000000..faf440b3dab --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/test.toml @@ -0,0 +1,9 @@ +Local = true +Cloud = true +RecordRequests = true +RequiresUnityCatalog = true + +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/direct-only/databricks.yml b/acceptance/bundle/resources/secrets/direct-only/databricks.yml new file mode 100644 index 00000000000..6edb6f3cf13 --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: test-bundle + +variables: + secret_value: + description: "Test secret value" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/secrets/direct-only/out.test.toml b/acceptance/bundle/resources/secrets/direct-only/out.test.toml new file mode 100644 index 00000000000..65156e0457c --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/secrets/direct-only/output.txt b/acceptance/bundle/resources/secrets/direct-only/output.txt new file mode 100644 index 00000000000..d3bef4d3b23 --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/output.txt @@ -0,0 +1,11 @@ + +=== Deploy should fail in terraform mode +>>> [CLI] bundle deploy --var secret_value=test-value +Error: Secret resources are only supported with direct deployment mode + in databricks.yml:10:5 + +Secret resources require direct deployment mode. Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use secret resources. +Learn more at https://docs.databricks.com/dev-tools/bundles/direct + + +Exit code: 1 diff --git a/acceptance/bundle/resources/secrets/direct-only/script b/acceptance/bundle/resources/secrets/direct-only/script new file mode 100755 index 00000000000..4ec9195eef9 --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/script @@ -0,0 +1,4 @@ +title "Deploy should fail in terraform mode" +trace $CLI bundle deploy --var secret_value=test-value 2>&1 | contains.py \ + "Secret resources are only supported with direct deployment mode" \ + "DATABRICKS_BUNDLE_ENGINE" diff --git a/acceptance/bundle/resources/secrets/direct-only/test.toml b/acceptance/bundle/resources/secrets/direct-only/test.toml new file mode 100644 index 00000000000..38ca093e146 --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false +RecordRequests = false + +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/secrets/update-value/databricks.yml b/acceptance/bundle/resources/secrets/update-value/databricks.yml new file mode 100644 index 00000000000..314e22db6ed --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +variables: + secret_value: + description: "Test secret value" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret + value: ${var.secret_value} + comment: "Test secret" diff --git a/acceptance/bundle/resources/secrets/update-value/out.test.toml b/acceptance/bundle/resources/secrets/update-value/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/update-value/output.txt b/acceptance/bundle/resources/secrets/update-value/output.txt new file mode 100644 index 00000000000..98f89e9d6e8 --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/output.txt @@ -0,0 +1,85 @@ + +>>> [CLI] bundle plan --var secret_value=initial-secret-value -o json +{ + "plan_version": 2, + "cli_version": "[CLI_VERSION]", + "plan": { + "resources.secrets.secret1": { + "action": "create", + "new_state": { + "value": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "" + } + } + } + } +} + +>>> [CLI] bundle deploy --var secret_value=initial-secret-value +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "initial-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret' value='' + +=== Update secret value by passing a different variable value +>>> [CLI] bundle deploy --var secret_value=updated-secret-value +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/main.default.test_secret", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "updated-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret' value='' + +=== Verify state does not contain actual secret value +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "state": { + "resources.secrets.secret1": { + "__id__": "main.default.test_secret", + "state": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "" + } + } + } +} diff --git a/acceptance/bundle/resources/secrets/update-value/script b/acceptance/bundle/resources/secrets/update-value/script new file mode 100755 index 00000000000..56ae4f03f94 --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/script @@ -0,0 +1,15 @@ +trace $CLI bundle plan --var secret_value=initial-secret-value -o json +trace $CLI bundle deploy --var secret_value=initial-secret-value + +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment value + +title "Update secret value by passing a different variable value" +trace $CLI bundle deploy --var secret_value=updated-secret-value +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment value + +title "Verify state does not contain actual secret value" +{ trace print_state.py | contains.py "!initial-secret-value" "!updated-secret-value"; } || true + +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/secrets/update-value/test.toml b/acceptance/bundle/resources/secrets/update-value/test.toml new file mode 100644 index 00000000000..331fda4ece2 --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false +RecordRequests = true + +Ignore = [".databricks", ".gitignore"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/databricks.yml b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/databricks.yml new file mode 100644 index 00000000000..aee031ac950 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +variables: + secret_value: + description: "Test secret value" + default: "plain-text-default-not-allowed" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/output.txt b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/output.txt new file mode 100644 index 00000000000..a8242e1fec1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/output.txt @@ -0,0 +1,14 @@ + +=== Deploy should fail when variable used for secret has a default value +>>> [CLI] bundle deploy +Error: Variable used for secret value must not have a default value + at resources.secrets.secret1.value + in databricks.yml:7:14 + databricks.yml:15:14 + +The variable "secret_value" used for the secret "secret1" must not have a default value. +A default value is stored in plain text in the configuration file, which defeats the purpose of using a variable reference for a secret. +Remove the default value and pass the secret value at deployment time using "--var", the BUNDLE_VAR_secret_value environment variable, or a variable overrides file. + + +Exit code: 1 diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/script b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/script new file mode 100644 index 00000000000..f4095fe7141 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/script @@ -0,0 +1,4 @@ +title "Deploy should fail when variable used for secret has a default value" +trace $CLI bundle deploy 2>&1 | contains.py \ + "Variable used for secret value must not have a default value" \ + "default value is stored in plain text" diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/test.toml new file mode 100644 index 00000000000..9a9e1194d6c --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false +RecordRequests = false + +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/databricks.yml b/acceptance/bundle/resources/secrets/validate-no-plain-text/databricks.yml new file mode 100644 index 00000000000..d183373d3b0 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: test-bundle + +resources: + catalogs: + test_catalog: + name: ${workspace.resource_prefix}_catalog + + schemas: + test_schema: + name: ${workspace.resource_prefix}_schema + catalog_name: ${resources.catalogs.test_catalog.name} + + secrets: + secret1: + catalog_name: ${resources.catalogs.test_catalog.name} + schema_name: ${resources.schemas.test_schema.name} + name: test_secret + value: "plain-text-secret-not-allowed" + comment: "This should fail validation" diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt b/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt new file mode 100644 index 00000000000..582ae14bb48 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt @@ -0,0 +1,13 @@ + +=== Deploy should fail due to plain text secret value +>>> [CLI] bundle deploy +Error: Secret value must be a variable reference + at resources.secrets.secret1.value + in databricks.yml:19:14 + +The secret value for "secret1" must be a variable reference (e.g., ${var.my_secret}). +Plain text secret values are not allowed to prevent leaking secrets in configuration files. +Use bundle variables to pass secret values at deployment time. + + +Exit code: 1 diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/script b/acceptance/bundle/resources/secrets/validate-no-plain-text/script new file mode 100755 index 00000000000..309005d41ab --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/script @@ -0,0 +1,4 @@ +title "Deploy should fail due to plain text secret value" +trace $CLI bundle deploy 2>&1 | contains.py \ + "Secret value must be a variable reference" \ + "Plain text secret values are not allowed" diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text/test.toml new file mode 100644 index 00000000000..9a9e1194d6c --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false +RecordRequests = false + +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index 08d3a757c83..1cd89cceda4 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,7 +9,7 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses === test auto-completion handler >>> [CLI] __complete experimental open , @@ -34,6 +34,7 @@ quality_monitors queries registered_models schemas +secrets synced_database_tables vector_search_endpoints vector_search_indexes diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 85f30c952f2..855878127e6 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -35,6 +35,7 @@ var unsupportedResources = []string{ "postgres_synced_tables", "vector_search_indexes", "job_runs", + "secrets", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index b97113028b8..73c76216c4e 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -179,6 +179,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Name: "secretScope1", }, }, + Secrets: map[string]*resources.Secret{ + "secret1": { + CatalogName: "main", + SchemaName: "default", + Name: "secret1", + }, + }, SqlWarehouses: map[string]*resources.SqlWarehouse{ "sql_warehouse1": { CreateWarehouseRequest: sql.CreateWarehouseRequest{ @@ -502,6 +509,7 @@ func TestAppropriateResourcesAreRenamed(t *testing.T) { notUserNamed := []string{ "Apps", "SecretScopes", + "Secrets", "DatabaseInstances", "DatabaseCatalogs", "SyncedDatabaseTables", diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index 61c2fed2592..c4b387b76c2 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -99,6 +99,13 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia v.SchemaName = resolveSchema(b, v.CatalogName, v.SchemaName) v.CatalogName = resolveCatalog(b, v.CatalogName) } + for _, s := range b.Config.Resources.Secrets { + if s == nil { + continue + } + s.SchemaName = resolveSchema(b, s.CatalogName, s.SchemaName) + s.CatalogName = resolveCatalog(b, s.CatalogName) + } for _, rm := range b.Config.Resources.RegisteredModels { if rm == nil { continue diff --git a/bundle/config/mutator/resourcemutator/merge_grants.go b/bundle/config/mutator/resourcemutator/merge_grants.go index a90e9a57507..3966190a5ee 100644 --- a/bundle/config/mutator/resourcemutator/merge_grants.go +++ b/bundle/config/mutator/resourcemutator/merge_grants.go @@ -14,6 +14,7 @@ var grantResourceTypes = []string{ "catalogs", "schemas", "external_locations", + "secrets", "volumes", "registered_models", "vector_search_indexes", diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 5faed5f7b1b..87312608de8 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -59,6 +59,7 @@ func allResourceTypes(t *testing.T) []string { "registered_models", "schemas", "secret_scopes", + "secrets", "sql_warehouses", "synced_database_tables", "vector_search_endpoints", @@ -195,6 +196,7 @@ var allowList = []string{ "job_runs", "schemas", "secret_scopes", + "secrets", "sql_warehouses", "vector_search_endpoints", "vector_search_indexes", diff --git a/bundle/config/mutator/validate_secret_value_is_variable.go b/bundle/config/mutator/validate_secret_value_is_variable.go new file mode 100644 index 00000000000..b077d9430e5 --- /dev/null +++ b/bundle/config/mutator/validate_secret_value_is_variable.go @@ -0,0 +1,97 @@ +package mutator + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" +) + +type validateSecretValueIsVariable struct{} + +func ValidateSecretValueIsVariable() bundle.Mutator { + return &validateSecretValueIsVariable{} +} + +func (v *validateSecretValueIsVariable) Name() string { + return "ValidateSecretValueIsVariable" +} + +func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + // Iterate over all secrets in the bundle + for key := range b.Config.Resources.Secrets { + p := dyn.NewPath(dyn.Key("resources"), dyn.Key("secrets"), dyn.Key(key), dyn.Key("value")) + val, err := dyn.GetByPath(b.Config.Value(), p) + if dyn.IsNoSuchKeyError(err) { + continue + } + if err != nil { + return diag.FromErr(err) + } + + valueStr, ok := val.AsString() + if !ok { + continue + } + + // Value must be a variable reference to prevent leaking secrets in config files + if !dynvar.IsPureVariableReference(valueStr) { + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "Secret value must be a variable reference", + Detail: fmt.Sprintf(`The secret value for "%s" must be a variable reference (e.g., ${var.my_secret}). +Plain text secret values are not allowed to prevent leaking secrets in configuration files. +Use bundle variables to pass secret values at deployment time.`, key), + Locations: val.Locations(), + Paths: []dyn.Path{p}, + }) + continue + } + + // The value is a variable reference. If it is a pure ${var.} reference, + // check that the referenced variable does not have a default value set. A default + // value would be stored in plain text in the config file, defeating the purpose + // of using a variable reference. + diags = append(diags, v.checkVariableDefault(b, key, valueStr, p, val)...) + } + + return diags +} + +// checkVariableDefault emits an error if valueStr is a pure ${var.} reference +// and the referenced variable has a default value set. +func (v *validateSecretValueIsVariable) checkVariableDefault(b *bundle.Bundle, secretKey, valueStr string, p dyn.Path, val dyn.Value) diag.Diagnostics { + refPath, ok := dynvar.PureReferenceToPath(valueStr) + if !ok || len(refPath) < 2 || refPath[0].Key() != "var" { + return nil + } + + varName := refPath[1].Key() + variable, exists := b.Config.Variables[varName] + if !exists || variable == nil || !variable.HasDefault() { + return nil + } + + // The default path in the dynamic config for the variable's default field. + defaultPath := dyn.NewPath(dyn.Key("variables"), dyn.Key(varName), dyn.Key("default")) + defaultVal, err := dyn.GetByPath(b.Config.Value(), defaultPath) + locations := val.Locations() + if err == nil { + locations = append(defaultVal.Locations(), locations...) + } + + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: "Variable used for secret value must not have a default value", + Detail: fmt.Sprintf(`The variable "%s" used for the secret "%s" must not have a default value. +A default value is stored in plain text in the configuration file, which defeats the purpose of using a variable reference for a secret. +Remove the default value and pass the secret value at deployment time using "--var", the BUNDLE_VAR_%s environment variable, or a variable overrides file.`, varName, secretKey, varName), + Locations: locations, + Paths: []dyn.Path{p}, + }} +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 3ab06f49918..ab12ec9f052 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -44,6 +44,7 @@ type Resources struct { VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` InstancePools map[string]*resources.InstancePool `json:"instance_pools,omitempty"` + Secrets map[string]*resources.Secret `json:"secrets,omitempty"` } type ConfigResource interface { @@ -129,6 +130,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), collectResourceMap(descriptions["instance_pools"], r.InstancePools), + collectResourceMap(descriptions["secrets"], r.Secrets), } } @@ -192,5 +194,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_synced_tables": (&resources.PostgresSyncedTable{}).ResourceDescription(), "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), + "secrets": (&resources.Secret{}).ResourceDescription(), } } diff --git a/bundle/config/resources/secret.go b/bundle/config/resources/secret.go new file mode 100644 index 00000000000..7f29ba1f235 --- /dev/null +++ b/bundle/config/resources/secret.go @@ -0,0 +1,98 @@ +package resources + +import ( + "context" + "fmt" + "net/url" + "time" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +type Secret struct { + BaseResource + + // The name of the catalog where the schema and the secret reside. + CatalogName string `json:"catalog_name"` + + // The name of the schema where the secret resides. + SchemaName string `json:"schema_name"` + + // The name of the secret, relative to its parent schema. + Name string `json:"name"` + + // The secret value to store. This field must be a variable reference (e.g., ${var.my_secret}) + // to prevent leaking secrets in configuration files. Plain text values are not allowed. + // The maximum size is 60 KiB (pre-encryption). + Value string `json:"value" bundle:"sensitive"` + + // User-provided free-form text description of the secret. + Comment string `json:"comment,omitempty"` + + // User-provided expiration time of the secret. This field indicates when + // the secret should no longer be used and may be displayed as a warning in + // the UI. It is purely informational and does not trigger any automatic + // actions or affect the secret's lifecycle. + ExpireTime *time.Time `json:"expire_time,omitempty"` + + // List of grants to apply on this secret. + Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` +} + +func (s *Secret) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s *Secret) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s *Secret) Exists(ctx context.Context, w *databricks.WorkspaceClient, fullName string) (bool, error) { + log.Tracef(ctx, "Checking if secret with fullName=%s exists", fullName) + + _, err := w.SecretsUc.GetSecret(ctx, catalog.GetSecretRequest{ + FullName: fullName, + }) + if err != nil { + log.Debugf(ctx, "secret with full name %s does not exist: %v", fullName, err) + + if apierr.IsMissing(err) { + return false, nil + } + + return false, err + } + return true, nil +} + +func (*Secret) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "secret", + PluralName: "secrets", + SingularTitle: "Secret", + PluralTitle: "Secrets", + } +} + +func (s *Secret) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + s.URL = workspaceurls.ResourceURL(baseURL, "secrets", s.ID) +} + +func (s *Secret) GetURL() string { + return s.URL +} + +func (s *Secret) GetName() string { + if s.ID != "" { + return s.ID + } + return fmt.Sprintf("%s.%s.%s", s.CatalogName, s.SchemaName, s.Name) +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index de9caa09dc2..b42c8dca170 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -240,6 +240,13 @@ func TestResourcesBindSupport(t *testing.T) { Name: "0", }, }, + Secrets: map[string]*resources.Secret{ + "my_secret": { + CatalogName: "main", + SchemaName: "default", + Name: "my_secret", + }, + }, SqlWarehouses: map[string]*resources.SqlWarehouse{ "my_sql_warehouse": { CreateWarehouseRequest: sql.CreateWarehouseRequest{}, @@ -381,6 +388,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPostgresAPI().EXPECT().GetRole(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchEndpointsAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchIndexesAPI().EXPECT().GetIndexByIndexName(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockSecretsUcAPI().EXPECT().GetSecret(mock.Anything, mock.Anything).Return(&catalog.Secret{FullName: "0"}, nil) allResources := supportedResources.AllResources() for _, group := range allResources { diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 9fb59329ffd..e44b86509ef 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -20,6 +20,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { "genie_spaces", "instance_pools", "job_runs", + "secrets", "vector_search_endpoints", "vector_search_indexes", } diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index d6bcd03f73c..ad310468da0 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -39,6 +39,7 @@ var SupportedResources = map[string]any{ "vector_search_endpoints": (*ResourceVectorSearchEndpoint)(nil), "vector_search_indexes": (*ResourceVectorSearchIndex)(nil), "instance_pools": (*ResourceInstancePool)(nil), + "secrets": (*ResourceSecret)(nil), // Permissions "jobs.permissions": (*ResourcePermissions)(nil), @@ -65,6 +66,7 @@ var SupportedResources = map[string]any{ "volumes.grants": (*ResourceGrants)(nil), "registered_models.grants": (*ResourceGrants)(nil), "vector_search_indexes.grants": (*ResourceGrants)(nil), + "secrets.grants": (*ResourceGrants)(nil), } func InitAll(client *databricks.WorkspaceClient) (map[string]*Adapter, error) { diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index a960e168fec..ad7a9cde532 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -129,6 +129,14 @@ var testConfig map[string]any = map[string]any{ }, }, + "secrets": &resources.Secret{ + CatalogName: "main", + SchemaName: "default", + Name: "my_secret", + Value: "my_secret_value", + Comment: "Test secret", + }, + "secret_scopes": &resources.SecretScope{ Name: "my_secret_scope", BackendType: workspace.ScopeBackendTypeAzureKeyvault, @@ -711,6 +719,17 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "secrets.grants": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + return &GrantsState{ + SecurableType: "secret", + FullName: "main.default.my_secret", + EmbeddedSlice: []catalog.PrivilegeAssignment{{ + Privileges: []catalog.Privilege{catalog.PrivilegeSelect}, + Principal: "user@example.com", + }}, + }, nil + }, + "secret_scopes.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { err := client.Secrets.CreateScope(ctx, workspace.CreateScope{ Scope: "permissions_test_scope", @@ -893,6 +912,9 @@ type testIgnoreFilter struct { } // newTestIgnoreFilter creates a filter from the adapter's resource configs. +// It also ignores fields that exist in StateType but not in RemoteType, because +// those are automatically suppressed by the planner (reason: missing_in_remote) +// and RemapState cannot populate them from remote state. func newTestIgnoreFilter(adapter *Adapter) *testIgnoreFilter { ignoreFields := make(map[string]bool) for _, cfg := range []*ResourceLifecycleConfig{adapter.ResourceConfig(), adapter.GeneratedResourceConfig()} { @@ -903,6 +925,17 @@ func newTestIgnoreFilter(adapter *Adapter) *testIgnoreFilter { ignoreFields[p.Field.String()] = true } } + // Auto-include fields present in StateType but absent from RemoteType. + _ = structwalk.WalkType(adapter.StateType(), func(path *structpath.PatternNode, typ reflect.Type, field *reflect.StructField) bool { + if path.IsRoot() { + return true + } + if structaccess.ValidatePattern(adapter.RemoteType(), path) != nil { + ignoreFields[path.String()] = true + return false + } + return true + }) return &testIgnoreFilter{ignoreFields: ignoreFields} } diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 6da6f3555ec..5e61d803183 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -54,6 +54,8 @@ schemas: catalog.CreateSchema secret_scopes: workspace.CreateScope +secrets: catalog.Secret + sql_warehouses: sql.EditWarehouseRequest synced_database_tables: database.SyncedDatabaseTable diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index bac70bc5615..f2cde9eb726 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -18,6 +18,7 @@ var grantResourceToSecurableType = map[string]string{ "external_locations": "external_location", "volumes": "volume", "registered_models": "function", + "secrets": "secret", "vector_search_indexes": "table", } diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index c25431c9037..8fcde464bc3 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -348,6 +348,39 @@ resources: # secret_scopes: no api field behaviors + secrets: + + recreate_on_changes: + - field: catalog_name + reason: spec:immutable + - field: name + reason: spec:immutable + - field: schema_name + reason: spec:immutable + + ignore_remote_changes: + - field: owner + reason: spec:input_only + - field: value + reason: spec:input_only + + - field: create_time + reason: spec:output_only + - field: created_by + reason: spec:output_only + - field: effective_owner + reason: spec:output_only + - field: effective_value + reason: spec:output_only + - field: full_name + reason: spec:output_only + - field: metastore_id + reason: spec:output_only + - field: update_time + reason: spec:output_only + - field: updated_by + reason: spec:output_only + # sql_warehouses: no api field behaviors synced_database_tables: diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go new file mode 100644 index 00000000000..e7f891bef14 --- /dev/null +++ b/bundle/direct/dresources/secret.go @@ -0,0 +1,163 @@ +package dresources + +import ( + "context" + "net/http" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/common/types/fieldmask" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +// Terraform provider implementation: +// https://github.com/databricks/terraform-provider-databricks/blob/main/catalog/resource_secret.go +type ResourceSecret struct { + client *databricks.WorkspaceClient +} + +// SecretState is the persisted state type for a UC secret. It extends the SDK +// Secret struct with a Fingerprint field so that value changes can be detected +// across deploys without storing the plaintext value on disk. The Value field +// is always cleared after the API call (see DoCreate/DoUpdate). +type SecretState struct { + catalog.Secret + + // SecretValue is the plaintext value of the secret. It is not stored in the state file. + // It is carried here so DoCreate/DoUpdate can send it to the API. + SecretValue string `json:"-" bundle:"sensitive"` +} + +func (*ResourceSecret) New(client *databricks.WorkspaceClient) *ResourceSecret { + return &ResourceSecret{client: client} +} + +func (*ResourceSecret) PrepareState(input *resources.Secret) *SecretState { + var expireTime *sdktime.Time + if input.ExpireTime != nil { + expireTime = sdktime.New(*input.ExpireTime) + } + return &SecretState{ + Secret: catalog.Secret{ + CatalogName: input.CatalogName, + SchemaName: input.SchemaName, + Name: input.Name, + Value: "", + Comment: input.Comment, + ExpireTime: expireTime, + Owner: "", + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + EffectiveValue: "", + FullName: "", + MetastoreId: "", + UpdateTime: nil, + UpdatedBy: "", + ForceSendFields: utils.FilterFields[catalog.Secret](nil), + }, + // Value is carried here so DoCreate/DoUpdate can send it to the API. + // It is cleared from state after the API call (see DoCreate/DoUpdate). + SecretValue: input.Value, + } +} + +func (*ResourceSecret) RemapState(remote *catalog.Secret) *SecretState { + return &SecretState{ + Secret: catalog.Secret{ + CatalogName: remote.CatalogName, + SchemaName: remote.SchemaName, + Name: remote.Name, + Comment: remote.Comment, + Owner: remote.Owner, + ExpireTime: remote.ExpireTime, + Value: "", + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + EffectiveValue: remote.EffectiveValue, + FullName: "", + MetastoreId: "", + UpdateTime: nil, + UpdatedBy: "", + ForceSendFields: utils.FilterFields[catalog.Secret](remote.ForceSendFields), + }, + SecretValue: remote.EffectiveValue, + } +} + +// DoRead fetches the secret by full name. +func (r *ResourceSecret) DoRead(ctx context.Context, id string) (*catalog.Secret, error) { + apiClient, err := client.New(r.client.Config) + if err != nil { + return nil, err + } + + // SDK does not support include_value in the GetSecretRequest, so we use the API directly. + var secret catalog.Secret + err = apiClient.Do(ctx, http.MethodGet, "/api/2.1/unity-catalog/secrets/"+id, auth.WorkspaceIDHeaders(r.client.Config), map[string]any{ + "include_value": true, + }, nil, &secret) + if err != nil { + return nil, err + } + return &secret, nil +} + +// DoCreate creates a new UC secret. +func (r *ResourceSecret) DoCreate(ctx context.Context, state *SecretState) (string, *catalog.Secret, error) { + state.Value = state.SecretValue + response, err := r.client.SecretsUc.CreateSecret(ctx, catalog.CreateSecretRequest{ + Secret: state.Secret, + }) + // Clear the plaintext so it is not written to the state file. + // Fingerprint already captures whether the value changed. + state.Value = "" + if err != nil || response == nil { + return "", nil, err + } + return response.FullName, response, nil +} + +// DoUpdate updates the secret in place and returns remote state. +func (r *ResourceSecret) DoUpdate(ctx context.Context, id string, state *SecretState, _ *PlanEntry) (*catalog.Secret, error) { + state.Value = state.SecretValue + response, err := r.client.SecretsUc.UpdateSecret(ctx, catalog.UpdateSecretRequest{ + FullName: id, + Secret: state.Secret, + UpdateMask: fieldmask.FieldMask{ + Paths: []string{"*"}, + }, + }) + // Clear the plaintext so it is not written to the state file. + state.Value = "" + if err != nil { + return nil, err + } + return response, nil +} + +// DoDelete deletes the secret. +func (r *ResourceSecret) DoDelete(ctx context.Context, id string, _ *SecretState) error { + return r.client.SecretsUc.DeleteSecret(ctx, catalog.DeleteSecretRequest{ + FullName: id, + }) +} + +// MarshalJSON serializes SecretState as a merged JSON object: the fields from +// catalog.Secret (via its own MarshalJSON) plus "fingerprint". Without this, +// the embedded catalog.Secret.MarshalJSON takes over and drops Fingerprint. +func (s SecretState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +// UnmarshalJSON deserializes SecretState, restoring both the embedded +// catalog.Secret fields and Fingerprint. +func (s *SecretState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index ac04be010b9..8067c706699 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -54,6 +54,12 @@ var knownMissingInRemoteType = map[string][]string{ "vector_search_endpoints": { "usage_policy_id", }, + // fingerprint is a locally-computed SHA-256 digest of the secret value used + // to detect value changes without storing the plaintext. The remote API never + // returns it, so it is intentionally absent from RemoteType. + "secrets": { + "fingerprint", + }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index f6c8fc8ba3c..7e92aa7c1d2 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -16,9 +16,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/internal/build" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/structs/structwalk" "github.com/google/uuid" ) @@ -129,10 +127,7 @@ func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []d db.Data.State = make(map[string]ResourceEntry) } - // Redact sensitive fields before persisting: secrets must not appear on disk - // in plaintext. The original struct is not modified; the plan uses the - // unredacted in-memory value for API calls. - jsonMessage, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + jsonMessage, err := json.Marshal(state) if err != nil { return err } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 10832fe04a6..978ec4bc1e0 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1859,6 +1859,36 @@ resources: "user_name": "description": |- The name of the user granted the permission level. This field translates to a `principal` field in secret scope ACL. + "secrets": + "description": |- + The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. + "markdown_description": |- + The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. See [\_](/dev-tools/bundles/resources.md#secrets). + "$fields": + "catalog_name": + "description": |- + The name of the catalog where the schema and the secret reside. + "comment": + "description": |- + User-provided free-form text description of the secret. + "expire_time": + "description": |- + User-provided expiration time of the secret. Purely informational; does not trigger automatic actions. + "grants": + "description": |- + The grants to apply on this secret. + "lifecycle": + "description": |- + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + "name": + "description": |- + The name of the secret, relative to its parent schema. + "schema_name": + "description": |- + The name of the schema where the secret resides. + "value": + "description": |- + The secret value to store. Must be a variable reference (e.g. ${var.my_secret}) to prevent plain-text secrets in configuration files. "sql_warehouses": "description": |- The SQL warehouse definitions for the bundle, where each key is the name of the warehouse. diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index ee87b6a7d71..ef90948b21f 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -228,6 +228,8 @@ var EnumFields = map[string][]string{ "resources.secret_scopes.*.backend_type": {"AZURE_KEYVAULT", "DATABRICKS"}, + "resources.secrets.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, + "resources.sql_warehouses.*.channel.name": {"CHANNEL_NAME_CURRENT", "CHANNEL_NAME_CUSTOM", "CHANNEL_NAME_PREVIEW", "CHANNEL_NAME_PREVIOUS"}, "resources.sql_warehouses.*.permissions[*].level": {"CAN_MANAGE", "CAN_MONITOR", "CAN_USE", "CAN_VIEW", "IS_OWNER"}, "resources.sql_warehouses.*.spot_instance_policy": {"COST_OPTIMIZED", "POLICY_UNSPECIFIED", "RELIABILITY_OPTIMIZED"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 37e0fb8d729..6faf1ff9f42 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -265,6 +265,8 @@ var RequiredFields = map[string][]string{ "resources.secret_scopes.*.keyvault_metadata": {"dns_name", "resource_id"}, "resources.secret_scopes.*.permissions[*]": {"level"}, + "resources.secrets.*": {"catalog_name", "schema_name", "name", "value"}, + "resources.sql_warehouses.*.permissions[*]": {"level"}, "resources.synced_database_tables.*": {"name"}, diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index bfa2af4124b..7fa462627bb 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -33,6 +33,7 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { validate.NoInterpolationInBundleName(), validate.ValidateEngine(), validate.Scripts(), + mutator.ValidateSecretValueIsVariable(), // Updates (dynamic): sync.{paths,include,exclude} (makes them relative to bundle root rather than to definition file) // Rewrites sync paths to be relative to the bundle root instead of the file they were defined in. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 4c78bd7c384..2987f37a9a3 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2340,6 +2340,58 @@ } ] }, + "resources.Secret": { + "oneOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog where the schema and the secret reside.", + "$ref": "#/$defs/string" + }, + "comment": { + "description": "User-provided free-form text description of the secret.", + "$ref": "#/$defs/string" + }, + "expire_time": { + "description": "User-provided expiration time of the secret. Purely informational; does not trigger automatic actions.", + "$ref": "#/$defs/time.Time" + }, + "grants": { + "description": "The grants to apply on this secret.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment" + }, + "lifecycle": { + "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "name": { + "description": "The name of the secret, relative to its parent schema.", + "$ref": "#/$defs/string" + }, + "schema_name": { + "description": "The name of the schema where the secret resides.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "The secret value to store. Must be a variable reference (e.g. ${var.my_secret}) to prevent plain-text secrets in configuration files.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_name", + "schema_name", + "name", + "value" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.SecretScope": { "oneOf": [ { @@ -3294,6 +3346,11 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SecretScope", "markdownDescription": "The secret scope definitions for the bundle, where each key is the name of the secret scope. See [secret_scopes](https://docs.databricks.com/dev-tools/bundles/resources.html#secret_scopes)." }, + "secrets": { + "description": "The Unity Catalog secret definitions for the bundle, where each key is the name of the secret.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Secret", + "markdownDescription": "The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. See [secrets](https://docs.databricks.com/dev-tools/bundles/resources.html#secrets)." + }, "sql_warehouses": { "description": "The SQL warehouse definitions for the bundle, where each key is the name of the warehouse.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SqlWarehouse", @@ -14452,6 +14509,20 @@ } ] }, + "resources.Secret": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Secret" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.SecretScope": { "oneOf": [ { @@ -15540,6 +15611,18 @@ }, "string": { "type": "string" + }, + "time.Time": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] } }, "type": "object", diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 1ec6dc82999..eea8b66de8c 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -58,6 +58,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, "resources.instance_pools.test_instance_pool": {ID: "1"}, + "resources.secrets.test_secret": {ID: "main.default.test_secret"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -153,6 +154,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.InstancePools["test_instance_pool"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.InstancePools["test_instance_pool"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret", config.Resources.Secrets["test_secret"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Secrets["test_secret"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -277,6 +281,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { Name: "test_secret_scope", }, }, + Secrets: map[string]*resources.Secret{ + "test_secret": { + CatalogName: "main", + SchemaName: "default", + Name: "test_secret", + }, + }, SqlWarehouses: map[string]*resources.SqlWarehouse{ "test_sql_warehouse": { CreateWarehouseRequest: sql.CreateWarehouseRequest{ @@ -446,6 +457,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.SecretScopes["test_secret_scope"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.SecretScopes["test_secret_scope"].ModifiedStatus) + assert.Empty(t, config.Resources.Secrets["test_secret"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Secrets["test_secret"].ModifiedStatus) + assert.Empty(t, config.Resources.SqlWarehouses["test_sql_warehouse"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.SqlWarehouses["test_sql_warehouse"].ModifiedStatus) @@ -693,6 +707,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { Name: "test_secret_scope_new", }, }, + Secrets: map[string]*resources.Secret{ + "test_secret": { + CatalogName: "main", + SchemaName: "default", + Name: "test_secret", + }, + "test_secret_new": { + CatalogName: "main", + SchemaName: "default", + Name: "test_secret_new", + }, + }, SqlWarehouses: map[string]*resources.SqlWarehouse{ "test_sql_warehouse": { CreateWarehouseRequest: sql.CreateWarehouseRequest{ @@ -941,6 +967,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.vector_search_indexes.test_vector_search_index_old": {ID: "vs-index-old"}, "resources.instance_pools.test_instance_pool": {ID: "1"}, "resources.instance_pools.test_instance_pool_old": {ID: "2"}, + "resources.secrets.test_secret": {ID: "main.default.test_secret"}, + "resources.secrets.test_secret_old": {ID: "main.default.test_secret_old"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -1143,6 +1171,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.InstancePools["test_instance_pool_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.InstancePools["test_instance_pool_new"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret", config.Resources.Secrets["test_secret"].ID) + assert.Empty(t, config.Resources.Secrets["test_secret"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret_old", config.Resources.Secrets["test_secret_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Secrets["test_secret_old"].ModifiedStatus) + assert.Empty(t, config.Resources.Secrets["test_secret_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Secrets["test_secret_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index 861dab1eda4..71904677781 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", "") assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") + assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -128,6 +128,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { "queries", "registered_models", "schemas", + "secrets", "synced_database_tables", "vector_search_endpoints", "vector_search_indexes", @@ -147,7 +148,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, instance_pools, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") diff --git a/libs/structs/structdiff/diff.go b/libs/structs/structdiff/diff.go index a852d347e42..4e9df593994 100644 --- a/libs/structs/structdiff/diff.go +++ b/libs/structs/structdiff/diff.go @@ -207,10 +207,13 @@ func diffStruct(ctx *diffContext, path *structpath.PathNode, s1, s2 reflect.Valu } jsonTag := structtag.JSONTag(sf.Tag.Get("json")) + bundleTag := structtag.BundleTag(sf.Tag.Get("bundle")) // Resolve field name from JSON tag or fall back to Go field name + // Sensitive fields are marked as "json:-" so they are not accidentally stored in the state file. + // But we still want to diff them to detect changes based on in-memory values (comes from config and remote) fieldName := jsonTag.Name() - if fieldName == "-" { + if fieldName == "-" && !bundleTag.Sensitive() { continue } diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 8d6e8ee0dd3..d970c6c90e6 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -196,6 +196,7 @@ type FakeWorkspace struct { SecretScopes map[string]workspace.SecretScope Secrets map[string]map[string]string // scope -> key -> value Acls map[string][]workspace.AclItem + UCSecrets map[string]catalog.Secret // full_name -> secret (Unity Catalog secrets) // Generic permissions storage: key is "{object_type}:{object_id}" Permissions map[string]iam.ObjectPermissions diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 0d7581ca117..4ce22b867e9 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -779,6 +779,30 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.SecretsAclsDelete(req) }) + // Unity Catalog base endpoint (used for UC availability check): + server.Handle("GET", "/api/2.1/unity-catalog", func(req Request) any { + return map[string]any{ + "metastore_id": "test-metastore-id", + } + }) + + // Unity Catalog Secrets: + server.Handle("POST", "/api/2.1/unity-catalog/secrets", func(req Request) any { + return req.Workspace.SecretsUcCreateSecret(req) + }) + + server.Handle("GET", "/api/2.1/unity-catalog/secrets/{full_name}", func(req Request) any { + return req.Workspace.SecretsUcGetSecret(req) + }) + + server.Handle("PATCH", "/api/2.1/unity-catalog/secrets/{full_name}", func(req Request) any { + return req.Workspace.SecretsUcUpdateSecret(req) + }) + + server.Handle("DELETE", "/api/2.1/unity-catalog/secrets/{full_name}", func(req Request) any { + return req.Workspace.SecretsUcDeleteSecret(req) + }) + // Groups: server.Handle("POST", "/api/2.0/preview/scim/v2/Groups", func(req Request) any { return req.Workspace.GroupsCreate(req) diff --git a/libs/testserver/uc_secrets.go b/libs/testserver/uc_secrets.go new file mode 100644 index 00000000000..15f8cecc796 --- /dev/null +++ b/libs/testserver/uc_secrets.go @@ -0,0 +1,198 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +// SecretsUcCreateSecret handles POST /api/2.1/unity-catalog/secrets +func (s *FakeWorkspace) SecretsUcCreateSecret(req Request) Response { + defer s.LockUnlock()() + + // The API accepts flat fields, not wrapped in a "secret" object + var inputSecret catalog.Secret + if err := json.Unmarshal(req.Body, &inputSecret); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{"message": fmt.Sprintf("Failed to parse request: %s", err)}, + } + } + + if s.UCSecrets == nil { + s.UCSecrets = make(map[string]catalog.Secret) + } + + fullName := fmt.Sprintf("%s.%s.%s", + inputSecret.CatalogName, + inputSecret.SchemaName, + inputSecret.Name) + + // Check if secret already exists + if _, exists := s.UCSecrets[fullName]; exists { + return Response{ + StatusCode: 409, + Body: map[string]any{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": fmt.Sprintf("Secret %s already exists", fullName), + }, + } + } + + now := sdktime.New(time.Now()) + secret := catalog.Secret{ + CatalogName: inputSecret.CatalogName, + SchemaName: inputSecret.SchemaName, + Name: inputSecret.Name, + FullName: fullName, + Value: inputSecret.Value, + Comment: inputSecret.Comment, + Owner: inputSecret.Owner, + ExpireTime: inputSecret.ExpireTime, + CreateTime: now, + UpdateTime: now, + CreatedBy: "test-user@databricks.com", + UpdatedBy: "test-user@databricks.com", + EffectiveOwner: inputSecret.Owner, + MetastoreId: "test-metastore-id", + } + + if secret.Owner == "" { + secret.Owner = "test-user@databricks.com" + secret.EffectiveOwner = "test-user@databricks.com" + } + + s.UCSecrets[fullName] = secret + + return Response{ + Body: secret, + } +} + +// SecretsUcGetSecret handles GET /api/2.1/unity-catalog/secrets/{full_name} +func (s *FakeWorkspace) SecretsUcGetSecret(req Request) Response { + defer s.LockUnlock()() + + // Extract full_name from path parameter + fullName := req.Vars["full_name"] + if fullName == "" { + // Fallback: extract from path + parts := strings.Split(req.URL.Path, "/") + if len(parts) >= 6 { + fullName = parts[5] + } + } + + secret, exists := s.UCSecrets[fullName] + if !exists { + return Response{ + StatusCode: 404, + Body: map[string]any{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": fmt.Sprintf("Secret %s not found", fullName), + }, + } + } + + // Return secret without the actual value (only metadata) + // The real API doesn't return the value unless specifically requested + returnSecret := secret + returnSecret.EffectiveValue = secret.Value + returnSecret.Value = "" + + return Response{ + Body: returnSecret, + } +} + +// SecretsUcUpdateSecret handles PATCH /api/2.1/unity-catalog/secrets/{full_name} +func (s *FakeWorkspace) SecretsUcUpdateSecret(req Request) Response { + defer s.LockUnlock()() + + // Extract full_name from path parameter + fullName := req.Vars["full_name"] + if fullName == "" { + // Fallback: extract from path + parts := strings.Split(req.URL.Path, "/") + if len(parts) >= 6 { + fullName = parts[5] + } + } + + // The API accepts flat fields + var updateSecret catalog.Secret + if err := json.Unmarshal(req.Body, &updateSecret); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{"message": fmt.Sprintf("Failed to parse request: %s", err)}, + } + } + + secret, exists := s.UCSecrets[fullName] + if !exists { + return Response{ + StatusCode: 404, + Body: map[string]any{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": fmt.Sprintf("Secret %s not found", fullName), + }, + } + } + + // Update fields based on update mask + if updateSecret.Value != "" { + secret.Value = updateSecret.Value + } + if updateSecret.Comment != "" { + secret.Comment = updateSecret.Comment + } + if updateSecret.Owner != "" { + secret.Owner = updateSecret.Owner + secret.EffectiveOwner = updateSecret.Owner + } + if updateSecret.ExpireTime != nil { + secret.ExpireTime = updateSecret.ExpireTime + } + + secret.UpdateTime = sdktime.New(time.Now()) + secret.UpdatedBy = "test-user@databricks.com" + + s.UCSecrets[fullName] = secret + + return Response{ + Body: secret, + } +} + +// SecretsUcDeleteSecret handles DELETE /api/2.1/unity-catalog/secrets/{full_name} +func (s *FakeWorkspace) SecretsUcDeleteSecret(req Request) Response { + defer s.LockUnlock()() + + // Extract full_name from path parameter + fullName := req.Vars["full_name"] + if fullName == "" { + // Fallback: extract from path + parts := strings.Split(req.URL.Path, "/") + if len(parts) >= 6 { + fullName = parts[5] + } + } + + if _, exists := s.UCSecrets[fullName]; !exists { + return Response{ + StatusCode: 404, + Body: map[string]any{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": fmt.Sprintf("Secret %s not found", fullName), + }, + } + } + + delete(s.UCSecrets, fullName) + + return Response{} +} diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index a1bf973801f..c3ae8fc60c5 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -28,6 +28,7 @@ var resourceURLPatterns = map[string]string{ "queries": "sql/editor/%s", "registered_models": "explore/data/models/%s", "schemas": "explore/data/%s", + "secrets": "explore/data/%s", "synced_database_tables": "explore/data/%s", "vector_search_endpoints": "compute/vector-search/%s", "vector_search_indexes": "explore/data/%s", @@ -54,6 +55,7 @@ var dotSeparatedResources = map[string]bool{ "quality_monitors": true, "registered_models": true, "schemas": true, + "secrets": true, "vector_search_indexes": true, "volumes": true, }