WIP minor refactoring of autogen code

.nextchanges/bundles/6553.md

@@ -0,0 +1 @@
+* Added PyDABs (Python) support for secrets: `Resources.add_secret` and the `secret_mutator` decorator. ([#6553](https://github.com/databricks/cli/pull/6553))

acceptance/bundle/python/secrets-support/databricks.yml

@@ -0,0 +1,23 @@
+bundle:
+  name: my_project
+
+sync: {paths: []} # don't need to copy files
+
+variables:
+  secret_value:
+    description: The value of the secret
+
+python:
+  resources:
+    - "resources:load_resources"
+  mutators:
+    - "mutators:update_secret"
+
+resources:
+  secrets:
+    my_secret_1:
+      catalog_name: main
+      schema_name: default
+      name: my_secret_1
+      value: ${var.secret_value}
+      expire_time: "2030-01-01T00:00:00Z"

acceptance/bundle/python/secrets-support/mutators.py

@@ -0,0 +1,11 @@
+from dataclasses import replace
+
+from databricks.bundles.secrets import Secret
+from databricks.bundles.core import secret_mutator
+
+
+@secret_mutator
+def update_secret(secret: Secret) -> Secret:
+    assert isinstance(secret.name, str)
+
+    return replace(secret, name=f"{secret.name} (updated)")

acceptance/bundle/python/secrets-support/output.txt

@@ -0,0 +1,32 @@
+
+>>> uv run [UV_ARGS] -q [CLI] bundle validate --var secret_value=my-secret-value --output json
+{
+  "experimental": {
+    "python": {
+      "mutators": [
+        "mutators:update_secret"
+      ],
+      "resources": [
+        "resources:load_resources"
+      ]
+    }
+  },
+  "resources": {
+    "secrets": {
+      "my_secret_1": {
+        "catalog_name": "main",
+        "expire_time": "[TIMESTAMP]",
+        "name": "my_secret_1 (updated)",
+        "schema_name": "default",
+        "value": "[redacted]"
+      },
+      "my_secret_2": {
+        "catalog_name": "main",
+        "expire_time": "[TIMESTAMP]",
+        "name": "my_secret_2 (updated)",
+        "schema_name": "default",
+        "value": "[redacted]"
+      }
+    }
+  }
+}

acceptance/bundle/python/secrets-support/resources.py

@@ -0,0 +1,18 @@
+from databricks.bundles.core import Resources
+
+
+def load_resources() -> Resources:
+    resources = Resources()
+
+    resources.add_secret(
+        "my_secret_2",
+        {
+            "catalog_name": "main",
+            "schema_name": "default",
+            "name": "my_secret_2",
+            "value": "${var.secret_value}",
+            "expire_time": "2030-01-01T00:00:00Z",
+        },
+    )
+
+    return resources

acceptance/bundle/python/secrets-support/script

@@ -0,0 +1,6 @@
+
+# secrets require value to be a variable reference, so supply it at validate time.
+trace uv run $UV_ARGS -q $CLI bundle validate --var secret_value=my-secret-value --output json | \
+  jq "pick(.experimental.python, .resources)"
+
+rm -fr .databricks __pycache__

acceptance/bundle/python/secrets-support/test.toml

@@ -0,0 +1,7 @@
+Cloud = false # tests don't interact with APIs
+
+# secrets are only supported in the current version of the wheel
+EnvMatrix.PYDAB_VERSION = ["current"]
+
+# secrets are only supported on the direct deployment engine
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

python/codegen/codegen/generated_dataclass.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from typing import TYPE_CHECKING, Optional
 
 if TYPE_CHECKING:
@@ -151,58 +151,38 @@ def generate_field(
     field_type = variable_or_type(field_type, is_required=is_required)
     param_type = variable_or_type(param_type, is_required=is_required)
 
+    field = GeneratedField(
+        field_name=field_name,
+        type_name=field_type,
+        param_type_name=param_type,
+        create_func_type_name=param_type,
+        description=prop.description,
+        default=None,
+        default_factory=None,
+        create_func_default="None",
+        experimental=is_experimental_stage(prop.stage),
+        deprecated=prop.deprecated or False,
+    )
+
+    # Collections default to an empty container (via a factory, to avoid a mutable
+    # default) and are made Optional in the "create" function; required scalars have
+    # no default; optional scalars default to None.
     if field_type.name == "VariableOrDict":
-        return GeneratedField(
-            field_name=field_name,
-            type_name=field_type,
-            param_type_name=param_type,
+        return replace(
+            field,
             create_func_type_name=optional_type(param_type),
-            description=prop.description,
-            default=None,
             default_factory="dict",
-            create_func_default="None",
-            experimental=is_experimental_stage(prop.stage),
-            deprecated=prop.deprecated or False,
         )
     elif field_type.name == "VariableOrList":
-        return GeneratedField(
-            field_name=field_name,
-            type_name=field_type,
-            param_type_name=param_type,
+        return replace(
+            field,
             create_func_type_name=optional_type(param_type),
-            description=prop.description,
-            default=None,
             default_factory="list",
-            create_func_default="None",
-            experimental=is_experimental_stage(prop.stage),
-            deprecated=prop.deprecated or False,
         )
     elif is_required:
-        return GeneratedField(
-            field_name=field_name,
-            type_name=field_type,
-            param_type_name=param_type,
-            create_func_type_name=param_type,
-            description=prop.description,
-            default=None,
-            default_factory=None,
-            create_func_default=None,
-            experimental=is_experimental_stage(prop.stage),
-            deprecated=prop.deprecated or False,
-        )
+        return replace(field, create_func_default=None)
     else:
-        return GeneratedField(
-            field_name=field_name,
-            type_name=field_type,
-            param_type_name=param_type,
-            create_func_type_name=param_type,
-            description=prop.description,
-            default="None",
-            default_factory=None,
-            create_func_default="None",
-            experimental=is_experimental_stage(prop.stage),
-            deprecated=prop.deprecated or False,
-        )
+        return replace(field, default="None")
 
 
 def optional_type(generated: GeneratedType) -> GeneratedType:

python/codegen/codegen/generated_test_cases.py

@@ -104,7 +104,7 @@ def _is_composite(ref: str) -> bool:
 
 
 def _synth_scalar(name: str, hint: str) -> _Scalar:
-    """Placeholder value for a primitive (str -> hint, int -> 0, float -> 0.0, bool -> True).
+    """Placeholder value for a primitive (str -> hint, int -> 0, float -> 0.0, bool -> True, time.Time -> RFC3339 string).
 
     :param name: the primitive's schema name, e.g. "string", "int", "boolean".
     :param hint: enclosing field name, used as the string placeholder so examples read meaningfully.
@@ -117,6 +117,9 @@ def _synth_scalar(name: str, hint: str) -> _Scalar:
         return _Scalar("0.0", "0.0")
     if name in ("boolean", "bool"):
         return _Scalar("True", "True")
+    # time.Time is generated as a str (see packages.RENAMES); serialized as RFC3339.
+    if name == "time.Time":
+        return _Scalar('"2020-01-01T00:00:00Z"', '"2020-01-01T00:00:00Z"')
 
     raise ValueError(f"Unknown primitive: {name}")
 
@@ -154,8 +157,7 @@ def _synth_ref(
 
     schema = schemas[name]
     class_name = packages.get_class_name(ref)
-    module = packages.get_package(namespace, ref)
-    assert module
+    module = _module_of(namespace, ref)
 
     if schema.type == openapi.SchemaType.STRING:
         value = schema.enum[0]
@@ -202,7 +204,7 @@ def _synth_object(
                 continue
             if (
                 prop.deprecated
-                or _STAGE_RANK[prop.stage]
+                or _STAGE_RANK.get(prop.stage, 0)
                 > _STAGE_RANK[openapi.LaunchStage.PUBLIC_PREVIEW]
             ):
                 continue
@@ -219,7 +221,7 @@ def _module_of(namespace: str, schema_name: str) -> str:
     """Python module a (non-primitive) schema's generated class lives in; asserts it exists.
 
     :param namespace: the resource's namespace; the type is generated under databricks.bundles.<namespace>._models.
-    :param schema_name: the object/enum schema name to resolve.
+    :param schema_name: the object/enum schema name (or full ref) to resolve.
     """
     module = packages.get_package(namespace, schema_name)
     assert module

python/codegen/codegen/packages.py

@@ -9,7 +9,6 @@
     "resources.ClusterPolicy",  # interface{}
     "resources.Dashboard",  # interface{}
     "resources.GenieSpace",  # interface{}
-    "resources.Secret",  # time.Time
 }
 
 # Only GA and public-preview resources are generated; later stages may still change.
@@ -63,6 +62,9 @@ def _load_resource_namespace() -> dict[str, str]:
 RESOURCE_TYPES = list(RESOURCE_NAMESPACE.keys())
 
 RENAMES = {
+    # time.Time is a scalar serialized as an RFC3339 string; the Go side models
+    # it as a string too (see libs/dyn/convert/sdk_native_types.go).
+    "time.Time": "str",
     "string": "str",
     "boolean": "bool",
     "integer": "int",
@@ -72,6 +74,8 @@ def _load_resource_namespace() -> dict[str, str]:
 }
 
 PRIMITIVES = [
+    # Treated as str
+    "time.Time",
     "string",
     "boolean",
     "integer",
@@ -85,6 +89,10 @@ def _load_resource_namespace() -> dict[str, str]:
 
 def get_class_name(ref: str) -> str:
     name = ref.split("/")[-1]
+
+    if name in RENAMES:
+        return RENAMES[name]
+
     name = name.split(".")[-1]
 
     return RENAMES.get(name, name)

python/codegen/codegen_tests/test_generated_test_cases.py

@@ -68,6 +68,7 @@ def test_is_composite():
         ("float64", _Scalar("0.0", "0.0")),
         ("boolean", _Scalar("True", "True")),
         ("bool", _Scalar("True", "True")),
+        ("time.Time", _Scalar('"2020-01-01T00:00:00Z"', '"2020-01-01T00:00:00Z"')),
     ],
 )
 def test_synth_scalar(name, expected):