From 3099afb6d6b4ab3d7767ae58cd3634146ff11664 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 12:25:26 +0000 Subject: [PATCH 01/60] job_runs: add lifecycle.triggers.on_file_change Re-fire a run when matched file contents change, using stable hash fingerprints so mtime-only updates (touch) do not recreate. --- .../bundles/job-runs-on-file-change.md | 1 + .../job_runs/on_file_change/databricks.yml | 18 ++ .../job_runs/on_file_change/out.test.toml | 2 + .../job_runs/on_file_change/output.txt | 90 +++++++++ .../resources/job_runs/on_file_change/script | 34 ++++ .../job_runs/on_file_change/seed.txt | 1 + .../mutator/resolve_job_run_file_triggers.go | 177 ++++++++++++++++++ ...lve_job_run_file_triggers_internal_test.go | 73 ++++++++ .../resolve_job_run_file_triggers_test.go | 143 ++++++++++++++ .../mutator/validate_job_run_triggers.go | 26 ++- .../mutator/validate_job_run_triggers_test.go | 50 ++++- bundle/config/resources/job_run.go | 29 +++ bundle/config/resources/lifecycle.go | 9 +- bundle/direct/dresources/job_run.go | 28 ++- bundle/direct/dresources/resources.yml | 6 + bundle/internal/schema/annotations.yml | 3 + bundle/phases/initialize.go | 3 + bundle/phases/plan.go | 4 +- bundle/schema/jsonschema.json | 4 + 19 files changed, 687 insertions(+), 14 deletions(-) create mode 100644 .nextchanges/bundles/job-runs-on-file-change.md create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/seed.txt create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers.go create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go create mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_test.go diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md new file mode 100644 index 00000000000..00682403a0d --- /dev/null +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -0,0 +1 @@ +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob to re-fire the run when matched file contents change. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml new file mode 100644 index 00000000000..0a6df75047a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: seed.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt new file mode 100644 index 00000000000..bcf5962f8fa --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -0,0 +1,90 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +job run [MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== redeploy with unchanged file plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== editing the file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +job run [MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== removing on_file_change rewrites state without a run +>>> [CLI] bundle plan +update job_runs.my_run + +Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Updated job_runs.my_run +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script new file mode 100644 index 00000000000..ff162e11f0b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -0,0 +1,34 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +# Name the first run so the recreated one becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "redeploy with unchanged file plans nothing" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "editing the file re-fires" +update_file.py seed.txt "v1" "v2" +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "removing on_file_change rewrites state without a run" +update_file.py databricks.yml " lifecycle: + triggers: + - on_file_change: seed.txt +" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change/seed.txt b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/seed.txt @@ -0,0 +1 @@ +v1 diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go new file mode 100644 index 00000000000..a82ed654be3 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -0,0 +1,177 @@ +package mutator + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" +) + +// missingFileSentinelSize marks a pattern with no matching file so the next +// plan can distinguish "still missing" from "file appeared". +const missingFileSentinelSize = int64(-1) + +type resolveJobRunFileTriggers struct{} + +// ResolveJobRunFileTriggers expands on_file_change globs and stores per-file +// fingerprints on each job_run for PrepareState to copy into local state. +func ResolveJobRunFileTriggers() bundle.Mutator { + return &resolveJobRunFileTriggers{} +} + +func (*resolveJobRunFileTriggers) Name() string { + return "ResolveJobRunFileTriggers" +} + +func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + for name, jr := range b.Config.Resources.JobRuns { + if jr == nil { + continue + } + patterns := jr.OnFileChangePatterns() + if len(patterns) == 0 { + continue + } + fps, d := resolveFileTriggers(b, name, patterns, previousFileTriggers(b, name)) + diags = diags.Extend(d) + jr.ResolvedFileTriggers = fps + } + return diags +} + +// previousFileTriggers reads on_file_change fingerprints from deployment state +// when it is open (plan/deploy after StatePull). Used so unchanged content keeps +// a stable fingerprint across mtime-only updates (e.g. touch). +func previousFileTriggers(b *bundle.Bundle, name string) map[string]resources.JobRunFileFingerprint { + if b.DeploymentBundle.StateDB.Path == "" { + return nil + } + entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry("resources.job_runs." + name) + if !ok || len(entry.State) == 0 { + return nil + } + var state struct { + Lifecycle *struct { + Triggers *struct { + OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change"` + } `json:"triggers"` + } `json:"lifecycle"` + } + if err := json.Unmarshal(entry.State, &state); err != nil { + return nil + } + if state.Lifecycle == nil || state.Lifecycle.Triggers == nil { + return nil + } + return state.Lifecycle.Triggers.OnFileChange +} + +func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev map[string]resources.JobRunFileFingerprint) (map[string]resources.JobRunFileFingerprint, diag.Diagnostics) { + var diags diag.Diagnostics + out := make(map[string]resources.JobRunFileFingerprint) + for _, pattern := range patterns { + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, filepath.FromSlash(pattern))) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + if len(matches) == 0 { + // Distinct state when the path/glob matches nothing (design doc). + out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ + Size: missingFileSentinelSize, + } + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(path), + }) + continue + } + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + if !info.Mode().IsRegular() { + continue + } + rel, err := filepath.Rel(b.SyncRootPath, match) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: relative path for %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + key := filepath.ToSlash(rel) + fp, err := fingerprintFile(match, info, prev[key]) + if err != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), + Locations: b.Config.GetLocations(path), + }) + continue + } + out[key] = fp + } + } + return out, diags +} + +// fingerprintFile builds a content fingerprint. If prev has the same size and +// mtime, it is reused without reading the file. If content hash matches prev, +// prev is reused so mtime-only updates (touch) do not change planned state. +func fingerprintFile(path string, info os.FileInfo, prev resources.JobRunFileFingerprint) (resources.JobRunFileFingerprint, error) { + size := info.Size() + mtime := info.ModTime().UnixNano() + if prev.Hash != "" && prev.Size == size && prev.MtimeNs == mtime { + return prev, nil + } + hash, err := hashFile(path) + if err != nil { + return resources.JobRunFileFingerprint{}, err + } + if prev.Hash != "" && prev.Hash == hash { + return prev, nil + } + return resources.JobRunFileFingerprint{ + Hash: hash, + Size: size, + MtimeNs: mtime, + }, nil +} + +func hashFile(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go new file mode 100644 index 00000000000..e82bd22945b --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go @@ -0,0 +1,73 @@ +package mutator + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFingerprintFileContentStableAcrossTouch(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) + require.NoError(t, err) + require.NotEmpty(t, first.Hash) + + // Advance mtime without changing contents (touch). + require.NoError(t, os.Chtimes(path, time.Now().Add(time.Minute), time.Now().Add(time.Minute))) + info, err = os.Stat(path) + require.NoError(t, err) + assert.NotEqual(t, first.MtimeNs, info.ModTime().UnixNano()) + + second, err := fingerprintFile(path, info, first) + require.NoError(t, err) + assert.Equal(t, first, second, "unchanged content must reuse the previous fingerprint") +} + +func TestFingerprintFileFastPathSkipsWhenMtimeAndSizeMatch(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + prev := resources.JobRunFileFingerprint{ + Hash: "not-the-real-hash", + Size: info.Size(), + MtimeNs: info.ModTime().UnixNano(), + } + + got, err := fingerprintFile(path, info, prev) + require.NoError(t, err) + assert.Equal(t, prev, got, "matching size+mtime must reuse prev without re-hashing") +} + +func TestFingerprintFileContentChange(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "seed.txt") + require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) + + info, err := os.Stat(path) + require.NoError(t, err) + first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) + require.NoError(t, err) + + // Different size so the mtime+size fast path cannot reuse prev. + require.NoError(t, os.WriteFile(path, []byte("v2-changed"), 0o644)) + info, err = os.Stat(path) + require.NoError(t, err) + second, err := fingerprintFile(path, info, first) + require.NoError(t, err) + assert.NotEqual(t, first.Hash, second.Hash) + assert.Equal(t, info.Size(), second.Size) + assert.Equal(t, info.ModTime().UnixNano(), second.MtimeNs) +} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go new file mode 100644 index 00000000000..9a9cf77bf53 --- /dev/null +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -0,0 +1,143 @@ +package mutator_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveJobRunFileTriggers(t *testing.T) { + t.Run("matches files and fills fingerprints", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) + + pattern := "*.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 2) + + assertFingerprint(t, fps["a.txt"], "hello") + assertFingerprint(t, fps["b.txt"], "world") + }) + + t.Run("no matches warns and stores sentinel", func(t *testing.T) { + dir := t.TempDir() + pattern := "missing.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Equal(t, diag.Warning, diags[0].Severity) + assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 1) + fp := fps["missing.txt"] + assert.Empty(t, fp.Hash) + assert.Equal(t, int64(-1), fp.Size) + assert.Zero(t, fp.MtimeNs) + }) + + t.Run("no file triggers is a no-op", func(t *testing.T) { + dir := t.TempDir() + on := true + b := &bundle.Bundle{ + SyncRootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &on}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + assert.Empty(t, diags) + assert.Nil(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("multiple patterns merge into one map", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("aaa"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "subdir", "x.py"), []byte("bbb"), 0o644)) + + patA := "a.txt" + patB := "subdir/*.py" + b := &bundle.Bundle{ + SyncRootPath: dir, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &patA}, + {OnFileChange: &patB}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, fps, 2) + assertFingerprint(t, fps["a.txt"], "aaa") + assertFingerprint(t, fps["subdir/x.py"], "bbb") + }) +} + +func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { + return &bundle.Bundle{ + SyncRootPath: syncRoot, + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + } +} + +func assertFingerprint(t *testing.T, fp resources.JobRunFileFingerprint, content string) { + t.Helper() + sum := sha256.Sum256([]byte(content)) + assert.Equal(t, hex.EncodeToString(sum[:]), fp.Hash) + assert.Equal(t, int64(len(content)), fp.Size) + assert.NotZero(t, fp.MtimeNs) +} diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index a7c5ada925e..c185d3b5781 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -3,6 +3,7 @@ package mutator import ( "context" "fmt" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" @@ -26,30 +27,45 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D continue } // Recreate-every-deploy cannot coexist with prevent_destroy. - if jr.HasOnBundleDeploy() && jr.Lifecycle.PreventDestroy { + if (jr.HasOnBundleDeploy() || jr.HasOnFileChange()) && jr.Lifecycle.PreventDestroy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + Summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), }) } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) - if t.OnBundleDeploy == nil { + if t.OnBundleDeploy == nil && t.OnFileChange == nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + Summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", Locations: b.Config.GetLocations(path), }) continue } - if !*t.OnBundleDeploy { + if t.OnBundleDeploy != nil && t.OnFileChange != nil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", + Locations: b.Config.GetLocations(path), + }) + continue + } + if t.OnBundleDeploy != nil && !*t.OnBundleDeploy { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "lifecycle.triggers.on_bundle_deploy must be true when set", Locations: b.Config.GetLocations(path + ".on_bundle_deploy"), }) } + if t.OnFileChange != nil && strings.TrimSpace(*t.OnFileChange) == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_file_change must be non-empty when set", + Locations: b.Config.GetLocations(path + ".on_file_change"), + }) + } } } return diags diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 9c225457471..7e8772d8d8b 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -14,6 +14,10 @@ func TestValidateJobRunTriggers(t *testing.T) { trueVal := true falseVal := false + fileChange := "seed.txt" + emptyFile := "" + whitespaceFile := " \t" + tests := []struct { name string triggers []resources.JobRunTrigger @@ -26,12 +30,32 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnBundleDeploy: &trueVal}, }, }, + { + name: "on_file_change set", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + }, + { + name: "both triggers as separate entries", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + {OnBundleDeploy: &trueVal}, + }, + }, { name: "empty entry", triggers: []resources.JobRunTrigger{ {}, }, - summary: "lifecycle.triggers entry must set on_bundle_deploy: true", + summary: "lifecycle.triggers entry must set on_bundle_deploy or on_file_change", + }, + { + name: "both keys on one entry", + triggers: []resources.JobRunTrigger{ + {OnBundleDeploy: &trueVal, OnFileChange: &fileChange}, + }, + summary: "lifecycle.triggers entry must set only one of on_bundle_deploy or on_file_change", }, { name: "on_bundle_deploy false", @@ -40,13 +64,35 @@ func TestValidateJobRunTriggers(t *testing.T) { }, summary: "lifecycle.triggers.on_bundle_deploy must be true when set", }, + { + name: "on_file_change empty", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &emptyFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, + { + name: "on_file_change whitespace", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &whitespaceFile}, + }, + summary: "lifecycle.triggers.on_file_change must be non-empty when set", + }, { name: "on_bundle_deploy with prevent_destroy", triggers: []resources.JobRunTrigger{ {OnBundleDeploy: &trueVal}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + }, + { + name: "on_file_change with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", }, { name: "prevent_destroy alone", diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 167530465e8..70d5433c71d 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -27,6 +27,10 @@ type JobRun struct { // the run URL. Keeping it separate from RunNow.JobId (a ${resources.jobs.*.id} // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` + + // ResolvedFileTriggers is the per-file fingerprint map for on_file_change, + // computed under SyncRoot before plan. bundle:"internal" keeps it out of schema. + ResolvedFileTriggers map[string]JobRunFileFingerprint `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. @@ -42,6 +46,31 @@ func (r *JobRun) HasOnBundleDeploy() bool { return false } +func (r *JobRun) HasOnFileChange() bool { + if r.Lifecycle == nil { + return false + } + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + return true + } + } + return false +} + +func (r *JobRun) OnFileChangePatterns() []string { + if r.Lifecycle == nil { + return nil + } + var patterns []string + for _, t := range r.Lifecycle.Triggers { + if t.OnFileChange != nil { + patterns = append(patterns, *t.OnFileChange) + } + } + return patterns +} + func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 88d130239ee..59ccb682b81 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -36,5 +36,12 @@ type JobRunLifecycle struct { // JobRunTrigger is one lifecycle.triggers entry. type JobRunTrigger struct { - OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root +} + +type JobRunFileFingerprint struct { + Hash string `json:"hash"` + Size int64 `json:"size"` + MtimeNs int64 `json:"mtime_ns"` } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9eb671f430c..4e57a4bcdf4 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,10 +26,20 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour +// jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. +var jobRunTriggerLocalPaths = []string{ + "lifecycle", + "lifecycle.triggers", + "lifecycle.triggers.on_bundle_deploy", + "lifecycle.triggers.on_file_change", +} + // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` + // Per-file fingerprints from ResolveJobRunFileTriggers; change to recreate. + OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change,omitempty"` } // JobRunLifecycleState holds local-only lifecycle fields persisted in state. @@ -97,12 +107,19 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: nil, } + var triggers *JobRunTriggersState if input.HasOnBundleDeploy() { - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{OnBundleDeploy: uuid.NewString()}, + triggers = &JobRunTriggersState{OnBundleDeploy: uuid.NewString()} + } + if len(input.ResolvedFileTriggers) > 0 { + if triggers == nil { + triggers = &JobRunTriggersState{} } + triggers.OnFileChange = input.ResolvedFileTriggers + } + if triggers != nil { + state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} } return state } @@ -371,8 +388,7 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - switch path.String() { - case "lifecycle", "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy": + if slices.Contains(jobRunTriggerLocalPaths, path.String()) { // A cleared trigger sets New empty; structdiff may report it at lifecycle, // lifecycle.triggers, or the leaf. if change.New == nil || change.New == "" { @@ -380,6 +396,8 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Reason = "trigger removed" } return nil + } + switch path.String() { case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 69bf2d93aef..78c1f3d2fa6 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -198,6 +198,12 @@ resources: recreate_on_changes: - reason: immutable + # mtime is stored for the resolve fast path; content identity is the hash. + # Ignoring mtime-only local drift keeps touch from recreating the run. + ignore_local_changes: + - field: lifecycle.triggers.on_file_change[*].mtime_ns + reason: mtime_only_fingerprint + pipelines: recreate_on_changes: - field: storage diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index f50d3f5bc98..a86f1dc6f5e 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1036,6 +1036,9 @@ resources: "on_bundle_deploy": "description": |- If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + "on_file_change": + "description": |- + Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 9d9c682ef62..ef3fe582528 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -194,6 +194,9 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). mutator.ValidateJobRunTriggers(), + // Expand on_file_change globs and hash matched files into ResolvedFileTriggers. + mutator.ResolveJobRunFileTriggers(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index aee481b2b14..c69074315dc 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -21,7 +21,8 @@ import ( // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". // Note, it is not run in "bundle migrate" so it must not modify the config func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.CheckDashboardsModifiedRemotely(isPlan, engine), resourcemutator.SecretScopeFixups(engine), deploy.StatePull(), @@ -29,6 +30,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateDirectOnlyResources(engine), mutator.ValidateLifecycleStarted(engine), mutator.ValidateCascadeOnDestroy(engine), + mutator.ResolveJobRunFileTriggers(), statemgmt.CheckRunningResource(engine), ) } diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 6db8c5cc7e2..086b31146bc 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1535,6 +1535,10 @@ "on_bundle_deploy": { "description": "If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/bool" + }, + "on_file_change": { + "description": "Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy.", + "$ref": "#/$defs/string" } }, "additionalProperties": false From a61e6623a0238f724e82d2bf38c7b80ae2ca34ea Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 12:41:58 +0000 Subject: [PATCH 02/60] job_runs: fix exhaustruct and regenerate refschema for on_file_change Satisfy the linter on PrepareState literals after adding OnFileChange, and refresh out.fields.txt so validate-generated stays green. --- bundle/direct/dresources/job_run.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 4e57a4bcdf4..f66991f1ca8 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -107,14 +107,21 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, + Lifecycle: nil, } var triggers *JobRunTriggersState if input.HasOnBundleDeploy() { - triggers = &JobRunTriggersState{OnBundleDeploy: uuid.NewString()} + triggers = &JobRunTriggersState{ + OnBundleDeploy: uuid.NewString(), + OnFileChange: nil, + } } if len(input.ResolvedFileTriggers) > 0 { if triggers == nil { - triggers = &JobRunTriggersState{} + triggers = &JobRunTriggersState{ + OnBundleDeploy: "", + OnFileChange: nil, + } } triggers.OnFileChange = input.ResolvedFileTriggers } From 568540f3b7a98d0bd0b53d75d2a2dc3c7536d5ea Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:03:41 +0000 Subject: [PATCH 03/60] job_runs: tighten on_file_change path and directory checks Reject patterns outside the sync root and directory-only matches so the trigger cannot fingerprint escaped paths or silently disarm. --- .../bundles/job-runs-on-file-change.md | 2 +- .../mutator/resolve_job_run_file_triggers.go | 31 ++++++++++++++++--- .../resolve_job_run_file_triggers_test.go | 25 +++++++++++++++ bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md index 00682403a0d..eadceda6268 100644 --- a/.nextchanges/bundles/job-runs-on-file-change.md +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -1 +1 @@ -direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob to re-fire the run when matched file contents change. Removing the trigger does not recreate the existing run. +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change, or when matches appear or disappear. Removing the trigger does not recreate the existing run. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index a82ed654be3..f0338b2d50a 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -80,7 +80,17 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev out := make(map[string]resources.JobRunFileFingerprint) for _, pattern := range patterns { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, filepath.FromSlash(pattern))) + localPattern := filepath.FromSlash(pattern) + // Keep fingerprints under SyncRoot; same IsLocal gate as translate_paths. + if !filepath.IsLocal(localPattern) { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(path), + }) + continue + } + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, @@ -90,7 +100,7 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev continue } if len(matches) == 0 { - // Distinct state when the path/glob matches nothing (design doc). + // Distinct state when the path/glob matches nothing so appear/disappear recreates. out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ Size: missingFileSentinelSize, } @@ -101,6 +111,8 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev }) continue } + regularMatches := 0 + sawNonRegular := false for _, match := range matches { info, err := os.Stat(match) if err != nil { @@ -112,13 +124,15 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev continue } if !info.Mode().IsRegular() { + sawNonRegular = true continue } + regularMatches++ rel, err := filepath.Rel(b.SyncRootPath, match) - if err != nil { + if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: relative path for %q: %s", match, err), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), Locations: b.Config.GetLocations(path), }) continue @@ -135,6 +149,15 @@ func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev } out[key] = fp } + // A directory-only match would otherwise leave ResolvedFileTriggers empty + // and silently disarm the trigger while config still sets on_file_change. + if regularMatches == 0 && sawNonRegular { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), + Locations: b.Config.GetLocations(path), + }) + } } return out, diags } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 9a9cf77bf53..06266d75894 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -113,6 +113,31 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assertFingerprint(t, fps["a.txt"], "aaa") assertFingerprint(t, fps["subdir/x.py"], "bbb") }) + + t.Run("pattern outside sync root is an error", func(t *testing.T) { + dir := t.TempDir() + pattern := "../outside.txt" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Contains(t, diags[0].Summary, `not under the sync root`) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("directory-only match is an error", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "migrations"), 0o755)) + pattern := "migrations" + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Len(t, diags, 1) + assert.Contains(t, diags[0].Summary, `matches no regular files`) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index a86f1dc6f5e..450f6f9520e 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1038,7 +1038,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 086b31146bc..fc03dfbc7df 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1537,7 +1537,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root. Re-fire the run when any matched file's content changes. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From fa6de6f9c6e2edc7a828a2bfd3ebcc4a36a8cc40 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:16:41 +0000 Subject: [PATCH 04/60] job_runs: store on_file_change as path-to-hash only Drop size/mtime fingerprints and the resolve fast path so content identity alone drives recreate, and touch no longer needs a planner exception. --- .../bundles/job-runs-on-file-change.md | 2 +- .../job_runs/on_file_change/output.txt | 11 + .../resources/job_runs/on_file_change/script | 6 + .../mutator/resolve_job_run_file_triggers.go | 204 +++++++----------- ...lve_job_run_file_triggers_internal_test.go | 73 ------- .../resolve_job_run_file_triggers_test.go | 48 +++-- bundle/config/resources/job_run.go | 19 +- bundle/config/resources/lifecycle.go | 6 - bundle/direct/dresources/job_run.go | 4 +- bundle/direct/dresources/resources.yml | 6 - bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 12 files changed, 127 insertions(+), 256 deletions(-) delete mode 100644 bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md index eadceda6268..95869a25ccf 100644 --- a/.nextchanges/bundles/job-runs-on-file-change.md +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -1 +1 @@ -direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change, or when matches appear or disappear. Removing the trigger does not recreate the existing run. +direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change (content hash), or when matches appear or disappear. Removing the trigger does not recreate the existing run. diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index bcf5962f8fa..f356fe7fb61 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -33,6 +33,17 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now +=== touch without content change plans nothing +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + === editing the file re-fires >>> [CLI] bundle plan recreate job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index ff162e11f0b..0506dd5e106 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -16,6 +16,12 @@ trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //jobs/run-now +title "touch without content change plans nothing" +python -c "import os; os.utime('seed.txt', None)" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + title "editing the file re-fires" update_file.py seed.txt "v1" "v2" trace $CLI bundle plan diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index f0338b2d50a..55a55a94b95 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -4,25 +4,23 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" "fmt" "io" "os" "path/filepath" + "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" ) -// missingFileSentinelSize marks a pattern with no matching file so the next -// plan can distinguish "still missing" from "file appeared". -const missingFileSentinelSize = int64(-1) +// missingFileHash marks a pattern with no matching file so appear/disappear recreates. +const missingFileHash = "" type resolveJobRunFileTriggers struct{} // ResolveJobRunFileTriggers expands on_file_change globs and stores per-file -// fingerprints on each job_run for PrepareState to copy into local state. +// content hashes on each job_run for PrepareState to copy into local state. func ResolveJobRunFileTriggers() bundle.Mutator { return &resolveJobRunFileTriggers{} } @@ -34,155 +32,105 @@ func (*resolveJobRunFileTriggers) Name() string { func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics for name, jr := range b.Config.Resources.JobRuns { - if jr == nil { + if jr == nil || jr.Lifecycle == nil { continue } - patterns := jr.OnFileChangePatterns() - if len(patterns) == 0 { - continue + out := make(map[string]string) + for i, t := range jr.Lifecycle.Triggers { + if t.OnFileChange == nil { + continue + } + path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) + hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) + diags = diags.Extend(d) + for k, v := range hashes { + out[k] = v + } + } + if len(out) == 0 { + jr.ResolvedFileTriggers = nil + } else { + jr.ResolvedFileTriggers = out } - fps, d := resolveFileTriggers(b, name, patterns, previousFileTriggers(b, name)) - diags = diags.Extend(d) - jr.ResolvedFileTriggers = fps } return diags } -// previousFileTriggers reads on_file_change fingerprints from deployment state -// when it is open (plan/deploy after StatePull). Used so unchanged content keeps -// a stable fingerprint across mtime-only updates (e.g. touch). -func previousFileTriggers(b *bundle.Bundle, name string) map[string]resources.JobRunFileFingerprint { - if b.DeploymentBundle.StateDB.Path == "" { - return nil - } - entry, ok := b.DeploymentBundle.StateDB.GetResourceEntry("resources.job_runs." + name) - if !ok || len(entry.State) == 0 { - return nil - } - var state struct { - Lifecycle *struct { - Triggers *struct { - OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change"` - } `json:"triggers"` - } `json:"lifecycle"` +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]string, diag.Diagnostics) { + var diags diag.Diagnostics + out := make(map[string]string) + localPattern := filepath.FromSlash(pattern) + // Keep hashes under SyncRoot; same IsLocal gate as translate_paths. + if !filepath.IsLocal(localPattern) { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(loc), + }) } - if err := json.Unmarshal(entry.State, &state); err != nil { - return nil + matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) + if err != nil { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(loc), + }) } - if state.Lifecycle == nil || state.Lifecycle.Triggers == nil { - return nil + if len(matches) == 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) } - return state.Lifecycle.Triggers.OnFileChange -} - -func resolveFileTriggers(b *bundle.Bundle, name string, patterns []string, prev map[string]resources.JobRunFileFingerprint) (map[string]resources.JobRunFileFingerprint, diag.Diagnostics) { - var diags diag.Diagnostics - out := make(map[string]resources.JobRunFileFingerprint) - for _, pattern := range patterns { - path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers", name) - localPattern := filepath.FromSlash(pattern) - // Keep fingerprints under SyncRoot; same IsLocal gate as translate_paths. - if !filepath.IsLocal(localPattern) { + regularMatches := 0 + sawNonRegular := false + for _, match := range matches { + info, err := os.Stat(match) + if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), - Locations: b.Config.GetLocations(path), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), + Locations: b.Config.GetLocations(loc), }) continue } - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), - Locations: b.Config.GetLocations(path), - }) + if !info.Mode().IsRegular() { + sawNonRegular = true continue } - if len(matches) == 0 { - // Distinct state when the path/glob matches nothing so appear/disappear recreates. - out[filepath.ToSlash(pattern)] = resources.JobRunFileFingerprint{ - Size: missingFileSentinelSize, - } + regularMatches++ + rel, err := filepath.Rel(b.SyncRootPath, match) + if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), - Locations: b.Config.GetLocations(path), + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), + Locations: b.Config.GetLocations(loc), }) continue } - regularMatches := 0 - sawNonRegular := false - for _, match := range matches { - info, err := os.Stat(match) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), - Locations: b.Config.GetLocations(path), - }) - continue - } - if !info.Mode().IsRegular() { - sawNonRegular = true - continue - } - regularMatches++ - rel, err := filepath.Rel(b.SyncRootPath, match) - if err != nil || !filepath.IsLocal(rel) { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), - Locations: b.Config.GetLocations(path), - }) - continue - } - key := filepath.ToSlash(rel) - fp, err := fingerprintFile(match, info, prev[key]) - if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), - Locations: b.Config.GetLocations(path), - }) - continue - } - out[key] = fp - } - // A directory-only match would otherwise leave ResolvedFileTriggers empty - // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular { + hash, err := hashFile(match) + if err != nil { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), - Locations: b.Config.GetLocations(path), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), + Locations: b.Config.GetLocations(loc), }) + continue } + out[filepath.ToSlash(rel)] = hash } - return out, diags -} - -// fingerprintFile builds a content fingerprint. If prev has the same size and -// mtime, it is reused without reading the file. If content hash matches prev, -// prev is reused so mtime-only updates (touch) do not change planned state. -func fingerprintFile(path string, info os.FileInfo, prev resources.JobRunFileFingerprint) (resources.JobRunFileFingerprint, error) { - size := info.Size() - mtime := info.ModTime().UnixNano() - if prev.Hash != "" && prev.Size == size && prev.MtimeNs == mtime { - return prev, nil + // A directory-only match would otherwise leave ResolvedFileTriggers empty + // and silently disarm the trigger while config still sets on_file_change. + if regularMatches == 0 && sawNonRegular { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), + Locations: b.Config.GetLocations(loc), + }) } - hash, err := hashFile(path) - if err != nil { - return resources.JobRunFileFingerprint{}, err - } - if prev.Hash != "" && prev.Hash == hash { - return prev, nil - } - return resources.JobRunFileFingerprint{ - Hash: hash, - Size: size, - MtimeNs: mtime, - }, nil + return out, diags } func hashFile(path string) (string, error) { diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go deleted file mode 100644 index e82bd22945b..00000000000 --- a/bundle/config/mutator/resolve_job_run_file_triggers_internal_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package mutator - -import ( - "os" - "path/filepath" - "testing" - "time" - - "github.com/databricks/cli/bundle/config/resources" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFingerprintFileContentStableAcrossTouch(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) - require.NoError(t, err) - require.NotEmpty(t, first.Hash) - - // Advance mtime without changing contents (touch). - require.NoError(t, os.Chtimes(path, time.Now().Add(time.Minute), time.Now().Add(time.Minute))) - info, err = os.Stat(path) - require.NoError(t, err) - assert.NotEqual(t, first.MtimeNs, info.ModTime().UnixNano()) - - second, err := fingerprintFile(path, info, first) - require.NoError(t, err) - assert.Equal(t, first, second, "unchanged content must reuse the previous fingerprint") -} - -func TestFingerprintFileFastPathSkipsWhenMtimeAndSizeMatch(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - prev := resources.JobRunFileFingerprint{ - Hash: "not-the-real-hash", - Size: info.Size(), - MtimeNs: info.ModTime().UnixNano(), - } - - got, err := fingerprintFile(path, info, prev) - require.NoError(t, err) - assert.Equal(t, prev, got, "matching size+mtime must reuse prev without re-hashing") -} - -func TestFingerprintFileContentChange(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "seed.txt") - require.NoError(t, os.WriteFile(path, []byte("v1"), 0o644)) - - info, err := os.Stat(path) - require.NoError(t, err) - first, err := fingerprintFile(path, info, resources.JobRunFileFingerprint{}) - require.NoError(t, err) - - // Different size so the mtime+size fast path cannot reuse prev. - require.NoError(t, os.WriteFile(path, []byte("v2-changed"), 0o644)) - info, err = os.Stat(path) - require.NoError(t, err) - second, err := fingerprintFile(path, info, first) - require.NoError(t, err) - assert.NotEqual(t, first.Hash, second.Hash) - assert.Equal(t, info.Size(), second.Size) - assert.Equal(t, info.ModTime().UnixNano(), second.MtimeNs) -} diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 06266d75894..3ec663c89bd 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -17,7 +17,7 @@ import ( ) func TestResolveJobRunFileTriggers(t *testing.T) { - t.Run("matches files and fills fingerprints", func(t *testing.T) { + t.Run("matches files and fills hashes", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) @@ -28,14 +28,13 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 2) - - assertFingerprint(t, fps["a.txt"], "hello") - assertFingerprint(t, fps["b.txt"], "world") + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 2) + assert.Equal(t, contentHash("hello"), hashes["a.txt"]) + assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("no matches warns and stores sentinel", func(t *testing.T) { + t.Run("no matches warns and stores empty hash", func(t *testing.T) { dir := t.TempDir() pattern := "missing.txt" b := bundleWithFileTrigger(dir, pattern) @@ -46,12 +45,9 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, diag.Warning, diags[0].Severity) assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 1) - fp := fps["missing.txt"] - assert.Empty(t, fp.Hash) - assert.Equal(t, int64(-1), fp.Size) - assert.Zero(t, fp.MtimeNs) + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Empty(t, hashes["missing.txt"]) }) t.Run("no file triggers is a no-op", func(t *testing.T) { @@ -108,10 +104,10 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - fps := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, fps, 2) - assertFingerprint(t, fps["a.txt"], "aaa") - assertFingerprint(t, fps["subdir/x.py"], "bbb") + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 2) + assert.Equal(t, contentHash("aaa"), hashes["a.txt"]) + assert.Equal(t, contentHash("bbb"), hashes["subdir/x.py"]) }) t.Run("pattern outside sync root is an error", func(t *testing.T) { @@ -138,6 +134,17 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Contains(t, diags[0].Summary, `matches no regular files`) assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) + + t.Run("trims pattern whitespace", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) + pattern := " seed.txt " + b := bundleWithFileTrigger(dir, pattern) + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, contentHash("v1"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["seed.txt"]) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { @@ -159,10 +166,7 @@ func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { } } -func assertFingerprint(t *testing.T, fp resources.JobRunFileFingerprint, content string) { - t.Helper() +func contentHash(content string) string { sum := sha256.Sum256([]byte(content)) - assert.Equal(t, hex.EncodeToString(sum[:]), fp.Hash) - assert.Equal(t, int64(len(content)), fp.Size) - assert.NotZero(t, fp.MtimeNs) + return hex.EncodeToString(sum[:]) } diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 70d5433c71d..b29594d3692 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -28,9 +28,9 @@ type JobRun struct { // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` - // ResolvedFileTriggers is the per-file fingerprint map for on_file_change, - // computed under SyncRoot before plan. bundle:"internal" keeps it out of schema. - ResolvedFileTriggers map[string]JobRunFileFingerprint `json:"resolved_file_triggers,omitempty" bundle:"internal"` + // ResolvedFileTriggers is path → content hash for on_file_change, computed + // under SyncRoot before plan. bundle:"internal" keeps it out of schema. + ResolvedFileTriggers map[string]string `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. @@ -58,19 +58,6 @@ func (r *JobRun) HasOnFileChange() bool { return false } -func (r *JobRun) OnFileChangePatterns() []string { - if r.Lifecycle == nil { - return nil - } - var patterns []string - for _, t := range r.Lifecycle.Triggers { - if t.OnFileChange != nil { - patterns = append(patterns, *t.OnFileChange) - } - } - return patterns -} - func (r *JobRun) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, r) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 59ccb682b81..a4e881d298c 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -39,9 +39,3 @@ type JobRunTrigger struct { OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root } - -type JobRunFileFingerprint struct { - Hash string `json:"hash"` - Size int64 `json:"size"` - MtimeNs int64 `json:"mtime_ns"` -} diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index f66991f1ca8..5b97cc9e22b 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -38,8 +38,8 @@ var jobRunTriggerLocalPaths = []string{ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - // Per-file fingerprints from ResolveJobRunFileTriggers; change to recreate. - OnFileChange map[string]resources.JobRunFileFingerprint `json:"on_file_change,omitempty"` + // Path → content hash from ResolveJobRunFileTriggers; change to recreate. + OnFileChange map[string]string `json:"on_file_change,omitempty"` } // JobRunLifecycleState holds local-only lifecycle fields persisted in state. diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 78c1f3d2fa6..69bf2d93aef 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -198,12 +198,6 @@ resources: recreate_on_changes: - reason: immutable - # mtime is stored for the resolve fast path; content identity is the hash. - # Ignoring mtime-only local drift keeps touch from recreating the run. - ignore_local_changes: - - field: lifecycle.triggers.on_file_change[*].mtime_ns - reason: mtime_only_fingerprint - pipelines: recreate_on_changes: - field: storage diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 450f6f9520e..4efb71bed1e 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1038,7 +1038,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index fc03dfbc7df..edfc0bc6390 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1537,7 +1537,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From ce24a87b63801535ec543fb3f4aa04c02f3b0a17 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 18 Aug 2026 13:23:44 +0000 Subject: [PATCH 05/60] job_runs: use maps.Copy in on_file_change hash merge The modernize linter rejects the manual m[k]=v copy loop, failing CI lint. --- bundle/config/mutator/resolve_job_run_file_triggers.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 55a55a94b95..ead17c59e17 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "fmt" "io" + "maps" "os" "path/filepath" "strings" @@ -43,9 +44,7 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) diags = diags.Extend(d) - for k, v := range hashes { - out[k] = v - } + maps.Copy(out, hashes) } if len(out) == 0 { jr.ResolvedFileTriggers = nil From eb34c418b4155bbc10c8487c4382a3715c62879a Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 08:39:23 +0000 Subject: [PATCH 06/60] job_runs: drop filepath.Glob jargon from on_file_change schema docs --- bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 4efb71bed1e..2229bab0b83 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1038,7 +1038,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index edfc0bc6390..147e0049306 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1537,7 +1537,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root (filepath.Glob; `**` is not supported). Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 3595e3acf409630a9096457411db339dc00dc724 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 11:38:20 +0000 Subject: [PATCH 07/60] acc: refresh on_file_change goldens for resource-key progress lines --- .../bundle/resources/job_runs/on_file_change/output.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index f356fe7fb61..a98eb3a6ca7 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -2,8 +2,8 @@ === first deploy triggers a run >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -job run [MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] -job run [MY_RUN_ID]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job Files: 5 uploaded, 0 deleted @@ -52,8 +52,8 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -job run [MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] -job run [MY_RUN_ID_2]: SUCCESS +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged From 58392c9b331450ed46eeaac71b264f464d94c0c3 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 14:50:14 +0000 Subject: [PATCH 08/60] acc: cover on_file_change globs, errors, and appear/disappear --- .../job_runs/on_file_change/out.test.toml | 1 + .../job_runs/on_file_change/output.txt | 131 +++++++++++++- .../resources/job_runs/on_file_change/script | 26 ++- .../job_runs/on_file_change/test.toml | 14 ++ .../on_file_change_errors/databricks.yml | 18 ++ .../dirs/nested/keep.txt | 1 + .../on_file_change_errors/out.test.toml | 2 + .../job_runs/on_file_change_errors/output.txt | 39 +++++ .../job_runs/on_file_change_errors/script | 12 ++ .../job_runs/on_file_change_errors/test.toml | 2 + .../on_file_change_glob/databricks.yml | 18 ++ .../on_file_change_glob/migrations/001.txt | 1 + .../on_file_change_glob/migrations/002.txt | 1 + .../migrations/archive/003.txt | 1 + .../on_file_change_glob/out.test.toml | 2 + .../job_runs/on_file_change_glob/output.txt | 164 ++++++++++++++++++ .../job_runs/on_file_change_glob/script | 47 +++++ .../job_runs/on_file_change_glob/test.toml | 9 + bundle/phases/plan.go | 4 +- 19 files changed, 486 insertions(+), 7 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index a98eb3a6ca7..d03307b90f2 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -6,7 +6,7 @@ Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_ Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> read_id.py my_job @@ -50,12 +50,46 @@ recreate job_runs.my_run Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged ->>> [CLI] bundle deploy +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][0]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS Recreated job_runs.my_run -Files: 3 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> print_requests.py --keep //jobs/runs/delete @@ -77,6 +111,97 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } +=== deleting the file warns and re-fires +>>> [CLI] bundle plan +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "seed.txt": "" + } + } + } + }, + "lifecycle.triggers.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +>>> [CLI] bundle deploy +Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" + in databricks.yml:18:29 + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== restoring the file re-fires again +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS +Recreated job_runs.my_run +Files: 3 uploaded, 1 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + === removing on_file_change rewrites state without a run >>> [CLI] bundle plan update job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index 0506dd5e106..fd1e8800c42 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -4,6 +4,10 @@ cleanup() { } trap cleanup EXIT +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + title "first deploy triggers a run" trace $CLI bundle deploy trace read_id.py my_job @@ -25,11 +29,31 @@ trace print_requests.py //jobs/run-now title "editing the file re-fires" update_file.py seed.txt "v1" "v2" trace $CLI bundle plan -trace $CLI bundle deploy +# Save the plan so the READPLAN=1 variant deploys the hashes computed here. +# The deploy is not traced: readplanarg makes the command line differ per variant. +$CLI bundle plan -o json > tmp.plan.json +trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now +title "deleting the file warns and re-fires" +mv seed.txt tmp.seed.txt +trace $CLI bundle plan +trace changes +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "restoring the file re-fires again" +mv tmp.seed.txt seed.txt +trace $CLI bundle plan +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + title "removing on_file_change rewrites state without a run" update_file.py databricks.yml " lifecycle: triggers: diff --git a/acceptance/bundle/resources/job_runs/on_file_change/test.toml b/acceptance/bundle/resources/job_runs/on_file_change/test.toml new file mode 100644 index 00000000000..7a950e9fd0c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/test.toml @@ -0,0 +1,14 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.seed.txt"] + +# Content hashes are stable but unreadable, and the parent's `\d{8,}` rule +# mangles any digit run inside them. Order=1 runs before it; Distinct keeps +# different hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml new file mode 100644 index 00000000000..44c5433e52a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-errors + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: ../shared/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt new file mode 100644 index 00000000000..79c53955ef8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/dirs/nested/keep.txt @@ -0,0 +1 @@ +nested diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt new file mode 100644 index 00000000000..c7d3a98e1cb --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -0,0 +1,39 @@ + +=== a pattern outside the sync root is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob that only matches directories is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches no regular files + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a pattern that matches nothing warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" + in databricks.yml:18:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 warning diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script new file mode 100644 index 00000000000..39596f0c772 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -0,0 +1,12 @@ +title "a pattern outside the sync root is rejected" +trace musterr $CLI bundle validate + +title "a glob that only matches directories is rejected" +# dirs/* matches dirs/nested, a directory: nothing is left to hash, so the +# trigger would silently never fire. +update_file.py databricks.yml "../shared/*.txt" "dirs/*" +trace musterr $CLI bundle validate + +title "a pattern that matches nothing warns" +update_file.py databricks.yml "dirs/*" "missing/*.txt" +trace $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml new file mode 100644 index 00000000000..7b88d613d4c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml @@ -0,0 +1,2 @@ +# The bundle is never deployed here, so there are no requests worth recording. +RecordRequests = false diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml new file mode 100644 index 00000000000..3b148538f38 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-glob + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: migrations/* diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt new file mode 100644 index 00000000000..5626abf0f72 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/001.txt @@ -0,0 +1 @@ +one diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt new file mode 100644 index 00000000000..f719efd430d --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/002.txt @@ -0,0 +1 @@ +two diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt new file mode 100644 index 00000000000..2bdf67abb16 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/archive/003.txt @@ -0,0 +1 @@ +three diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt new file mode 100644 index 00000000000..798b459a9bd --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -0,0 +1,164 @@ + +=== deploy hashes the matched files +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 8 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== editing one matched file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][2]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][2]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][3]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "action": "recreate", + "reason": "immutable", + "new": "[FILE_HASH][3]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== removing a matched file re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> changes +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][0]" + } + } + } + }, + "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a file below the glob does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== ** is not recursive: it matches the same files as * +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script new file mode 100644 index 00000000000..8feeebcd067 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -0,0 +1,47 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +changes() { + $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + +title "deploy hashes the matched files" +trace $CLI bundle deploy +trace read_id.py my_job +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +# Each phase below plans against the state deployed above and then restores the +# tree, so every plan is compared against the same baseline. + +title "editing one matched file re-fires" +update_file.py migrations/001.txt "one" "one-edited" +trace $CLI bundle plan +trace changes +update_file.py migrations/001.txt "one-edited" "one" + +title "a new matching file re-fires" +printf 'three\n' > migrations/003.txt +trace $CLI bundle plan +trace changes +rm migrations/003.txt + +title "removing a matched file re-fires" +mv migrations/002.txt tmp.002.txt +trace $CLI bundle plan +trace changes +mv tmp.002.txt migrations/002.txt + +title "editing a file below the glob does not re-fire" +# migrations/* matches the archive directory itself, which is skipped: the glob +# does not descend into it. +update_file.py migrations/archive/003.txt "three" "three-edited" +trace $CLI bundle plan +update_file.py migrations/archive/003.txt "three-edited" "three" + +title "** is not recursive: it matches the same files as *" +update_file.py databricks.yml "migrations/*" "migrations/**" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml new file mode 100644 index 00000000000..1c349066b94 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -0,0 +1,9 @@ +Ignore = ["tmp.002.txt"] + +# See on_file_change/test.toml: mask content hashes before the parent's +# `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index c69074315dc..aee481b2b14 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -21,8 +21,7 @@ import ( // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". // Note, it is not run in "bundle migrate" so it must not modify the config func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) { - bundle.ApplySeqContext( - ctx, b, + bundle.ApplySeqContext(ctx, b, terraform.CheckDashboardsModifiedRemotely(isPlan, engine), resourcemutator.SecretScopeFixups(engine), deploy.StatePull(), @@ -30,7 +29,6 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine mutator.ValidateDirectOnlyResources(engine), mutator.ValidateLifecycleStarted(engine), mutator.ValidateCascadeOnDestroy(engine), - mutator.ResolveJobRunFileTriggers(), statemgmt.CheckRunningResource(engine), ) } From ca1f5d4a0ff7409582752586065bc8d8cbaa85c1 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 19:58:11 +0000 Subject: [PATCH 09/60] acc: expand on_file_change glob coverage and skip ignored files Honor .gitignore and sync.exclude when hashing matches. Cover mysubdir globs, READPLAN, and keep LF on Windows so restored files hash cleanly. --- .../job_runs/on_file_change_glob/.gitignore | 2 + .../on_file_change_glob/databricks.yml | 1 + .../on_file_change_glob/mysubdir/a.txt | 1 + .../on_file_change_glob/out.test.toml | 1 + .../job_runs/on_file_change_glob/output.txt | 183 +++++++++++++++--- .../job_runs/on_file_change_glob/script | 39 ++-- .../job_runs/on_file_change_glob/test.toml | 6 +- .../mutator/resolve_job_run_file_triggers.go | 65 ++++++- .../resolve_job_run_file_triggers_test.go | 40 ++++ 9 files changed, 287 insertions(+), 51 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore new file mode 100644 index 00000000000..57433d43f47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore @@ -0,0 +1,2 @@ +# Matches migrations/* but sync skips it, so a content edit must not re-fire. +migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml index 3b148538f38..d47fcf44049 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/databricks.yml @@ -16,3 +16,4 @@ resources: lifecycle: triggers: - on_file_change: migrations/* + - on_file_change: mysubdir/*.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt new file mode 100644 index 00000000000..62e0af52c19 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/mysubdir/a.txt @@ -0,0 +1 @@ +sub diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 798b459a9bd..d424482b961 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -6,7 +6,7 @@ Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_ Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 8 uploaded, 0 deleted +Files: 10 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> read_id.py my_job @@ -23,12 +23,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing one matched file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> changes +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -37,15 +32,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "triggers": { "on_file_change": { "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][2]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -54,7 +51,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", - "new": "[FILE_HASH][2]" + "new": "[FILE_HASH][3]" }, "result_state": { "action": "skip", @@ -64,13 +61,35 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } } -=== a new matching file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py --keep //jobs/runs/delete +{ + "method": "POST", + "path": "/api/2.2/jobs/runs/delete", + "body": { + "run_id": [MY_RUN_ID] + } +} -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} ->>> changes +=== editing a file in a subdirectory re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -78,17 +97,58 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "old": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", + "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", - "migrations/003.txt": "[FILE_HASH][3]" + "mysubdir/a.txt": "[FILE_HASH][4]" + } + } + } + }, + "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][2]", + "new": "[FILE_HASH][4]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== a new matching file re-fires +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "recreate", + "reason": "immutable", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][5]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -96,7 +156,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "lifecycle.triggers.on_file_change['migrations/003.txt']": { "action": "recreate", "reason": "immutable", - "new": "[FILE_HASH][3]" + "new": "[FILE_HASH][5]" }, "result_state": { "action": "skip", @@ -107,12 +167,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } === removing a matched file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> changes +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle": { "action": "recreate", @@ -120,15 +175,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "old": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]" + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } }, "new": { "triggers": { "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]" + "migrations/001.txt": "[FILE_HASH][3]", + "mysubdir/a.txt": "[FILE_HASH][2]" } } } @@ -147,12 +204,74 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged } === editing a file below the glob does not re-fire +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing a gitignored file does not re-fire >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +>>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +{ + "lifecycle": { + "action": "skip", + "reason": "missing_in_remote", + "old": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + }, + "new": { + "triggers": { + "on_file_change": { + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" + } + } + } + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 8feeebcd067..983cb9ac401 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,8 +4,10 @@ cleanup() { } trap cleanup EXIT +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { - $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' + $CLI bundle plan -o json > tmp.plan.json + trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json } title "deploy hashes the matched files" @@ -14,34 +16,45 @@ trace read_id.py my_job read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -# Each phase below plans against the state deployed above and then restores the -# tree, so every plan is compared against the same baseline. - title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" -trace $CLI bundle plan -trace changes -update_file.py migrations/001.txt "one-edited" "one" +changes +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +read_id.py my_run > /dev/null +trace print_requests.py --keep //jobs/runs/delete +trace print_requests.py //jobs/run-now + +title "editing a file in a subdirectory re-fires" +# printf keeps LF on Windows, where update_file.py's text-mode rewrite would +# restore CRLF and leave the file hashing differently than the deploy above. +printf 'sub-edited\n' > mysubdir/a.txt +changes +printf 'sub\n' > mysubdir/a.txt title "a new matching file re-fires" printf 'three\n' > migrations/003.txt -trace $CLI bundle plan -trace changes +changes rm migrations/003.txt title "removing a matched file re-fires" mv migrations/002.txt tmp.002.txt -trace $CLI bundle plan -trace changes +changes mv tmp.002.txt migrations/002.txt title "editing a file below the glob does not re-fire" # migrations/* matches the archive directory itself, which is skipped: the glob # does not descend into it. update_file.py migrations/archive/003.txt "three" "three-edited" -trace $CLI bundle plan +changes update_file.py migrations/archive/003.txt "three-edited" "three" +title "editing a gitignored file does not re-fire" +update_file.py migrations/ignored.txt "ignored" "ignored-edited" +trace $CLI bundle plan +update_file.py migrations/ignored.txt "ignored-edited" "ignored" + title "** is not recursive: it matches the same files as *" update_file.py databricks.yml "migrations/*" "migrations/**" -trace $CLI bundle plan +changes diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 1c349066b94..3516572564d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,4 +1,8 @@ -Ignore = ["tmp.002.txt"] +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + +Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's # `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index ead17c59e17..d74b9841c57 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -13,6 +13,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" + libsync "github.com/databricks/cli/libs/sync" ) // missingFileHash marks a pattern with no matching file so appear/disappear recreates. @@ -30,8 +31,13 @@ func (*resolveJobRunFileTriggers) Name() string { return "ResolveJobRunFileTriggers" } -func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics + syncable, d := syncableRelPaths(ctx, b) + if d.HasError() { + return d + } + diags = diags.Extend(d) for name, jr := range b.Config.Resources.JobRuns { if jr == nil || jr.Lifecycle == nil { continue @@ -42,7 +48,7 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia continue } path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) - hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange)) + hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange), syncable) diags = diags.Extend(d) maps.Copy(out, hashes) } @@ -55,7 +61,42 @@ func (*resolveJobRunFileTriggers) Apply(_ context.Context, b *bundle.Bundle) dia return diags } -func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]string, diag.Diagnostics) { +// syncableRelPaths is the set of relative paths sync would upload. +func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{}, diag.Diagnostics) { + var diags diag.Diagnostics + needs := false + for _, jr := range b.Config.Resources.JobRuns { + if jr != nil && jr.HasOnFileChange() { + needs = true + break + } + } + if !needs { + return nil, diags + } + + fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + files, err := fl.Files(ctx) + if err != nil { + return nil, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + out := make(map[string]struct{}, len(files)) + for _, f := range files { + out[filepath.ToSlash(f.Relative)] = struct{}{} + } + return out, diags +} + +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { var diags diag.Diagnostics out := make(map[string]string) localPattern := filepath.FromSlash(pattern) @@ -84,6 +125,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin }) } regularMatches := 0 + ignoredMatches := 0 sawNonRegular := false for _, match := range matches { info, err := os.Stat(match) @@ -99,7 +141,6 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin sawNonRegular = true continue } - regularMatches++ rel, err := filepath.Rel(b.SyncRootPath, match) if err != nil || !filepath.IsLocal(rel) { diags = diags.Append(diag.Diagnostic{ @@ -109,6 +150,12 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin }) continue } + // Same membership as sync: .gitignore and sync.exclude drop a glob match. + if _, ok := syncable[filepath.ToSlash(rel)]; !ok { + ignoredMatches++ + continue + } + regularMatches++ hash, err := hashFile(match) if err != nil { diags = diags.Append(diag.Diagnostic{ @@ -122,13 +169,21 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string) (map[string]strin } // A directory-only match would otherwise leave ResolvedFileTriggers empty // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular { + if regularMatches == 0 && sawNonRegular && ignoredMatches == 0 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), Locations: b.Config.GetLocations(loc), }) } + if len(out) == 0 && ignoredMatches > 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) + } return out, diags } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 3ec663c89bd..344fed25657 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -85,7 +86,10 @@ func TestResolveJobRunFileTriggers(t *testing.T) { patB := "subdir/*.py" b := &bundle.Bundle{ SyncRootPath: dir, + SyncRoot: vfs.MustNew(dir), + WorktreeRoot: vfs.MustNew(dir), Config: config.Root{ + Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ "my_run": { @@ -135,6 +139,38 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) + t.Run("skips gitignored files", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("skip.txt\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) + + b := bundleWithFileTrigger(dir, "*.txt") + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) + assert.NotContains(t, hashes, "skip.txt") + }) + + t.Run("skips sync.exclude files", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) + + b := bundleWithFileTrigger(dir, "*.txt") + b.Config.Sync.Exclude = []string{"skip.txt"} + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + + hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + require.Len(t, hashes, 1) + assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) + assert.NotContains(t, hashes, "skip.txt") + }) + t.Run("trims pattern whitespace", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) @@ -148,9 +184,13 @@ func TestResolveJobRunFileTriggers(t *testing.T) { } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { + root := vfs.MustNew(syncRoot) return &bundle.Bundle{ SyncRootPath: syncRoot, + SyncRoot: root, + WorktreeRoot: root, Config: config.Root{ + Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ "my_run": { From 426a9f30adefc52a102988f70cd30d999a9f0645 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Wed, 19 Aug 2026 20:10:03 +0000 Subject: [PATCH 10/60] acc: force-add gitignored on_file_change fixture The glob test edits migrations/ignored.txt, but the test .gitignore excluded it from the commit so CI clones would miss the file. --- .../job_runs/on_file_change_glob/migrations/ignored.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt new file mode 100644 index 00000000000..ea10ec85c10 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/migrations/ignored.txt @@ -0,0 +1 @@ +ignored From 1b3f797dc559942200c8c83e66fdab6a3cbcd4ea Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 08:11:43 +0000 Subject: [PATCH 11/60] acc: drop READPLAN from the on_file_change glob test Plan serialization is already covered by on_file_change; the glob cases only assert the hash map. --- .../resources/job_runs/on_file_change_glob/out.test.toml | 1 - .../bundle/resources/job_runs/on_file_change_glob/output.txt | 2 +- .../bundle/resources/job_runs/on_file_change_glob/script | 5 +---- .../bundle/resources/job_runs/on_file_change_glob/test.toml | 4 ---- 4 files changed, 2 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 57b0f616850..0938e678987 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,3 +1,2 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index d424482b961..bd7d08557f5 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } -=== bundle deploy +>>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 983cb9ac401..23390d23a55 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,7 +4,6 @@ cleanup() { } trap cleanup EXIT -# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { $CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json @@ -19,9 +18,7 @@ trace print_requests.py //jobs/run-now title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" changes -# The deploy is not traced: readplanarg makes the command line differ per variant. -title "bundle deploy\n" -$CLI bundle deploy $(readplanarg tmp.plan.json) +trace $CLI bundle deploy read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 3516572564d..d372fd54eae 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,7 +1,3 @@ -# Deploy both by re-planning and from a plan saved on disk, so the hashes -# computed during planning survive plan serialization. -EnvMatrix.READPLAN = ["", "1"] - Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's From 13bf6f643b8f00fda9a221f011372e053fd50033 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 12:34:44 +0000 Subject: [PATCH 12/60] acc: restore READPLAN on the on_file_change glob test Cover deploying glob hashes from a saved plan, not only from a live re-plan. --- .../resources/job_runs/on_file_change_glob/out.test.toml | 1 + .../bundle/resources/job_runs/on_file_change_glob/output.txt | 2 +- .../bundle/resources/job_runs/on_file_change_glob/script | 5 ++++- .../bundle/resources/job_runs/on_file_change_glob/test.toml | 4 ++++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 0938e678987..57b0f616850 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index bd7d08557f5..d424482b961 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -61,7 +61,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } ->>> [CLI] bundle deploy +=== bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 23390d23a55..983cb9ac401 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -4,6 +4,7 @@ cleanup() { } trap cleanup EXIT +# Plan once, print JSON changes, leave tmp.plan.json for readplanarg. changes() { $CLI bundle plan -o json > tmp.plan.json trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json @@ -18,7 +19,9 @@ trace print_requests.py //jobs/run-now title "editing one matched file re-fires" update_file.py migrations/001.txt "one" "one-edited" changes -trace $CLI bundle deploy +# The deploy is not traced: readplanarg makes the command line differ per variant. +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index d372fd54eae..3516572564d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -1,3 +1,7 @@ +# Deploy both by re-planning and from a plan saved on disk, so the hashes +# computed during planning survive plan serialization. +EnvMatrix.READPLAN = ["", "1"] + Ignore = ["tmp.plan.json", "tmp.002.txt"] # See on_file_change/test.toml: mask content hashes before the parent's From 41fc58bec215e4383c1c90a75b84f0662220f1d7 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 14:08:42 +0000 Subject: [PATCH 13/60] job_runs: address on_file_change review comments Keep the gitignore fixture as dot_gitignore so it does not affect the repo while developing, drop unit tests that duplicate acceptance coverage, and allocate trigger state once when a trigger is armed. --- .../{.gitignore => dot_gitignore} | 0 .../job_runs/on_file_change_glob/script | 3 + .../job_runs/on_file_change_glob/test.toml | 2 +- .../resolve_job_run_file_triggers_test.go | 139 +----------------- bundle/direct/dresources/job_run.go | 23 +-- 5 files changed, 11 insertions(+), 156 deletions(-) rename acceptance/bundle/resources/job_runs/on_file_change_glob/{.gitignore => dot_gitignore} (100%) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore b/acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore similarity index 100% rename from acceptance/bundle/resources/job_runs/on_file_change_glob/.gitignore rename to acceptance/bundle/resources/job_runs/on_file_change_glob/dot_gitignore diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 983cb9ac401..dd737f242b5 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -1,3 +1,6 @@ +# Fixture is named dot_gitignore so a real .gitignore does not affect git while developing. +mv dot_gitignore .gitignore + cleanup() { trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 3516572564d..98c98631e5b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -2,7 +2,7 @@ # computed during planning survive plan serialization. EnvMatrix.READPLAN = ["", "1"] -Ignore = ["tmp.plan.json", "tmp.002.txt"] +Ignore = ["tmp.plan.json", "tmp.002.txt", ".gitignore"] # See on_file_change/test.toml: mask content hashes before the parent's # `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 344fed25657..1cc54eecf4f 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -11,14 +11,13 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/vfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestResolveJobRunFileTriggers(t *testing.T) { - t.Run("matches files and fills hashes", func(t *testing.T) { + t.Run("hashes file contents with sha256", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) @@ -35,142 +34,6 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("no matches warns and stores empty hash", func(t *testing.T) { - dir := t.TempDir() - pattern := "missing.txt" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Equal(t, diag.Warning, diags[0].Severity) - assert.Contains(t, diags[0].Summary, `no files match "missing.txt"`) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Empty(t, hashes["missing.txt"]) - }) - - t.Run("no file triggers is a no-op", func(t *testing.T) { - dir := t.TempDir() - on := true - b := &bundle.Bundle{ - SyncRootPath: dir, - Config: config.Root{ - Resources: config.Resources{ - JobRuns: map[string]*resources.JobRun{ - "my_run": { - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{ - {OnBundleDeploy: &on}, - }, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - assert.Empty(t, diags) - assert.Nil(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("multiple patterns merge into one map", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("aaa"), 0o644)) - require.NoError(t, os.MkdirAll(filepath.Join(dir, "subdir"), 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "subdir", "x.py"), []byte("bbb"), 0o644)) - - patA := "a.txt" - patB := "subdir/*.py" - b := &bundle.Bundle{ - SyncRootPath: dir, - SyncRoot: vfs.MustNew(dir), - WorktreeRoot: vfs.MustNew(dir), - Config: config.Root{ - Sync: config.Sync{Paths: []string{"."}}, - Resources: config.Resources{ - JobRuns: map[string]*resources.JobRun{ - "my_run": { - Lifecycle: &resources.JobRunLifecycle{ - Triggers: []resources.JobRunTrigger{ - {OnFileChange: &patA}, - {OnFileChange: &patB}, - }, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 2) - assert.Equal(t, contentHash("aaa"), hashes["a.txt"]) - assert.Equal(t, contentHash("bbb"), hashes["subdir/x.py"]) - }) - - t.Run("pattern outside sync root is an error", func(t *testing.T) { - dir := t.TempDir() - pattern := "../outside.txt" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.True(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Contains(t, diags[0].Summary, `not under the sync root`) - assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("directory-only match is an error", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.MkdirAll(filepath.Join(dir, "migrations"), 0o755)) - pattern := "migrations" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.True(t, diags.HasError()) - require.Len(t, diags, 1) - assert.Contains(t, diags[0].Summary, `matches no regular files`) - assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) - }) - - t.Run("skips gitignored files", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, ".gitignore"), []byte("skip.txt\n"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) - - b := bundleWithFileTrigger(dir, "*.txt") - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) - assert.NotContains(t, hashes, "skip.txt") - }) - - t.Run("skips sync.exclude files", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "skip.txt"), []byte("skip"), 0o644)) - - b := bundleWithFileTrigger(dir, "*.txt") - b.Config.Sync.Exclude = []string{"skip.txt"} - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers - require.Len(t, hashes, 1) - assert.Equal(t, contentHash("keep"), hashes["keep.txt"]) - assert.NotContains(t, hashes, "skip.txt") - }) - t.Run("trims pattern whitespace", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 5b97cc9e22b..8117af9aa22 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -109,25 +109,14 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { ResultState: jobs.RunResultStateSuccess, Lifecycle: nil, } - var triggers *JobRunTriggersState - if input.HasOnBundleDeploy() { - triggers = &JobRunTriggersState{ - OnBundleDeploy: uuid.NewString(), - OnFileChange: nil, - } + if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { + return state } - if len(input.ResolvedFileTriggers) > 0 { - if triggers == nil { - triggers = &JobRunTriggersState{ - OnBundleDeploy: "", - OnFileChange: nil, - } - } - triggers.OnFileChange = input.ResolvedFileTriggers - } - if triggers != nil { - state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} + triggers := &JobRunTriggersState{OnFileChange: input.ResolvedFileTriggers} + if input.HasOnBundleDeploy() { + triggers.OnBundleDeploy = uuid.NewString() } + state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} return state } From 756b44663f41d480098a7acb682ac12d4b970c0b Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 20 Aug 2026 14:16:35 +0000 Subject: [PATCH 14/60] job_runs: list every trigger field to satisfy exhaustruct The previous simplification omitted OnBundleDeploy from the struct literal, which the linter rejects. --- bundle/direct/dresources/job_run.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 8117af9aa22..ac65e5db697 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -112,11 +112,16 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { return state } - triggers := &JobRunTriggersState{OnFileChange: input.ResolvedFileTriggers} + onBundleDeploy := "" if input.HasOnBundleDeploy() { - triggers.OnBundleDeploy = uuid.NewString() + onBundleDeploy = uuid.NewString() + } + state.Lifecycle = &JobRunLifecycleState{ + Triggers: &JobRunTriggersState{ + OnBundleDeploy: onBundleDeploy, + OnFileChange: input.ResolvedFileTriggers, + }, } - state.Lifecycle = &JobRunLifecycleState{Triggers: triggers} return state } From c2d364d1fda041b8a5a0785cbb1229c02a5e299f Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 21 Aug 2026 12:31:38 +0000 Subject: [PATCH 15/60] job_runs: keep empty nested lifecycle on state and remote Always persist the same lifecycle.triggers shape so trigger fingerprints compare at the same paths. Wrap on_file_change hashes in files so dropping the trigger updates instead of recreating. --- .../resources/job_runs/failed_run/output.txt | 10 ++ .../job_runs/interrupted_run/output.txt | 5 + .../job_runs/on_bundle_deploy/output.txt | 22 +-- .../job_runs/on_file_change/output.txt | 32 +--- .../job_runs/on_file_change_glob/output.txt | 144 ++++++------------ .../resources/job_runs/redeploy/output.txt | 10 ++ bundle/direct/dresources/job_run.go | 86 +++++++---- bundle/direct/dresources/job_run_test.go | 16 +- bundle/direct/dresources/resources.yml | 3 + bundle/direct/dresources/type_test.go | 4 - 10 files changed, 161 insertions(+), 171 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index f8b8dbf4398..0c6bddb0d9b 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -27,11 +27,21 @@ Files: 7 uploaded, 0 deleted "new_state": { "value": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS" } }, "remote_state": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "FAILED", "run_id": [MY_RUN_ID], "run_name": "test-job-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index 2e1d9cacd7a..da185c3704d 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -42,6 +42,11 @@ Exit code: 1 "action": "skip", "remote_state": { "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "run_id": [MY_RUN_ID], "run_name": "my-job", "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt index 39123f3370f..6cd8d44a418 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -30,20 +30,6 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { - "action": "recreate", - "reason": "immutable", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - }, - "new": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } - }, "lifecycle.triggers.on_bundle_deploy": { "action": "recreate", "reason": "immutable", @@ -108,14 +94,10 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_bundle_deploy": { "action": "skip", "reason": "trigger removed", - "old": { - "triggers": { - "on_bundle_deploy": "[UUID]" - } - } + "old": "[UUID]" }, "result_state": { "action": "skip", diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index d03307b90f2..9b662e1936d 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,25 +52,17 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][0]" - } - } + "seed.txt": "[FILE_HASH][0]" }, "new": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][1]" - } - } + "seed.txt": "[FILE_HASH][1]" } }, - "lifecycle.triggers.on_file_change['seed.txt']": { + "lifecycle.triggers.on_file_change.files['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -125,25 +117,17 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" in databricks.yml:18:29 { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "seed.txt": "[FILE_HASH][1]" - } - } + "seed.txt": "[FILE_HASH][1]" }, "new": { - "triggers": { - "on_file_change": { - "seed.txt": "" - } - } + "seed.txt": "" } }, - "lifecycle.triggers.on_file_change['seed.txt']": { + "lifecycle.triggers.on_file_change.files['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index d424482b961..5b4ef23178c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -25,29 +25,21 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing one matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][0]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -91,29 +83,21 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file in a subdirectory re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][4]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][4]" } }, - "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "lifecycle.triggers.on_file_change.files['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -130,30 +114,22 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === a new matching file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "migrations/003.txt": "[FILE_HASH][5]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "migrations/003.txt": "[FILE_HASH][5]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/003.txt']": { "action": "recreate", "reason": "immutable", "new": "[FILE_HASH][5]" @@ -169,28 +145,20 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === removing a matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "recreate", "reason": "immutable", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "lifecycle.triggers.on_file_change.files['migrations/002.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]" @@ -206,26 +174,18 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file below the glob does not re-fire >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "missing_in_remote", + "reason": "not_returned_by_api", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, "result_state": { @@ -243,26 +203,18 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { + "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "missing_in_remote", + "reason": "not_returned_by_api", "old": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" }, "new": { - "triggers": { - "on_file_change": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - } + "migrations/001.txt": "[FILE_HASH][3]", + "migrations/002.txt": "[FILE_HASH][1]", + "mysubdir/a.txt": "[FILE_HASH][2]" } }, "result_state": { diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 06129f79373..60111c98a93 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -59,6 +59,11 @@ Resources: "job_parameters": { "env": "prod" }, + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS" } }, @@ -67,6 +72,11 @@ Resources: "job_parameters": { "env": "dev" }, + "lifecycle": { + "triggers": { + "on_file_change": {} + } + }, "result_state": "SUCCESS", "run_id": [MY_RUN_ID], "run_name": "my-job", diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index ac65e5db697..815091a721a 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -3,6 +3,7 @@ package dresources import ( "cmp" "context" + "encoding/json" "errors" "fmt" "slices" @@ -28,10 +29,12 @@ const jobRunTimeout = 24 * time.Hour // jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. var jobRunTriggerLocalPaths = []string{ - "lifecycle", - "lifecycle.triggers", "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change", + "lifecycle.triggers.on_file_change.files", +} + +func isJobRunTriggerPath(path string) bool { + return slices.Contains(jobRunTriggerLocalPaths, path) } // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. @@ -39,12 +42,44 @@ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Path → content hash from ResolveJobRunFileTriggers; change to recreate. - OnFileChange map[string]string `json:"on_file_change,omitempty"` + OnFileChange *JobRunFileTriggerState `json:"on_file_change"` +} + +// JobRunFileTriggerState is always present so remote and state share one shape. +// Files is nil when the trigger is off, so dropping the trigger diffs here and +// a file appear/disappear diffs inside the map. +type JobRunFileTriggerState struct { + Files map[string]string `json:"files,omitempty"` +} + +// UnmarshalJSON accepts the wrapped map and the older path-to-hash map. +func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { + var raw map[string]json.RawMessage + if err := json.Unmarshal(b, &raw); err != nil { + return err + } + if files, ok := raw["files"]; ok && len(files) > 0 && files[0] == '{' { + return json.Unmarshal(files, &s.Files) + } + if len(raw) == 0 { + s.Files = nil + return nil + } + return json.Unmarshal(b, &s.Files) } -// JobRunLifecycleState holds local-only lifecycle fields persisted in state. +// JobRunLifecycleState is the local-only trigger fingerprint, also present (empty) on remote. type JobRunLifecycleState struct { - Triggers *JobRunTriggersState `json:"triggers,omitempty"` + Triggers *JobRunTriggersState `json:"triggers"` +} + +func newJobRunLifecycleState() *JobRunLifecycleState { + return &JobRunLifecycleState{ + Triggers: &JobRunTriggersState{ + OnBundleDeploy: "", + OnFileChange: &JobRunFileTriggerState{Files: nil}, + }, + } } // JobRunState is the RunNow request plus the outcome required for planning. @@ -54,12 +89,13 @@ type JobRunState struct { // Always SUCCESS during planning and cleared before persistence. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Local-only; listed in knownMissingInRemoteType. Nested under lifecycle to - // mirror config and avoid colliding with a future Jobs API field. - Lifecycle *JobRunLifecycleState `json:"lifecycle,omitempty"` + // Local-only. Nested under lifecycle to mirror config and avoid colliding + // with a future Jobs API field. + Lifecycle *JobRunLifecycleState `json:"lifecycle"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { + s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -76,6 +112,9 @@ type JobRunRemote struct { // compare the two. See "RemapState is a dumb copy" in README.md. ResultState jobs.RunResultState `json:"result_state,omitempty"` + // Always the empty fingerprint: GetRun does not return triggers. + Lifecycle *JobRunLifecycleState `json:"lifecycle"` + RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` @@ -86,6 +125,7 @@ type JobRunRemote struct { // Custom marshaler needed because embedded RunNow's MarshalJSON would otherwise // take over and drop the additional fields. func (s *JobRunRemote) UnmarshalJSON(b []byte) error { + s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -107,20 +147,13 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: nil, - } - if !input.HasOnBundleDeploy() && len(input.ResolvedFileTriggers) == 0 { - return state + Lifecycle: newJobRunLifecycleState(), } - onBundleDeploy := "" if input.HasOnBundleDeploy() { - onBundleDeploy = uuid.NewString() + state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() } - state.Lifecycle = &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{ - OnBundleDeploy: onBundleDeploy, - OnFileChange: input.ResolvedFileTriggers, - }, + if len(input.ResolvedFileTriggers) > 0 { + state.Lifecycle.Triggers.OnFileChange.Files = input.ResolvedFileTriggers } return state } @@ -161,6 +194,7 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { ForceSendFields: nil, }, ResultState: run.State.ResultState, + Lifecycle: newJobRunLifecycleState(), RunId: run.RunId, RunName: run.RunName, // Rebuilt, not copied: the SDK records explicitly-sent fields in @@ -201,8 +235,7 @@ func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - // Local-only lifecycle fingerprints stay unset on the remapped remote. - Lifecycle: nil, + Lifecycle: remote.Lifecycle, } } @@ -389,10 +422,11 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - if slices.Contains(jobRunTriggerLocalPaths, path.String()) { - // A cleared trigger sets New empty; structdiff may report it at lifecycle, - // lifecycle.triggers, or the leaf. - if change.New == nil || change.New == "" { + pathString := path.String() + if isJobRunTriggerPath(pathString) { + removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") + removed = removed || pathString == "lifecycle.triggers.on_file_change.files" && change.New == nil + if removed { change.Action = deployplan.Skip change.Reason = "trigger removed" } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index a2efdc0e6d3..5f2c184d406 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -255,6 +255,18 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } +// State written before lifecycle existed has no such key, and must still load. +func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { + var state JobRunState + + require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) + + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.Triggers) + require.NotNil(t, state.Lifecycle.Triggers.OnFileChange) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) +} + // The planner diffs RemapState(remote) against PrepareState(config), so a run // that did not end in SUCCESS has to surface as a difference on result_state. func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { @@ -265,11 +277,13 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { "", } { t.Run(string(outcome), func(t *testing.T) { - remote := &JobRunRemote{RunId: 123, ResultState: outcome} + lifecycle := newJobRunLifecycleState() + remote := &JobRunRemote{RunId: 123, ResultState: outcome, Lifecycle: lifecycle} state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) + assert.Same(t, lifecycle, state.Lifecycle) }) } } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 69bf2d93aef..1e08f9e78dd 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -191,6 +191,9 @@ resources: reason: not_returned_by_api - field: queue reason: not_returned_by_api + # Local-only trigger fingerprints; GetRun does not return them. + - field: lifecycle + reason: not_returned_by_api # A run is immutable and fire-once, so any change recreates it. Omitting # `field` matches every field (root; see TestFieldRuleOmittedIsRoot). # `field: ""` would instead match nothing. The one exception is a run that is diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 1cba568d103..3aedb80dad3 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -61,10 +61,6 @@ var knownMissingInRemoteType = map[string][]string{ "acl", "zip_path", }, - "job_runs": { - // Local-only trigger fingerprints under lifecycle. - "lifecycle", - }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. From c2e04c9955460f509a1229149fa95bc420220652 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 21 Aug 2026 13:02:42 +0000 Subject: [PATCH 16/60] job_runs: keep the lifecycle fingerprint out of the remote type Nest lifecycle by value so structdiff descends to the leaf that changed instead of reporting the whole subtree, and let RemapState supply the empty shape. GetRun never returns the fingerprints, so the remote type no longer advertises a lifecycle it cannot fill. --- .../resources/job_runs/failed_run/output.txt | 5 --- .../job_runs/interrupted_run/output.txt | 5 --- .../job_runs/on_file_change_glob/output.txt | 4 +- .../resources/job_runs/redeploy/output.txt | 5 --- bundle/direct/dresources/job_run.go | 45 +++++++++---------- bundle/direct/dresources/job_run_test.go | 8 +--- bundle/direct/dresources/resources.yml | 3 -- bundle/direct/dresources/type_test.go | 5 +++ 8 files changed, 31 insertions(+), 49 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 0c6bddb0d9b..37f1a831681 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -37,11 +37,6 @@ Files: 7 uploaded, 0 deleted }, "remote_state": { "job_id": [MY_JOB_ID], - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "result_state": "FAILED", "run_id": [MY_RUN_ID], "run_name": "test-job-[UNIQUE_NAME]", diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt index da185c3704d..2e1d9cacd7a 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/output.txt +++ b/acceptance/bundle/resources/job_runs/interrupted_run/output.txt @@ -42,11 +42,6 @@ Exit code: 1 "action": "skip", "remote_state": { "job_id": [MY_JOB_ID], - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "run_id": [MY_RUN_ID], "run_name": "my-job", "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 5b4ef23178c..69a7e5ece94 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -176,7 +176,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged { "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "missing_in_remote", "old": { "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", @@ -205,7 +205,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged { "lifecycle.triggers.on_file_change.files": { "action": "skip", - "reason": "not_returned_by_api", + "reason": "missing_in_remote", "old": { "migrations/001.txt": "[FILE_HASH][3]", "migrations/002.txt": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 60111c98a93..a80685733ad 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -72,11 +72,6 @@ Resources: "job_parameters": { "env": "dev" }, - "lifecycle": { - "triggers": { - "on_file_change": {} - } - }, "result_state": "SUCCESS", "run_id": [MY_RUN_ID], "run_name": "my-job", diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 815091a721a..9a937787e02 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -41,13 +41,12 @@ func isJobRunTriggerPath(path string) bool { type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - // Path → content hash from ResolveJobRunFileTriggers; change to recreate. - OnFileChange *JobRunFileTriggerState `json:"on_file_change"` + // Content hashes from ResolveJobRunFileTriggers; any change recreates. + OnFileChange JobRunFileTriggerState `json:"on_file_change"` } -// JobRunFileTriggerState is always present so remote and state share one shape. -// Files is nil when the trigger is off, so dropping the trigger diffs here and -// a file appear/disappear diffs inside the map. +// JobRunFileTriggerState wraps the hashes so turning the trigger off (Files nil) +// diffs at one path, distinct from the per-file entries a changed file produces. type JobRunFileTriggerState struct { Files map[string]string `json:"files,omitempty"` } @@ -68,16 +67,21 @@ func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { return json.Unmarshal(b, &s.Files) } -// JobRunLifecycleState is the local-only trigger fingerprint, also present (empty) on remote. +// JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, +// not by pointer: structdiff cannot descend into a nil pointer and would report +// the whole subtree at "lifecycle" instead of the leaf that actually changed. type JobRunLifecycleState struct { - Triggers *JobRunTriggersState `json:"triggers"` + Triggers JobRunTriggersState `json:"triggers"` } -func newJobRunLifecycleState() *JobRunLifecycleState { - return &JobRunLifecycleState{ - Triggers: &JobRunTriggersState{ +// Zero value spelled out field by field, as exhaustruct requires. +func emptyJobRunLifecycleState() JobRunLifecycleState { + return JobRunLifecycleState{ + Triggers: JobRunTriggersState{ OnBundleDeploy: "", - OnFileChange: &JobRunFileTriggerState{Files: nil}, + OnFileChange: JobRunFileTriggerState{ + Files: nil, + }, }, } } @@ -91,11 +95,10 @@ type JobRunState struct { // Local-only. Nested under lifecycle to mirror config and avoid colliding // with a future Jobs API field. - Lifecycle *JobRunLifecycleState `json:"lifecycle"` + Lifecycle JobRunLifecycleState `json:"lifecycle"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { - s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -103,8 +106,8 @@ func (s JobRunState) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } -// JobRunRemote embeds RunNow so every StateType path is a valid RemoteType path -// (see TestRemoteSuperset), plus the run's output-only fields for a faithful view. +// JobRunRemote is the RunNow request plus the run's output-only fields. It has no +// lifecycle: GetRun never returns the fingerprints (see knownMissingInRemoteType). type JobRunRemote struct { jobs.RunNow @@ -112,9 +115,6 @@ type JobRunRemote struct { // compare the two. See "RemapState is a dumb copy" in README.md. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Always the empty fingerprint: GetRun does not return triggers. - Lifecycle *JobRunLifecycleState `json:"lifecycle"` - RunId int64 `json:"run_id,omitempty"` RunName string `json:"run_name,omitempty"` State *jobs.RunState `json:"state,omitempty"` @@ -125,7 +125,6 @@ type JobRunRemote struct { // Custom marshaler needed because embedded RunNow's MarshalJSON would otherwise // take over and drop the additional fields. func (s *JobRunRemote) UnmarshalJSON(b []byte) error { - s.Lifecycle = newJobRunLifecycleState() return marshal.Unmarshal(b, s) } @@ -147,7 +146,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: newJobRunLifecycleState(), + Lifecycle: emptyJobRunLifecycleState(), } if input.HasOnBundleDeploy() { state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() @@ -194,7 +193,6 @@ func makeJobRunRemote(run *jobs.Run) *JobRunRemote { ForceSendFields: nil, }, ResultState: run.State.ResultState, - Lifecycle: newJobRunLifecycleState(), RunId: run.RunId, RunName: run.RunName, // Rebuilt, not copied: the SDK records explicitly-sent fields in @@ -230,12 +228,13 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, } // RemapState extracts the fields used for diffing: the RunNow request and the -// outcome the run reached. +// outcome the run reached. Lifecycle has no remote counterpart, so it stays empty +// and the planner skips it as missing_in_remote. func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - Lifecycle: remote.Lifecycle, + Lifecycle: emptyJobRunLifecycleState(), } } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 5f2c184d406..301e119f88f 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -261,9 +261,6 @@ func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - require.NotNil(t, state.Lifecycle) - require.NotNil(t, state.Lifecycle.Triggers) - require.NotNil(t, state.Lifecycle.Triggers.OnFileChange) assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) } @@ -277,13 +274,12 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { "", } { t.Run(string(outcome), func(t *testing.T) { - lifecycle := newJobRunLifecycleState() - remote := &JobRunRemote{RunId: 123, ResultState: outcome, Lifecycle: lifecycle} + remote := &JobRunRemote{RunId: 123, ResultState: outcome} state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) - assert.Same(t, lifecycle, state.Lifecycle) + assert.Equal(t, emptyJobRunLifecycleState(), state.Lifecycle) }) } } diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 1e08f9e78dd..69bf2d93aef 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -191,9 +191,6 @@ resources: reason: not_returned_by_api - field: queue reason: not_returned_by_api - # Local-only trigger fingerprints; GetRun does not return them. - - field: lifecycle - reason: not_returned_by_api # A run is immutable and fire-once, so any change recreates it. Omitting # `field` matches every field (root; see TestFieldRuleOmittedIsRoot). # `field: ""` would instead match nothing. The one exception is a run that is diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 3aedb80dad3..b79314dce31 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -19,6 +19,11 @@ var knownMissingInRemoteType = map[string][]string{ "external_locations": { "skip_validation", }, + "job_runs": { + // Deliberate omission: the trigger fingerprints are client-side only, so + // GetRun has nothing to report them under. + "lifecycle", + }, "model_serving_endpoints": { "rate_limits", }, From 575f26650eaf3707670605e1bdb945909e3a427d Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 07:55:05 +0000 Subject: [PATCH 17/60] job_runs: store on_file_change hashes on the trigger itself Drop the files wrapper so plan diffs land on lifecycle.triggers.on_file_change, matching the config path without an extra state-only level. --- .../job_runs/on_file_change/output.txt | 8 ++--- .../job_runs/on_file_change_glob/output.txt | 20 +++++------ bundle/direct/dresources/job_run.go | 35 +++---------------- bundle/direct/dresources/job_run_test.go | 2 +- 4 files changed, 20 insertions(+), 45 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 9b662e1936d..28deeee38a4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,7 +52,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -62,7 +62,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "seed.txt": "[FILE_HASH][1]" } }, - "lifecycle.triggers.on_file_change.files['seed.txt']": { + "lifecycle.triggers.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -117,7 +117,7 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" in databricks.yml:18:29 { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -127,7 +127,7 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" "seed.txt": "" } }, - "lifecycle.triggers.on_file_change.files['seed.txt']": { + "lifecycle.triggers.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 69a7e5ece94..a04e3710fb4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -25,7 +25,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing one matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -39,7 +39,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/001.txt']": { + "lifecycle.triggers.on_file_change['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -83,7 +83,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file in a subdirectory re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -97,7 +97,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][4]" } }, - "lifecycle.triggers.on_file_change.files['mysubdir/a.txt']": { + "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -114,7 +114,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === a new matching file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -129,7 +129,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/003.txt']": { + "lifecycle.triggers.on_file_change['migrations/003.txt']": { "action": "recreate", "reason": "immutable", "new": "[FILE_HASH][5]" @@ -145,7 +145,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === removing a matched file re-fires >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "recreate", "reason": "immutable", "old": { @@ -158,7 +158,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged "mysubdir/a.txt": "[FILE_HASH][2]" } }, - "lifecycle.triggers.on_file_change.files['migrations/002.txt']": { + "lifecycle.triggers.on_file_change['migrations/002.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]" @@ -174,7 +174,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a file below the glob does not re-fire >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "skip", "reason": "missing_in_remote", "old": { @@ -203,7 +203,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged === ** is not recursive: it matches the same files as * >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change.files": { + "lifecycle.triggers.on_file_change": { "action": "skip", "reason": "missing_in_remote", "old": { diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9a937787e02..9d1a272a7a3 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -3,7 +3,6 @@ package dresources import ( "cmp" "context" - "encoding/json" "errors" "fmt" "slices" @@ -30,7 +29,7 @@ const jobRunTimeout = 24 * time.Hour // jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. var jobRunTriggerLocalPaths = []string{ "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change.files", + "lifecycle.triggers.on_file_change", } func isJobRunTriggerPath(path string) bool { @@ -42,29 +41,7 @@ type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` // Content hashes from ResolveJobRunFileTriggers; any change recreates. - OnFileChange JobRunFileTriggerState `json:"on_file_change"` -} - -// JobRunFileTriggerState wraps the hashes so turning the trigger off (Files nil) -// diffs at one path, distinct from the per-file entries a changed file produces. -type JobRunFileTriggerState struct { - Files map[string]string `json:"files,omitempty"` -} - -// UnmarshalJSON accepts the wrapped map and the older path-to-hash map. -func (s *JobRunFileTriggerState) UnmarshalJSON(b []byte) error { - var raw map[string]json.RawMessage - if err := json.Unmarshal(b, &raw); err != nil { - return err - } - if files, ok := raw["files"]; ok && len(files) > 0 && files[0] == '{' { - return json.Unmarshal(files, &s.Files) - } - if len(raw) == 0 { - s.Files = nil - return nil - } - return json.Unmarshal(b, &s.Files) + OnFileChange map[string]string `json:"on_file_change,omitempty"` } // JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, @@ -79,9 +56,7 @@ func emptyJobRunLifecycleState() JobRunLifecycleState { return JobRunLifecycleState{ Triggers: JobRunTriggersState{ OnBundleDeploy: "", - OnFileChange: JobRunFileTriggerState{ - Files: nil, - }, + OnFileChange: nil, }, } } @@ -152,7 +127,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() } if len(input.ResolvedFileTriggers) > 0 { - state.Lifecycle.Triggers.OnFileChange.Files = input.ResolvedFileTriggers + state.Lifecycle.Triggers.OnFileChange = input.ResolvedFileTriggers } return state } @@ -424,7 +399,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa pathString := path.String() if isJobRunTriggerPath(pathString) { removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") - removed = removed || pathString == "lifecycle.triggers.on_file_change.files" && change.New == nil + removed = removed || pathString == "lifecycle.triggers.on_file_change" && change.New == nil if removed { change.Action = deployplan.Skip change.Reason = "trigger removed" diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 301e119f88f..9e26e369123 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -261,7 +261,7 @@ func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - assert.Nil(t, state.Lifecycle.Triggers.OnFileChange.Files) + assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) } // The planner diffs RemapState(remote) against PrepareState(config), so a run From 2892947111b0c2cb6e1c08865c92b5fac5f38514 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 08:27:10 +0000 Subject: [PATCH 18/60] acc: refresh job_runs goldens for the omitted empty trigger Flattening on_file_change to a map made an unarmed trigger drop out of the serialized state, so the three plans that print an unarmed job_run now report an empty triggers object. --- acceptance/bundle/resources/job_runs/failed_run/output.txt | 4 +--- acceptance/bundle/resources/job_runs/redeploy/output.txt | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 37f1a831681..5b8b6dbcd5f 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -28,9 +28,7 @@ Files: 7 uploaded, 0 deleted "value": { "job_id": [MY_JOB_ID], "lifecycle": { - "triggers": { - "on_file_change": {} - } + "triggers": {} }, "result_state": "SUCCESS" } diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index a80685733ad..8532692bdd9 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -60,9 +60,7 @@ Resources: "env": "prod" }, "lifecycle": { - "triggers": { - "on_file_change": {} - } + "triggers": {} }, "result_state": "SUCCESS" } From 19b4448e22fc5584fc7aa5f6d1d8ce8affd52417 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 09:37:16 +0000 Subject: [PATCH 19/60] job_runs: error when on_file_change hashes nothing Directory-only and sync-excluded matches silently disarmed the trigger. Treat them as errors, warn that ** is not recursive, and cover both in acc. --- .../on_file_change_errors/allexcluded/a.txt | 1 + .../on_file_change_errors/databricks.yml | 6 ++++ .../on_file_change_errors/mixed/excluded.txt | 1 + .../mixed/nested/keep.txt | 1 + .../job_runs/on_file_change_errors/output.txt | 34 ++++++++++++++++--- .../job_runs/on_file_change_errors/script | 15 ++++++-- .../job_runs/on_file_change_glob/output.txt | 6 +++- .../job_runs/on_file_change_glob/script | 4 ++- .../mutator/resolve_job_run_file_triggers.go | 28 +++++++-------- 9 files changed, 73 insertions(+), 23 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt new file mode 100644 index 00000000000..78981922613 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/allexcluded/a.txt @@ -0,0 +1 @@ +a diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml index 44c5433e52a..0ce9819887c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/databricks.yml @@ -1,6 +1,12 @@ bundle: name: job-runs-on-file-change-errors +sync: + exclude: + - mixed/excluded.txt + # Literal path: the script rewrites the trigger by substring, and allexcluded/* would hit this line. + - allexcluded/a.txt + resources: jobs: my_job: diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt new file mode 100644 index 00000000000..bbde3dc9de9 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/excluded.txt @@ -0,0 +1 @@ +excluded diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt new file mode 100644 index 00000000000..2fa992c0b8b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/mixed/nested/keep.txt @@ -0,0 +1 @@ +keep diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index c7d3a98e1cb..381dc3e114f 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -2,7 +2,7 @@ === a pattern outside the sync root is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root - in databricks.yml:18:29 + in databricks.yml:25:29 Name: job-runs-on-file-change-errors Target: default @@ -14,8 +14,34 @@ Found 1 error === a glob that only matches directories is rejected >>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches no regular files - in databricks.yml:18:29 +Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, leaving nothing to hash + in databricks.yml:25:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob matching only a directory and an excluded file is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, leaving nothing to hash + in databricks.yml:25:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error + +=== a glob matching only excluded files is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, leaving nothing to hash + in databricks.yml:25:29 Name: job-runs-on-file-change-errors Target: default @@ -28,7 +54,7 @@ Found 1 error === a pattern that matches nothing warns >>> [CLI] bundle validate Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" - in databricks.yml:18:29 + in databricks.yml:25:29 Name: job-runs-on-file-change-errors Target: default diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index 39596f0c772..48b9c5c7fc4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -2,11 +2,20 @@ title "a pattern outside the sync root is rejected" trace musterr $CLI bundle validate title "a glob that only matches directories is rejected" -# dirs/* matches dirs/nested, a directory: nothing is left to hash, so the -# trigger would silently never fire. +# dirs/* matches dirs/nested, a directory, so nothing is left to hash. update_file.py databricks.yml "../shared/*.txt" "dirs/*" trace musterr $CLI bundle validate +title "a glob matching only a directory and an excluded file is rejected" +# mixed/excluded.txt is dropped by sync.exclude; mixed/nested is a directory. +update_file.py databricks.yml "dirs/*" "mixed/*" +trace musterr $CLI bundle validate + +title "a glob matching only excluded files is rejected" +# Every match is dropped by sync.exclude; unlike a missing pattern, this cannot re-arm later. +update_file.py databricks.yml "mixed/*" "allexcluded/*" +trace musterr $CLI bundle validate + title "a pattern that matches nothing warns" -update_file.py databricks.yml "dirs/*" "missing/*.txt" +update_file.py databricks.yml "allexcluded/*" "missing/*.txt" trace $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index a04e3710fb4..2f66f33e664 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -200,7 +200,11 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged -=== ** is not recursive: it matches the same files as * +=== ** is not recursive: it warns and matches the same files as * +Warning: lifecycle.triggers.on_file_change: ** in "migrations/**" is not recursive and matches the same files as * + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index dd737f242b5..0f7107a43db 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -58,6 +58,8 @@ update_file.py migrations/ignored.txt "ignored" "ignored-edited" trace $CLI bundle plan update_file.py migrations/ignored.txt "ignored-edited" "ignored" -title "** is not recursive: it matches the same files as *" +# Warning is on stderr of the untraced plan in changes(); trailing newline keeps it off the title. +title "** is not recursive: it warns and matches the same files as *\n" update_file.py databricks.yml "migrations/*" "migrations/**" changes +update_file.py databricks.yml "migrations/**" "migrations/*" diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index d74b9841c57..fbbabb60815 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -108,6 +108,14 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri Locations: b.Config.GetLocations(loc), }) } + // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. + if strings.Contains(pattern, "**") { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: ** in %q is not recursive and matches the same files as *", pattern), + Locations: b.Config.GetLocations(loc), + }) + } matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) if err != nil { return out, diags.Append(diag.Diagnostic{ @@ -150,7 +158,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri }) continue } - // Same membership as sync: .gitignore and sync.exclude drop a glob match. + // Honor .gitignore and sync.exclude the same way sync does. if _, ok := syncable[filepath.ToSlash(rel)]; !ok { ignoredMatches++ continue @@ -167,20 +175,12 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri } out[filepath.ToSlash(rel)] = hash } - // A directory-only match would otherwise leave ResolvedFileTriggers empty - // and silently disarm the trigger while config still sets on_file_change. - if regularMatches == 0 && sawNonRegular && ignoredMatches == 0 { - diags = diags.Append(diag.Diagnostic{ + // Directories or excluded files would leave hashes empty and disarm the trigger. + // Unlike a missing-file warning, this cannot re-arm when a file appears later. + if regularMatches == 0 && (sawNonRegular || ignoredMatches > 0) { + return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches no regular files", pattern), - Locations: b.Config.GetLocations(loc), - }) - } - if len(out) == 0 && ignoredMatches > 0 { - out[filepath.ToSlash(pattern)] = missingFileHash - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories or files excluded from sync, leaving nothing to hash", pattern), Locations: b.Config.GetLocations(loc), }) } From ae5133972ec1681ea86a849b06e01c952573378c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 11:58:36 +0000 Subject: [PATCH 20/60] acc: fix on_file_change_errors diagnostic line numbers The goldens were captured against a one-line-taller databricks.yml, so CI failed after the source settled on line 24. --- .../job_runs/on_file_change_errors/output.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 381dc3e114f..15934e8eeb6 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -2,7 +2,7 @@ === a pattern outside the sync root is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -15,7 +15,7 @@ Found 1 error === a glob that only matches directories is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, leaving nothing to hash - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -28,7 +28,7 @@ Found 1 error === a glob matching only a directory and an excluded file is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, leaving nothing to hash - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -41,7 +41,7 @@ Found 1 error === a glob matching only excluded files is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, leaving nothing to hash - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default @@ -54,7 +54,7 @@ Found 1 error === a pattern that matches nothing warns >>> [CLI] bundle validate Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" - in databricks.yml:25:29 + in databricks.yml:24:29 Name: job-runs-on-file-change-errors Target: default From ad85edf1434070fb4df65ad838831dc1241abef2 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 13:11:35 +0000 Subject: [PATCH 21/60] acc: removing on_file_change is unchanged, not an update Clearing a trigger is now a skip, so the plan reports the run as unchanged and the deploy makes no API call. --- .../bundle/resources/job_runs/on_file_change/output.txt | 9 +++------ .../bundle/resources/job_runs/on_file_change/script | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 28deeee38a4..978e4b4394a 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -186,17 +186,14 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } -=== removing on_file_change rewrites state without a run +=== removing on_file_change is unchanged and does not run >>> [CLI] bundle plan -update job_runs.my_run - -Plan: 0 to add, 1 to change, 0 to delete, 1 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -Updated job_runs.my_run Files: 3 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 1 unchanged +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index fd1e8800c42..ef432d76033 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -54,7 +54,7 @@ trace $CLI bundle deploy read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -title "removing on_file_change rewrites state without a run" +title "removing on_file_change is unchanged and does not run" update_file.py databricks.yml " lifecycle: triggers: - on_file_change: seed.txt From 7b715fc62908300d5bc2372698e5921c5b4f980d Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Mon, 24 Aug 2026 16:56:03 +0000 Subject: [PATCH 22/60] job_runs: resolve on_file_change relative to the defining YAML file Patterns must still land under the sync root. Reject **, warn when a glob matches directories, and skip only when the trigger itself is cleared. --- .../job_runs/on_file_change_errors/output.txt | 29 ++++++--- .../job_runs/on_file_change_errors/script | 12 ++-- .../job_runs/on_file_change_glob/output.txt | 59 ++++++++++--------- .../job_runs/on_file_change_glob/script | 16 ++--- .../on_file_change_include/databricks.yml | 14 +++++ .../on_file_change_include/out.test.toml | 2 + .../on_file_change_include/output.txt | 29 +++++++++ .../resources/job_run.yml | 7 +++ .../resources/watched.txt | 1 + .../job_runs/on_file_change_include/script | 17 ++++++ .../job_runs/on_file_change_include/test.toml | 3 + .../on_file_change_include/watched.txt | 1 + .../bundle/databricks.yml | 19 ++++++ .../bundle/resources/job_run.yml | 7 +++ .../bundle/resources/watched.txt | 1 + .../bundle/watched.txt | 1 + .../out.test.toml | 2 + .../output.txt | 33 +++++++++++ .../on_file_change_include_sync_root/script | 22 +++++++ .../shared.txt | 1 + .../test.toml | 2 + .../bundle/databricks.yml | 23 ++++++++ .../bundle/shared.txt | 1 + .../on_file_change_sync_root/out.test.toml | 2 + .../on_file_change_sync_root/output.txt | 29 +++++++++ .../job_runs/on_file_change_sync_root/script | 17 ++++++ .../on_file_change_sync_root/shared.txt | 1 + .../on_file_change_sync_root/test.toml | 2 + bundle/config/mutator/normalize_paths_test.go | 38 ++++++++++++ .../mutator/paths/job_run_paths_visitor.go | 24 ++++++++ .../paths/job_run_paths_visitor_test.go | 34 +++++++++++ bundle/config/mutator/paths/visitor.go | 1 + .../mutator/resolve_job_run_file_triggers.go | 46 +++++++++------ .../resolve_job_run_file_triggers_test.go | 22 ++++++- bundle/config/resources/lifecycle.go | 2 +- bundle/direct/dresources/job_run.go | 34 +++++++---- bundle/direct/dresources/job_run_test.go | 27 +++++++++ bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 39 files changed, 499 insertions(+), 86 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml create mode 100644 bundle/config/mutator/paths/job_run_paths_visitor.go create mode 100644 bundle/config/mutator/paths/job_run_paths_visitor_test.go diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 15934e8eeb6..12e4730d8d9 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -1,5 +1,5 @@ -=== a pattern outside the sync root is rejected +=== a pattern that escapes the sync root is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "../shared/*.txt" is not under the sync root in databricks.yml:24:29 @@ -12,9 +12,9 @@ Workspace: Found 1 error -=== a glob that only matches directories is rejected ->>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, leaving nothing to hash +=== a glob that only matches directories warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -23,11 +23,11 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default -Found 1 error +Found 1 warning -=== a glob matching only a directory and an excluded file is rejected +=== a glob matching a directory and an excluded file is rejected >>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, leaving nothing to hash +Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only files excluded from sync, leaving nothing to hash in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -40,7 +40,7 @@ Found 1 error === a glob matching only excluded files is rejected >>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, leaving nothing to hash +Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only files excluded from sync, leaving nothing to hash in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -63,3 +63,16 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default Found 1 warning + +=== ** is not supported +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: ** in "src/**" is not supported; use * for a single directory level + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index 48b9c5c7fc4..4b6e4ead73b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -1,12 +1,12 @@ -title "a pattern outside the sync root is rejected" +title "a pattern that escapes the sync root is rejected" trace musterr $CLI bundle validate -title "a glob that only matches directories is rejected" +title "a glob that only matches directories warns" # dirs/* matches dirs/nested, a directory, so nothing is left to hash. update_file.py databricks.yml "../shared/*.txt" "dirs/*" -trace musterr $CLI bundle validate +trace $CLI bundle validate -title "a glob matching only a directory and an excluded file is rejected" +title "a glob matching a directory and an excluded file is rejected" # mixed/excluded.txt is dropped by sync.exclude; mixed/nested is a directory. update_file.py databricks.yml "dirs/*" "mixed/*" trace musterr $CLI bundle validate @@ -19,3 +19,7 @@ trace musterr $CLI bundle validate title "a pattern that matches nothing warns" update_file.py databricks.yml "allexcluded/*" "missing/*.txt" trace $CLI bundle validate + +title "** is not supported" +update_file.py databricks.yml "missing/*.txt" "src/**" +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 2f66f33e664..390dda9f513 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -1,6 +1,9 @@ === deploy hashes the matched files >>> [CLI] bundle deploy +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS @@ -23,6 +26,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing one matched file re-fires +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -54,6 +61,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === bundle deploy +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS @@ -81,6 +91,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === editing a file in a subdirectory re-fires +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -112,6 +126,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === a new matching file re-fires +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -143,6 +161,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === removing a matched file re-fires +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -172,6 +194,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === editing a file below the glob does not re-fire +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + + >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { "lifecycle.triggers.on_file_change": { @@ -198,38 +224,15 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === editing a gitignored file does not re-fire >>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - -=== ** is not recursive: it warns and matches the same files as * -Warning: lifecycle.triggers.on_file_change: ** in "migrations/**" is not recursive and matches the same files as * +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed in databricks.yml:18:29 - ->>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json -{ - "lifecycle.triggers.on_file_change": { - "action": "skip", - "reason": "missing_in_remote", - "old": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - }, - "new": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - }, - "result_state": { - "action": "skip", - "reason": "remote_already_set", - "new": "SUCCESS", - "remote": "SUCCESS" - } -} +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle destroy --auto-approve +Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed + in databricks.yml:18:29 + The following resources will be deleted: delete resources.job_runs.my_run delete resources.jobs.my_job diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index 0f7107a43db..f0130f2637b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -19,7 +19,7 @@ trace read_id.py my_job read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -title "editing one matched file re-fires" +title "editing one matched file re-fires\n" update_file.py migrations/001.txt "one" "one-edited" changes # The deploy is not traced: readplanarg makes the command line differ per variant. @@ -29,24 +29,24 @@ read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now -title "editing a file in a subdirectory re-fires" +title "editing a file in a subdirectory re-fires\n" # printf keeps LF on Windows, where update_file.py's text-mode rewrite would # restore CRLF and leave the file hashing differently than the deploy above. printf 'sub-edited\n' > mysubdir/a.txt changes printf 'sub\n' > mysubdir/a.txt -title "a new matching file re-fires" +title "a new matching file re-fires\n" printf 'three\n' > migrations/003.txt changes rm migrations/003.txt -title "removing a matched file re-fires" +title "removing a matched file re-fires\n" mv migrations/002.txt tmp.002.txt changes mv tmp.002.txt migrations/002.txt -title "editing a file below the glob does not re-fire" +title "editing a file below the glob does not re-fire\n" # migrations/* matches the archive directory itself, which is skipped: the glob # does not descend into it. update_file.py migrations/archive/003.txt "three" "three-edited" @@ -57,9 +57,3 @@ title "editing a gitignored file does not re-fire" update_file.py migrations/ignored.txt "ignored" "ignored-edited" trace $CLI bundle plan update_file.py migrations/ignored.txt "ignored-edited" "ignored" - -# Warning is on stderr of the untraced plan in changes(); trailing newline keeps it off the title. -title "** is not recursive: it warns and matches the same files as *\n" -update_file.py databricks.yml "migrations/*" "migrations/**" -changes -update_file.py databricks.yml "migrations/**" "migrations/*" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml new file mode 100644 index 00000000000..3d8b74d28f8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: job-runs-on-file-change-include + +include: + - resources/job_run.yml + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt new file mode 100644 index 00000000000..52de054e734 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt @@ -0,0 +1,29 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 7 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== editing a same-named file at the bundle root does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing the file next to the included YAML re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml new file mode 100644 index 00000000000..091489142f5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml @@ -0,0 +1,7 @@ +resources: + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt new file mode 100644 index 00000000000..8b4e79d3c47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt @@ -0,0 +1 @@ +include-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/script b/acceptance/bundle/resources/job_runs/on_file_change_include/script new file mode 100644 index 00000000000..65a8cea7a5b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/script @@ -0,0 +1,17 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "first deploy" +trace $CLI bundle deploy + +title "editing a same-named file at the bundle root does not re-fire" +update_file.py watched.txt "root-v1" "root-v2" +trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py watched.txt "root-v2" "root-v1" + +title "editing the file next to the included YAML re-fires" +update_file.py resources/watched.txt "include-v1" "include-v2" +trace $CLI bundle plan | contains.py "recreate job_runs.my_run" +update_file.py resources/watched.txt "include-v2" "include-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml new file mode 100644 index 00000000000..c81bf4f943a --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml @@ -0,0 +1,3 @@ +# Path resolution only; the HTTP traffic is the same as on_file_change. +RecordRequests = false +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt new file mode 100644 index 00000000000..d1688530992 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt @@ -0,0 +1 @@ +root-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml new file mode 100644 index 00000000000..2bdd1c06d13 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: job-runs-on-file-change-include-sync-root + +include: + - resources/job_run.yml + +sync: + paths: + - . + - ../shared.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml new file mode 100644 index 00000000000..091489142f5 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml @@ -0,0 +1,7 @@ +resources: + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt new file mode 100644 index 00000000000..8b4e79d3c47 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt @@ -0,0 +1 @@ +include-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt new file mode 100644 index 00000000000..6f22941eacf --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt @@ -0,0 +1 @@ +bundle-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt new file mode 100644 index 00000000000..6c475c0dbfc --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt @@ -0,0 +1,33 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include-sync-root/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== editing a same-named file in the bundle does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing a file at the sync root does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing the file next to the included YAML re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include-sync-root/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script new file mode 100644 index 00000000000..5a608ea9188 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script @@ -0,0 +1,22 @@ +cleanup() { + withdir bundle trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "first deploy" +withdir bundle trace $CLI bundle deploy + +title "editing a same-named file in the bundle does not re-fire" +update_file.py bundle/watched.txt "bundle-v1" "bundle-v2" +withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py bundle/watched.txt "bundle-v2" "bundle-v1" + +title "editing a file at the sync root does not re-fire" +update_file.py shared.txt "parent-v1" "parent-v2" +withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py shared.txt "parent-v2" "parent-v1" + +title "editing the file next to the included YAML re-fires" +update_file.py bundle/resources/watched.txt "include-v1" "include-v2" +withdir bundle trace $CLI bundle plan | contains.py "recreate job_runs.my_run" +update_file.py bundle/resources/watched.txt "include-v2" "include-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt new file mode 100644 index 00000000000..4ddcc71c374 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/shared.txt @@ -0,0 +1 @@ +parent-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml new file mode 100644 index 00000000000..e1290bad9f4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml @@ -0,0 +1,2 @@ +RecordRequests = false +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml new file mode 100644 index 00000000000..772a7ef92e7 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: job-runs-on-file-change-sync-root + +sync: + paths: + - . + - ../shared.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: ../shared.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt new file mode 100644 index 00000000000..6f22941eacf --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt @@ -0,0 +1 @@ +bundle-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt new file mode 100644 index 00000000000..6a52f127ce8 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt @@ -0,0 +1,29 @@ + +=== first deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-sync-root/default/files... +Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] +Output from job_runs.my_run: id=[NUMID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 3 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== editing a same-named file in the bundle does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== editing the file at the sync root re-fires +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-sync-root/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script new file mode 100644 index 00000000000..dcbc475840b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script @@ -0,0 +1,17 @@ +cleanup() { + withdir bundle trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "first deploy" +withdir bundle trace $CLI bundle deploy + +title "editing a same-named file in the bundle does not re-fire" +update_file.py bundle/shared.txt "bundle-v1" "bundle-v2" +withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +update_file.py bundle/shared.txt "bundle-v2" "bundle-v1" + +title "editing the file at the sync root re-fires" +update_file.py shared.txt "sync-v1" "sync-v2" +withdir bundle trace $CLI bundle plan | contains.py "recreate job_runs.my_run" +update_file.py shared.txt "sync-v2" "sync-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt new file mode 100644 index 00000000000..04f175f2a1c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt @@ -0,0 +1 @@ +sync-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml new file mode 100644 index 00000000000..e1290bad9f4 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml @@ -0,0 +1,2 @@ +RecordRequests = false +Ignore = [".databricks"] diff --git a/bundle/config/mutator/normalize_paths_test.go b/bundle/config/mutator/normalize_paths_test.go index 8bee6c5064b..2399a03aa83 100644 --- a/bundle/config/mutator/normalize_paths_test.go +++ b/bundle/config/mutator/normalize_paths_test.go @@ -53,6 +53,44 @@ func TestNormalizePaths(t *testing.T) { require.Equal(t, "src/notebook.py", newValue.MustString()) } +func TestNormalizePaths_jobRunOnFileChange(t *testing.T) { + tmpDir := t.TempDir() + pattern := "../data/*.txt" + m := NormalizePaths() + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "run1": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + BundleRootPath: tmpDir, + } + + location := dyn.Location{File: filepath.Join(tmpDir, "resources", "run.yml")} + path := dyn.MustPathFromString("resources.job_runs.run1.lifecycle.triggers[0].on_file_change") + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPath(v, path, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return dyn.NewValue(value.MustString(), []dyn.Location{location}), nil + }) + }) + require.NoError(t, err) + + diags := bundle.Apply(t.Context(), b, m) + require.NoError(t, diags.Error()) + + newValue, err := dyn.GetByPath(b.Config.Value(), path) + require.NoError(t, err) + require.Equal(t, "data/*.txt", newValue.MustString()) +} + func TestNormalizePath_absolutePath(t *testing.T) { value, err := normalizePath("/notebook.py", dyn.Location{}, "/tmp") assert.NoError(t, err) diff --git a/bundle/config/mutator/paths/job_run_paths_visitor.go b/bundle/config/mutator/paths/job_run_paths_visitor.go new file mode 100644 index 00000000000..3bc56bacf72 --- /dev/null +++ b/bundle/config/mutator/paths/job_run_paths_visitor.go @@ -0,0 +1,24 @@ +package paths + +import ( + "github.com/databricks/cli/libs/dyn" +) + +// VisitJobRunPaths visits local paths on job_runs so NormalizePaths can rewrite +// them relative to the bundle root. Not used by TranslatePaths: hashing still +// needs a local glob, not a workspace path. +func VisitJobRunPaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("job_runs"), + dyn.AnyKey(), + dyn.Key("lifecycle"), + dyn.Key("triggers"), + dyn.AnyIndex(), + dyn.Key("on_file_change"), + ) + + return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return fn(path, TranslateModeLocalRelative, value) + }) +} diff --git a/bundle/config/mutator/paths/job_run_paths_visitor_test.go b/bundle/config/mutator/paths/job_run_paths_visitor_test.go new file mode 100644 index 00000000000..0a006fd9df1 --- /dev/null +++ b/bundle/config/mutator/paths/job_run_paths_visitor_test.go @@ -0,0 +1,34 @@ +package paths + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" +) + +func TestVisitJobRunPaths(t *testing.T) { + watched := "watched.txt" + root := config.Root{ + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "run0": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &watched}, + }, + }, + }, + }, + }, + } + + actual := collectVisitedPaths(t, root, VisitJobRunPaths) + expected := []dyn.Path{ + dyn.MustPathFromString("resources.job_runs.run0.lifecycle.triggers[0].on_file_change"), + } + + assert.ElementsMatch(t, expected, actual) +} diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go index bdf42188fde..cadc52b75fe 100644 --- a/bundle/config/mutator/paths/visitor.go +++ b/bundle/config/mutator/paths/visitor.go @@ -10,6 +10,7 @@ type VisitFunc func(path dyn.Path, mode TranslateMode, value dyn.Value) (dyn.Val func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { visitors := []func(dyn.Value, VisitFunc) (dyn.Value, error){ VisitJobPaths, + VisitJobRunPaths, VisitJobLibrariesPaths, VisitAppPaths, VisitArtifactPaths, diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index fbbabb60815..2b3f11a1a88 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -52,11 +52,7 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d diags = diags.Extend(d) maps.Copy(out, hashes) } - if len(out) == 0 { - jr.ResolvedFileTriggers = nil - } else { - jr.ResolvedFileTriggers = out - } + jr.ResolvedFileTriggers = out } return diags } @@ -100,23 +96,27 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri var diags diag.Diagnostics out := make(map[string]string) localPattern := filepath.FromSlash(pattern) - // Keep hashes under SyncRoot; same IsLocal gate as translate_paths. - if !filepath.IsLocal(localPattern) { + // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. + if strings.Contains(pattern, "**") { return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: ** in %q is not supported; use * for a single directory level", pattern), Locations: b.Config.GetLocations(loc), }) } - // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. - if strings.Contains(pattern, "**") { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: ** in %q is not recursive and matches the same files as *", pattern), + // NormalizePaths has already rewritten YAML-relative globs to be bundle-root + // relative. Join that onto the bundle root, then require the result stay + // under the sync root (an ancestor of the bundle when sync.paths uses ..). + joined := filepath.Join(b.BundleRootPath, localPattern) + relPattern, err := filepath.Rel(b.SyncRootPath, joined) + if err != nil || !filepath.IsLocal(relPattern) { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), Locations: b.Config.GetLocations(loc), }) } - matches, err := filepath.Glob(filepath.Join(b.SyncRootPath, localPattern)) + matches, err := filepath.Glob(joined) if err != nil { return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, @@ -175,12 +175,22 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri } out[filepath.ToSlash(rel)] = hash } - // Directories or excluded files would leave hashes empty and disarm the trigger. - // Unlike a missing-file warning, this cannot re-arm when a file appears later. - if regularMatches == 0 && (sawNonRegular || ignoredMatches > 0) { + if regularMatches == 0 && ignoredMatches > 0 { return out, diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories or files excluded from sync, leaving nothing to hash", pattern), + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only files excluded from sync, leaving nothing to hash", pattern), + Locations: b.Config.GetLocations(loc), + }) + } + if sawNonRegular { + msg := fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q also matched directories, which are not hashed", pattern) + if regularMatches == 0 { + out[filepath.ToSlash(pattern)] = missingFileHash + msg = fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories", pattern) + } + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: msg, Locations: b.Config.GetLocations(loc), }) } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 1cc54eecf4f..0630a4b35f6 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -44,14 +44,30 @@ func TestResolveJobRunFileTriggers(t *testing.T) { require.False(t, diags.HasError()) assert.Equal(t, contentHash("v1"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["seed.txt"]) }) + + t.Run("globs from the bundle root when the sync root is an ancestor", func(t *testing.T) { + parent := t.TempDir() + bundleDir := filepath.Join(parent, "bundle") + require.NoError(t, os.Mkdir(bundleDir, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(parent, "shared.txt"), []byte("from-sync-root"), 0o644)) + + pattern := "../shared.txt" + b := bundleWithFileTrigger(parent, pattern) + b.BundleRootPath = bundleDir + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, contentHash("from-sync-root"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["shared.txt"]) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { root := vfs.MustNew(syncRoot) return &bundle.Bundle{ - SyncRootPath: syncRoot, - SyncRoot: root, - WorktreeRoot: root, + BundleRootPath: syncRoot, + SyncRootPath: syncRoot, + SyncRoot: root, + WorktreeRoot: root, Config: config.Root{ Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index a4e881d298c..1f5d576d835 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -37,5 +37,5 @@ type JobRunLifecycle struct { // JobRunTrigger is one lifecycle.triggers entry. type JobRunTrigger struct { OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` - OnFileChange *string `json:"on_file_change,omitempty"` // path or glob under sync root + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob relative to the defining YAML file; must resolve under the sync root } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 9d1a272a7a3..872720ffcd4 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,14 +26,29 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour -// jobRunTriggerLocalPaths are local-only fingerprints: clearing one is skip, not recreate. -var jobRunTriggerLocalPaths = []string{ - "lifecycle.triggers.on_bundle_deploy", - "lifecycle.triggers.on_file_change", +// Parsed so HasPrefix catches parent and child paths; skip is only when the +// trigger itself is cleared, not when a file hash under on_file_change changes. +var ( + jobRunTriggersPath = structpath.MustParsePath("lifecycle.triggers") + jobRunOnBundleDeployPath = structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy") + jobRunOnFileChangePath = structpath.MustParsePath("lifecycle.triggers.on_file_change") +) + +func samePath(a, b *structpath.PathNode) bool { + return a.HasPrefix(b) && b.HasPrefix(a) } -func isJobRunTriggerPath(path string) bool { - return slices.Contains(jobRunTriggerLocalPaths, path) +func jobRunTriggerRemoved(path *structpath.PathNode, change *ChangeDesc) bool { + switch { + case samePath(path, jobRunOnBundleDeployPath): + return change.New == nil || change.New == "" + case samePath(path, jobRunOnFileChangePath): + return change.New == nil + case samePath(path, jobRunTriggersPath): + return change.New == nil + default: + return false + } } // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. @@ -396,11 +411,8 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - pathString := path.String() - if isJobRunTriggerPath(pathString) { - removed := pathString == "lifecycle.triggers.on_bundle_deploy" && (change.New == nil || change.New == "") - removed = removed || pathString == "lifecycle.triggers.on_file_change" && change.New == nil - if removed { + if path.HasPrefix(jobRunTriggersPath) { + if jobRunTriggerRemoved(path, change) { change.Action = deployplan.Skip change.Reason = "trigger removed" } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 9e26e369123..fd6417f48e1 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "testing" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" "github.com/databricks/databricks-sdk-go" @@ -378,3 +379,29 @@ func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { assert.False(t, cancelled.Load(), "a run that already finished has nothing to cancel") } + +func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { + r := &ResourceJobRun{} + for _, tt := range []struct { + name string + path string + new any + action deployplan.ActionType + }{ + {"cleared on_bundle_deploy string", "lifecycle.triggers.on_bundle_deploy", "", deployplan.Skip}, + {"nil on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", nil, deployplan.Skip}, + {"rotated on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "uuid", deployplan.Recreate}, + {"cleared on_file_change", "lifecycle.triggers.on_file_change", nil, deployplan.Skip}, + {"changed on_file_change map", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h"}, deployplan.Recreate}, + {"cleared on_file_change child", "lifecycle.triggers.on_file_change['a.txt']", nil, deployplan.Recreate}, + {"cleared triggers parent", "lifecycle.triggers", nil, deployplan.Skip}, + {"changed triggers parent", "lifecycle.triggers", JobRunTriggersState{OnBundleDeploy: "uuid"}, deployplan.Recreate}, + {"result_state unchanged", "result_state", nil, deployplan.Recreate}, + } { + t.Run(tt.name, func(t *testing.T) { + change := &ChangeDesc{Action: deployplan.Recreate, New: tt.new} + require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath(tt.path), change, nil)) + assert.Equal(t, tt.action, change.Action) + }) + } +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 2229bab0b83..8894ff1e4fd 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1038,7 +1038,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 147e0049306..5f6e60d44c3 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1537,7 +1537,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 8e61fa1ad8f801cc4e4fa46d03ceb4cd941eb43c Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 08:30:03 +0000 Subject: [PATCH 23/60] job_runs: reject absolute on_file_change patterns filepath.Join rebases an absolute glob under the bundle root, so reject those up front. Key missing-file placeholders to the sync root, and skip a cleared trigger from path.String() instead of prefix matching. --- .../job_runs/on_file_change_errors/output.txt | 13 ++++++ .../job_runs/on_file_change_errors/script | 4 ++ .../mutator/resolve_job_run_file_triggers.go | 20 ++++++--- .../resolve_job_run_file_triggers_test.go | 22 ++++++++-- .../mutator/validate_job_run_triggers.go | 28 +++++++++---- .../mutator/validate_job_run_triggers_test.go | 13 +++++- bundle/config/resources/job_run.go | 1 + bundle/direct/dresources/job_run.go | 42 ++++--------------- 8 files changed, 92 insertions(+), 51 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 12e4730d8d9..37ea391136b 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -76,3 +76,16 @@ Workspace: Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default Found 1 error + +=== an absolute pattern is rejected +>>> musterr [CLI] bundle validate +Error: lifecycle.triggers.on_file_change: pattern "/tmp/seed.txt" must be relative to the defining YAML file + in databricks.yml:24:29 + +Name: job-runs-on-file-change-errors +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default + +Found 1 error diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index 4b6e4ead73b..86b10b70bf0 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -23,3 +23,7 @@ trace $CLI bundle validate title "** is not supported" update_file.py databricks.yml "missing/*.txt" "src/**" trace musterr $CLI bundle validate + +title "an absolute pattern is rejected" +update_file.py databricks.yml "src/**" "/tmp/seed.txt" +trace musterr $CLI bundle validate diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 2b3f11a1a88..dc615f030c8 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -8,6 +8,7 @@ import ( "io" "maps" "os" + pathlib "path" "path/filepath" "strings" @@ -48,7 +49,7 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d continue } path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) - hashes, d := resolveFileTrigger(b, path, strings.TrimSpace(*t.OnFileChange), syncable) + hashes, d := resolveFileTrigger(b, path, *t.OnFileChange, syncable) diags = diags.Extend(d) maps.Copy(out, hashes) } @@ -95,7 +96,6 @@ func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { var diags diag.Diagnostics out := make(map[string]string) - localPattern := filepath.FromSlash(pattern) // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. if strings.Contains(pattern, "**") { return out, diags.Append(diag.Diagnostic{ @@ -104,10 +104,19 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri Locations: b.Config.GetLocations(loc), }) } + // filepath.Join would otherwise rebase an absolute pattern under the bundle + // root (Join("/bundle", "/etc/passwd") is "/bundle/etc/passwd"). + if filepath.IsAbs(filepath.FromSlash(pattern)) || pathlib.IsAbs(pattern) { + return out, diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q must be relative to the defining YAML file", pattern), + Locations: b.Config.GetLocations(loc), + }) + } // NormalizePaths has already rewritten YAML-relative globs to be bundle-root // relative. Join that onto the bundle root, then require the result stay // under the sync root (an ancestor of the bundle when sync.paths uses ..). - joined := filepath.Join(b.BundleRootPath, localPattern) + joined := filepath.Join(b.BundleRootPath, filepath.FromSlash(pattern)) relPattern, err := filepath.Rel(b.SyncRootPath, joined) if err != nil || !filepath.IsLocal(relPattern) { return out, diags.Append(diag.Diagnostic{ @@ -116,6 +125,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri Locations: b.Config.GetLocations(loc), }) } + key := filepath.ToSlash(relPattern) matches, err := filepath.Glob(joined) if err != nil { return out, diags.Append(diag.Diagnostic{ @@ -125,7 +135,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri }) } if len(matches) == 0 { - out[filepath.ToSlash(pattern)] = missingFileHash + out[key] = missingFileHash return out, diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), @@ -185,7 +195,7 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri if sawNonRegular { msg := fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q also matched directories, which are not hashed", pattern) if regularMatches == 0 { - out[filepath.ToSlash(pattern)] = missingFileHash + out[key] = missingFileHash msg = fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories", pattern) } diags = diags.Append(diag.Diagnostic{ diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 0630a4b35f6..9271589bb48 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -34,15 +34,29 @@ func TestResolveJobRunFileTriggers(t *testing.T) { assert.Equal(t, contentHash("world"), hashes["b.txt"]) }) - t.Run("trims pattern whitespace", func(t *testing.T) { + t.Run("rejects an absolute pattern", func(t *testing.T) { dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "seed.txt"), []byte("v1"), 0o644)) - pattern := " seed.txt " + pattern := "/etc/passwd" b := bundleWithFileTrigger(dir, pattern) + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError()) + require.Equal(t, `lifecycle.triggers.on_file_change: pattern "/etc/passwd" must be relative to the defining YAML file`, diags[0].Summary) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + }) + + t.Run("missing pattern is keyed relative to the sync root", func(t *testing.T) { + parent := t.TempDir() + bundleDir := filepath.Join(parent, "bundle") + require.NoError(t, os.Mkdir(bundleDir, 0o755)) + + pattern := "../missing.txt" + b := bundleWithFileTrigger(parent, pattern) + b.BundleRootPath = bundleDir + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - assert.Equal(t, contentHash("v1"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["seed.txt"]) + assert.Equal(t, map[string]string{"missing.txt": ""}, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) }) t.Run("globs from the bundle root when the sync root is an ancestor", func(t *testing.T) { diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index c185d3b5781..fa31032c689 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -26,13 +26,14 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D if jr == nil || jr.Lifecycle == nil { continue } - // Recreate-every-deploy cannot coexist with prevent_destroy. - if (jr.HasOnBundleDeploy() || jr.HasOnFileChange()) && jr.Lifecycle.PreventDestroy { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", - Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), - }) + if jr.Lifecycle.PreventDestroy { + if summary := preventDestroyError(jr.HasOnBundleDeploy(), jr.HasOnFileChange()); summary != "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: summary, + Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), + }) + } } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) @@ -70,3 +71,16 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D } return diags } + +func preventDestroyError(onBundleDeploy, onFileChange bool) string { + switch { + case onBundleDeploy && onFileChange: + return "lifecycle.triggers.on_bundle_deploy and on_file_change are incompatible with lifecycle.prevent_destroy" + case onBundleDeploy: + return "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy" + case onFileChange: + return "lifecycle.triggers.on_file_change is incompatible with lifecycle.prevent_destroy" + default: + return "" + } +} diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 7e8772d8d8b..07a7276a41c 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -84,7 +84,7 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnBundleDeploy: &trueVal}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", }, { name: "on_file_change with prevent_destroy", @@ -92,7 +92,16 @@ func TestValidateJobRunTriggers(t *testing.T) { {OnFileChange: &fileChange}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy or on_file_change is incompatible with lifecycle.prevent_destroy", + summary: "lifecycle.triggers.on_file_change is incompatible with lifecycle.prevent_destroy", + }, + { + name: "both triggers with prevent_destroy", + triggers: []resources.JobRunTrigger{ + {OnFileChange: &fileChange}, + {OnBundleDeploy: &trueVal}, + }, + preventDestroy: true, + summary: "lifecycle.triggers.on_bundle_deploy and on_file_change are incompatible with lifecycle.prevent_destroy", }, { name: "prevent_destroy alone", diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index b29594d3692..48820950e83 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -46,6 +46,7 @@ func (r *JobRun) HasOnBundleDeploy() bool { return false } +// HasOnFileChange reports whether any trigger re-fires when matched files change. func (r *JobRun) HasOnFileChange() bool { if r.Lifecycle == nil { return false diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 872720ffcd4..d7574bdc2ac 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,31 +26,6 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour -// Parsed so HasPrefix catches parent and child paths; skip is only when the -// trigger itself is cleared, not when a file hash under on_file_change changes. -var ( - jobRunTriggersPath = structpath.MustParsePath("lifecycle.triggers") - jobRunOnBundleDeployPath = structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy") - jobRunOnFileChangePath = structpath.MustParsePath("lifecycle.triggers.on_file_change") -) - -func samePath(a, b *structpath.PathNode) bool { - return a.HasPrefix(b) && b.HasPrefix(a) -} - -func jobRunTriggerRemoved(path *structpath.PathNode, change *ChangeDesc) bool { - switch { - case samePath(path, jobRunOnBundleDeployPath): - return change.New == nil || change.New == "" - case samePath(path, jobRunOnFileChangePath): - return change.New == nil - case samePath(path, jobRunTriggersPath): - return change.New == nil - default: - return false - } -} - // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. @@ -411,14 +386,17 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - if path.HasPrefix(jobRunTriggersPath) { - if jobRunTriggerRemoved(path, change) { + switch path.String() { + case "lifecycle.triggers.on_bundle_deploy": + if change.New == nil || change.New == "" { + change.Action = deployplan.Skip + change.Reason = "trigger removed" + } + case "lifecycle.triggers.on_file_change", "lifecycle.triggers": + if change.New == nil { change.Action = deployplan.Skip change.Reason = "trigger removed" } - return nil - } - switch path.String() { case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { @@ -426,10 +404,8 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa } change.Action = deployplan.Skip change.Reason = "run in progress" - return nil - default: - return nil } + return nil } // DoDelete deletes the run via jobs/runs/delete, on both destroy and the From 0934d61b3beb6a4c7a2428a87cdbcd5081aab5a8 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 09:13:00 +0000 Subject: [PATCH 24/60] job_runs: warn instead of erroring when on_file_change hashes nothing Every "nothing to hash" case re-arms once a matching file appears, now that the placeholder is always recorded, so none of them justify failing the deploy. Directories matched alongside real files are skipped silently: a glob like migrations/* routinely matches a subdirectory, and warning about it on every command was noise the user cannot act on. Also drop the unreachable "lifecycle.triggers" branch in OverrideChangeDesc (structdiff descends into the value struct and never reports a change at that path) and document the ** and sync-skipping rules in the schema. --- .../job_runs/on_file_change_errors/output.txt | 18 +-- .../job_runs/on_file_change_errors/script | 10 +- .../job_runs/on_file_change_glob/output.txt | 27 ---- .../mutator/resolve_job_run_file_triggers.go | 126 +++++++----------- bundle/direct/dresources/job_run.go | 11 +- bundle/direct/dresources/job_run_test.go | 6 +- bundle/internal/schema/annotations.yml | 2 +- bundle/schema/jsonschema.json | 2 +- 8 files changed, 68 insertions(+), 134 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 37ea391136b..246a713e321 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -14,7 +14,7 @@ Found 1 error === a glob that only matches directories warns >>> [CLI] bundle validate -Warning: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories +Warning: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, so nothing is hashed in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -25,9 +25,9 @@ Workspace: Found 1 warning -=== a glob matching a directory and an excluded file is rejected ->>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only files excluded from sync, leaving nothing to hash +=== a glob matching a directory and an excluded file warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, so nothing is hashed in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -36,11 +36,11 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default -Found 1 error +Found 1 warning -=== a glob matching only excluded files is rejected ->>> musterr [CLI] bundle validate -Error: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only files excluded from sync, leaving nothing to hash +=== a glob matching only excluded files warns +>>> [CLI] bundle validate +Warning: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, so nothing is hashed in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -49,7 +49,7 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-errors/default -Found 1 error +Found 1 warning === a pattern that matches nothing warns >>> [CLI] bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index 86b10b70bf0..bca06ed87d4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -6,15 +6,15 @@ title "a glob that only matches directories warns" update_file.py databricks.yml "../shared/*.txt" "dirs/*" trace $CLI bundle validate -title "a glob matching a directory and an excluded file is rejected" +title "a glob matching a directory and an excluded file warns" # mixed/excluded.txt is dropped by sync.exclude; mixed/nested is a directory. update_file.py databricks.yml "dirs/*" "mixed/*" -trace musterr $CLI bundle validate +trace $CLI bundle validate -title "a glob matching only excluded files is rejected" -# Every match is dropped by sync.exclude; unlike a missing pattern, this cannot re-arm later. +title "a glob matching only excluded files warns" +# Every match is dropped by sync.exclude. update_file.py databricks.yml "mixed/*" "allexcluded/*" -trace musterr $CLI bundle validate +trace $CLI bundle validate title "a pattern that matches nothing warns" update_file.py databricks.yml "allexcluded/*" "missing/*.txt" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index 390dda9f513..ffd878ba0cd 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -1,9 +1,6 @@ === deploy hashes the matched files >>> [CLI] bundle deploy -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS @@ -26,9 +23,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === editing one matched file re-fires -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { @@ -61,9 +55,6 @@ Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched } === bundle deploy -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS @@ -91,9 +82,6 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } === editing a file in a subdirectory re-fires -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { @@ -126,9 +114,6 @@ Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched } === a new matching file re-fires -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { @@ -161,9 +146,6 @@ Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched } === removing a matched file re-fires -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { @@ -194,9 +176,6 @@ Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched } === editing a file below the glob does not re-fire -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { @@ -224,15 +203,9 @@ Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched === editing a gitignored file does not re-fire >>> [CLI] bundle plan -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle destroy --auto-approve -Warning: lifecycle.triggers.on_file_change: pattern "migrations/*" also matched directories, which are not hashed - in databricks.yml:18:29 - The following resources will be deleted: delete resources.job_runs.my_run delete resources.jobs.my_job diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index dc615f030c8..3cfb7a42ee1 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -33,12 +33,10 @@ func (*resolveJobRunFileTriggers) Name() string { } func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - syncable, d := syncableRelPaths(ctx, b) - if d.HasError() { - return d + syncable, diags := syncableRelPaths(ctx, b) + if diags.HasError() { + return diags } - diags = diags.Extend(d) for name, jr := range b.Config.Resources.JobRuns { if jr == nil || jr.Lifecycle == nil { continue @@ -72,25 +70,39 @@ func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{ return nil, diags } - fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) + out, err := listSyncableRelPaths(ctx, b) if err != nil { return nil, diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), }) } + return out, diags +} + +func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{}, error) { + fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) + if err != nil { + return nil, err + } files, err := fl.Files(ctx) if err != nil { - return nil, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), - }) + return nil, err } out := make(map[string]struct{}, len(files)) for _, f := range files { out[filepath.ToSlash(f.Relative)] = struct{}{} } - return out, diags + return out, nil +} + +// fileTriggerDiag reports a diagnostic against the on_file_change entry at loc. +func fileTriggerDiag(b *bundle.Bundle, loc string, severity diag.Severity, format string, args ...any) diag.Diagnostic { + return diag.Diagnostic{ + Severity: severity, + Summary: "lifecycle.triggers.on_file_change: " + fmt.Sprintf(format, args...), + Locations: b.Config.GetLocations(loc), + } } func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { @@ -98,20 +110,13 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri out := make(map[string]string) // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. if strings.Contains(pattern, "**") { - return out, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: ** in %q is not supported; use * for a single directory level", pattern), - Locations: b.Config.GetLocations(loc), - }) + return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "** in %q is not supported; use * for a single directory level", pattern)) } // filepath.Join would otherwise rebase an absolute pattern under the bundle - // root (Join("/bundle", "/etc/passwd") is "/bundle/etc/passwd"). - if filepath.IsAbs(filepath.FromSlash(pattern)) || pathlib.IsAbs(pattern) { - return out, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q must be relative to the defining YAML file", pattern), - Locations: b.Config.GetLocations(loc), - }) + // root (Join("/bundle", "/etc/passwd") is "/bundle/etc/passwd"). A POSIX path + // is absolute on Windows too, so check both flavours like NormalizePaths does. + if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) { + return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q must be relative to the defining YAML file", pattern)) } // NormalizePaths has already rewritten YAML-relative globs to be bundle-root // relative. Join that onto the bundle root, then require the result stay @@ -119,90 +124,51 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri joined := filepath.Join(b.BundleRootPath, filepath.FromSlash(pattern)) relPattern, err := filepath.Rel(b.SyncRootPath, joined) if err != nil || !filepath.IsLocal(relPattern) { - return out, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q is not under the sync root", pattern), - Locations: b.Config.GetLocations(loc), - }) + return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q is not under the sync root", pattern)) } - key := filepath.ToSlash(relPattern) matches, err := filepath.Glob(joined) if err != nil { - return out, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: invalid pattern %q: %s", pattern, err), - Locations: b.Config.GetLocations(loc), - }) - } - if len(matches) == 0 { - out[key] = missingFileHash - return out, diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: no files match %q", pattern), - Locations: b.Config.GetLocations(loc), - }) + return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) } - regularMatches := 0 - ignoredMatches := 0 - sawNonRegular := false for _, match := range matches { info, err := os.Stat(match) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: stat %q: %s", match, err), - Locations: b.Config.GetLocations(loc), - }) + diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "stat %q: %s", match, err)) continue } + // A glob like migrations/* routinely matches subdirectories; there is + // nothing to hash and nothing for the user to fix, so skip them quietly. if !info.Mode().IsRegular() { - sawNonRegular = true continue } rel, err := filepath.Rel(b.SyncRootPath, match) if err != nil || !filepath.IsLocal(rel) { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: matched path %q is not under the sync root", match), - Locations: b.Config.GetLocations(loc), - }) + diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "matched path %q is not under the sync root", match)) continue } // Honor .gitignore and sync.exclude the same way sync does. if _, ok := syncable[filepath.ToSlash(rel)]; !ok { - ignoredMatches++ continue } - regularMatches++ hash, err := hashFile(match) if err != nil { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: hash %q: %s", match, err), - Locations: b.Config.GetLocations(loc), - }) + diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "hash %q: %s", match, err)) continue } out[filepath.ToSlash(rel)] = hash } - if regularMatches == 0 && ignoredMatches > 0 { - return out, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only files excluded from sync, leaving nothing to hash", pattern), - Locations: b.Config.GetLocations(loc), - }) - } - if sawNonRegular { - msg := fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q also matched directories, which are not hashed", pattern) - if regularMatches == 0 { - out[key] = missingFileHash - msg = fmt.Sprintf("lifecycle.triggers.on_file_change: pattern %q matches only directories", pattern) + // A pattern that hashes nothing is a warning, not an error: every such case + // re-arms once a matching file appears. Record the placeholder under the + // pattern's own sync-root-relative key so that appearance is a hash change + // rather than a key swap. Skip it when a match failed to be read, since the + // error already says the fingerprint is incomplete. + if len(out) == 0 && !diags.HasError() { + out[filepath.ToSlash(relPattern)] = missingFileHash + if len(matches) == 0 { + diags = diags.Append(fileTriggerDiag(b, loc, diag.Warning, "no files match %q", pattern)) + } else { + diags = diags.Append(fileTriggerDiag(b, loc, diag.Warning, "pattern %q matches only directories or files excluded from sync, so nothing is hashed", pattern)) } - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Warning, - Summary: msg, - Locations: b.Config.GetLocations(loc), - }) } return out, diags } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index d7574bdc2ac..078ae1b1aa6 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -41,14 +41,9 @@ type JobRunLifecycleState struct { Triggers JobRunTriggersState `json:"triggers"` } -// Zero value spelled out field by field, as exhaustruct requires. func emptyJobRunLifecycleState() JobRunLifecycleState { - return JobRunLifecycleState{ - Triggers: JobRunTriggersState{ - OnBundleDeploy: "", - OnFileChange: nil, - }, - } + var empty JobRunLifecycleState + return empty } // JobRunState is the RunNow request plus the outcome required for planning. @@ -392,7 +387,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Action = deployplan.Skip change.Reason = "trigger removed" } - case "lifecycle.triggers.on_file_change", "lifecycle.triggers": + case "lifecycle.triggers.on_file_change": if change.New == nil { change.Action = deployplan.Skip change.Reason = "trigger removed" diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index fd6417f48e1..f1fac2ca1f9 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -393,10 +393,10 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { {"rotated on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "uuid", deployplan.Recreate}, {"cleared on_file_change", "lifecycle.triggers.on_file_change", nil, deployplan.Skip}, {"changed on_file_change map", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h"}, deployplan.Recreate}, + // A file dropping out of the map is a real change, so the skip must not + // extend to paths below on_file_change. {"cleared on_file_change child", "lifecycle.triggers.on_file_change['a.txt']", nil, deployplan.Recreate}, - {"cleared triggers parent", "lifecycle.triggers", nil, deployplan.Skip}, - {"changed triggers parent", "lifecycle.triggers", JobRunTriggersState{OnBundleDeploy: "uuid"}, deployplan.Recreate}, - {"result_state unchanged", "result_state", nil, deployplan.Recreate}, + {"result_state with unreadable remote", "result_state", nil, deployplan.Recreate}, } { t.Run(tt.name, func(t *testing.T) { change := &ChangeDesc{Action: deployplan.Recreate, New: tt.new} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 8894ff1e4fd..05b9b8a5925 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1038,7 +1038,7 @@ resources: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. "on_file_change": "description": |- - Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy. + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 5f6e60d44c3..b438956e0fe 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1537,7 +1537,7 @@ "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy.", "$ref": "#/$defs/string" } }, From 8670de6b465262c0fda27c7f1e2ecc945638cb49 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 09:40:05 +0000 Subject: [PATCH 25/60] acc: stop Git Bash rewriting the on_file_change absolute pattern Windows CI converted /tmp/seed.txt to %TEMP%/seed.txt, so the golden expecting the POSIX path failed. MSYS_NO_PATHCONV keeps the argument intact. --- .../bundle/resources/job_runs/on_file_change_errors/test.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml index 7b88d613d4c..af98ba376da 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml @@ -1,2 +1,5 @@ +# Git Bash otherwise rewrites /tmp/seed.txt to %TEMP%/seed.txt. +Env.MSYS_NO_PATHCONV = "1" + # The bundle is never deployed here, so there are no requests worth recording. RecordRequests = false From 94fb2e58007a7610004774f13251662bda0d5a12 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Tue, 25 Aug 2026 10:09:43 +0000 Subject: [PATCH 26/60] acc: read the absolute on_file_change pattern from a committed file MSYS_NO_PATHCONV also stops Git Bash converting the path of a shebang helper, so update_file.py could not be found on Windows. Keep the leading-slash pattern out of argv entirely instead. --- .../on_file_change_errors/absolute_pattern.yml | 18 ++++++++++++++++++ .../job_runs/on_file_change_errors/output.txt | 2 +- .../job_runs/on_file_change_errors/script | 4 +++- .../job_runs/on_file_change_errors/test.toml | 3 --- 4 files changed, 22 insertions(+), 5 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml b/acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml new file mode 100644 index 00000000000..e0d709d3fac --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/absolute_pattern.yml @@ -0,0 +1,18 @@ +bundle: + name: job-runs-on-file-change-errors + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: /tmp/seed.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index 246a713e321..efcd8f002b4 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -80,7 +80,7 @@ Found 1 error === an absolute pattern is rejected >>> musterr [CLI] bundle validate Error: lifecycle.triggers.on_file_change: pattern "/tmp/seed.txt" must be relative to the defining YAML file - in databricks.yml:24:29 + in databricks.yml:18:29 Name: job-runs-on-file-change-errors Target: default diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/script b/acceptance/bundle/resources/job_runs/on_file_change_errors/script index bca06ed87d4..faaa5de8d3c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/script @@ -25,5 +25,7 @@ update_file.py databricks.yml "missing/*.txt" "src/**" trace musterr $CLI bundle validate title "an absolute pattern is rejected" -update_file.py databricks.yml "src/**" "/tmp/seed.txt" +# Copied from a committed file rather than passed to update_file.py: Git Bash on +# Windows rewrites a leading-slash argument into a Windows path. +cp absolute_pattern.yml databricks.yml trace musterr $CLI bundle validate diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml index af98ba376da..7b88d613d4c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/test.toml @@ -1,5 +1,2 @@ -# Git Bash otherwise rewrites /tmp/seed.txt to %TEMP%/seed.txt. -Env.MSYS_NO_PATHCONV = "1" - # The bundle is never deployed here, so there are no requests worth recording. RecordRequests = false From f24efe8f63ec07c5eaa322e1f82277f386f9f6ea Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 27 Aug 2026 08:23:30 +0000 Subject: [PATCH 27/60] job_runs: match on_file_change patterns against synced files and drop duplicate plan entry Iterate the sync file list with path.Match instead of filepath.Glob so on_file_change honors .gitignore and sync.exclude directly, and drop the whole-map diff when per-file entries already report the change. --- .../job_runs/on_file_change/output.txt | 26 +------ .../job_runs/on_file_change_errors/output.txt | 8 +- .../job_runs/on_file_change_glob/output.txt | 78 +------------------ .../mutator/resolve_job_run_file_triggers.go | 60 ++++++-------- bundle/direct/dresources/job_run.go | 17 +++- bundle/direct/dresources/job_run_test.go | 20 +++-- 6 files changed, 58 insertions(+), 151 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 978e4b4394a..998f0bbf0a1 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,16 +52,6 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change": { - "action": "recreate", - "reason": "immutable", - "old": { - "seed.txt": "[FILE_HASH][0]" - }, - "new": { - "seed.txt": "[FILE_HASH][1]" - } - }, "lifecycle.triggers.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", @@ -105,7 +95,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged === deleting the file warns and re-fires >>> [CLI] bundle plan -Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" +Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" in databricks.yml:18:29 recreate job_runs.my_run @@ -113,20 +103,10 @@ recreate job_runs.my_run Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> changes -Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" +Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" in databricks.yml:18:29 { - "lifecycle.triggers.on_file_change": { - "action": "recreate", - "reason": "immutable", - "old": { - "seed.txt": "[FILE_HASH][1]" - }, - "new": { - "seed.txt": "" - } - }, "lifecycle.triggers.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", @@ -142,7 +122,7 @@ Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" } >>> [CLI] bundle deploy -Warning: lifecycle.triggers.on_file_change: no files match "seed.txt" +Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" in databricks.yml:18:29 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt index efcd8f002b4..73f170fe8e9 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/output.txt @@ -14,7 +14,7 @@ Found 1 error === a glob that only matches directories warns >>> [CLI] bundle validate -Warning: lifecycle.triggers.on_file_change: pattern "dirs/*" matches only directories or files excluded from sync, so nothing is hashed +Warning: lifecycle.triggers.on_file_change: no synced files match "dirs/*" in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -27,7 +27,7 @@ Found 1 warning === a glob matching a directory and an excluded file warns >>> [CLI] bundle validate -Warning: lifecycle.triggers.on_file_change: pattern "mixed/*" matches only directories or files excluded from sync, so nothing is hashed +Warning: lifecycle.triggers.on_file_change: no synced files match "mixed/*" in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -40,7 +40,7 @@ Found 1 warning === a glob matching only excluded files warns >>> [CLI] bundle validate -Warning: lifecycle.triggers.on_file_change: pattern "allexcluded/*" matches only directories or files excluded from sync, so nothing is hashed +Warning: lifecycle.triggers.on_file_change: no synced files match "allexcluded/*" in databricks.yml:24:29 Name: job-runs-on-file-change-errors @@ -53,7 +53,7 @@ Found 1 warning === a pattern that matches nothing warns >>> [CLI] bundle validate -Warning: lifecycle.triggers.on_file_change: no files match "missing/*.txt" +Warning: lifecycle.triggers.on_file_change: no synced files match "missing/*.txt" in databricks.yml:24:29 Name: job-runs-on-file-change-errors diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index ffd878ba0cd..db61edb23db 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -26,25 +26,11 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change": { - "action": "recreate", - "reason": "immutable", - "old": { - "migrations/001.txt": "[FILE_HASH][0]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - }, - "new": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - }, "lifecycle.triggers.on_file_change['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", - "new": "[FILE_HASH][3]" + "new": "[FILE_HASH][1]" }, "result_state": { "action": "skip", @@ -85,25 +71,11 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change": { - "action": "recreate", - "reason": "immutable", - "old": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - }, - "new": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][4]" - } - }, "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", - "new": "[FILE_HASH][4]" + "new": "[FILE_HASH][3]" }, "result_state": { "action": "skip", @@ -117,25 +89,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change": { - "action": "recreate", - "reason": "immutable", - "old": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - }, - "new": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "migrations/003.txt": "[FILE_HASH][5]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - }, "lifecycle.triggers.on_file_change['migrations/003.txt']": { "action": "recreate", "reason": "immutable", - "new": "[FILE_HASH][5]" + "new": "[FILE_HASH][4]" }, "result_state": { "action": "skip", @@ -149,23 +106,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change": { - "action": "recreate", - "reason": "immutable", - "old": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - }, - "new": { - "migrations/001.txt": "[FILE_HASH][3]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - }, "lifecycle.triggers.on_file_change['migrations/002.txt']": { "action": "recreate", "reason": "immutable", - "old": "[FILE_HASH][1]" + "old": "[FILE_HASH][5]" }, "result_state": { "action": "skip", @@ -179,20 +123,6 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change": { - "action": "skip", - "reason": "missing_in_remote", - "old": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - }, - "new": { - "migrations/001.txt": "[FILE_HASH][3]", - "migrations/002.txt": "[FILE_HASH][1]", - "mysubdir/a.txt": "[FILE_HASH][2]" - } - }, "result_state": { "action": "skip", "reason": "remote_already_set", diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 3cfb7a42ee1..e597085563a 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -10,6 +10,7 @@ import ( "os" pathlib "path" "path/filepath" + "slices" "strings" "github.com/databricks/cli/bundle" @@ -56,8 +57,8 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d return diags } -// syncableRelPaths is the set of relative paths sync would upload. -func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{}, diag.Diagnostics) { +// syncableRelPaths lists the relative paths sync would upload. +func syncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, diag.Diagnostics) { var diags diag.Diagnostics needs := false for _, jr := range b.Config.Resources.JobRuns { @@ -80,7 +81,7 @@ func syncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{ return out, diags } -func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]struct{}, error) { +func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, error) { fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) if err != nil { return nil, err @@ -89,10 +90,11 @@ func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) (map[string]str if err != nil { return nil, err } - out := make(map[string]struct{}, len(files)) + out := make([]string, 0, len(files)) for _, f := range files { - out[filepath.ToSlash(f.Relative)] = struct{}{} + out = append(out, filepath.ToSlash(f.Relative)) } + slices.Sort(out) return out, nil } @@ -105,16 +107,14 @@ func fileTriggerDiag(b *bundle.Bundle, loc string, severity diag.Severity, forma } } -func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[string]struct{}) (map[string]string, diag.Diagnostics) { +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string) (map[string]string, diag.Diagnostics) { var diags diag.Diagnostics out := make(map[string]string) - // filepath.Glob treats ** as two *, so doublestar-style patterns match less than expected. + // A double star looks recursive but path.Match treats it as two ordinary stars. if strings.Contains(pattern, "**") { return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "** in %q is not supported; use * for a single directory level", pattern)) } - // filepath.Join would otherwise rebase an absolute pattern under the bundle - // root (Join("/bundle", "/etc/passwd") is "/bundle/etc/passwd"). A POSIX path - // is absolute on Windows too, so check both flavours like NormalizePaths does. + // A POSIX path is absolute on Windows too, so check both flavours like NormalizePaths does. if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) { return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q must be relative to the defining YAML file", pattern)) } @@ -126,49 +126,33 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable map[stri if err != nil || !filepath.IsLocal(relPattern) { return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q is not under the sync root", pattern)) } - matches, err := filepath.Glob(joined) + relPattern = filepath.ToSlash(relPattern) + _, err = pathlib.Match(relPattern, "") if err != nil { return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) } - for _, match := range matches { - info, err := os.Stat(match) + for _, rel := range syncable { + matched, err := pathlib.Match(relPattern, rel) if err != nil { - diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "stat %q: %s", match, err)) + diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) continue } - // A glob like migrations/* routinely matches subdirectories; there is - // nothing to hash and nothing for the user to fix, so skip them quietly. - if !info.Mode().IsRegular() { - continue - } - rel, err := filepath.Rel(b.SyncRootPath, match) - if err != nil || !filepath.IsLocal(rel) { - diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "matched path %q is not under the sync root", match)) - continue - } - // Honor .gitignore and sync.exclude the same way sync does. - if _, ok := syncable[filepath.ToSlash(rel)]; !ok { + if !matched { continue } + match := filepath.Join(b.SyncRootPath, filepath.FromSlash(rel)) hash, err := hashFile(match) if err != nil { diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "hash %q: %s", match, err)) continue } - out[filepath.ToSlash(rel)] = hash + out[rel] = hash } - // A pattern that hashes nothing is a warning, not an error: every such case - // re-arms once a matching file appears. Record the placeholder under the - // pattern's own sync-root-relative key so that appearance is a hash change - // rather than a key swap. Skip it when a match failed to be read, since the - // error already says the fingerprint is incomplete. + // Record the pattern with an empty hash so a later match re-arms the trigger. + // A read error already reports an incomplete fingerprint, so skip it then. if len(out) == 0 && !diags.HasError() { - out[filepath.ToSlash(relPattern)] = missingFileHash - if len(matches) == 0 { - diags = diags.Append(fileTriggerDiag(b, loc, diag.Warning, "no files match %q", pattern)) - } else { - diags = diags.Append(fileTriggerDiag(b, loc, diag.Warning, "pattern %q matches only directories or files excluded from sync, so nothing is hashed", pattern)) - } + out[relPattern] = missingFileHash + diags = diags.Append(fileTriggerDiag(b, loc, diag.Warning, "no synced files match %q", pattern)) } return out, diags } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 078ae1b1aa6..01ad6f72ac9 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,6 +26,12 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour +var ( + jobRunOnBundleDeployPath = structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy") + jobRunOnFileChangePath = structpath.MustParsePath("lifecycle.triggers.on_file_change") + jobRunResultStatePath = structpath.MustParsePath("result_state") +) + // JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. type JobRunTriggersState struct { // Fresh UUID each plan while armed so Old!=New forces recreate. @@ -381,18 +387,21 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - switch path.String() { - case "lifecycle.triggers.on_bundle_deploy": + switch { + case path.Len() == jobRunOnBundleDeployPath.Len() && path.HasPrefix(jobRunOnBundleDeployPath): if change.New == nil || change.New == "" { change.Action = deployplan.Skip change.Reason = "trigger removed" } - case "lifecycle.triggers.on_file_change": + case path.Len() == jobRunOnFileChangePath.Len() && path.HasPrefix(jobRunOnFileChangePath): if change.New == nil { change.Action = deployplan.Skip change.Reason = "trigger removed" + } else if change.Old != nil { + // Per-file entries already report the change; drop the whole-map duplicate. + change.Reason = deployplan.ReasonDrop } - case "result_state": + case path.Len() == jobRunResultStatePath.Len() && path.HasPrefix(jobRunResultStatePath): // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { return nil diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index f1fac2ca1f9..7c4d6da7012 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -385,23 +385,27 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { for _, tt := range []struct { name string path string + old any new any action deployplan.ActionType + reason string }{ - {"cleared on_bundle_deploy string", "lifecycle.triggers.on_bundle_deploy", "", deployplan.Skip}, - {"nil on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", nil, deployplan.Skip}, - {"rotated on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "uuid", deployplan.Recreate}, - {"cleared on_file_change", "lifecycle.triggers.on_file_change", nil, deployplan.Skip}, - {"changed on_file_change map", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h"}, deployplan.Recreate}, + {"cleared on_bundle_deploy string", "lifecycle.triggers.on_bundle_deploy", "uuid", "", deployplan.Skip, "trigger removed"}, + {"nil on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "uuid", nil, deployplan.Skip, "trigger removed"}, + {"rotated on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "old-uuid", "new-uuid", deployplan.Recreate, ""}, + {"cleared on_file_change", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h"}, nil, deployplan.Skip, "trigger removed"}, + {"added on_file_change map", "lifecycle.triggers.on_file_change", nil, map[string]string{"a.txt": "h"}, deployplan.Recreate, ""}, + {"changed on_file_change map", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "old"}, map[string]string{"a.txt": "new"}, deployplan.Recreate, deployplan.ReasonDrop}, // A file dropping out of the map is a real change, so the skip must not // extend to paths below on_file_change. - {"cleared on_file_change child", "lifecycle.triggers.on_file_change['a.txt']", nil, deployplan.Recreate}, - {"result_state with unreadable remote", "result_state", nil, deployplan.Recreate}, + {"cleared on_file_change child", "lifecycle.triggers.on_file_change['a.txt']", "h", nil, deployplan.Recreate, ""}, + {"result_state with unreadable remote", "result_state", jobs.RunResultStateSuccess, nil, deployplan.Recreate, ""}, } { t.Run(tt.name, func(t *testing.T) { - change := &ChangeDesc{Action: deployplan.Recreate, New: tt.new} + change := &ChangeDesc{Action: deployplan.Recreate, Old: tt.old, New: tt.new} require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath(tt.path), change, nil)) assert.Equal(t, tt.action, change.Action) + assert.Equal(t, tt.reason, change.Reason) }) } } From 380cda9ab8911a3eec1fcb02d885a0a1333698c5 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 27 Aug 2026 13:46:21 +0000 Subject: [PATCH 28/60] job_runs: persist trigger fingerprints as lifecycle.triggers_state Keep resolved hashes off user config, omit empty lifecycle from state, and fold include/sync-root path coverage into one acceptance test. --- .../resources/job_runs/failed_run/output.txt | 3 - .../job_runs/on_bundle_deploy/output.txt | 10 ++- .../job_runs/on_file_change/output.txt | 4 +- .../job_runs/on_file_change_glob/output.txt | 8 +- .../on_file_change_include/databricks.yml | 14 ---- .../on_file_change_include/out.test.toml | 2 - .../on_file_change_include/output.txt | 29 ------- .../resources/job_run.yml | 7 -- .../job_runs/on_file_change_include/script | 17 ---- .../job_runs/on_file_change_include/test.toml | 3 - .../on_file_change_include/watched.txt | 1 - .../bundle/resources/job_run.yml | 4 +- .../bundle/resources/watched-resources.txt} | 0 .../bundle/resources/watched.txt | 1 - .../{watched.txt => watched-bundle.txt} | 0 .../output.txt | 64 ++++++++++++--- .../on_file_change_include_sync_root/script | 31 +++++--- .../test.toml | 7 ++ .../bundle/databricks.yml | 23 ------ .../bundle/shared.txt | 1 - .../on_file_change_sync_root/out.test.toml | 2 - .../on_file_change_sync_root/output.txt | 29 ------- .../job_runs/on_file_change_sync_root/script | 17 ---- .../on_file_change_sync_root/shared.txt | 1 - .../on_file_change_sync_root/test.toml | 2 - .../resources/job_runs/redeploy/output.txt | 3 - bundle/config/mutator/normalize_paths_test.go | 1 + .../paths/job_run_paths_visitor_test.go | 1 + .../mutator/resolve_job_run_file_triggers.go | 8 +- .../resolve_job_run_file_triggers_test.go | 9 ++- .../mutator/validate_job_run_triggers_test.go | 5 +- bundle/config/resources/job_run.go | 4 - bundle/config/resources/lifecycle.go | 9 +++ bundle/direct/dresources/job_run.go | 79 +++++++++++-------- bundle/direct/dresources/job_run_test.go | 29 ++++--- bundle/direct/dresources/type_test.go | 2 +- bundle/phases/initialize.go | 2 +- 37 files changed, 187 insertions(+), 245 deletions(-) delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/output.txt delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/script delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/test.toml delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt rename acceptance/bundle/resources/job_runs/{on_file_change_include/resources/watched.txt => on_file_change_include_sync_root/bundle/resources/watched-resources.txt} (100%) delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt rename acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/{watched.txt => watched-bundle.txt} (100%) delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/script delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt delete mode 100644 acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml diff --git a/acceptance/bundle/resources/job_runs/failed_run/output.txt b/acceptance/bundle/resources/job_runs/failed_run/output.txt index 5b8b6dbcd5f..f8b8dbf4398 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/output.txt +++ b/acceptance/bundle/resources/job_runs/failed_run/output.txt @@ -27,9 +27,6 @@ Files: 7 uploaded, 0 deleted "new_state": { "value": { "job_id": [MY_JOB_ID], - "lifecycle": { - "triggers": {} - }, "result_state": "SUCCESS" } }, diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt index 6cd8d44a418..8b4ed98cfb7 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -30,7 +30,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_bundle_deploy": { + "lifecycle.triggers_state.on_bundle_deploy": { "action": "recreate", "reason": "immutable", "old": "[UUID]", @@ -94,10 +94,14 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_bundle_deploy": { + "lifecycle": { "action": "skip", "reason": "trigger removed", - "old": "[UUID]" + "old": { + "triggers_state": { + "on_bundle_deploy": "[UUID]" + } + } }, "result_state": { "action": "skip", diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 998f0bbf0a1..0aec568a78a 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,7 +52,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change['seed.txt']": { + "lifecycle.triggers_state.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -107,7 +107,7 @@ Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" in databricks.yml:18:29 { - "lifecycle.triggers.on_file_change['seed.txt']": { + "lifecycle.triggers_state.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index db61edb23db..c77aa5c7c97 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -26,7 +26,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change['migrations/001.txt']": { + "lifecycle.triggers_state.on_file_change['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -71,7 +71,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change['mysubdir/a.txt']": { + "lifecycle.triggers_state.on_file_change['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -89,7 +89,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change['migrations/003.txt']": { + "lifecycle.triggers_state.on_file_change['migrations/003.txt']": { "action": "recreate", "reason": "immutable", "new": "[FILE_HASH][4]" @@ -106,7 +106,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers.on_file_change['migrations/002.txt']": { + "lifecycle.triggers_state.on_file_change['migrations/002.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][5]" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml deleted file mode 100644 index 3d8b74d28f8..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_include/databricks.yml +++ /dev/null @@ -1,14 +0,0 @@ -bundle: - name: job-runs-on-file-change-include - -include: - - resources/job_run.yml - -resources: - jobs: - my_job: - name: my-job - tasks: - - task_key: main - notebook_task: - notebook_path: /Workspace/test diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml deleted file mode 100644 index 0938e678987..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_include/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt deleted file mode 100644 index 52de054e734..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_include/output.txt +++ /dev/null @@ -1,29 +0,0 @@ - -=== first deploy ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include/default/files... -Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -Output from job_runs.my_run: id=[NUMID]: SUCCESS -Created job_runs.my_run -Created jobs.my_job -Files: 7 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged - -=== editing a same-named file at the bundle root does not re-fire ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - -=== editing the file next to the included YAML re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.job_runs.my_run - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-include/default - -Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml b/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml deleted file mode 100644 index 091489142f5..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_include/resources/job_run.yml +++ /dev/null @@ -1,7 +0,0 @@ -resources: - job_runs: - my_run: - job_id: ${resources.jobs.my_job.id} - lifecycle: - triggers: - - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/script b/acceptance/bundle/resources/job_runs/on_file_change_include/script deleted file mode 100644 index 65a8cea7a5b..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_include/script +++ /dev/null @@ -1,17 +0,0 @@ -cleanup() { - trace $CLI bundle destroy --auto-approve -} -trap cleanup EXIT - -title "first deploy" -trace $CLI bundle deploy - -title "editing a same-named file at the bundle root does not re-fire" -update_file.py watched.txt "root-v1" "root-v2" -trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" -update_file.py watched.txt "root-v2" "root-v1" - -title "editing the file next to the included YAML re-fires" -update_file.py resources/watched.txt "include-v1" "include-v2" -trace $CLI bundle plan | contains.py "recreate job_runs.my_run" -update_file.py resources/watched.txt "include-v2" "include-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml deleted file mode 100644 index c81bf4f943a..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_include/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Path resolution only; the HTTP traffic is the same as on_file_change. -RecordRequests = false -Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt deleted file mode 100644 index d1688530992..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_include/watched.txt +++ /dev/null @@ -1 +0,0 @@ -root-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml index 091489142f5..f6761be7e15 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/job_run.yml @@ -4,4 +4,6 @@ resources: job_id: ${resources.jobs.my_job.id} lifecycle: triggers: - - on_file_change: watched.txt + - on_file_change: watched-resources.txt + - on_file_change: ../watched-bundle.txt + - on_file_change: ../../shared.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched-resources.txt similarity index 100% rename from acceptance/bundle/resources/job_runs/on_file_change_include/resources/watched.txt rename to acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched-resources.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt deleted file mode 100644 index 8b4e79d3c47..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/resources/watched.txt +++ /dev/null @@ -1 +0,0 @@ -include-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched-bundle.txt similarity index 100% rename from acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched.txt rename to acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/bundle/watched-bundle.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt index 6c475c0dbfc..015ce22241c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt @@ -9,19 +9,63 @@ Created jobs.my_job Files: 5 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged -=== editing a same-named file in the bundle does not re-fire ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - -=== editing a file at the sync root does not re-fire ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +>>> file_triggers +{ + "bundle/resources/watched-resources.txt": "[FILE_HASH][0]", + "bundle/watched-bundle.txt": "[FILE_HASH][1]", + "shared.txt": "[FILE_HASH][2]" +} === editing the file next to the included YAML re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run +>>> changes +{ + "lifecycle.triggers_state.on_file_change['bundle/resources/watched-resources.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][3]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} + +=== editing the file at the bundle root re-fires +>>> changes +{ + "lifecycle.triggers_state.on_file_change['bundle/watched-bundle.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "[FILE_HASH][4]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged +=== editing the file at the sync root re-fires +>>> changes +{ + "lifecycle.triggers_state.on_file_change['shared.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][2]", + "new": "[FILE_HASH][5]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script index 5a608ea9188..a843cc605ca 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script @@ -3,20 +3,29 @@ cleanup() { } trap cleanup EXIT +changes() { + withdir bundle $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +} + +file_triggers() { + withdir bundle print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle.triggers_state.on_file_change' +} + title "first deploy" withdir bundle trace $CLI bundle deploy +trace file_triggers -title "editing a same-named file in the bundle does not re-fire" -update_file.py bundle/watched.txt "bundle-v1" "bundle-v2" -withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" -update_file.py bundle/watched.txt "bundle-v2" "bundle-v1" +title "editing the file next to the included YAML re-fires" +update_file.py bundle/resources/watched-resources.txt "include-v1" "include-v2" +trace changes +update_file.py bundle/resources/watched-resources.txt "include-v2" "include-v1" + +title "editing the file at the bundle root re-fires" +update_file.py bundle/watched-bundle.txt "bundle-v1" "bundle-v2" +trace changes +update_file.py bundle/watched-bundle.txt "bundle-v2" "bundle-v1" -title "editing a file at the sync root does not re-fire" +title "editing the file at the sync root re-fires" update_file.py shared.txt "parent-v1" "parent-v2" -withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" +trace changes update_file.py shared.txt "parent-v2" "parent-v1" - -title "editing the file next to the included YAML re-fires" -update_file.py bundle/resources/watched.txt "include-v1" "include-v2" -withdir bundle trace $CLI bundle plan | contains.py "recreate job_runs.my_run" -update_file.py bundle/resources/watched.txt "include-v2" "include-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml index e1290bad9f4..b0362c07bc0 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/test.toml @@ -1,2 +1,9 @@ RecordRequests = false Ignore = [".databricks"] + +# Mask hashes before the parent's `\d{8,}` rule; Distinct keeps them distinguishable. +[[Repls]] +Old = '[0-9a-f]{64}' +New = '[FILE_HASH]' +Order = 1 +Distinct = true diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml deleted file mode 100644 index 772a7ef92e7..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/databricks.yml +++ /dev/null @@ -1,23 +0,0 @@ -bundle: - name: job-runs-on-file-change-sync-root - -sync: - paths: - - . - - ../shared.txt - -resources: - jobs: - my_job: - name: my-job - tasks: - - task_key: main - notebook_task: - notebook_path: /Workspace/test - - job_runs: - my_run: - job_id: ${resources.jobs.my_job.id} - lifecycle: - triggers: - - on_file_change: ../shared.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt deleted file mode 100644 index 6f22941eacf..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/bundle/shared.txt +++ /dev/null @@ -1 +0,0 @@ -bundle-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml deleted file mode 100644 index 0938e678987..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/out.test.toml +++ /dev/null @@ -1,2 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt deleted file mode 100644 index 6a52f127ce8..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/output.txt +++ /dev/null @@ -1,29 +0,0 @@ - -=== first deploy ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-sync-root/default/files... -Output from job_runs.my_run: id=[NUMID]: Run URL: [DATABRICKS_URL]/jobs/[NUMID]/runs/[NUMID]?o=[NUMID] -Output from job_runs.my_run: id=[NUMID]: SUCCESS -Created job_runs.my_run -Created jobs.my_job -Files: 3 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged - -=== editing a same-named file in the bundle does not re-fire ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - -=== editing the file at the sync root re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.job_runs.my_run - delete resources.jobs.my_job - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-sync-root/default - -Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script deleted file mode 100644 index dcbc475840b..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/script +++ /dev/null @@ -1,17 +0,0 @@ -cleanup() { - withdir bundle trace $CLI bundle destroy --auto-approve -} -trap cleanup EXIT - -title "first deploy" -withdir bundle trace $CLI bundle deploy - -title "editing a same-named file in the bundle does not re-fire" -update_file.py bundle/shared.txt "bundle-v1" "bundle-v2" -withdir bundle trace $CLI bundle plan | contains.py "2 unchanged" "!recreate" -update_file.py bundle/shared.txt "bundle-v2" "bundle-v1" - -title "editing the file at the sync root re-fires" -update_file.py shared.txt "sync-v1" "sync-v2" -withdir bundle trace $CLI bundle plan | contains.py "recreate job_runs.my_run" -update_file.py shared.txt "sync-v2" "sync-v1" diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt deleted file mode 100644 index 04f175f2a1c..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/shared.txt +++ /dev/null @@ -1 +0,0 @@ -sync-v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml deleted file mode 100644 index e1290bad9f4..00000000000 --- a/acceptance/bundle/resources/job_runs/on_file_change_sync_root/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -RecordRequests = false -Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/job_runs/redeploy/output.txt b/acceptance/bundle/resources/job_runs/redeploy/output.txt index 8532692bdd9..06129f79373 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/output.txt +++ b/acceptance/bundle/resources/job_runs/redeploy/output.txt @@ -59,9 +59,6 @@ Resources: "job_parameters": { "env": "prod" }, - "lifecycle": { - "triggers": {} - }, "result_state": "SUCCESS" } }, diff --git a/bundle/config/mutator/normalize_paths_test.go b/bundle/config/mutator/normalize_paths_test.go index 2399a03aa83..9ad1dcad128 100644 --- a/bundle/config/mutator/normalize_paths_test.go +++ b/bundle/config/mutator/normalize_paths_test.go @@ -66,6 +66,7 @@ func TestNormalizePaths_jobRunOnFileChange(t *testing.T) { Triggers: []resources.JobRunTrigger{ {OnFileChange: &pattern}, }, + TriggersState: nil, }, }, }, diff --git a/bundle/config/mutator/paths/job_run_paths_visitor_test.go b/bundle/config/mutator/paths/job_run_paths_visitor_test.go index 0a006fd9df1..93df73d3c9e 100644 --- a/bundle/config/mutator/paths/job_run_paths_visitor_test.go +++ b/bundle/config/mutator/paths/job_run_paths_visitor_test.go @@ -19,6 +19,7 @@ func TestVisitJobRunPaths(t *testing.T) { Triggers: []resources.JobRunTrigger{ {OnFileChange: &watched}, }, + TriggersState: nil, }, }, }, diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index e597085563a..2a745d8b288 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" libsync "github.com/databricks/cli/libs/sync" ) @@ -23,8 +24,7 @@ const missingFileHash = "" type resolveJobRunFileTriggers struct{} -// ResolveJobRunFileTriggers expands on_file_change globs and stores per-file -// content hashes on each job_run for PrepareState to copy into local state. +// ResolveJobRunFileTriggers expands on_file_change globs into trigger state. func ResolveJobRunFileTriggers() bundle.Mutator { return &resolveJobRunFileTriggers{} } @@ -39,7 +39,7 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d return diags } for name, jr := range b.Config.Resources.JobRuns { - if jr == nil || jr.Lifecycle == nil { + if jr == nil || !jr.HasOnFileChange() { continue } out := make(map[string]string) @@ -52,7 +52,7 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d diags = diags.Extend(d) maps.Copy(out, hashes) } - jr.ResolvedFileTriggers = out + jr.Lifecycle.TriggersState = &resources.JobRunTriggersState{OnFileChange: out} } return diags } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 9271589bb48..48f0278f656 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -28,7 +28,7 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - hashes := b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers + hashes := b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange require.Len(t, hashes, 2) assert.Equal(t, contentHash("hello"), hashes["a.txt"]) assert.Equal(t, contentHash("world"), hashes["b.txt"]) @@ -42,7 +42,7 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.True(t, diags.HasError()) require.Equal(t, `lifecycle.triggers.on_file_change: pattern "/etc/passwd" must be relative to the defining YAML file`, diags[0].Summary) - assert.Empty(t, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + assert.Empty(t, b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange) }) t.Run("missing pattern is keyed relative to the sync root", func(t *testing.T) { @@ -56,7 +56,7 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - assert.Equal(t, map[string]string{"missing.txt": ""}, b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers) + assert.Equal(t, map[string]string{"missing.txt": ""}, b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange) }) t.Run("globs from the bundle root when the sync root is an ancestor", func(t *testing.T) { @@ -71,7 +71,7 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - assert.Equal(t, contentHash("from-sync-root"), b.Config.Resources.JobRuns["my_run"].ResolvedFileTriggers["shared.txt"]) + assert.Equal(t, contentHash("from-sync-root"), b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange["shared.txt"]) }) } @@ -91,6 +91,7 @@ func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { Triggers: []resources.JobRunTrigger{ {OnFileChange: &pattern}, }, + TriggersState: nil, }, }, }, diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 07a7276a41c..eadab7812e5 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -117,8 +117,9 @@ func TestValidateJobRunTriggers(t *testing.T) { JobRuns: map[string]*resources.JobRun{ "my_run": { Lifecycle: &resources.JobRunLifecycle{ - Lifecycle: resources.Lifecycle{PreventDestroy: tt.preventDestroy}, - Triggers: tt.triggers, + Lifecycle: resources.Lifecycle{PreventDestroy: tt.preventDestroy}, + Triggers: tt.triggers, + TriggersState: nil, }, }, }, diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 48820950e83..670a4b23622 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -27,10 +27,6 @@ type JobRun struct { // the run URL. Keeping it separate from RunNow.JobId (a ${resources.jobs.*.id} // reference) lets state loading preserve that reference and its plan dependency. ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` - - // ResolvedFileTriggers is path → content hash for on_file_change, computed - // under SyncRoot before plan. bundle:"internal" keeps it out of schema. - ResolvedFileTriggers map[string]string `json:"resolved_file_triggers,omitempty" bundle:"internal"` } // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 1f5d576d835..f9b7380dcf2 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -32,6 +32,9 @@ type JobRunLifecycle struct { // Triggers that cause the run to re-fire (in addition to config changes). Triggers []JobRunTrigger `json:"triggers,omitempty"` + + // Resolved fingerprint for the planner; not user config. + TriggersState *JobRunTriggersState `json:"triggers_state,omitempty" bundle:"internal"` } // JobRunTrigger is one lifecycle.triggers entry. @@ -39,3 +42,9 @@ type JobRunTrigger struct { OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` OnFileChange *string `json:"on_file_change,omitempty"` // path or glob relative to the defining YAML file; must resolve under the sync root } + +// JobRunTriggersState is the resolved fingerprint of lifecycle.triggers. +type JobRunTriggersState struct { + OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` + OnFileChange map[string]string `json:"on_file_change,omitempty"` +} diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 01ad6f72ac9..f8a104bdba3 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -27,29 +27,15 @@ import ( const jobRunTimeout = 24 * time.Hour var ( - jobRunOnBundleDeployPath = structpath.MustParsePath("lifecycle.triggers.on_bundle_deploy") - jobRunOnFileChangePath = structpath.MustParsePath("lifecycle.triggers.on_file_change") + jobRunLifecyclePath = structpath.MustParsePath("lifecycle") + jobRunOnBundleDeployPath = structpath.MustParsePath("lifecycle.triggers_state.on_bundle_deploy") + jobRunOnFileChangePath = structpath.MustParsePath("lifecycle.triggers_state.on_file_change") jobRunResultStatePath = structpath.MustParsePath("result_state") ) -// JobRunTriggersState is the persisted fingerprint of lifecycle.triggers. -type JobRunTriggersState struct { - // Fresh UUID each plan while armed so Old!=New forces recreate. - OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - // Content hashes from ResolveJobRunFileTriggers; any change recreates. - OnFileChange map[string]string `json:"on_file_change,omitempty"` -} - -// JobRunLifecycleState is the local-only trigger fingerprint. Nested by value, -// not by pointer: structdiff cannot descend into a nil pointer and would report -// the whole subtree at "lifecycle" instead of the leaf that actually changed. +// JobRunLifecycleState is the local-only trigger fingerprint. type JobRunLifecycleState struct { - Triggers JobRunTriggersState `json:"triggers"` -} - -func emptyJobRunLifecycleState() JobRunLifecycleState { - var empty JobRunLifecycleState - return empty + TriggersState resources.JobRunTriggersState `json:"triggers_state"` } // JobRunState is the RunNow request plus the outcome required for planning. @@ -59,9 +45,9 @@ type JobRunState struct { // Always SUCCESS during planning and cleared before persistence. ResultState jobs.RunResultState `json:"result_state,omitempty"` - // Local-only. Nested under lifecycle to mirror config and avoid colliding - // with a future Jobs API field. - Lifecycle JobRunLifecycleState `json:"lifecycle"` + // Local-only fingerprint. Nested under lifecycle to avoid colliding with a + // future Jobs API field; nil when no trigger is armed so the key is omitted. + Lifecycle *JobRunLifecycleState `json:"lifecycle,omitempty"` } func (s *JobRunState) UnmarshalJSON(b []byte) error { @@ -112,14 +98,23 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { state := &JobRunState{ RunNow: input.RunNow, ResultState: jobs.RunResultStateSuccess, - Lifecycle: emptyJobRunLifecycleState(), + Lifecycle: nil, + } + var fileTriggers map[string]string + if input.Lifecycle != nil && input.Lifecycle.TriggersState != nil { + fileTriggers = input.Lifecycle.TriggersState.OnFileChange } + if !input.HasOnBundleDeploy() && len(fileTriggers) == 0 { + return state + } + var ts resources.JobRunTriggersState if input.HasOnBundleDeploy() { - state.Lifecycle.Triggers.OnBundleDeploy = uuid.NewString() + ts.OnBundleDeploy = uuid.NewString() } - if len(input.ResolvedFileTriggers) > 0 { - state.Lifecycle.Triggers.OnFileChange = input.ResolvedFileTriggers + if len(fileTriggers) > 0 { + ts.OnFileChange = fileTriggers } + state.Lifecycle = &JobRunLifecycleState{TriggersState: ts} return state } @@ -194,13 +189,12 @@ func (r *ResourceJobRun) DoRead(ctx context.Context, id string) (*JobRunRemote, } // RemapState extracts the fields used for diffing: the RunNow request and the -// outcome the run reached. Lifecycle has no remote counterpart, so it stays empty -// and the planner skips it as missing_in_remote. +// outcome the run reached. Lifecycle has no remote counterpart. func (*ResourceJobRun) RemapState(remote *JobRunRemote) *JobRunState { return &JobRunState{ RunNow: remote.RunNow, ResultState: remote.ResultState, - Lifecycle: emptyJobRunLifecycleState(), + Lifecycle: nil, } } @@ -388,15 +382,28 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // Clearing a trigger skips its local-only fingerprint without re-firing the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { switch { + case path.Len() == jobRunLifecyclePath.Len() && path.HasPrefix(jobRunLifecyclePath): + if change.New == nil { + change.Action = deployplan.Skip + change.Reason = "trigger removed" + } else if change.Old != nil { + // Trigger fields already report the change. + change.Reason = deployplan.ReasonDrop + } case path.Len() == jobRunOnBundleDeployPath.Len() && path.HasPrefix(jobRunOnBundleDeployPath): if change.New == nil || change.New == "" { change.Action = deployplan.Skip change.Reason = "trigger removed" } case path.Len() == jobRunOnFileChangePath.Len() && path.HasPrefix(jobRunOnFileChangePath): - if change.New == nil { - change.Action = deployplan.Skip - change.Reason = "trigger removed" + if isEmptyFileTriggerMap(change.New) { + if isEmptyFileTriggerMap(change.Old) { + // Dyn may materialize a missing map as {}; omit the phantom skip. + change.Reason = deployplan.ReasonDrop + } else { + change.Action = deployplan.Skip + change.Reason = "trigger removed" + } } else if change.Old != nil { // Per-file entries already report the change; drop the whole-map duplicate. change.Reason = deployplan.ReasonDrop @@ -447,6 +454,14 @@ func (r *ResourceJobRun) cancelRun(ctx context.Context, runID int64) error { return nil } +func isEmptyFileTriggerMap(v any) bool { + if v == nil { + return true + } + m, ok := v.(map[string]string) + return ok && len(m) == 0 +} + func parseRunID(id string) (int64, error) { result, err := strconv.ParseInt(id, 10, 64) if err != nil { diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 7c4d6da7012..02e7ae73fb0 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -256,13 +256,15 @@ func TestJobRunWaitAbandonedLinksTheRun(t *testing.T) { require.ErrorContains(t, err, testRunPageLink) } -// State written before lifecycle existed has no such key, and must still load. -func TestJobRunStateUnmarshalWithoutLifecycle(t *testing.T) { +func TestJobRunStateOmitsEmptyLifecycle(t *testing.T) { var state JobRunState require.NoError(t, json.Unmarshal([]byte(`{}`), &state)) - assert.Nil(t, state.Lifecycle.Triggers.OnFileChange) + assert.Nil(t, state.Lifecycle) + serialized, err := json.Marshal(state) + require.NoError(t, err) + assert.NotContains(t, string(serialized), `"lifecycle"`) } // The planner diffs RemapState(remote) against PrepareState(config), so a run @@ -280,7 +282,7 @@ func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { state := (&ResourceJobRun{}).RemapState(remote) assert.Equal(t, outcome, state.ResultState) - assert.Equal(t, emptyJobRunLifecycleState(), state.Lifecycle) + assert.Nil(t, state.Lifecycle) }) } } @@ -382,6 +384,7 @@ func TestJobRunDeleteLeavesFinishedRunAlone(t *testing.T) { func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { r := &ResourceJobRun{} + var lifecycle JobRunLifecycleState for _, tt := range []struct { name string path string @@ -390,15 +393,19 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { action deployplan.ActionType reason string }{ - {"cleared on_bundle_deploy string", "lifecycle.triggers.on_bundle_deploy", "uuid", "", deployplan.Skip, "trigger removed"}, - {"nil on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "uuid", nil, deployplan.Skip, "trigger removed"}, - {"rotated on_bundle_deploy", "lifecycle.triggers.on_bundle_deploy", "old-uuid", "new-uuid", deployplan.Recreate, ""}, - {"cleared on_file_change", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "h"}, nil, deployplan.Skip, "trigger removed"}, - {"added on_file_change map", "lifecycle.triggers.on_file_change", nil, map[string]string{"a.txt": "h"}, deployplan.Recreate, ""}, - {"changed on_file_change map", "lifecycle.triggers.on_file_change", map[string]string{"a.txt": "old"}, map[string]string{"a.txt": "new"}, deployplan.Recreate, deployplan.ReasonDrop}, + {"cleared lifecycle", "lifecycle", lifecycle, nil, deployplan.Skip, "trigger removed"}, + {"added lifecycle", "lifecycle", nil, lifecycle, deployplan.Recreate, ""}, + {"changed lifecycle", "lifecycle", lifecycle, lifecycle, deployplan.Recreate, deployplan.ReasonDrop}, + {"cleared on_bundle_deploy string", "lifecycle.triggers_state.on_bundle_deploy", "uuid", "", deployplan.Skip, "trigger removed"}, + {"nil on_bundle_deploy", "lifecycle.triggers_state.on_bundle_deploy", "uuid", nil, deployplan.Skip, "trigger removed"}, + {"rotated on_bundle_deploy", "lifecycle.triggers_state.on_bundle_deploy", "old-uuid", "new-uuid", deployplan.Recreate, ""}, + {"cleared on_file_change", "lifecycle.triggers_state.on_file_change", map[string]string{"a.txt": "h"}, nil, deployplan.Skip, "trigger removed"}, + {"empty on_file_change maps", "lifecycle.triggers_state.on_file_change", map[string]string{}, map[string]string{}, deployplan.Recreate, deployplan.ReasonDrop}, + {"added on_file_change map", "lifecycle.triggers_state.on_file_change", nil, map[string]string{"a.txt": "h"}, deployplan.Recreate, ""}, + {"changed on_file_change map", "lifecycle.triggers_state.on_file_change", map[string]string{"a.txt": "old"}, map[string]string{"a.txt": "new"}, deployplan.Recreate, deployplan.ReasonDrop}, // A file dropping out of the map is a real change, so the skip must not // extend to paths below on_file_change. - {"cleared on_file_change child", "lifecycle.triggers.on_file_change['a.txt']", "h", nil, deployplan.Recreate, ""}, + {"cleared on_file_change child", "lifecycle.triggers_state.on_file_change['a.txt']", "h", nil, deployplan.Recreate, ""}, {"result_state with unreadable remote", "result_state", jobs.RunResultStateSuccess, nil, deployplan.Recreate, ""}, } { t.Run(tt.name, func(t *testing.T) { diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index b79314dce31..6bdf973278b 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -95,7 +95,7 @@ var knownMissingInStateType = map[string][]string{ "job_runs": { // State stores trigger fingerprints, not the config trigger list / prevent_destroy. "lifecycle.prevent_destroy", - "lifecycle.triggers[*]", + "lifecycle.triggers", }, "dashboards": { "file_path", diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index ef3fe582528..76965182dce 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -194,7 +194,7 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). mutator.ValidateJobRunTriggers(), - // Expand on_file_change globs and hash matched files into ResolvedFileTriggers. + // Expand on_file_change globs and hash matched files into triggers_state. mutator.ResolveJobRunFileTriggers(), // Reads (dynamic): * (strings) (searches for ${resources.*} references) From aa2622330ed5b853e26f22f5d6bd5c71ae6cd079 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 27 Aug 2026 14:21:17 +0000 Subject: [PATCH 29/60] job_runs: store triggers_state as a pointer on state Match the INPUT type so refschema reports one INPUT STATE row instead of splitting pointer and value. --- bundle/direct/dresources/job_run.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index f8a104bdba3..2b3e6288812 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -35,7 +35,7 @@ var ( // JobRunLifecycleState is the local-only trigger fingerprint. type JobRunLifecycleState struct { - TriggersState resources.JobRunTriggersState `json:"triggers_state"` + TriggersState *resources.JobRunTriggersState `json:"triggers_state,omitempty"` } // JobRunState is the RunNow request plus the outcome required for planning. @@ -114,7 +114,7 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { if len(fileTriggers) > 0 { ts.OnFileChange = fileTriggers } - state.Lifecycle = &JobRunLifecycleState{TriggersState: ts} + state.Lifecycle = &JobRunLifecycleState{TriggersState: &ts} return state } From 427b3203dfbad3552ba8cb291375d9e78d7ebb9f Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 27 Aug 2026 14:21:19 +0000 Subject: [PATCH 30/60] acc: keep LF when rewriting on_file_change include fixtures update_file.py uses text mode, so Windows rewrites committed LF files as CRLF and later plans still see those files as changed. --- .../on_file_change_include_sync_root/script | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script index a843cc605ca..7f7934d00cf 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/script @@ -16,16 +16,17 @@ withdir bundle trace $CLI bundle deploy trace file_triggers title "editing the file next to the included YAML re-fires" -update_file.py bundle/resources/watched-resources.txt "include-v1" "include-v2" +# printf keeps LF on Windows, where update_file.py's text-mode rewrite uses CRLF. +printf 'include-v2\n' > bundle/resources/watched-resources.txt trace changes -update_file.py bundle/resources/watched-resources.txt "include-v2" "include-v1" +printf 'include-v1\n' > bundle/resources/watched-resources.txt title "editing the file at the bundle root re-fires" -update_file.py bundle/watched-bundle.txt "bundle-v1" "bundle-v2" +printf 'bundle-v2\n' > bundle/watched-bundle.txt trace changes -update_file.py bundle/watched-bundle.txt "bundle-v2" "bundle-v1" +printf 'bundle-v1\n' > bundle/watched-bundle.txt title "editing the file at the sync root re-fires" -update_file.py shared.txt "parent-v1" "parent-v2" +printf 'parent-v2\n' > shared.txt trace changes -update_file.py shared.txt "parent-v2" "parent-v1" +printf 'parent-v1\n' > shared.txt From 3dcb9d4c600752735589ba988bdbecd7c22f37a0 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 27 Aug 2026 14:43:36 +0000 Subject: [PATCH 31/60] job_runs: key on_file_change fingerprints by pattern Group resolved hashes as pattern -> file -> hash so the planner can tell a removed trigger from a file dropping out of a glob, replacing the flat map and the empty-hash placeholder. PrepareState now copies the resolved trigger state instead of rebuilding it. --- .../job_runs/on_file_change/output.txt | 7 ++-- .../job_runs/on_file_change_glob/output.txt | 19 ++++++++-- .../job_runs/on_file_change_glob/script | 8 ++++ .../job_runs/on_file_change_glob/test.toml | 2 +- .../output.txt | 18 ++++++--- .../mutator/resolve_job_run_file_triggers.go | 28 ++++++-------- .../resolve_job_run_file_triggers_test.go | 12 +++--- bundle/config/resources/lifecycle.go | 4 +- bundle/direct/dresources/job_run.go | 38 +++++++++---------- bundle/direct/dresources/job_run_test.go | 38 +++++++++++++++---- 10 files changed, 108 insertions(+), 66 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 0aec568a78a..df8319a0da0 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,7 +52,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers_state.on_file_change['seed.txt']": { + "lifecycle.triggers_state.on_file_change['seed.txt']['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -107,11 +107,10 @@ Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" in databricks.yml:18:29 { - "lifecycle.triggers_state.on_file_change['seed.txt']": { + "lifecycle.triggers_state.on_file_change['seed.txt']['seed.txt']": { "action": "recreate", "reason": "immutable", - "old": "[FILE_HASH][1]", - "new": "" + "old": "[FILE_HASH][1]" }, "result_state": { "action": "skip", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index c77aa5c7c97..e48fd5df52a 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -26,7 +26,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers_state.on_file_change['migrations/001.txt']": { + "lifecycle.triggers_state.on_file_change['migrations/*']['migrations/001.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -71,7 +71,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers_state.on_file_change['mysubdir/a.txt']": { + "lifecycle.triggers_state.on_file_change['mysubdir/*.txt']['mysubdir/a.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -89,7 +89,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers_state.on_file_change['migrations/003.txt']": { + "lifecycle.triggers_state.on_file_change['migrations/*']['migrations/003.txt']": { "action": "recreate", "reason": "immutable", "new": "[FILE_HASH][4]" @@ -106,7 +106,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers_state.on_file_change['migrations/002.txt']": { + "lifecycle.triggers_state.on_file_change['migrations/*']['migrations/002.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][5]" @@ -135,6 +135,17 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +=== removing one pattern does not re-fire +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-glob/default/files... +Files: 7 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/script b/acceptance/bundle/resources/job_runs/on_file_change_glob/script index f0130f2637b..324efdb49a9 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/script +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/script @@ -57,3 +57,11 @@ title "editing a gitignored file does not re-fire" update_file.py migrations/ignored.txt "ignored" "ignored-edited" trace $CLI bundle plan update_file.py migrations/ignored.txt "ignored-edited" "ignored" + +title "removing one pattern does not re-fire" +cp databricks.yml tmp.databricks.yml +update_file.py databricks.yml " - on_file_change: mysubdir/*.txt" "" +trace $CLI bundle plan +trace $CLI bundle deploy +trace print_requests.py //jobs/run-now +mv tmp.databricks.yml databricks.yml diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml index 98c98631e5b..45b345e15a7 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/test.toml @@ -2,7 +2,7 @@ # computed during planning survive plan serialization. EnvMatrix.READPLAN = ["", "1"] -Ignore = ["tmp.plan.json", "tmp.002.txt", ".gitignore"] +Ignore = ["tmp.plan.json", "tmp.002.txt", "tmp.databricks.yml", ".gitignore"] # See on_file_change/test.toml: mask content hashes before the parent's # `\d{8,}` rule can mangle them, keeping distinct hashes distinguishable. diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt index 015ce22241c..7a376ded314 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt @@ -11,15 +11,21 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> file_triggers { - "bundle/resources/watched-resources.txt": "[FILE_HASH][0]", - "bundle/watched-bundle.txt": "[FILE_HASH][1]", - "shared.txt": "[FILE_HASH][2]" + "bundle/resources/watched-resources.txt": { + "bundle/resources/watched-resources.txt": "[FILE_HASH][0]" + }, + "bundle/watched-bundle.txt": { + "bundle/watched-bundle.txt": "[FILE_HASH][1]" + }, + "shared.txt": { + "shared.txt": "[FILE_HASH][2]" + } } === editing the file next to the included YAML re-fires >>> changes { - "lifecycle.triggers_state.on_file_change['bundle/resources/watched-resources.txt']": { + "lifecycle.triggers_state.on_file_change['bundle/resources/watched-resources.txt']['bundle/resources/watched-resources.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -36,7 +42,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing the file at the bundle root re-fires >>> changes { - "lifecycle.triggers_state.on_file_change['bundle/watched-bundle.txt']": { + "lifecycle.triggers_state.on_file_change['bundle/watched-bundle.txt']['bundle/watched-bundle.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", @@ -53,7 +59,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing the file at the sync root re-fires >>> changes { - "lifecycle.triggers_state.on_file_change['shared.txt']": { + "lifecycle.triggers_state.on_file_change['shared.txt']['shared.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 2a745d8b288..24a90904000 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "fmt" "io" - "maps" "os" pathlib "path" "path/filepath" @@ -19,9 +18,6 @@ import ( libsync "github.com/databricks/cli/libs/sync" ) -// missingFileHash marks a pattern with no matching file so appear/disappear recreates. -const missingFileHash = "" - type resolveJobRunFileTriggers struct{} // ResolveJobRunFileTriggers expands on_file_change globs into trigger state. @@ -42,15 +38,17 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d if jr == nil || !jr.HasOnFileChange() { continue } - out := make(map[string]string) + out := make(map[string]map[string]string) for i, t := range jr.Lifecycle.Triggers { if t.OnFileChange == nil { continue } path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) - hashes, d := resolveFileTrigger(b, path, *t.OnFileChange, syncable) + pattern, hashes, d := resolveFileTrigger(b, path, *t.OnFileChange, syncable) diags = diags.Extend(d) - maps.Copy(out, hashes) + if !d.HasError() { + out[pattern] = hashes + } } jr.Lifecycle.TriggersState = &resources.JobRunTriggersState{OnFileChange: out} } @@ -107,16 +105,16 @@ func fileTriggerDiag(b *bundle.Bundle, loc string, severity diag.Severity, forma } } -func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string) (map[string]string, diag.Diagnostics) { +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string) (string, map[string]string, diag.Diagnostics) { var diags diag.Diagnostics out := make(map[string]string) // A double star looks recursive but path.Match treats it as two ordinary stars. if strings.Contains(pattern, "**") { - return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "** in %q is not supported; use * for a single directory level", pattern)) + return "", out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "** in %q is not supported; use * for a single directory level", pattern)) } // A POSIX path is absolute on Windows too, so check both flavours like NormalizePaths does. if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) { - return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q must be relative to the defining YAML file", pattern)) + return "", out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q must be relative to the defining YAML file", pattern)) } // NormalizePaths has already rewritten YAML-relative globs to be bundle-root // relative. Join that onto the bundle root, then require the result stay @@ -124,12 +122,12 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string joined := filepath.Join(b.BundleRootPath, filepath.FromSlash(pattern)) relPattern, err := filepath.Rel(b.SyncRootPath, joined) if err != nil || !filepath.IsLocal(relPattern) { - return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q is not under the sync root", pattern)) + return "", out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q is not under the sync root", pattern)) } relPattern = filepath.ToSlash(relPattern) _, err = pathlib.Match(relPattern, "") if err != nil { - return out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) + return "", out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) } for _, rel := range syncable { matched, err := pathlib.Match(relPattern, rel) @@ -148,13 +146,11 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string } out[rel] = hash } - // Record the pattern with an empty hash so a later match re-arms the trigger. - // A read error already reports an incomplete fingerprint, so skip it then. + // The empty map is still stored under the pattern, so a later match re-arms it. if len(out) == 0 && !diags.HasError() { - out[relPattern] = missingFileHash diags = diags.Append(fileTriggerDiag(b, loc, diag.Warning, "no synced files match %q", pattern)) } - return out, diags + return relPattern, out, diags } func hashFile(path string) (string, error) { diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 48f0278f656..2ee0c0f60a0 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -29,9 +29,11 @@ func TestResolveJobRunFileTriggers(t *testing.T) { require.False(t, diags.HasError()) hashes := b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange - require.Len(t, hashes, 2) - assert.Equal(t, contentHash("hello"), hashes["a.txt"]) - assert.Equal(t, contentHash("world"), hashes["b.txt"]) + require.Len(t, hashes, 1) + assert.Equal(t, map[string]string{ + "a.txt": contentHash("hello"), + "b.txt": contentHash("world"), + }, hashes["*.txt"]) }) t.Run("rejects an absolute pattern", func(t *testing.T) { @@ -56,7 +58,7 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - assert.Equal(t, map[string]string{"missing.txt": ""}, b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange) + assert.Equal(t, map[string]map[string]string{"missing.txt": {}}, b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange) }) t.Run("globs from the bundle root when the sync root is an ancestor", func(t *testing.T) { @@ -71,7 +73,7 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - assert.Equal(t, contentHash("from-sync-root"), b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange["shared.txt"]) + assert.Equal(t, contentHash("from-sync-root"), b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange["shared.txt"]["shared.txt"]) }) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index f9b7380dcf2..83af467b2c1 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -45,6 +45,6 @@ type JobRunTrigger struct { // JobRunTriggersState is the resolved fingerprint of lifecycle.triggers. type JobRunTriggersState struct { - OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - OnFileChange map[string]string `json:"on_file_change,omitempty"` + OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` + OnFileChange map[string]map[string]string `json:"on_file_change,omitempty"` } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 2b3e6288812..f923dca5076 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -100,19 +100,15 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { ResultState: jobs.RunResultStateSuccess, Lifecycle: nil, } - var fileTriggers map[string]string + var ts resources.JobRunTriggersState if input.Lifecycle != nil && input.Lifecycle.TriggersState != nil { - fileTriggers = input.Lifecycle.TriggersState.OnFileChange - } - if !input.HasOnBundleDeploy() && len(fileTriggers) == 0 { - return state + ts = *input.Lifecycle.TriggersState } - var ts resources.JobRunTriggersState if input.HasOnBundleDeploy() { ts.OnBundleDeploy = uuid.NewString() } - if len(fileTriggers) > 0 { - ts.OnFileChange = fileTriggers + if ts.OnBundleDeploy == "" && len(ts.OnFileChange) == 0 { + return state } state.Lifecycle = &JobRunLifecycleState{TriggersState: &ts} return state @@ -380,6 +376,7 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // stopped without succeeding keeps its recreate. A SKIPPED run reports no // result_state either, so the lifecycle state is what tells the two apart. // Clearing a trigger skips its local-only fingerprint without re-firing the run. +// All other trigger changes retain the resource's default recreate action. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { switch { case path.Len() == jobRunLifecyclePath.Len() && path.HasPrefix(jobRunLifecyclePath): @@ -387,7 +384,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Action = deployplan.Skip change.Reason = "trigger removed" } else if change.Old != nil { - // Trigger fields already report the change. + // Trigger fields classify the change, including a removed pattern. change.Reason = deployplan.ReasonDrop } case path.Len() == jobRunOnBundleDeployPath.Len() && path.HasPrefix(jobRunOnBundleDeployPath): @@ -396,17 +393,16 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Reason = "trigger removed" } case path.Len() == jobRunOnFileChangePath.Len() && path.HasPrefix(jobRunOnFileChangePath): - if isEmptyFileTriggerMap(change.New) { - if isEmptyFileTriggerMap(change.Old) { - // Dyn may materialize a missing map as {}; omit the phantom skip. - change.Reason = deployplan.ReasonDrop - } else { - change.Action = deployplan.Skip - change.Reason = "trigger removed" - } - } else if change.Old != nil { - // Per-file entries already report the change; drop the whole-map duplicate. - change.Reason = deployplan.ReasonDrop + if isEmptyFileTriggerMap(change.New) && !isEmptyFileTriggerMap(change.Old) { + change.Action = deployplan.Skip + change.Reason = "trigger removed" + } + case path.Len() == jobRunOnFileChangePath.Len()+1 && path.HasPrefix(jobRunOnFileChangePath): + // A whole pattern dropping out means the user removed that trigger; a file + // dropping out of a pattern is a real change and keeps the default recreate. + if change.New == nil { + change.Action = deployplan.Skip + change.Reason = "trigger removed" } case path.Len() == jobRunResultStatePath.Len() && path.HasPrefix(jobRunResultStatePath): // The planner passes no remote state when the run could not be read. @@ -458,7 +454,7 @@ func isEmptyFileTriggerMap(v any) bool { if v == nil { return true } - m, ok := v.(map[string]string) + m, ok := v.(map[string]map[string]string) return ok && len(m) == 0 } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 02e7ae73fb0..2104406c91e 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -9,6 +9,7 @@ import ( "sync/atomic" "testing" + "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/testserver" @@ -267,6 +268,28 @@ func TestJobRunStateOmitsEmptyLifecycle(t *testing.T) { assert.NotContains(t, string(serialized), `"lifecycle"`) } +func TestJobRunPrepareStateCopiesResolvedTriggers(t *testing.T) { + enabled := true + triggers := &resources.JobRunTriggersState{ + OnFileChange: map[string]map[string]string{"*.txt": {"a.txt": "hash"}}, + } + input := &resources.JobRun{ + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{{OnBundleDeploy: &enabled}}, + TriggersState: triggers, + }, + } + + state := (&ResourceJobRun{}).PrepareState(input) + + require.NotNil(t, state.Lifecycle) + require.NotNil(t, state.Lifecycle.TriggersState) + assert.NotSame(t, triggers, state.Lifecycle.TriggersState) + assert.Equal(t, triggers.OnFileChange, state.Lifecycle.TriggersState.OnFileChange) + assert.NotEmpty(t, state.Lifecycle.TriggersState.OnBundleDeploy) + assert.Empty(t, triggers.OnBundleDeploy) +} + // The planner diffs RemapState(remote) against PrepareState(config), so a run // that did not end in SUCCESS has to surface as a difference on result_state. func TestJobRunRemapStateCarriesTheOutcome(t *testing.T) { @@ -399,13 +422,14 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { {"cleared on_bundle_deploy string", "lifecycle.triggers_state.on_bundle_deploy", "uuid", "", deployplan.Skip, "trigger removed"}, {"nil on_bundle_deploy", "lifecycle.triggers_state.on_bundle_deploy", "uuid", nil, deployplan.Skip, "trigger removed"}, {"rotated on_bundle_deploy", "lifecycle.triggers_state.on_bundle_deploy", "old-uuid", "new-uuid", deployplan.Recreate, ""}, - {"cleared on_file_change", "lifecycle.triggers_state.on_file_change", map[string]string{"a.txt": "h"}, nil, deployplan.Skip, "trigger removed"}, - {"empty on_file_change maps", "lifecycle.triggers_state.on_file_change", map[string]string{}, map[string]string{}, deployplan.Recreate, deployplan.ReasonDrop}, - {"added on_file_change map", "lifecycle.triggers_state.on_file_change", nil, map[string]string{"a.txt": "h"}, deployplan.Recreate, ""}, - {"changed on_file_change map", "lifecycle.triggers_state.on_file_change", map[string]string{"a.txt": "old"}, map[string]string{"a.txt": "new"}, deployplan.Recreate, deployplan.ReasonDrop}, - // A file dropping out of the map is a real change, so the skip must not - // extend to paths below on_file_change. - {"cleared on_file_change child", "lifecycle.triggers_state.on_file_change['a.txt']", "h", nil, deployplan.Recreate, ""}, + {"cleared on_file_change", "lifecycle.triggers_state.on_file_change", map[string]map[string]string{"*.txt": {"a.txt": "h"}}, nil, deployplan.Skip, "trigger removed"}, + {"empty on_file_change maps", "lifecycle.triggers_state.on_file_change", map[string]map[string]string{}, map[string]map[string]string{}, deployplan.Recreate, ""}, + {"added on_file_change map", "lifecycle.triggers_state.on_file_change", nil, map[string]map[string]string{"*.txt": {"a.txt": "h"}}, deployplan.Recreate, ""}, + {"changed on_file_change map", "lifecycle.triggers_state.on_file_change", map[string]map[string]string{"*.txt": {"a.txt": "old"}}, map[string]map[string]string{"*.txt": {"a.txt": "new"}}, deployplan.Recreate, ""}, + {"cleared on_file_change pattern", "lifecycle.triggers_state.on_file_change['*.txt']", map[string]string{"a.txt": "h"}, nil, deployplan.Skip, "trigger removed"}, + // A file dropping out of a pattern is a real change, so the skip must not + // extend to paths below the pattern. + {"cleared on_file_change file", "lifecycle.triggers_state.on_file_change['*.txt']['a.txt']", "h", nil, deployplan.Recreate, ""}, {"result_state with unreadable remote", "result_state", jobs.RunResultStateSuccess, nil, deployplan.Recreate, ""}, } { t.Run(tt.name, func(t *testing.T) { From 123ffa4ae3aea6257d13e76260966503fd95000f Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 27 Aug 2026 14:49:00 +0000 Subject: [PATCH 32/60] job_runs: aggregate file trigger fingerprints by pattern Store one deterministic fingerprint per pattern so plan paths remain concise while file additions, removals, and content changes still re-fire the run. --- .../job_runs/on_file_change/output.txt | 7 +++-- .../job_runs/on_file_change_glob/output.txt | 12 ++++---- .../output.txt | 18 ++++------- .../mutator/resolve_job_run_file_triggers.go | 30 +++++++++++-------- .../resolve_job_run_file_triggers_test.go | 21 +++++++------ bundle/config/resources/lifecycle.go | 4 +-- bundle/direct/dresources/job_run.go | 17 +++++++---- bundle/direct/dresources/job_run_test.go | 16 +++++----- 8 files changed, 66 insertions(+), 59 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index df8319a0da0..0aeb9e6828e 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -52,7 +52,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers_state.on_file_change['seed.txt']['seed.txt']": { + "lifecycle.triggers_state.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -107,10 +107,11 @@ Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" in databricks.yml:18:29 { - "lifecycle.triggers_state.on_file_change['seed.txt']['seed.txt']": { + "lifecycle.triggers_state.on_file_change['seed.txt']": { "action": "recreate", "reason": "immutable", - "old": "[FILE_HASH][1]" + "old": "[FILE_HASH][1]", + "new": "[FILE_HASH][2]" }, "result_state": { "action": "skip", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt index e48fd5df52a..5349b686287 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/output.txt @@ -26,7 +26,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers_state.on_file_change['migrations/*']['migrations/001.txt']": { + "lifecycle.triggers_state.on_file_change['migrations/*']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -71,7 +71,7 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers_state.on_file_change['mysubdir/*.txt']['mysubdir/a.txt']": { + "lifecycle.triggers_state.on_file_change['mysubdir/*.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", @@ -89,9 +89,10 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers_state.on_file_change['migrations/*']['migrations/003.txt']": { + "lifecycle.triggers_state.on_file_change['migrations/*']": { "action": "recreate", "reason": "immutable", + "old": "[FILE_HASH][1]", "new": "[FILE_HASH][4]" }, "result_state": { @@ -106,10 +107,11 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle.triggers_state.on_file_change['migrations/*']['migrations/002.txt']": { + "lifecycle.triggers_state.on_file_change['migrations/*']": { "action": "recreate", "reason": "immutable", - "old": "[FILE_HASH][5]" + "old": "[FILE_HASH][1]", + "new": "[FILE_HASH][5]" }, "result_state": { "action": "skip", diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt index 7a376ded314..015ce22241c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/output.txt @@ -11,21 +11,15 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> file_triggers { - "bundle/resources/watched-resources.txt": { - "bundle/resources/watched-resources.txt": "[FILE_HASH][0]" - }, - "bundle/watched-bundle.txt": { - "bundle/watched-bundle.txt": "[FILE_HASH][1]" - }, - "shared.txt": { - "shared.txt": "[FILE_HASH][2]" - } + "bundle/resources/watched-resources.txt": "[FILE_HASH][0]", + "bundle/watched-bundle.txt": "[FILE_HASH][1]", + "shared.txt": "[FILE_HASH][2]" } === editing the file next to the included YAML re-fires >>> changes { - "lifecycle.triggers_state.on_file_change['bundle/resources/watched-resources.txt']['bundle/resources/watched-resources.txt']": { + "lifecycle.triggers_state.on_file_change['bundle/resources/watched-resources.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][0]", @@ -42,7 +36,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing the file at the bundle root re-fires >>> changes { - "lifecycle.triggers_state.on_file_change['bundle/watched-bundle.txt']['bundle/watched-bundle.txt']": { + "lifecycle.triggers_state.on_file_change['bundle/watched-bundle.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][1]", @@ -59,7 +53,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged === editing the file at the sync root re-fires >>> changes { - "lifecycle.triggers_state.on_file_change['shared.txt']['shared.txt']": { + "lifecycle.triggers_state.on_file_change['shared.txt']": { "action": "recreate", "reason": "immutable", "old": "[FILE_HASH][2]", diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 24a90904000..b951632b538 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -38,16 +38,16 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d if jr == nil || !jr.HasOnFileChange() { continue } - out := make(map[string]map[string]string) + out := make(map[string]string) for i, t := range jr.Lifecycle.Triggers { if t.OnFileChange == nil { continue } path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d].on_file_change", name, i) - pattern, hashes, d := resolveFileTrigger(b, path, *t.OnFileChange, syncable) + pattern, fingerprint, d := resolveFileTrigger(b, path, *t.OnFileChange, syncable) diags = diags.Extend(d) if !d.HasError() { - out[pattern] = hashes + out[pattern] = fingerprint } } jr.Lifecycle.TriggersState = &resources.JobRunTriggersState{OnFileChange: out} @@ -105,16 +105,15 @@ func fileTriggerDiag(b *bundle.Bundle, loc string, severity diag.Severity, forma } } -func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string) (string, map[string]string, diag.Diagnostics) { +func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string) (string, string, diag.Diagnostics) { var diags diag.Diagnostics - out := make(map[string]string) // A double star looks recursive but path.Match treats it as two ordinary stars. if strings.Contains(pattern, "**") { - return "", out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "** in %q is not supported; use * for a single directory level", pattern)) + return "", "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "** in %q is not supported; use * for a single directory level", pattern)) } // A POSIX path is absolute on Windows too, so check both flavours like NormalizePaths does. if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) { - return "", out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q must be relative to the defining YAML file", pattern)) + return "", "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q must be relative to the defining YAML file", pattern)) } // NormalizePaths has already rewritten YAML-relative globs to be bundle-root // relative. Join that onto the bundle root, then require the result stay @@ -122,13 +121,15 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string joined := filepath.Join(b.BundleRootPath, filepath.FromSlash(pattern)) relPattern, err := filepath.Rel(b.SyncRootPath, joined) if err != nil || !filepath.IsLocal(relPattern) { - return "", out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q is not under the sync root", pattern)) + return "", "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q is not under the sync root", pattern)) } relPattern = filepath.ToSlash(relPattern) _, err = pathlib.Match(relPattern, "") if err != nil { - return "", out, diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) + return "", "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) } + h := sha256.New() + matches := 0 for _, rel := range syncable { matched, err := pathlib.Match(relPattern, rel) if err != nil { @@ -144,13 +145,16 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "hash %q: %s", match, err)) continue } - out[rel] = hash + h.Write([]byte(rel)) + h.Write([]byte{0}) + h.Write([]byte(hash)) + h.Write([]byte{0}) + matches++ } - // The empty map is still stored under the pattern, so a later match re-arms it. - if len(out) == 0 && !diags.HasError() { + if matches == 0 && !diags.HasError() { diags = diags.Append(fileTriggerDiag(b, loc, diag.Warning, "no synced files match %q", pattern)) } - return relPattern, out, diags + return relPattern, hex.EncodeToString(h.Sum(nil)), diags } func hashFile(path string) (string, error) { diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 2ee0c0f60a0..2f083def1ea 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -17,7 +17,7 @@ import ( ) func TestResolveJobRunFileTriggers(t *testing.T) { - t.Run("hashes file contents with sha256", func(t *testing.T) { + t.Run("fingerprints the matched file set", func(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) @@ -28,12 +28,12 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - hashes := b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange - require.Len(t, hashes, 1) - assert.Equal(t, map[string]string{ - "a.txt": contentHash("hello"), - "b.txt": contentHash("world"), - }, hashes["*.txt"]) + fingerprints := b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange + require.Len(t, fingerprints, 1) + assert.Equal(t, contentHash( + "a.txt\x00"+contentHash("hello")+"\x00"+ + "b.txt\x00"+contentHash("world")+"\x00", + ), fingerprints["*.txt"]) }) t.Run("rejects an absolute pattern", func(t *testing.T) { @@ -58,7 +58,7 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - assert.Equal(t, map[string]map[string]string{"missing.txt": {}}, b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange) + assert.Equal(t, map[string]string{"missing.txt": contentHash("")}, b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange) }) t.Run("globs from the bundle root when the sync root is an ancestor", func(t *testing.T) { @@ -73,7 +73,10 @@ func TestResolveJobRunFileTriggers(t *testing.T) { diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) - assert.Equal(t, contentHash("from-sync-root"), b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange["shared.txt"]["shared.txt"]) + assert.Equal(t, + contentHash("shared.txt\x00"+contentHash("from-sync-root")+"\x00"), + b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange["shared.txt"], + ) }) } diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 83af467b2c1..f9b7380dcf2 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -45,6 +45,6 @@ type JobRunTrigger struct { // JobRunTriggersState is the resolved fingerprint of lifecycle.triggers. type JobRunTriggersState struct { - OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - OnFileChange map[string]map[string]string `json:"on_file_change,omitempty"` + OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` + OnFileChange map[string]string `json:"on_file_change,omitempty"` } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index f923dca5076..5d0847b8773 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -393,13 +393,18 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa change.Reason = "trigger removed" } case path.Len() == jobRunOnFileChangePath.Len() && path.HasPrefix(jobRunOnFileChangePath): - if isEmptyFileTriggerMap(change.New) && !isEmptyFileTriggerMap(change.Old) { - change.Action = deployplan.Skip - change.Reason = "trigger removed" + if isEmptyFileTriggerMap(change.New) { + if isEmptyFileTriggerMap(change.Old) { + change.Reason = deployplan.ReasonDrop + } else { + change.Action = deployplan.Skip + change.Reason = "trigger removed" + } + } else if change.Old != nil { + // Pattern entries classify the change. + change.Reason = deployplan.ReasonDrop } case path.Len() == jobRunOnFileChangePath.Len()+1 && path.HasPrefix(jobRunOnFileChangePath): - // A whole pattern dropping out means the user removed that trigger; a file - // dropping out of a pattern is a real change and keeps the default recreate. if change.New == nil { change.Action = deployplan.Skip change.Reason = "trigger removed" @@ -454,7 +459,7 @@ func isEmptyFileTriggerMap(v any) bool { if v == nil { return true } - m, ok := v.(map[string]map[string]string) + m, ok := v.(map[string]string) return ok && len(m) == 0 } diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index 2104406c91e..cb22ad7c55c 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -271,7 +271,7 @@ func TestJobRunStateOmitsEmptyLifecycle(t *testing.T) { func TestJobRunPrepareStateCopiesResolvedTriggers(t *testing.T) { enabled := true triggers := &resources.JobRunTriggersState{ - OnFileChange: map[string]map[string]string{"*.txt": {"a.txt": "hash"}}, + OnFileChange: map[string]string{"*.txt": "hash"}, } input := &resources.JobRun{ Lifecycle: &resources.JobRunLifecycle{ @@ -422,14 +422,12 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { {"cleared on_bundle_deploy string", "lifecycle.triggers_state.on_bundle_deploy", "uuid", "", deployplan.Skip, "trigger removed"}, {"nil on_bundle_deploy", "lifecycle.triggers_state.on_bundle_deploy", "uuid", nil, deployplan.Skip, "trigger removed"}, {"rotated on_bundle_deploy", "lifecycle.triggers_state.on_bundle_deploy", "old-uuid", "new-uuid", deployplan.Recreate, ""}, - {"cleared on_file_change", "lifecycle.triggers_state.on_file_change", map[string]map[string]string{"*.txt": {"a.txt": "h"}}, nil, deployplan.Skip, "trigger removed"}, - {"empty on_file_change maps", "lifecycle.triggers_state.on_file_change", map[string]map[string]string{}, map[string]map[string]string{}, deployplan.Recreate, ""}, - {"added on_file_change map", "lifecycle.triggers_state.on_file_change", nil, map[string]map[string]string{"*.txt": {"a.txt": "h"}}, deployplan.Recreate, ""}, - {"changed on_file_change map", "lifecycle.triggers_state.on_file_change", map[string]map[string]string{"*.txt": {"a.txt": "old"}}, map[string]map[string]string{"*.txt": {"a.txt": "new"}}, deployplan.Recreate, ""}, - {"cleared on_file_change pattern", "lifecycle.triggers_state.on_file_change['*.txt']", map[string]string{"a.txt": "h"}, nil, deployplan.Skip, "trigger removed"}, - // A file dropping out of a pattern is a real change, so the skip must not - // extend to paths below the pattern. - {"cleared on_file_change file", "lifecycle.triggers_state.on_file_change['*.txt']['a.txt']", "h", nil, deployplan.Recreate, ""}, + {"cleared on_file_change", "lifecycle.triggers_state.on_file_change", map[string]string{"*.txt": "hash"}, nil, deployplan.Skip, "trigger removed"}, + {"empty on_file_change maps", "lifecycle.triggers_state.on_file_change", map[string]string{}, map[string]string{}, deployplan.Recreate, deployplan.ReasonDrop}, + {"added on_file_change map", "lifecycle.triggers_state.on_file_change", nil, map[string]string{"*.txt": "hash"}, deployplan.Recreate, ""}, + {"changed on_file_change map", "lifecycle.triggers_state.on_file_change", map[string]string{"*.txt": "old"}, map[string]string{"*.txt": "new"}, deployplan.Recreate, deployplan.ReasonDrop}, + {"cleared on_file_change pattern", "lifecycle.triggers_state.on_file_change['*.txt']", "hash", nil, deployplan.Skip, "trigger removed"}, + {"changed on_file_change pattern", "lifecycle.triggers_state.on_file_change['*.txt']", "old", "new", deployplan.Recreate, ""}, {"result_state with unreadable remote", "result_state", jobs.RunResultStateSuccess, nil, deployplan.Recreate, ""}, } { t.Run(tt.name, func(t *testing.T) { From abe08c163f21f6f7d64d6d8f986dedd8ce7a37b6 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Thu, 27 Aug 2026 16:08:58 +0000 Subject: [PATCH 33/60] job_runs: fingerprint on_file_change after build and persist cleared triggers Resolve on_file_change fingerprints after the build/predeploy scripts have run and hash matched files through the SyncRoot VFS, so build outputs and WSFS notebooks are captured. Persist a cleared trigger's state without invoking the Jobs API, so removing then re-adding a trigger re-fires the run. --- .../job_runs/on_file_change/output.txt | 29 +++++++- .../resources/job_runs/on_file_change/script | 16 +++++ .../on_file_change_build/databricks.yml | 22 ++++++ .../on_file_change_build/out.test.toml | 2 + .../job_runs/on_file_change_build/output.txt | 53 ++++++++++++++ .../job_runs/on_file_change_build/script | 18 +++++ .../job_runs/on_file_change_build/source.txt | 1 + .../job_runs/on_file_change_build/watched.txt | 1 + .../mutator/resolve_job_run_file_triggers.go | 70 +++++++++++-------- .../resolve_job_run_file_triggers_test.go | 19 ++++- .../mutator/validate_job_run_triggers.go | 18 +++-- .../mutator/validate_job_run_triggers_test.go | 3 + bundle/deployplan/plan.go | 13 ++-- bundle/direct/bundle_apply.go | 29 +++++--- bundle/direct/bundle_plan.go | 14 +++- bundle/direct/dresources/job_run.go | 8 ++- bundle/direct/dresources/job_run_test.go | 1 + bundle/phases/deploy.go | 22 ++++-- bundle/phases/initialize.go | 3 - cmd/bundle/utils/process.go | 10 +++ 20 files changed, 292 insertions(+), 60 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_build/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_build/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_build/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_build/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_build/source.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_build/watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index 0aeb9e6828e..bced8e0b19a 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -170,12 +170,39 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged +=== bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 4 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now + +=== re-adding on_file_change re-fires +>>> print_state.py +null + +>>> [CLI] bundle plan +recreate job_runs.my_run + +Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged + >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_5]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_5]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_5]: SUCCESS +Recreated job_runs.my_run Files: 3 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 2 unchanged +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index ef432d76033..cf392f1e575 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -60,5 +60,21 @@ update_file.py databricks.yml " lifecycle: - on_file_change: seed.txt " "" trace $CLI bundle plan +# Save the removal too, so READPLAN=1 verifies state-only updates survive serialization. +$CLI bundle plan -o json > tmp.plan.json +title "bundle deploy\n" +$CLI bundle deploy $(readplanarg tmp.plan.json) +trace print_requests.py //jobs/run-now + +title "re-adding on_file_change re-fires" +trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle' +update_file.py databricks.yml ' job_id: ${resources.jobs.my_job.id} +' ' job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: seed.txt +' +trace $CLI bundle plan trace $CLI bundle deploy +read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_build/databricks.yml new file mode 100644 index 00000000000..ddc7a4d8896 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: job-runs-on-file-change-build + +experimental: + scripts: + prebuild: cp source.txt watched.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_build/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt new file mode 100644 index 00000000000..b674e4063a7 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/output.txt @@ -0,0 +1,53 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Executing 'prebuild' script +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-build/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== a prebuild file change re-fires in the same deploy +>>> [CLI] bundle deploy +Executing 'prebuild' script +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-build/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-build/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/script b/acceptance/bundle/resources/job_runs/on_file_change_build/script new file mode 100644 index 00000000000..e350d2ba6e6 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/script @@ -0,0 +1,18 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +# Name the first run so the recreated one becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "a prebuild file change re-fires in the same deploy" +update_file.py source.txt "v1" "v2" +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/source.txt b/acceptance/bundle/resources/job_runs/on_file_change_build/source.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/source.txt @@ -0,0 +1 @@ +v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_build/watched.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/watched.txt @@ -0,0 +1 @@ +v1 diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index b951632b538..81ccc8fc80e 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -6,7 +6,7 @@ import ( "encoding/hex" "fmt" "io" - "os" + "io/fs" pathlib "path" "path/filepath" "slices" @@ -80,7 +80,13 @@ func syncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, diag.Dia } func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, error) { - fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, b.Config.Sync.Include, b.Config.Sync.Exclude) + // Match sync's effective include set, not just Sync.Include, so a pattern can + // hash the internal and AI-snapshot dirs sync force-includes. + includes, err := b.GetSyncIncludePatterns(ctx) + if err != nil { + return nil, err + } + fl, err := libsync.NewFileList(ctx, b.WorktreeRoot, b.SyncRoot, b.Config.Sync.Paths, includes, b.Config.Sync.Exclude) if err != nil { return nil, err } @@ -106,28 +112,11 @@ func fileTriggerDiag(b *bundle.Bundle, loc string, severity diag.Severity, forma } func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string) (string, string, diag.Diagnostics) { - var diags diag.Diagnostics - // A double star looks recursive but path.Match treats it as two ordinary stars. - if strings.Contains(pattern, "**") { - return "", "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "** in %q is not supported; use * for a single directory level", pattern)) - } - // A POSIX path is absolute on Windows too, so check both flavours like NormalizePaths does. - if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) { - return "", "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q must be relative to the defining YAML file", pattern)) - } - // NormalizePaths has already rewritten YAML-relative globs to be bundle-root - // relative. Join that onto the bundle root, then require the result stay - // under the sync root (an ancestor of the bundle when sync.paths uses ..). - joined := filepath.Join(b.BundleRootPath, filepath.FromSlash(pattern)) - relPattern, err := filepath.Rel(b.SyncRootPath, joined) - if err != nil || !filepath.IsLocal(relPattern) { - return "", "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q is not under the sync root", pattern)) - } - relPattern = filepath.ToSlash(relPattern) - _, err = pathlib.Match(relPattern, "") - if err != nil { - return "", "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) + relPattern, diags := validateFileTriggerPattern(b, loc, pattern) + if diags.HasError() { + return "", "", diags } + h := sha256.New() matches := 0 for _, rel := range syncable { @@ -139,10 +128,9 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string if !matched { continue } - match := filepath.Join(b.SyncRootPath, filepath.FromSlash(rel)) - hash, err := hashFile(match) + hash, err := hashFile(b.SyncRoot, rel) if err != nil { - diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "hash %q: %s", match, err)) + diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "hash %q: %s", rel, err)) continue } h.Write([]byte(rel)) @@ -157,8 +145,34 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string return relPattern, hex.EncodeToString(h.Sum(nil)), diags } -func hashFile(path string) (string, error) { - f, err := os.Open(path) +func validateFileTriggerPattern(b *bundle.Bundle, loc, pattern string) (string, diag.Diagnostics) { + var diags diag.Diagnostics + // A double star looks recursive but path.Match treats it as two ordinary stars. + if strings.Contains(pattern, "**") { + return "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "** in %q is not supported; use * for a single directory level", pattern)) + } + // A POSIX path is absolute on Windows too, so check both flavours like NormalizePaths does. + if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) { + return "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q must be relative to the defining YAML file", pattern)) + } + // NormalizePaths has already rewritten YAML-relative globs to be bundle-root + // relative. Join that onto the bundle root, then require the result stay + // under the sync root (an ancestor of the bundle when sync.paths uses ..). + joined := filepath.Join(b.BundleRootPath, filepath.FromSlash(pattern)) + relPattern, err := filepath.Rel(b.SyncRootPath, joined) + if err != nil || !filepath.IsLocal(relPattern) { + return "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q is not under the sync root", pattern)) + } + relPattern = filepath.ToSlash(relPattern) + _, err = pathlib.Match(relPattern, "") + if err != nil { + return "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) + } + return relPattern, diags +} + +func hashFile(root fs.FS, path string) (string, error) { + f, err := root.Open(path) if err != nil { return "", err } diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 2f083def1ea..e4e96152f3f 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -78,6 +78,22 @@ func TestResolveJobRunFileTriggers(t *testing.T) { b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange["shared.txt"], ) }) + + t.Run("hashes through the sync root", func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "watched.txt"), []byte("native"), 0o644)) + b := bundleWithFileTrigger(dir, "watched.txt") + overlay, err := vfs.Overlay(b.SyncRoot, map[string][]byte{"watched.txt": []byte("overlay")}) + require.NoError(t, err) + b.SyncRoot = overlay + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, + contentHash("watched.txt\x00"+contentHash("overlay")+"\x00"), + b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange["watched.txt"], + ) + }) } func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { @@ -88,7 +104,8 @@ func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { SyncRoot: root, WorktreeRoot: root, Config: config.Root{ - Sync: config.Sync{Paths: []string{"."}}, + Bundle: config.Bundle{Target: "default"}, + Sync: config.Sync{Paths: []string{"."}}, Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ "my_run": { diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index fa31032c689..eff70b05e5f 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -60,12 +60,18 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D Locations: b.Config.GetLocations(path + ".on_bundle_deploy"), }) } - if t.OnFileChange != nil && strings.TrimSpace(*t.OnFileChange) == "" { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.triggers.on_file_change must be non-empty when set", - Locations: b.Config.GetLocations(path + ".on_file_change"), - }) + if t.OnFileChange != nil { + if strings.TrimSpace(*t.OnFileChange) == "" { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "lifecycle.triggers.on_file_change must be non-empty when set", + Locations: b.Config.GetLocations(path + ".on_file_change"), + }) + continue + } + // Report bad patterns at validate time; hashing only runs on deploy. + _, patternDiags := validateFileTriggerPattern(b, path+".on_file_change", *t.OnFileChange) + diags = diags.Extend(patternDiags) } } } diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index eadab7812e5..0b14e8cfbd0 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -111,7 +111,10 @@ func TestValidateJobRunTriggers(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + root := t.TempDir() b := &bundle.Bundle{ + BundleRootPath: root, + SyncRootPath: root, Config: config.Root{ Resources: config.Resources{ JobRuns: map[string]*resources.JobRun{ diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 67236f74766..23d3ca3c6a3 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -122,10 +122,12 @@ type PlanEntry struct { // Gone is set on Delete entries when planning confirmed the resource no longer // exists remotely. Applying such an entry only removes it from the state, without // calling the delete API, and approval prompts do not list it as a deletion. - Gone bool `json:"gone,omitempty"` - NewState *structvar.StructVarJSON `json:"new_state,omitempty"` - RemoteState any `json:"remote_state,omitempty"` - Changes Changes `json:"changes,omitempty"` + Gone bool `json:"gone,omitempty"` + // PersistState applies NewState without invoking the resource when Action is Skip. + PersistState bool `json:"persist_state,omitempty"` + NewState *structvar.StructVarJSON `json:"new_state,omitempty"` + RemoteState any `json:"remote_state,omitempty"` + Changes Changes `json:"changes,omitempty"` } type DependsOnEntry struct { @@ -141,6 +143,9 @@ type ChangeDesc struct { Old any `json:"old,omitempty"` New any `json:"new,omitempty"` Remote any `json:"remote,omitempty"` + + // PersistState keeps the desired state when this change is skipped. + PersistState bool `json:"-"` } // Possible values for Reason field diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index db0a7ab4b3c..5680fb75452 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -62,6 +62,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa action := entry.Action errorPrefix := fmt.Sprintf("cannot %s %s", action, resourceKey) + if action == deployplan.Skip && entry.PersistState { + errorPrefix = "cannot persist state for " + resourceKey + } if action == deployplan.Undefined { logdiag.LogError(ctx, fmt.Errorf("cannot deploy %s: unknown action %q", resourceKey, action)) @@ -123,9 +126,8 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true } - // We don't keep NewState around for 'skip' nodes - - if action != deployplan.Skip { + // Skip nodes carry no new state, unless the plan asked to persist it. + if action != deployplan.Skip || entry.PersistState { if !b.resolveReferences(ctx, resourceKey, entry, errorPrefix, false) { return false } @@ -142,11 +144,22 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } - // TODO: redo calcDiff to downgrade planned action if possible (?) - // - // Success is recorded by the state writes inside Deploy, so a recreate reports - // each of its steps. - err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) + if action == deployplan.Skip { + // Persist-state skip: write the new state under the existing id + // without calling the resource API. + id := b.StateDB.GetResourceID(resourceKey) + if id == "" { + logdiag.LogError(ctx, fmt.Errorf("%s: internal error: missing entry in state", errorPrefix)) + return false + } + err = b.StateDB.SaveState(resourceKey, id, sv.Value, entry.DependsOn) + } else { + // TODO: redo calcDiff to downgrade planned action if possible (?) + // + // Success is recorded by the state writes inside Deploy, so a recreate reports + // each of its steps. + err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) + } if err != nil { // Empty for a create that never got an ID, and for a recreate whose delete // step already dropped it. diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 48d76abe97f..c50bcddeb8d 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -343,6 +343,16 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } else { action = getMaxAction(entry.Changes) } + // A skipped entry with a persist-state change (e.g. a cleared trigger) + // still writes its new state; propagate the flag to the entry. + if action == deployplan.Skip { + for _, change := range entry.Changes { + if change.PersistState { + entry.PersistState = true + break + } + } + } // Note, this unconditionally stores remoteState. However, it may updated post-deploy, so whether // it can be used for variable resolution depends on several factors, see canReadRemoteCache in LookupReferencePreDeploy @@ -362,8 +372,10 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, errors.New("planning failed") } + // Skipped entries drop their state, unless it must be persisted (e.g. clearing + // a trigger fingerprint) so re-adding it later is a real change. for _, entry := range plan.Plan { - if entry.Action == deployplan.Skip { + if entry.Action == deployplan.Skip && !entry.PersistState { entry.NewState = nil } } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 5d0847b8773..ed191aab0af 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -375,14 +375,15 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // still going, so a run that may yet succeed is not recreated. A run that // stopped without succeeding keeps its recreate. A SKIPPED run reports no // result_state either, so the lifecycle state is what tells the two apart. -// Clearing a trigger skips its local-only fingerprint without re-firing the run. -// All other trigger changes retain the resource's default recreate action. +// Clearing a trigger skips the run but persists the cleared fingerprint, so +// re-adding it later re-fires. All other trigger changes recreate the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { switch { case path.Len() == jobRunLifecyclePath.Len() && path.HasPrefix(jobRunLifecyclePath): if change.New == nil { change.Action = deployplan.Skip change.Reason = "trigger removed" + change.PersistState = true } else if change.Old != nil { // Trigger fields classify the change, including a removed pattern. change.Reason = deployplan.ReasonDrop @@ -391,6 +392,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa if change.New == nil || change.New == "" { change.Action = deployplan.Skip change.Reason = "trigger removed" + change.PersistState = true } case path.Len() == jobRunOnFileChangePath.Len() && path.HasPrefix(jobRunOnFileChangePath): if isEmptyFileTriggerMap(change.New) { @@ -399,6 +401,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa } else { change.Action = deployplan.Skip change.Reason = "trigger removed" + change.PersistState = true } } else if change.Old != nil { // Pattern entries classify the change. @@ -408,6 +411,7 @@ func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.Pa if change.New == nil { change.Action = deployplan.Skip change.Reason = "trigger removed" + change.PersistState = true } case path.Len() == jobRunResultStatePath.Len() && path.HasPrefix(jobRunResultStatePath): // The planner passes no remote state when the run could not be read. diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index cb22ad7c55c..a9b936cc91c 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -435,6 +435,7 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath(tt.path), change, nil)) assert.Equal(t, tt.action, change.Action) assert.Equal(t, tt.reason, change.Reason) + assert.Equal(t, tt.reason == "trigger removed", change.PersistState) }) } } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 4025c55c40b..19ad1c2124e 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -184,14 +184,24 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. - bundle.ApplySeqContext( - ctx, b, - scripts.Execute(config.ScriptPreDeploy), - lock.Acquire(lock.GoalDeploy), - ) + bundle.ApplyContext( + ctx, b, scripts.Execute(config.ScriptPreDeploy)) + + if logdiag.HasError(ctx) { + return + } + + // Fingerprint after predeploy so build and script outputs are seen. A loaded + // plan already carries its fingerprints, so only recompute when planning here. + if plan == nil { + bundle.ApplyContext(ctx, b, mutator.ResolveJobRunFileTriggers()) + if logdiag.HasError(ctx) { + return + } + } + bundle.ApplyContext(ctx, b, lock.Acquire(lock.GoalDeploy)) if logdiag.HasError(ctx) { - // lock is not acquired here return } diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 76965182dce..9d9c682ef62 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -194,9 +194,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Reject invalid job_runs.lifecycle.triggers (empty, false, prevent_destroy). mutator.ValidateJobRunTriggers(), - // Expand on_file_change globs and hash matched files into triggers_state. - mutator.ResolveJobRunFileTriggers(), - // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index f273e576814..da4a7af9171 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -422,6 +422,16 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } } + // Resolve file-trigger fingerprints after build scripts and generated overlays + // have produced the exact files that a plan would deploy. Deploy resolves them + // after predeploy instead; a loaded plan already contains its fingerprints. + if !opts.SkipInitialize && !opts.Deploy { + bundle.ApplyContext(ctx, b, mutator.ResolveJobRunFileTriggers()) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } + if opts.PreDeployChecks { downgradeWarningToError := !opts.Deploy phases.PreDeployChecks(ctx, b, downgradeWarningToError, stateDesc.Engine) From 85f00b710593c2ec0ed00096e16fec313d5bda74 Mon Sep 17 00:00:00 2001 From: Rada Kamysheva Date: Fri, 28 Aug 2026 11:08:07 +0000 Subject: [PATCH 34/60] job_runs: drop redundant on_file_change validation and unit tests ValidateJobRunTriggers already runs during initialize, and acceptance covers the pattern cases that duplicated the mutator unit tests. --- .../resolve_job_run_file_triggers_test.go | 104 +++--------------- 1 file changed, 18 insertions(+), 86 deletions(-) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index e4e96152f3f..bb4676be531 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -16,91 +16,14 @@ import ( "github.com/stretchr/testify/require" ) -func TestResolveJobRunFileTriggers(t *testing.T) { - t.Run("fingerprints the matched file set", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "a.txt"), []byte("hello"), 0o644)) - require.NoError(t, os.WriteFile(filepath.Join(dir, "b.txt"), []byte("world"), 0o644)) - - pattern := "*.txt" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - - fingerprints := b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange - require.Len(t, fingerprints, 1) - assert.Equal(t, contentHash( - "a.txt\x00"+contentHash("hello")+"\x00"+ - "b.txt\x00"+contentHash("world")+"\x00", - ), fingerprints["*.txt"]) - }) - - t.Run("rejects an absolute pattern", func(t *testing.T) { - dir := t.TempDir() - pattern := "/etc/passwd" - b := bundleWithFileTrigger(dir, pattern) - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.True(t, diags.HasError()) - require.Equal(t, `lifecycle.triggers.on_file_change: pattern "/etc/passwd" must be relative to the defining YAML file`, diags[0].Summary) - assert.Empty(t, b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange) - }) - - t.Run("missing pattern is keyed relative to the sync root", func(t *testing.T) { - parent := t.TempDir() - bundleDir := filepath.Join(parent, "bundle") - require.NoError(t, os.Mkdir(bundleDir, 0o755)) - - pattern := "../missing.txt" - b := bundleWithFileTrigger(parent, pattern) - b.BundleRootPath = bundleDir - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - assert.Equal(t, map[string]string{"missing.txt": contentHash("")}, b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange) - }) - - t.Run("globs from the bundle root when the sync root is an ancestor", func(t *testing.T) { - parent := t.TempDir() - bundleDir := filepath.Join(parent, "bundle") - require.NoError(t, os.Mkdir(bundleDir, 0o755)) - require.NoError(t, os.WriteFile(filepath.Join(parent, "shared.txt"), []byte("from-sync-root"), 0o644)) - - pattern := "../shared.txt" - b := bundleWithFileTrigger(parent, pattern) - b.BundleRootPath = bundleDir - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - assert.Equal(t, - contentHash("shared.txt\x00"+contentHash("from-sync-root")+"\x00"), - b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange["shared.txt"], - ) - }) - - t.Run("hashes through the sync root", func(t *testing.T) { - dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "watched.txt"), []byte("native"), 0o644)) - b := bundleWithFileTrigger(dir, "watched.txt") - overlay, err := vfs.Overlay(b.SyncRoot, map[string][]byte{"watched.txt": []byte("overlay")}) - require.NoError(t, err) - b.SyncRoot = overlay - - diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) - require.False(t, diags.HasError()) - assert.Equal(t, - contentHash("watched.txt\x00"+contentHash("overlay")+"\x00"), - b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange["watched.txt"], - ) - }) -} - -func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { - root := vfs.MustNew(syncRoot) - return &bundle.Bundle{ - BundleRootPath: syncRoot, - SyncRootPath: syncRoot, +func TestResolveJobRunFileTriggersHashesThroughSyncRoot(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "watched.txt"), []byte("native"), 0o644)) + root := vfs.MustNew(dir) + pattern := "watched.txt" + b := &bundle.Bundle{ + BundleRootPath: dir, + SyncRootPath: dir, SyncRoot: root, WorktreeRoot: root, Config: config.Root{ @@ -113,13 +36,22 @@ func bundleWithFileTrigger(syncRoot, pattern string) *bundle.Bundle { Triggers: []resources.JobRunTrigger{ {OnFileChange: &pattern}, }, - TriggersState: nil, }, }, }, }, }, } + overlay, err := vfs.Overlay(b.SyncRoot, map[string][]byte{"watched.txt": []byte("overlay")}) + require.NoError(t, err) + b.SyncRoot = overlay + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.False(t, diags.HasError()) + assert.Equal(t, + contentHash("watched.txt\x00"+contentHash("overlay")+"\x00"), + b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange["watched.txt"], + ) } func contentHash(content string) string { From d097bf9c21fa60b09764ba9b7715bad07ccafc7d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 3 Sep 2026 09:48:10 +0200 Subject: [PATCH 35/60] nextchanges: conform on_file_change fragment to the bullet+PR-link format Co-authored-by: Isaac --- .nextchanges/bundles/job-runs-on-file-change.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md index 95869a25ccf..6b48c6f76a0 100644 --- a/.nextchanges/bundles/job-runs-on-file-change.md +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -1 +1 @@ -direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change (content hash), or when matches appear or disappear. Removing the trigger does not recreate the existing run. +* direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change (content hash), or when matches appear or disappear. Removing the trigger does not recreate the existing run. ([#6309](https://github.com/databricks/cli/pull/6309)) From 318fb127bb74a9c2ce91b446e4e6a5d85faaa7df Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 3 Sep 2026 10:56:12 +0200 Subject: [PATCH 36/60] acc: record the inherited DMS matrix in the new on_file_change goldens Co-authored-by: Isaac --- .../bundle/resources/job_runs/on_file_change/out.test.toml | 1 + .../bundle/resources/job_runs/on_file_change_build/out.test.toml | 1 + .../resources/job_runs/on_file_change_errors/out.test.toml | 1 + .../bundle/resources/job_runs/on_file_change_glob/out.test.toml | 1 + .../job_runs/on_file_change_include_sync_root/out.test.toml | 1 + 5 files changed, 5 insertions(+) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml index 57b0f616850..dd03baaabb6 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.test.toml @@ -1,3 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_build/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_build/out.test.toml index 0938e678987..59b56a2037c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_build/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_build/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml index 0938e678987..59b56a2037c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_errors/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml index 57b0f616850..dd03baaabb6 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_glob/out.test.toml @@ -1,3 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml index 0938e678987..59b56a2037c 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change_include_sync_root/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] From 5fda71291a8741fefde030a9d958113fcce8f4d5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 3 Sep 2026 11:52:40 +0200 Subject: [PATCH 37/60] job_runs: fingerprint on_file_change once, after the predeploy script Move the predeploy script hook out of phases.Deploy so ProcessBundleRet runs predeploy checks, the predeploy script and then trigger resolution in one sequence. This drops the second ResolveJobRunFileTriggers call site, which existed only because the script ran inside the deploy phase. Add on_file_change_predeploy to cover a predeploy script that rewrites a watched file; only the prebuild hook was covered before. Co-authored-by: Isaac --- .../on_file_change_predeploy/databricks.yml | 22 ++++++++ .../on_file_change_predeploy/out.test.toml | 3 ++ .../on_file_change_predeploy/output.txt | 53 +++++++++++++++++++ .../job_runs/on_file_change_predeploy/script | 18 +++++++ .../on_file_change_predeploy/source.txt | 1 + .../on_file_change_predeploy/watched.txt | 1 + bundle/phases/deploy.go | 20 ++----- cmd/bundle/utils/process.go | 30 ++++++++--- 8 files changed, 123 insertions(+), 25 deletions(-) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_predeploy/databricks.yml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_predeploy/out.test.toml create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_predeploy/script create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_predeploy/source.txt create mode 100644 acceptance/bundle/resources/job_runs/on_file_change_predeploy/watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/databricks.yml b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/databricks.yml new file mode 100644 index 00000000000..ca80131494b --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/databricks.yml @@ -0,0 +1,22 @@ +bundle: + name: job-runs-on-file-change-predeploy + +experimental: + scripts: + predeploy: cp source.txt watched.txt + +resources: + jobs: + my_job: + name: my-job + tasks: + - task_key: main + notebook_task: + notebook_path: /Workspace/test + + job_runs: + my_run: + job_id: ${resources.jobs.my_job.id} + lifecycle: + triggers: + - on_file_change: watched.txt diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/out.test.toml b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/out.test.toml new file mode 100644 index 00000000000..59b56a2037c --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt new file mode 100644 index 00000000000..4778b7e19d7 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/output.txt @@ -0,0 +1,53 @@ + +=== first deploy triggers a run +>>> [CLI] bundle deploy +Executing 'predeploy' script +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-predeploy/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS +Created job_runs.my_run +Created jobs.my_job +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> read_id.py my_job +[MY_JOB_ID] + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +=== a predeploy file change re-fires in the same deploy +>>> [CLI] bundle deploy +Executing 'predeploy' script +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-predeploy/default/files... +Output from job_runs.my_run: id=[MY_RUN_ID_2]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID] +Output from job_runs.my_run: id=[MY_RUN_ID_2]: SUCCESS +Recreated job_runs.my_run +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> print_requests.py //jobs/run-now +{ + "method": "POST", + "path": "/api/2.2/jobs/run-now", + "body": { + "idempotency_token": "[UUID]", + "job_id": [MY_JOB_ID] + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.job_runs.my_run + delete resources.jobs.my_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change-predeploy/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/script b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/script new file mode 100644 index 00000000000..80661900eab --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/script @@ -0,0 +1,18 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "first deploy triggers a run" +trace $CLI bundle deploy +trace read_id.py my_job +# Name the first run so the recreated one becomes [MY_RUN_ID_2]. +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now + +title "a predeploy file change re-fires in the same deploy" +update_file.py source.txt "v1" "v2" +trace $CLI bundle deploy +read_id.py my_run > /dev/null +trace print_requests.py //jobs/run-now diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/source.txt b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/source.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/source.txt @@ -0,0 +1 @@ +v1 diff --git a/acceptance/bundle/resources/job_runs/on_file_change_predeploy/watched.txt b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/watched.txt new file mode 100644 index 00000000000..626799f0f85 --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change_predeploy/watched.txt @@ -0,0 +1 @@ +v1 diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 19ad1c2124e..659537a54f9 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -183,25 +183,11 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand log.Info(ctx, "Phase: deploy") // Core mutators that CRUD resources and modify deployment state. These - // mutators need informed consent if they are potentially destructive. - bundle.ApplyContext( - ctx, b, scripts.Execute(config.ScriptPreDeploy)) - - if logdiag.HasError(ctx) { - return - } - - // Fingerprint after predeploy so build and script outputs are seen. A loaded - // plan already carries its fingerprints, so only recompute when planning here. - if plan == nil { - bundle.ApplyContext(ctx, b, mutator.ResolveJobRunFileTriggers()) - if logdiag.HasError(ctx) { - return - } - } - + // mutators need informed consent if they are potentially destructive. The + // predeploy script ran in ProcessBundleRet, ahead of this phase. bundle.ApplyContext(ctx, b, lock.Acquire(lock.GoalDeploy)) if logdiag.HasError(ctx) { + // lock is not acquired here return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index da4a7af9171..810bd11ef5e 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -10,6 +10,7 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" @@ -19,6 +20,7 @@ import ( "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/scripts" "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/build" @@ -422,20 +424,32 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } } - // Resolve file-trigger fingerprints after build scripts and generated overlays - // have produced the exact files that a plan would deploy. Deploy resolves them - // after predeploy instead; a loaded plan already contains its fingerprints. - if !opts.SkipInitialize && !opts.Deploy { - bundle.ApplyContext(ctx, b, mutator.ResolveJobRunFileTriggers()) + if opts.PreDeployChecks { + downgradeWarningToError := !opts.Deploy + phases.PreDeployChecks(ctx, b, downgradeWarningToError, stateDesc.Engine) + if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } } - if opts.PreDeployChecks { - downgradeWarningToError := !opts.Deploy - phases.PreDeployChecks(ctx, b, downgradeWarningToError, stateDesc.Engine) + // The predeploy script can generate or rewrite files that on_file_change + // watches, so it has to run before those files are fingerprinted below. It + // stays ahead of the deployment lock, as it was when phases.Deploy ran it. + if opts.Deploy { + bundle.ApplyContext(ctx, b, scripts.Execute(config.ScriptPreDeploy)) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } + // Fingerprint on_file_change triggers once, after every step that can produce + // a watched file: build, generated overlays and the predeploy script. Reads + // the sync root that phases.Initialize resolves, so it is skipped along with + // it; `bundle deploy --plan` recomputes fingerprints that the loaded plan then + // overrides with the ones it recorded. + if !opts.SkipInitialize { + bundle.ApplyContext(ctx, b, mutator.ResolveJobRunFileTriggers()) if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } From a66e3580738202f01cdd75dbfbbc0ead59b306c5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 3 Sep 2026 11:52:41 +0200 Subject: [PATCH 38/60] job_runs: drop the duplicate on_file_change pre-scan syncableRelPaths re-derived HasOnFileChange over the job_runs map purely so the mutator stayed cheap to call from more than one place. With a single call site, collect the matching names once and list the sync tree only when that set is non-empty. Sorting the names also keeps diagnostics from several job_runs out of Go map order. Co-authored-by: Isaac --- .../mutator/resolve_job_run_file_triggers.go | 55 +++++++++---------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 81ccc8fc80e..dd59756d823 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -30,14 +30,33 @@ func (*resolveJobRunFileTriggers) Name() string { } func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - syncable, diags := syncableRelPaths(ctx, b) - if diags.HasError() { - return diags - } + var diags diag.Diagnostics + + // Sorted so diagnostics from several job_runs come out in a stable order. + names := make([]string, 0, len(b.Config.Resources.JobRuns)) for name, jr := range b.Config.Resources.JobRuns { - if jr == nil || !jr.HasOnFileChange() { - continue + // A job_run declared with an empty YAML body is a nil entry here. + if jr != nil && jr.HasOnFileChange() { + names = append(names, name) } + } + if len(names) == 0 { + return diags + } + slices.Sort(names) + + // Listing the sync files walks the tree, so only do it once the loop above + // found a pattern that needs matching against it. + syncable, err := listSyncableRelPaths(ctx, b) + if err != nil { + return diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + }) + } + + for _, name := range names { + jr := b.Config.Resources.JobRuns[name] out := make(map[string]string) for i, t := range jr.Lifecycle.Triggers { if t.OnFileChange == nil { @@ -55,30 +74,6 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d return diags } -// syncableRelPaths lists the relative paths sync would upload. -func syncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, diag.Diagnostics) { - var diags diag.Diagnostics - needs := false - for _, jr := range b.Config.Resources.JobRuns { - if jr != nil && jr.HasOnFileChange() { - needs = true - break - } - } - if !needs { - return nil, diags - } - - out, err := listSyncableRelPaths(ctx, b) - if err != nil { - return nil, diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), - }) - } - return out, diags -} - func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, error) { // Match sync's effective include set, not just Sync.Include, so a pattern can // hash the internal and AI-snapshot dirs sync force-includes. From 2eb2468954e5b5d181b1736c730e20a1abbde3fa Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 3 Sep 2026 23:24:16 +0200 Subject: [PATCH 39/60] job_runs: drop removed-trigger changes instead of persisting cleared state Removing a trigger downgraded the change to skip and wrote the cleared fingerprint, so re-adding the trigger re-fired the run even when the watched file had not changed since it last ran. Drop the change instead: state keeps the last hash, and re-adding compares against it, so the run fires only when the file changed while the trigger was disarmed. That removes the only producer of PersistState, so the plan entry field, the change field and the persist-state branches in plan and apply all go away. lifecycle.triggers state no longer needs anything from the shared engine: deployplan, bundle_plan.go and bundle_apply.go are back to their state on main, and skip once again always means no state write. Co-authored-by: Isaac --- .../job_runs/on_bundle_deploy/output.txt | 9 ----- .../job_runs/on_file_change/output.txt | 21 +++++++++-- .../resources/job_runs/on_file_change/script | 10 +++++- bundle/deployplan/plan.go | 13 +++---- bundle/direct/bundle_apply.go | 29 +++++---------- bundle/direct/bundle_plan.go | 14 +------- bundle/direct/dresources/job_run.go | 36 +++++++------------ bundle/direct/dresources/job_run_test.go | 11 +++--- 8 files changed, 58 insertions(+), 85 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt index 8b4ed98cfb7..f69c5934d83 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/output.txt @@ -94,15 +94,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json { - "lifecycle": { - "action": "skip", - "reason": "trigger removed", - "old": { - "triggers_state": { - "on_bundle_deploy": "[UUID]" - } - } - }, "result_state": { "action": "skip", "reason": "remote_already_set", diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index bced8e0b19a..dfd940b4721 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -177,10 +177,27 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now -=== re-adding on_file_change re-fires +=== re-adding on_file_change does not re-fire an unchanged file >>> print_state.py -null +{ + "triggers_state": { + "on_file_change": { + "seed.txt": "[FILE_HASH][1]" + } + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... +Files: 3 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //jobs/run-now +=== editing the file while re-armed re-fires >>> [CLI] bundle plan recreate job_runs.my_run diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index cf392f1e575..b14acdc1944 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -66,7 +66,9 @@ title "bundle deploy\n" $CLI bundle deploy $(readplanarg tmp.plan.json) trace print_requests.py //jobs/run-now -title "re-adding on_file_change re-fires" +title "re-adding on_file_change does not re-fire an unchanged file" +# Removing the trigger left the last hash in state, so re-adding it compares +# against that and only re-fires when the file changed while disarmed. trace print_state.py | jq '.state["resources.job_runs.my_run"].state.lifecycle' update_file.py databricks.yml ' job_id: ${resources.jobs.my_job.id} ' ' job_id: ${resources.jobs.my_job.id} @@ -76,5 +78,11 @@ update_file.py databricks.yml ' job_id: ${resources.jobs.my_job.id} ' trace $CLI bundle plan trace $CLI bundle deploy +trace print_requests.py //jobs/run-now + +title "editing the file while re-armed re-fires" +update_file.py seed.txt "v2" "v3" +trace $CLI bundle plan +trace $CLI bundle deploy read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 23d3ca3c6a3..67236f74766 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -122,12 +122,10 @@ type PlanEntry struct { // Gone is set on Delete entries when planning confirmed the resource no longer // exists remotely. Applying such an entry only removes it from the state, without // calling the delete API, and approval prompts do not list it as a deletion. - Gone bool `json:"gone,omitempty"` - // PersistState applies NewState without invoking the resource when Action is Skip. - PersistState bool `json:"persist_state,omitempty"` - NewState *structvar.StructVarJSON `json:"new_state,omitempty"` - RemoteState any `json:"remote_state,omitempty"` - Changes Changes `json:"changes,omitempty"` + Gone bool `json:"gone,omitempty"` + NewState *structvar.StructVarJSON `json:"new_state,omitempty"` + RemoteState any `json:"remote_state,omitempty"` + Changes Changes `json:"changes,omitempty"` } type DependsOnEntry struct { @@ -143,9 +141,6 @@ type ChangeDesc struct { Old any `json:"old,omitempty"` New any `json:"new,omitempty"` Remote any `json:"remote,omitempty"` - - // PersistState keeps the desired state when this change is skipped. - PersistState bool `json:"-"` } // Possible values for Reason field diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 5680fb75452..db0a7ab4b3c 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -62,9 +62,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa action := entry.Action errorPrefix := fmt.Sprintf("cannot %s %s", action, resourceKey) - if action == deployplan.Skip && entry.PersistState { - errorPrefix = "cannot persist state for " + resourceKey - } if action == deployplan.Undefined { logdiag.LogError(ctx, fmt.Errorf("cannot deploy %s: unknown action %q", resourceKey, action)) @@ -126,8 +123,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true } - // Skip nodes carry no new state, unless the plan asked to persist it. - if action != deployplan.Skip || entry.PersistState { + // We don't keep NewState around for 'skip' nodes + + if action != deployplan.Skip { if !b.resolveReferences(ctx, resourceKey, entry, errorPrefix, false) { return false } @@ -144,22 +142,11 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } - if action == deployplan.Skip { - // Persist-state skip: write the new state under the existing id - // without calling the resource API. - id := b.StateDB.GetResourceID(resourceKey) - if id == "" { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: missing entry in state", errorPrefix)) - return false - } - err = b.StateDB.SaveState(resourceKey, id, sv.Value, entry.DependsOn) - } else { - // TODO: redo calcDiff to downgrade planned action if possible (?) - // - // Success is recorded by the state writes inside Deploy, so a recreate reports - // each of its steps. - err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) - } + // TODO: redo calcDiff to downgrade planned action if possible (?) + // + // Success is recorded by the state writes inside Deploy, so a recreate reports + // each of its steps. + err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { // Empty for a create that never got an ID, and for a recreate whose delete // step already dropped it. diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index c50bcddeb8d..48d76abe97f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -343,16 +343,6 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } else { action = getMaxAction(entry.Changes) } - // A skipped entry with a persist-state change (e.g. a cleared trigger) - // still writes its new state; propagate the flag to the entry. - if action == deployplan.Skip { - for _, change := range entry.Changes { - if change.PersistState { - entry.PersistState = true - break - } - } - } // Note, this unconditionally stores remoteState. However, it may updated post-deploy, so whether // it can be used for variable resolution depends on several factors, see canReadRemoteCache in LookupReferencePreDeploy @@ -372,10 +362,8 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, errors.New("planning failed") } - // Skipped entries drop their state, unless it must be persisted (e.g. clearing - // a trigger fingerprint) so re-adding it later is a real change. for _, entry := range plan.Plan { - if entry.Action == deployplan.Skip && !entry.PersistState { + if entry.Action == deployplan.Skip { entry.NewState = nil } } diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index ed191aab0af..6640b5d726e 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -375,43 +375,31 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // still going, so a run that may yet succeed is not recreated. A run that // stopped without succeeding keeps its recreate. A SKIPPED run reports no // result_state either, so the lifecycle state is what tells the two apart. -// Clearing a trigger skips the run but persists the cleared fingerprint, so -// re-adding it later re-fires. All other trigger changes recreate the run. +// Removing a trigger drops the change and leaves the last fingerprint in state, +// so re-adding the trigger only re-fires when the watched files changed +// meanwhile. All other trigger changes recreate the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { switch { case path.Len() == jobRunLifecyclePath.Len() && path.HasPrefix(jobRunLifecyclePath): - if change.New == nil { - change.Action = deployplan.Skip - change.Reason = "trigger removed" - change.PersistState = true - } else if change.Old != nil { - // Trigger fields classify the change, including a removed pattern. + // Dropped when the trigger is removed (New nil) and when both sides are + // present, where the trigger fields below classify the change instead. + // Arming from no lifecycle at all keeps its recreate. + if change.New == nil || change.Old != nil { change.Reason = deployplan.ReasonDrop } case path.Len() == jobRunOnBundleDeployPath.Len() && path.HasPrefix(jobRunOnBundleDeployPath): if change.New == nil || change.New == "" { - change.Action = deployplan.Skip - change.Reason = "trigger removed" - change.PersistState = true + change.Reason = deployplan.ReasonDrop } case path.Len() == jobRunOnFileChangePath.Len() && path.HasPrefix(jobRunOnFileChangePath): - if isEmptyFileTriggerMap(change.New) { - if isEmptyFileTriggerMap(change.Old) { - change.Reason = deployplan.ReasonDrop - } else { - change.Action = deployplan.Skip - change.Reason = "trigger removed" - change.PersistState = true - } - } else if change.Old != nil { - // Pattern entries classify the change. + // As above: an emptied map is a removal, and pattern entries classify a map + // that still has both sides. + if isEmptyFileTriggerMap(change.New) || change.Old != nil { change.Reason = deployplan.ReasonDrop } case path.Len() == jobRunOnFileChangePath.Len()+1 && path.HasPrefix(jobRunOnFileChangePath): if change.New == nil { - change.Action = deployplan.Skip - change.Reason = "trigger removed" - change.PersistState = true + change.Reason = deployplan.ReasonDrop } case path.Len() == jobRunResultStatePath.Len() && path.HasPrefix(jobRunResultStatePath): // The planner passes no remote state when the run could not be read. diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index a9b936cc91c..d7727b3327e 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -416,17 +416,17 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { action deployplan.ActionType reason string }{ - {"cleared lifecycle", "lifecycle", lifecycle, nil, deployplan.Skip, "trigger removed"}, + {"cleared lifecycle", "lifecycle", lifecycle, nil, deployplan.Recreate, deployplan.ReasonDrop}, {"added lifecycle", "lifecycle", nil, lifecycle, deployplan.Recreate, ""}, {"changed lifecycle", "lifecycle", lifecycle, lifecycle, deployplan.Recreate, deployplan.ReasonDrop}, - {"cleared on_bundle_deploy string", "lifecycle.triggers_state.on_bundle_deploy", "uuid", "", deployplan.Skip, "trigger removed"}, - {"nil on_bundle_deploy", "lifecycle.triggers_state.on_bundle_deploy", "uuid", nil, deployplan.Skip, "trigger removed"}, + {"cleared on_bundle_deploy string", "lifecycle.triggers_state.on_bundle_deploy", "uuid", "", deployplan.Recreate, deployplan.ReasonDrop}, + {"nil on_bundle_deploy", "lifecycle.triggers_state.on_bundle_deploy", "uuid", nil, deployplan.Recreate, deployplan.ReasonDrop}, {"rotated on_bundle_deploy", "lifecycle.triggers_state.on_bundle_deploy", "old-uuid", "new-uuid", deployplan.Recreate, ""}, - {"cleared on_file_change", "lifecycle.triggers_state.on_file_change", map[string]string{"*.txt": "hash"}, nil, deployplan.Skip, "trigger removed"}, + {"cleared on_file_change", "lifecycle.triggers_state.on_file_change", map[string]string{"*.txt": "hash"}, nil, deployplan.Recreate, deployplan.ReasonDrop}, {"empty on_file_change maps", "lifecycle.triggers_state.on_file_change", map[string]string{}, map[string]string{}, deployplan.Recreate, deployplan.ReasonDrop}, {"added on_file_change map", "lifecycle.triggers_state.on_file_change", nil, map[string]string{"*.txt": "hash"}, deployplan.Recreate, ""}, {"changed on_file_change map", "lifecycle.triggers_state.on_file_change", map[string]string{"*.txt": "old"}, map[string]string{"*.txt": "new"}, deployplan.Recreate, deployplan.ReasonDrop}, - {"cleared on_file_change pattern", "lifecycle.triggers_state.on_file_change['*.txt']", "hash", nil, deployplan.Skip, "trigger removed"}, + {"cleared on_file_change pattern", "lifecycle.triggers_state.on_file_change['*.txt']", "hash", nil, deployplan.Recreate, deployplan.ReasonDrop}, {"changed on_file_change pattern", "lifecycle.triggers_state.on_file_change['*.txt']", "old", "new", deployplan.Recreate, ""}, {"result_state with unreadable remote", "result_state", jobs.RunResultStateSuccess, nil, deployplan.Recreate, ""}, } { @@ -435,7 +435,6 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { require.NoError(t, r.OverrideChangeDesc(t.Context(), structpath.MustParsePath(tt.path), change, nil)) assert.Equal(t, tt.action, change.Action) assert.Equal(t, tt.reason, change.Reason) - assert.Equal(t, tt.reason == "trigger removed", change.PersistState) }) } } From 4f39ed8825cbfad57324574d047a508c82cd5418 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 09:51:18 +0200 Subject: [PATCH 40/60] job_runs: revert unrelated churn in type_test.go and expand the override table The knownMissingInRemoteType entry for job_runs was moved and its comment reworded for no reason; the map is not sorted either way, so put it back untouched. Only the knownMissingInStateType path has to change: state renamed triggers to triggers_state, so the whole lifecycle.triggers subtree is now missing from StateType rather than just its elements. Write each TestJobRunOverrideChangeDescTriggerRemoved case one field per line with named fields. Co-authored-by: Isaac --- bundle/direct/dresources/job_run_test.go | 117 ++++++++++++++++++++--- bundle/direct/dresources/type_test.go | 9 +- 2 files changed, 108 insertions(+), 18 deletions(-) diff --git a/bundle/direct/dresources/job_run_test.go b/bundle/direct/dresources/job_run_test.go index d7727b3327e..ecda3243f50 100644 --- a/bundle/direct/dresources/job_run_test.go +++ b/bundle/direct/dresources/job_run_test.go @@ -416,19 +416,110 @@ func TestJobRunOverrideChangeDescTriggerRemoved(t *testing.T) { action deployplan.ActionType reason string }{ - {"cleared lifecycle", "lifecycle", lifecycle, nil, deployplan.Recreate, deployplan.ReasonDrop}, - {"added lifecycle", "lifecycle", nil, lifecycle, deployplan.Recreate, ""}, - {"changed lifecycle", "lifecycle", lifecycle, lifecycle, deployplan.Recreate, deployplan.ReasonDrop}, - {"cleared on_bundle_deploy string", "lifecycle.triggers_state.on_bundle_deploy", "uuid", "", deployplan.Recreate, deployplan.ReasonDrop}, - {"nil on_bundle_deploy", "lifecycle.triggers_state.on_bundle_deploy", "uuid", nil, deployplan.Recreate, deployplan.ReasonDrop}, - {"rotated on_bundle_deploy", "lifecycle.triggers_state.on_bundle_deploy", "old-uuid", "new-uuid", deployplan.Recreate, ""}, - {"cleared on_file_change", "lifecycle.triggers_state.on_file_change", map[string]string{"*.txt": "hash"}, nil, deployplan.Recreate, deployplan.ReasonDrop}, - {"empty on_file_change maps", "lifecycle.triggers_state.on_file_change", map[string]string{}, map[string]string{}, deployplan.Recreate, deployplan.ReasonDrop}, - {"added on_file_change map", "lifecycle.triggers_state.on_file_change", nil, map[string]string{"*.txt": "hash"}, deployplan.Recreate, ""}, - {"changed on_file_change map", "lifecycle.triggers_state.on_file_change", map[string]string{"*.txt": "old"}, map[string]string{"*.txt": "new"}, deployplan.Recreate, deployplan.ReasonDrop}, - {"cleared on_file_change pattern", "lifecycle.triggers_state.on_file_change['*.txt']", "hash", nil, deployplan.Recreate, deployplan.ReasonDrop}, - {"changed on_file_change pattern", "lifecycle.triggers_state.on_file_change['*.txt']", "old", "new", deployplan.Recreate, ""}, - {"result_state with unreadable remote", "result_state", jobs.RunResultStateSuccess, nil, deployplan.Recreate, ""}, + { + name: "cleared lifecycle", + path: "lifecycle", + old: lifecycle, + new: nil, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "added lifecycle", + path: "lifecycle", + old: nil, + new: lifecycle, + action: deployplan.Recreate, + reason: "", + }, + { + name: "changed lifecycle", + path: "lifecycle", + old: lifecycle, + new: lifecycle, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "cleared on_bundle_deploy string", + path: "lifecycle.triggers_state.on_bundle_deploy", + old: "uuid", + new: "", + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "nil on_bundle_deploy", + path: "lifecycle.triggers_state.on_bundle_deploy", + old: "uuid", + new: nil, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "rotated on_bundle_deploy", + path: "lifecycle.triggers_state.on_bundle_deploy", + old: "old-uuid", + new: "new-uuid", + action: deployplan.Recreate, + reason: "", + }, + { + name: "cleared on_file_change", + path: "lifecycle.triggers_state.on_file_change", + old: map[string]string{"*.txt": "hash"}, + new: nil, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "empty on_file_change maps", + path: "lifecycle.triggers_state.on_file_change", + old: map[string]string{}, + new: map[string]string{}, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "added on_file_change map", + path: "lifecycle.triggers_state.on_file_change", + old: nil, + new: map[string]string{"*.txt": "hash"}, + action: deployplan.Recreate, + reason: "", + }, + { + name: "changed on_file_change map", + path: "lifecycle.triggers_state.on_file_change", + old: map[string]string{"*.txt": "old"}, + new: map[string]string{"*.txt": "new"}, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "cleared on_file_change pattern", + path: "lifecycle.triggers_state.on_file_change['*.txt']", + old: "hash", + new: nil, + action: deployplan.Recreate, + reason: deployplan.ReasonDrop, + }, + { + name: "changed on_file_change pattern", + path: "lifecycle.triggers_state.on_file_change['*.txt']", + old: "old", + new: "new", + action: deployplan.Recreate, + reason: "", + }, + { + name: "result_state with unreadable remote", + path: "result_state", + old: jobs.RunResultStateSuccess, + new: nil, + action: deployplan.Recreate, + reason: "", + }, } { t.Run(tt.name, func(t *testing.T) { change := &ChangeDesc{Action: deployplan.Recreate, Old: tt.old, New: tt.new} diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 6bdf973278b..a82a3447e42 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -19,11 +19,6 @@ var knownMissingInRemoteType = map[string][]string{ "external_locations": { "skip_validation", }, - "job_runs": { - // Deliberate omission: the trigger fingerprints are client-side only, so - // GetRun has nothing to report them under. - "lifecycle", - }, "model_serving_endpoints": { "rate_limits", }, @@ -61,6 +56,10 @@ var knownMissingInRemoteType = map[string][]string{ "vector_search_endpoints": { "usage_policy_id", }, + "job_runs": { + // Local-only trigger fingerprints under lifecycle. + "lifecycle", + }, "internal_immutable_snapshots": { "bundle_id", "acl", From 4d2480b08e956d0e39fed3d387f8908ea7ee9765 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 10:46:58 +0200 Subject: [PATCH 41/60] update test --- .../job_runs/on_file_change/output.txt | 193 ++++++++++-------- .../resources/job_runs/on_file_change/script | 50 ++--- .../job_runs/on_file_change/test.toml | 2 +- 3 files changed, 135 insertions(+), 110 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index dfd940b4721..f1527678167 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -1,12 +1,12 @@ === first deploy triggers a run ->>> [CLI] bundle deploy +>>> [CLI] bundle plan -o json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 6 uploaded, 0 deleted +Files: 7 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> read_id.py my_job @@ -23,9 +23,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === redeploy with unchanged file plans nothing ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Files: 2 uploaded, 0 deleted @@ -34,9 +31,6 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now === touch without content change plans nothing ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Files: 3 uploaded, 0 deleted @@ -45,24 +39,53 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now === editing the file re-fires ->>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> jq .plan["resources.job_runs.my_run"].changes tmp.plan.json +>>> jq .plan["resources.job_runs.my_run"] tmp.plan1.json { - "lifecycle.triggers_state.on_file_change['seed.txt']": { - "action": "recreate", - "reason": "immutable", - "old": "[FILE_HASH][0]", - "new": "[FILE_HASH][1]" + "depends_on": [ + { + "node": "resources.jobs.my_job", + "label": "${resources.jobs.my_job.id}" + } + ], + "action": "recreate", + "new_state": { + "value": { + "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers_state": { + "on_file_change": { + "seed.txt": "[FILE_HASH][0]" + } + } + }, + "result_state": "SUCCESS" + } + }, + "remote_state": { + "job_id": [MY_JOB_ID], + "result_state": "SUCCESS", + "run_id": [MY_RUN_ID], + "run_name": "my-job", + "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID]?o=[NUMID]", + "run_type": "JOB_RUN", + "state": { + "life_cycle_state": "TERMINATED", + "result_state": "SUCCESS" + } }, - "result_state": { - "action": "skip", - "reason": "remote_already_set", - "new": "SUCCESS", - "remote": "SUCCESS" + "changes": { + "lifecycle.triggers_state.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][1]", + "new": "[FILE_HASH][0]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } } } @@ -93,35 +116,64 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } -=== deleting the file warns and re-fires >>> [CLI] bundle plan -Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" - in databricks.yml:18:29 - -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged ->>> changes +=== deleting the file warns and re-fires +>>> [CLI] bundle plan -o json Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" in databricks.yml:18:29 + +>>> jq .plan["resources.job_runs.my_run"] tmp.plan2.json { - "lifecycle.triggers_state.on_file_change['seed.txt']": { - "action": "recreate", - "reason": "immutable", - "old": "[FILE_HASH][1]", - "new": "[FILE_HASH][2]" + "depends_on": [ + { + "node": "resources.jobs.my_job", + "label": "${resources.jobs.my_job.id}" + } + ], + "action": "recreate", + "new_state": { + "value": { + "job_id": [MY_JOB_ID], + "lifecycle": { + "triggers_state": { + "on_file_change": { + "seed.txt": "[FILE_HASH][2]" + } + } + }, + "result_state": "SUCCESS" + } }, - "result_state": { - "action": "skip", - "reason": "remote_already_set", - "new": "SUCCESS", - "remote": "SUCCESS" + "remote_state": { + "job_id": [MY_JOB_ID], + "result_state": "SUCCESS", + "run_id": [MY_RUN_ID_2], + "run_name": "my-job", + "run_page_url": "[DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_2]?o=[NUMID]", + "run_type": "JOB_RUN", + "state": { + "life_cycle_state": "TERMINATED", + "result_state": "SUCCESS" + } + }, + "changes": { + "lifecycle.triggers_state.on_file_change['seed.txt']": { + "action": "recreate", + "reason": "immutable", + "old": "[FILE_HASH][0]", + "new": "[FILE_HASH][2]" + }, + "result_state": { + "action": "skip", + "reason": "remote_already_set", + "new": "SUCCESS", + "remote": "SUCCESS" + } } } - ->>> [CLI] bundle deploy Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" in databricks.yml:18:29 @@ -129,7 +181,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-c Output from job_runs.my_run: id=[MY_RUN_ID_3]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_3]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_3]: SUCCESS Recreated job_runs.my_run -Files: 3 uploaded, 1 deleted +Files: 4 uploaded, 1 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> print_requests.py //jobs/run-now @@ -142,18 +194,19 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } -=== restoring the file re-fires again >>> [CLI] bundle plan -recreate job_runs.my_run +Warning: lifecycle.triggers.on_file_change: no synced files match "seed.txt" + in databricks.yml:18:29 -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged ->>> [CLI] bundle deploy +=== restoring the file re-fires again +>>> [CLI] bundle plan -o json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... Output from job_runs.my_run: id=[MY_RUN_ID_4]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_4]?o=[NUMID] Output from job_runs.my_run: id=[MY_RUN_ID_4]: SUCCESS Recreated job_runs.my_run -Files: 3 uploaded, 1 deleted +Files: 4 uploaded, 1 deleted Resources: 1 created, 0 changed, 1 deleted, 1 unchanged >>> print_requests.py //jobs/run-now @@ -166,6 +219,9 @@ Resources: 1 created, 0 changed, 1 deleted, 1 unchanged } } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + === removing on_file_change is unchanged and does not run >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -177,49 +233,26 @@ Resources: 0 created, 0 changed, 0 deleted, 2 unchanged >>> print_requests.py //jobs/run-now +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + === re-adding on_file_change does not re-fire an unchanged file >>> print_state.py { "triggers_state": { "on_file_change": { - "seed.txt": "[FILE_HASH][1]" + "seed.txt": "[FILE_HASH][0]" } } } ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - ->>> [CLI] bundle deploy +>>> [CLI] bundle plan -o json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -Files: 3 uploaded, 0 deleted +Files: 4 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged ->>> print_requests.py //jobs/run-now - -=== editing the file while re-armed re-fires >>> [CLI] bundle plan -recreate job_runs.my_run - -Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/files... -Output from job_runs.my_run: id=[MY_RUN_ID_5]: Run URL: [DATABRICKS_URL]/jobs/[MY_JOB_ID]/runs/[MY_RUN_ID_5]?o=[NUMID] -Output from job_runs.my_run: id=[MY_RUN_ID_5]: SUCCESS -Recreated job_runs.my_run -Files: 3 uploaded, 0 deleted -Resources: 1 created, 0 changed, 1 deleted, 1 unchanged - ->>> print_requests.py //jobs/run-now -{ - "method": "POST", - "path": "/api/2.2/jobs/run-now", - "body": { - "idempotency_token": "[UUID]", - "job_id": [MY_JOB_ID] - } -} +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index b14acdc1944..7569a172e9e 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -4,55 +4,54 @@ cleanup() { } trap cleanup EXIT -changes() { - $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"].changes' +plan_no_changes() { + trace $CLI bundle plan | contains.py "0 to add, 0 to change" } title "first deploy triggers a run" -trace $CLI bundle deploy +trace $CLI bundle plan -o json > out.plan.create.json +$CLI bundle deploy $(readplanarg out.plan.create.json) trace read_id.py my_job # Name the first run so the recreated one becomes [MY_RUN_ID_2]. read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now title "redeploy with unchanged file plans nothing" -trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //jobs/run-now title "touch without content change plans nothing" python -c "import os; os.utime('seed.txt', None)" -trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py //jobs/run-now title "editing the file re-fires" update_file.py seed.txt "v1" "v2" -trace $CLI bundle plan -# Save the plan so the READPLAN=1 variant deploys the hashes computed here. -# The deploy is not traced: readplanarg makes the command line differ per variant. -$CLI bundle plan -o json > tmp.plan.json -trace jq '.plan["resources.job_runs.my_run"].changes' tmp.plan.json +$CLI bundle plan -o json > tmp.plan1.json +trace jq '.plan["resources.job_runs.my_run"]' tmp.plan1.json title "bundle deploy\n" -$CLI bundle deploy $(readplanarg tmp.plan.json) +$CLI bundle deploy $(readplanarg tmp.plan1.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now +plan_no_changes title "deleting the file warns and re-fires" mv seed.txt tmp.seed.txt -trace $CLI bundle plan -trace changes -trace $CLI bundle deploy +trace $CLI bundle plan -o json > tmp.plan2.json +trace jq '.plan["resources.job_runs.my_run"]' tmp.plan2.json +$CLI bundle deploy $(readplanarg tmp.plan2.json) read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now +plan_no_changes title "restoring the file re-fires again" mv tmp.seed.txt seed.txt -trace $CLI bundle plan -trace $CLI bundle deploy +trace $CLI bundle plan -o json > tmp.plan3.json +$CLI bundle deploy $(readplanarg tmp.plan3.json) read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now +plan_no_changes title "removing on_file_change is unchanged and does not run" update_file.py databricks.yml " lifecycle: @@ -60,11 +59,11 @@ update_file.py databricks.yml " lifecycle: - on_file_change: seed.txt " "" trace $CLI bundle plan -# Save the removal too, so READPLAN=1 verifies state-only updates survive serialization. -$CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json > tmp.plan4.json title "bundle deploy\n" -$CLI bundle deploy $(readplanarg tmp.plan.json) +$CLI bundle deploy $(readplanarg tmp.plan4.json) trace print_requests.py //jobs/run-now +plan_no_changes title "re-adding on_file_change does not re-fire an unchanged file" # Removing the trigger left the last hash in state, so re-adding it compares @@ -76,13 +75,6 @@ update_file.py databricks.yml ' job_id: ${resources.jobs.my_job.id} triggers: - on_file_change: seed.txt ' -trace $CLI bundle plan -trace $CLI bundle deploy -trace print_requests.py //jobs/run-now - -title "editing the file while re-armed re-fires" -update_file.py seed.txt "v2" "v3" -trace $CLI bundle plan -trace $CLI bundle deploy -read_id.py my_run > /dev/null -trace print_requests.py //jobs/run-now +trace $CLI bundle plan -o json > tmp.plan5.json +$CLI bundle deploy $(readplanarg tmp.plan5.json) +plan_no_changes diff --git a/acceptance/bundle/resources/job_runs/on_file_change/test.toml b/acceptance/bundle/resources/job_runs/on_file_change/test.toml index 7a950e9fd0c..f5010a202d7 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/test.toml +++ b/acceptance/bundle/resources/job_runs/on_file_change/test.toml @@ -2,7 +2,7 @@ # computed during planning survive plan serialization. EnvMatrix.READPLAN = ["", "1"] -Ignore = ["tmp.plan.json", "tmp.seed.txt"] +Ignore = ["tmp.*.*"] # Content hashes are stable but unreadable, and the parent's `\d{8,}` rule # mangles any digit run inside them. Order=1 runs before it; Distinct keeps From 0dbf58cb34f98e0c26087a142a4fe87794dff3ff Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 10:53:03 +0200 Subject: [PATCH 42/60] acc: inline plan_no_changes in the on_file_change script Keep the assertion at each call site and hoist only the expected substring, which is the one literal inlining would have repeated five times. Co-authored-by: Isaac --- .../resources/job_runs/on_file_change/script | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index 7569a172e9e..cfe201342dd 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -4,9 +4,9 @@ cleanup() { } trap cleanup EXIT -plan_no_changes() { - trace $CLI bundle plan | contains.py "0 to add, 0 to change" -} +# Asserted after every deploy: the fingerprints just written match the files on +# disk, so a second plan has nothing left to do. +NO_CHANGES="0 to add, 0 to change" title "first deploy triggers a run" trace $CLI bundle plan -o json > out.plan.create.json @@ -34,7 +34,7 @@ $CLI bundle deploy $(readplanarg tmp.plan1.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now -plan_no_changes +trace $CLI bundle plan | contains.py "$NO_CHANGES" title "deleting the file warns and re-fires" mv seed.txt tmp.seed.txt @@ -43,7 +43,7 @@ trace jq '.plan["resources.job_runs.my_run"]' tmp.plan2.json $CLI bundle deploy $(readplanarg tmp.plan2.json) read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -plan_no_changes +trace $CLI bundle plan | contains.py "$NO_CHANGES" title "restoring the file re-fires again" mv tmp.seed.txt seed.txt @@ -51,7 +51,7 @@ trace $CLI bundle plan -o json > tmp.plan3.json $CLI bundle deploy $(readplanarg tmp.plan3.json) read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -plan_no_changes +trace $CLI bundle plan | contains.py "$NO_CHANGES" title "removing on_file_change is unchanged and does not run" update_file.py databricks.yml " lifecycle: @@ -63,7 +63,7 @@ $CLI bundle plan -o json > tmp.plan4.json title "bundle deploy\n" $CLI bundle deploy $(readplanarg tmp.plan4.json) trace print_requests.py //jobs/run-now -plan_no_changes +trace $CLI bundle plan | contains.py "$NO_CHANGES" title "re-adding on_file_change does not re-fire an unchanged file" # Removing the trigger left the last hash in state, so re-adding it compares @@ -77,4 +77,4 @@ update_file.py databricks.yml ' job_id: ${resources.jobs.my_job.id} ' trace $CLI bundle plan -o json > tmp.plan5.json $CLI bundle deploy $(readplanarg tmp.plan5.json) -plan_no_changes +trace $CLI bundle plan | contains.py "$NO_CHANGES" From 04e72c83b33fdf53fc7ad3a0035bf15d51c521f0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 11:06:17 +0200 Subject: [PATCH 43/60] Revert "acc: inline plan_no_changes in the on_file_change script" This reverts 364ff5762. plan_no_changes was not the helper meant to be inlined. Co-authored-by: Isaac --- .../resources/job_runs/on_file_change/script | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index cfe201342dd..7569a172e9e 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -4,9 +4,9 @@ cleanup() { } trap cleanup EXIT -# Asserted after every deploy: the fingerprints just written match the files on -# disk, so a second plan has nothing left to do. -NO_CHANGES="0 to add, 0 to change" +plan_no_changes() { + trace $CLI bundle plan | contains.py "0 to add, 0 to change" +} title "first deploy triggers a run" trace $CLI bundle plan -o json > out.plan.create.json @@ -34,7 +34,7 @@ $CLI bundle deploy $(readplanarg tmp.plan1.json) read_id.py my_run > /dev/null trace print_requests.py --keep //jobs/runs/delete trace print_requests.py //jobs/run-now -trace $CLI bundle plan | contains.py "$NO_CHANGES" +plan_no_changes title "deleting the file warns and re-fires" mv seed.txt tmp.seed.txt @@ -43,7 +43,7 @@ trace jq '.plan["resources.job_runs.my_run"]' tmp.plan2.json $CLI bundle deploy $(readplanarg tmp.plan2.json) read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -trace $CLI bundle plan | contains.py "$NO_CHANGES" +plan_no_changes title "restoring the file re-fires again" mv tmp.seed.txt seed.txt @@ -51,7 +51,7 @@ trace $CLI bundle plan -o json > tmp.plan3.json $CLI bundle deploy $(readplanarg tmp.plan3.json) read_id.py my_run > /dev/null trace print_requests.py //jobs/run-now -trace $CLI bundle plan | contains.py "$NO_CHANGES" +plan_no_changes title "removing on_file_change is unchanged and does not run" update_file.py databricks.yml " lifecycle: @@ -63,7 +63,7 @@ $CLI bundle plan -o json > tmp.plan4.json title "bundle deploy\n" $CLI bundle deploy $(readplanarg tmp.plan4.json) trace print_requests.py //jobs/run-now -trace $CLI bundle plan | contains.py "$NO_CHANGES" +plan_no_changes title "re-adding on_file_change does not re-fire an unchanged file" # Removing the trigger left the last hash in state, so re-adding it compares @@ -77,4 +77,4 @@ update_file.py databricks.yml ' job_id: ${resources.jobs.my_job.id} ' trace $CLI bundle plan -o json > tmp.plan5.json $CLI bundle deploy $(readplanarg tmp.plan5.json) -trace $CLI bundle plan | contains.py "$NO_CHANGES" +plan_no_changes From 1fdcae89b1f8597329bfe64130b88829df86d298 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 11:06:18 +0200 Subject: [PATCH 44/60] acc: commit the on_file_change create-plan golden The script writes out.plan.create.json, so the file has to be checked in: with it missing the run fails with "Test produced unexpected files". Co-authored-by: Isaac --- .../on_file_change/out.plan.create.json | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 acceptance/bundle/resources/job_runs/on_file_change/out.plan.create.json diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.plan.create.json b/acceptance/bundle/resources/job_runs/on_file_change/out.plan.create.json new file mode 100644 index 00000000000..50ba69a707f --- /dev/null +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.plan.create.json @@ -0,0 +1,57 @@ +{ + "plan_version": 2, + "cli_version": "[CLI_VERSION]", + "plan": { + "resources.job_runs.my_run": { + "depends_on": [ + { + "node": "resources.jobs.my_job", + "label": "${resources.jobs.my_job.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "job_id": 0, + "lifecycle": { + "triggers_state": { + "on_file_change": { + "seed.txt": "[FILE_HASH]" + } + } + }, + "result_state": "SUCCESS" + }, + "vars": { + "job_id": "${resources.jobs.my_job.id}" + } + } + }, + "resources.jobs.my_job": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/job-runs-on-file-change/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "my-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "notebook_task": { + "notebook_path": "/Workspace/test" + }, + "task_key": "main" + } + ] + } + } + } + } +} From df0746e42355071831bb636e2408171e1c632143 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 11:06:18 +0200 Subject: [PATCH 45/60] job_runs: inline fileTriggerDiag Build each diagnostic at its call site so severity and locations are visible there, keeping only the shared summary prefix as a constant. Co-authored-by: Isaac --- .../mutator/resolve_job_run_file_triggers.go | 56 +++++++++++++------ 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index dd59756d823..eff981b7df2 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -18,6 +18,9 @@ import ( libsync "github.com/databricks/cli/libs/sync" ) +// Every diagnostic below is reported against an on_file_change entry. +const fileTriggerPrefix = "lifecycle.triggers.on_file_change: " + type resolveJobRunFileTriggers struct{} // ResolveJobRunFileTriggers expands on_file_change globs into trigger state. @@ -51,7 +54,7 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d if err != nil { return diags.Append(diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("lifecycle.triggers.on_file_change: list sync files: %s", err), + Summary: fileTriggerPrefix + fmt.Sprintf("list sync files: %s", err), }) } @@ -97,15 +100,6 @@ func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, erro return out, nil } -// fileTriggerDiag reports a diagnostic against the on_file_change entry at loc. -func fileTriggerDiag(b *bundle.Bundle, loc string, severity diag.Severity, format string, args ...any) diag.Diagnostic { - return diag.Diagnostic{ - Severity: severity, - Summary: "lifecycle.triggers.on_file_change: " + fmt.Sprintf(format, args...), - Locations: b.Config.GetLocations(loc), - } -} - func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string) (string, string, diag.Diagnostics) { relPattern, diags := validateFileTriggerPattern(b, loc, pattern) if diags.HasError() { @@ -117,7 +111,11 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string for _, rel := range syncable { matched, err := pathlib.Match(relPattern, rel) if err != nil { - diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(loc), + }) continue } if !matched { @@ -125,7 +123,11 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string } hash, err := hashFile(b.SyncRoot, rel) if err != nil { - diags = diags.Append(fileTriggerDiag(b, loc, diag.Error, "hash %q: %s", rel, err)) + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("hash %q: %s", rel, err), + Locations: b.Config.GetLocations(loc), + }) continue } h.Write([]byte(rel)) @@ -135,7 +137,11 @@ func resolveFileTrigger(b *bundle.Bundle, loc, pattern string, syncable []string matches++ } if matches == 0 && !diags.HasError() { - diags = diags.Append(fileTriggerDiag(b, loc, diag.Warning, "no synced files match %q", pattern)) + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: fileTriggerPrefix + fmt.Sprintf("no synced files match %q", pattern), + Locations: b.Config.GetLocations(loc), + }) } return relPattern, hex.EncodeToString(h.Sum(nil)), diags } @@ -144,11 +150,19 @@ func validateFileTriggerPattern(b *bundle.Bundle, loc, pattern string) (string, var diags diag.Diagnostics // A double star looks recursive but path.Match treats it as two ordinary stars. if strings.Contains(pattern, "**") { - return "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "** in %q is not supported; use * for a single directory level", pattern)) + return "", diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("** in %q is not supported; use * for a single directory level", pattern), + Locations: b.Config.GetLocations(loc), + }) } // A POSIX path is absolute on Windows too, so check both flavours like NormalizePaths does. if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) { - return "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q must be relative to the defining YAML file", pattern)) + return "", diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("pattern %q must be relative to the defining YAML file", pattern), + Locations: b.Config.GetLocations(loc), + }) } // NormalizePaths has already rewritten YAML-relative globs to be bundle-root // relative. Join that onto the bundle root, then require the result stay @@ -156,12 +170,20 @@ func validateFileTriggerPattern(b *bundle.Bundle, loc, pattern string) (string, joined := filepath.Join(b.BundleRootPath, filepath.FromSlash(pattern)) relPattern, err := filepath.Rel(b.SyncRootPath, joined) if err != nil || !filepath.IsLocal(relPattern) { - return "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "pattern %q is not under the sync root", pattern)) + return "", diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("pattern %q is not under the sync root", pattern), + Locations: b.Config.GetLocations(loc), + }) } relPattern = filepath.ToSlash(relPattern) _, err = pathlib.Match(relPattern, "") if err != nil { - return "", diags.Append(fileTriggerDiag(b, loc, diag.Error, "invalid pattern %q: %s", pattern, err)) + return "", diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fileTriggerPrefix + fmt.Sprintf("invalid pattern %q: %s", pattern, err), + Locations: b.Config.GetLocations(loc), + }) } return relPattern, diags } From dcd60240f3f790df78c7eef54c2957dbe86e1e83 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 11:40:53 +0200 Subject: [PATCH 46/60] job_runs: reject Windows-rooted on_file_change patterns on every host filepath.IsAbs only recognises the host's flavour, so "C:\watched.txt" was rejected on Windows but treated as a relative "C:" directory on macOS and Linux, where it then warned about matching no files. The same bundle now fails validation the same way wherever it is deployed from. Co-authored-by: Isaac --- .../mutator/resolve_job_run_file_triggers.go | 21 ++++++++- .../resolve_job_run_file_triggers_test.go | 45 +++++++++++++++++++ 2 files changed, 64 insertions(+), 2 deletions(-) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index eff981b7df2..9864a7ad145 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -156,8 +156,11 @@ func validateFileTriggerPattern(b *bundle.Bundle, loc, pattern string) (string, Locations: b.Config.GetLocations(loc), }) } - // A POSIX path is absolute on Windows too, so check both flavours like NormalizePaths does. - if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) { + // filepath.IsAbs only recognises the host's flavour, so check both plus the + // Windows forms lexically. Otherwise "C:\watched.txt" is rejected on Windows + // but silently treated as a relative "C:" directory elsewhere, and the same + // bundle validates differently depending on where it is deployed from. + if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) || isWindowsAbs(pattern) { return "", diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fileTriggerPrefix + fmt.Sprintf("pattern %q must be relative to the defining YAML file", pattern), @@ -188,6 +191,20 @@ func validateFileTriggerPattern(b *bundle.Bundle, loc, pattern string) (string, return relPattern, diags } +// isWindowsAbs reports whether pattern is rooted in Windows terms - a drive +// letter, a UNC share, or a leading separator - whatever the host OS is. +func isWindowsAbs(pattern string) bool { + if strings.HasPrefix(pattern, `\`) { + return true + } + // "C:", "C:/x" and "C:\x", plus the drive-relative "C:x". + if len(pattern) < 2 || pattern[1] != ':' { + return false + } + c := pattern[0] + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' +} + func hashFile(root fs.FS, path string) (string, error) { f, err := root.Open(path) if err != nil { diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index bb4676be531..d9770117ccc 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -54,6 +54,51 @@ func TestResolveJobRunFileTriggersHashesThroughSyncRoot(t *testing.T) { ) } +// Rooted patterns must be rejected the same way on every host: filepath.IsAbs +// alone only knows the local flavour, so a Windows-rooted pattern used to pass +// validation on POSIX and fail on Windows. +func TestResolveJobRunFileTriggersRejectsRootedPatterns(t *testing.T) { + for _, pattern := range []string{ + "/abs/watched.txt", + `C:\watched.txt`, + "C:/watched.txt", + "c:watched.txt", + `\\server\share\watched.txt`, + `\rooted.txt`, + } { + t.Run(pattern, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "watched.txt"), []byte("native"), 0o644)) + root := vfs.MustNew(dir) + b := &bundle.Bundle{ + BundleRootPath: dir, + SyncRootPath: dir, + SyncRoot: root, + WorktreeRoot: root, + Config: config.Root{ + Bundle: config.Bundle{Target: "default"}, + Sync: config.Sync{Paths: []string{"."}}, + Resources: config.Resources{ + JobRuns: map[string]*resources.JobRun{ + "my_run": { + Lifecycle: &resources.JobRunLifecycle{ + Triggers: []resources.JobRunTrigger{ + {OnFileChange: &pattern}, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) + require.True(t, diags.HasError(), "expected %q to be rejected", pattern) + assert.Contains(t, diags[0].Summary, "must be relative to the defining YAML file") + }) + } +} + func contentHash(content string) string { sum := sha256.Sum256([]byte(content)) return hex.EncodeToString(sum[:]) From 6961c6a938781ea75a987fb3785237e11fadd395 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 4 Sep 2026 17:58:27 +0200 Subject: [PATCH 47/60] job_runs: give JobRunTriggersState an IsEmpty method The emptiness rule enumerates every field of the struct, so keep it next to the struct rather than in PrepareState in another package: a fingerprint added later without extending it would be silently dropped instead of persisted. Co-authored-by: Isaac --- bundle/config/resources/lifecycle.go | 7 +++++++ bundle/direct/dresources/job_run.go | 5 ++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index f9b7380dcf2..66e50d4ecf0 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -48,3 +48,10 @@ type JobRunTriggersState struct { OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` OnFileChange map[string]string `json:"on_file_change,omitempty"` } + +// IsEmpty reports whether no trigger is armed. An empty state is left off the +// job run entirely, so this has to cover every field above: a new fingerprint +// added without extending it would be dropped instead of persisted. +func (s JobRunTriggersState) IsEmpty() bool { + return s.OnBundleDeploy == "" && len(s.OnFileChange) == 0 +} diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index 6640b5d726e..a1c9a71d584 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -107,10 +107,9 @@ func (*ResourceJobRun) PrepareState(input *resources.JobRun) *JobRunState { if input.HasOnBundleDeploy() { ts.OnBundleDeploy = uuid.NewString() } - if ts.OnBundleDeploy == "" && len(ts.OnFileChange) == 0 { - return state + if !ts.IsEmpty() { + state.Lifecycle = &JobRunLifecycleState{TriggersState: &ts} } - state.Lifecycle = &JobRunLifecycleState{TriggersState: &ts} return state } From 73d4865bf8a15a049e01943ee79c21d6aba01ed4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 7 Sep 2026 17:03:45 +0200 Subject: [PATCH 48/60] job_runs: note why on_file_change hashes the candidate sync set sync drops files whose notebook type it cannot determine, so a malformed .ipynb is hashed but never uploaded. That is intended - editing one is when the run should re-fire - but it is worth saying so where the set is chosen. Co-authored-by: Isaac --- bundle/config/mutator/resolve_job_run_file_triggers.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 9864a7ad145..0d3248c6e72 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -80,6 +80,11 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, error) { // Match sync's effective include set, not just Sync.Include, so a pattern can // hash the internal and AI-snapshot dirs sync force-includes. + // + // This is the candidate set, not sync's post-snapshot upload set: NewSnapshotState + // additionally drops files whose notebook type it cannot determine, such as a + // malformed .ipynb. Those stay watched on purpose - editing one is exactly when the + // run should re-fire - so a pattern can hash a file that this deploy will not upload. includes, err := b.GetSyncIncludePatterns(ctx) if err != nil { return nil, err From ea1c78914f9c8337d33eab49a050c49e3a4f1583 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 8 Sep 2026 11:05:38 +0200 Subject: [PATCH 49/60] job_runs: match OverrideChangeDesc paths by string like other resources The four fixed cases were spelled as parsed-path Len()+HasPrefix comparisons, which is exact equality written the long way; every other resource's OverrideChangeDesc switches on path.String(). Do the same and drop the four package-level path vars. The single on_file_change pattern entry has no fixed string, so it stays a Parent() check in the default branch. Co-authored-by: Isaac --- bundle/direct/dresources/job_run.go | 30 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/bundle/direct/dresources/job_run.go b/bundle/direct/dresources/job_run.go index a1c9a71d584..9174c6255b6 100644 --- a/bundle/direct/dresources/job_run.go +++ b/bundle/direct/dresources/job_run.go @@ -26,13 +26,6 @@ import ( // jobRunTimeout matches the timeout `bundle run` allows a run (bundle/run/job.go). const jobRunTimeout = 24 * time.Hour -var ( - jobRunLifecyclePath = structpath.MustParsePath("lifecycle") - jobRunOnBundleDeployPath = structpath.MustParsePath("lifecycle.triggers_state.on_bundle_deploy") - jobRunOnFileChangePath = structpath.MustParsePath("lifecycle.triggers_state.on_file_change") - jobRunResultStatePath = structpath.MustParsePath("result_state") -) - // JobRunLifecycleState is the local-only trigger fingerprint. type JobRunLifecycleState struct { TriggersState *resources.JobRunTriggersState `json:"triggers_state,omitempty"` @@ -378,35 +371,40 @@ func reportRunLine(ctx context.Context, runID int64, msg string) { // so re-adding the trigger only re-fires when the watched files changed // meanwhile. All other trigger changes recreate the run. func (*ResourceJobRun) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, remote *JobRunRemote) error { - switch { - case path.Len() == jobRunLifecyclePath.Len() && path.HasPrefix(jobRunLifecyclePath): + switch path.String() { + case "lifecycle": // Dropped when the trigger is removed (New nil) and when both sides are // present, where the trigger fields below classify the change instead. // Arming from no lifecycle at all keeps its recreate. if change.New == nil || change.Old != nil { change.Reason = deployplan.ReasonDrop } - case path.Len() == jobRunOnBundleDeployPath.Len() && path.HasPrefix(jobRunOnBundleDeployPath): + case "lifecycle.triggers_state.on_bundle_deploy": if change.New == nil || change.New == "" { change.Reason = deployplan.ReasonDrop } - case path.Len() == jobRunOnFileChangePath.Len() && path.HasPrefix(jobRunOnFileChangePath): + case "lifecycle.triggers_state.on_file_change": // As above: an emptied map is a removal, and pattern entries classify a map // that still has both sides. if isEmptyFileTriggerMap(change.New) || change.Old != nil { change.Reason = deployplan.ReasonDrop } - case path.Len() == jobRunOnFileChangePath.Len()+1 && path.HasPrefix(jobRunOnFileChangePath): - if change.New == nil { - change.Reason = deployplan.ReasonDrop - } - case path.Len() == jobRunResultStatePath.Len() && path.HasPrefix(jobRunResultStatePath): + case "result_state": // The planner passes no remote state when the run could not be read. if remote == nil || runIsTerminal(remote.State.LifeCycleState) { return nil } change.Action = deployplan.Skip change.Reason = "run in progress" + default: + // A single on_file_change pattern entry, e.g. + // lifecycle.triggers_state.on_file_change['seed.txt']. Removing one pattern + // drops its change and leaves the last fingerprint in state. + if parent := path.Parent(); parent != nil && parent.String() == "lifecycle.triggers_state.on_file_change" { + if change.New == nil { + change.Reason = deployplan.ReasonDrop + } + } } return nil } From 6c87810c5fe5b784a61cc36de0cb80f5b73c8359 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 8 Sep 2026 20:12:46 +0200 Subject: [PATCH 50/60] job_runs: move JobRunLifecycle types to job_run.go lifecycle.go is for lifecycle settings common to all resources; the job-run trigger types belong next to JobRun. Pure move, no behaviour change. Co-authored-by: Isaac --- bundle/config/resources/job_run.go | 30 ++++++++++++++++++++++++++++ bundle/config/resources/lifecycle.go | 30 ---------------------------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/bundle/config/resources/job_run.go b/bundle/config/resources/job_run.go index 670a4b23622..8b7428e8fb8 100644 --- a/bundle/config/resources/job_run.go +++ b/bundle/config/resources/job_run.go @@ -29,6 +29,36 @@ type JobRun struct { ResolvedJobID int64 `json:"resolved_job_id,omitempty" bundle:"internal"` } +// JobRunLifecycle extends Lifecycle with run-fire triggers. +type JobRunLifecycle struct { + Lifecycle + + // Triggers that cause the run to re-fire (in addition to config changes). + Triggers []JobRunTrigger `json:"triggers,omitempty"` + + // Resolved fingerprint for the planner; not user config. + TriggersState *JobRunTriggersState `json:"triggers_state,omitempty" bundle:"internal"` +} + +// JobRunTrigger is one lifecycle.triggers entry. +type JobRunTrigger struct { + OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` + OnFileChange *string `json:"on_file_change,omitempty"` // path or glob relative to the defining YAML file; must resolve under the sync root +} + +// JobRunTriggersState is the resolved fingerprint of lifecycle.triggers. +type JobRunTriggersState struct { + OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` + OnFileChange map[string]string `json:"on_file_change,omitempty"` +} + +// IsEmpty reports whether no trigger is armed. An empty state is left off the +// job run entirely, so this has to cover every field above: a new fingerprint +// added without extending it would be dropped instead of persisted. +func (s JobRunTriggersState) IsEmpty() bool { + return s.OnBundleDeploy == "" && len(s.OnFileChange) == 0 +} + // HasOnBundleDeploy reports whether any trigger re-fires on every deploy. func (r *JobRun) HasOnBundleDeploy() bool { if r.Lifecycle == nil { diff --git a/bundle/config/resources/lifecycle.go b/bundle/config/resources/lifecycle.go index 66e50d4ecf0..db2b130d313 100644 --- a/bundle/config/resources/lifecycle.go +++ b/bundle/config/resources/lifecycle.go @@ -25,33 +25,3 @@ type LifecycleWithStarted struct { // Supported only for apps, clusters, and sql_warehouses. Started *bool `json:"started,omitempty"` } - -// JobRunLifecycle extends Lifecycle with run-fire triggers. -type JobRunLifecycle struct { - Lifecycle - - // Triggers that cause the run to re-fire (in addition to config changes). - Triggers []JobRunTrigger `json:"triggers,omitempty"` - - // Resolved fingerprint for the planner; not user config. - TriggersState *JobRunTriggersState `json:"triggers_state,omitempty" bundle:"internal"` -} - -// JobRunTrigger is one lifecycle.triggers entry. -type JobRunTrigger struct { - OnBundleDeploy *bool `json:"on_bundle_deploy,omitempty"` - OnFileChange *string `json:"on_file_change,omitempty"` // path or glob relative to the defining YAML file; must resolve under the sync root -} - -// JobRunTriggersState is the resolved fingerprint of lifecycle.triggers. -type JobRunTriggersState struct { - OnBundleDeploy string `json:"on_bundle_deploy,omitempty"` - OnFileChange map[string]string `json:"on_file_change,omitempty"` -} - -// IsEmpty reports whether no trigger is armed. An empty state is left off the -// job run entirely, so this has to cover every field above: a new fingerprint -// added without extending it would be dropped instead of persisted. -func (s JobRunTriggersState) IsEmpty() bool { - return s.OnBundleDeploy == "" && len(s.OnFileChange) == 0 -} From f894190d9044648d60846e008af032b19b243d5a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 8 Sep 2026 20:33:07 +0200 Subject: [PATCH 51/60] job_runs: regenerate pydabs and refschema for on_file_change main now autogenerates the job_runs pydabs models, so on_file_change has to be regenerated into job_run_trigger.py and the refschema golden. Also update the knownMissingInStateType entry: state renamed triggers to triggers_state, so the whole lifecycle.triggers subtree is missing from StateType, not just its elements. Co-authored-by: Isaac --- bundle/direct/dresources/type_test.go | 8 ++++---- .../bundles/job_runs/_models/job_run_trigger.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index a82a3447e42..9d7f6770180 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -56,15 +56,15 @@ var knownMissingInRemoteType = map[string][]string{ "vector_search_endpoints": { "usage_policy_id", }, - "job_runs": { - // Local-only trigger fingerprints under lifecycle. - "lifecycle", - }, "internal_immutable_snapshots": { "bundle_id", "acl", "zip_path", }, + "job_runs": { + // Local-only trigger fingerprints under lifecycle. + "lifecycle", + }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. diff --git a/python/databricks/bundles/job_runs/_models/job_run_trigger.py b/python/databricks/bundles/job_runs/_models/job_run_trigger.py index 50e7c0fe12c..9878235db6e 100644 --- a/python/databricks/bundles/job_runs/_models/job_run_trigger.py +++ b/python/databricks/bundles/job_runs/_models/job_run_trigger.py @@ -20,6 +20,11 @@ class JobRunTrigger: If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. """ + on_file_change: VariableOrOptional[str] = None + """ + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy. + """ + @classmethod def from_dict(cls, value: "JobRunTriggerDict") -> "Self": return _transform(cls, value) @@ -36,5 +41,10 @@ class JobRunTriggerDict(TypedDict, total=False): If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. """ + on_file_change: VariableOrOptional[str] + """ + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy. + """ + JobRunTriggerParam = JobRunTriggerDict | JobRunTrigger From 59f58176dd8836fe30b77563a5fa7286b2c05041 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 8 Sep 2026 20:46:51 +0200 Subject: [PATCH 52/60] acc: mask plan_version in the on_file_change create-plan golden main added a [PLAN_VERSION] replacement, so the committed out.plan.create.json needs the masked value in place of the literal. Co-authored-by: Isaac --- .../resources/job_runs/on_file_change/out.plan.create.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/resources/job_runs/on_file_change/out.plan.create.json b/acceptance/bundle/resources/job_runs/on_file_change/out.plan.create.json index 50ba69a707f..8fb3fcdfd59 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/out.plan.create.json +++ b/acceptance/bundle/resources/job_runs/on_file_change/out.plan.create.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": [PLAN_VERSION], "cli_version": "[CLI_VERSION]", "plan": { "resources.job_runs.my_run": { From b39cfedcdbece72e14ed3bd2628e5679aacca098 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 14:28:27 +0200 Subject: [PATCH 53/60] shorten desc --- .nextchanges/bundles/job-runs-on-file-change.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nextchanges/bundles/job-runs-on-file-change.md b/.nextchanges/bundles/job-runs-on-file-change.md index 6b48c6f76a0..2357fefcb61 100644 --- a/.nextchanges/bundles/job-runs-on-file-change.md +++ b/.nextchanges/bundles/job-runs-on-file-change.md @@ -1 +1 @@ -* direct: `resources.job_runs` can set `lifecycle.triggers.on_file_change` to a path or glob under the sync root to re-fire the run when matched file contents change (content hash), or when matches appear or disappear. Removing the trigger does not recreate the existing run. ([#6309](https://github.com/databricks/cli/pull/6309)) +* direct: resources.job\_runs: new lifecycle.triggers.on\_file\_change setting to restart the run when monitored files change. Can be set to a series of paths or globs. ([#6309](https://github.com/databricks/cli/pull/6309)) From c27a0b7766300e557dbf8afbd91a08ab877f2358 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 16:11:50 +0200 Subject: [PATCH 54/60] job_runs: regenerate for main rebase (DMS version_id, refschema) Rebasing onto main pulled in DMS state recording (#6094), which stamps a version_id into the deployment block under DMS. out.plan.create.json served double duty as both the compared golden and the --plan deploy input, so it could not simply be masked: deploy from the raw plan (keeps version_id) and nostamp a separate copy for the golden. Also regenerate the refschema for the on_file_change / triggers_state fields. --- acceptance/bundle/refschema/out.fields.txt | 7 +++++-- .../bundle/resources/job_runs/on_file_change/output.txt | 2 +- acceptance/bundle/resources/job_runs/on_file_change/script | 7 +++++-- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 158fcc5d1c0..411d7b29e49 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -925,11 +925,14 @@ resources.job_runs.*.lifecycle *dresources.JobRunLifecycleState STATE resources.job_runs.*.lifecycle *resources.JobRunLifecycle INPUT resources.job_runs.*.lifecycle resources.Lifecycle INPUT resources.job_runs.*.lifecycle.prevent_destroy bool INPUT -resources.job_runs.*.lifecycle.triggers *dresources.JobRunTriggersState STATE resources.job_runs.*.lifecycle.triggers []resources.JobRunTrigger INPUT -resources.job_runs.*.lifecycle.triggers.on_bundle_deploy string STATE resources.job_runs.*.lifecycle.triggers[*] resources.JobRunTrigger INPUT resources.job_runs.*.lifecycle.triggers[*].on_bundle_deploy *bool INPUT +resources.job_runs.*.lifecycle.triggers[*].on_file_change *string INPUT +resources.job_runs.*.lifecycle.triggers_state *resources.JobRunTriggersState INPUT STATE +resources.job_runs.*.lifecycle.triggers_state.on_bundle_deploy string INPUT STATE +resources.job_runs.*.lifecycle.triggers_state.on_file_change map[string]string INPUT STATE +resources.job_runs.*.lifecycle.triggers_state.on_file_change.* string INPUT STATE resources.job_runs.*.modified_status string INPUT resources.job_runs.*.notebook_params map[string]string ALL resources.job_runs.*.notebook_params.* string ALL diff --git a/acceptance/bundle/resources/job_runs/on_file_change/output.txt b/acceptance/bundle/resources/job_runs/on_file_change/output.txt index f1527678167..37e4b63a2bc 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/output.txt +++ b/acceptance/bundle/resources/job_runs/on_file_change/output.txt @@ -6,7 +6,7 @@ Output from job_runs.my_run: id=[MY_RUN_ID]: Run URL: [DATABRICKS_URL]/jobs/[MY_ Output from job_runs.my_run: id=[MY_RUN_ID]: SUCCESS Created job_runs.my_run Created jobs.my_job -Files: 7 uploaded, 0 deleted +Files: 8 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> read_id.py my_job diff --git a/acceptance/bundle/resources/job_runs/on_file_change/script b/acceptance/bundle/resources/job_runs/on_file_change/script index 7569a172e9e..53758e64a30 100644 --- a/acceptance/bundle/resources/job_runs/on_file_change/script +++ b/acceptance/bundle/resources/job_runs/on_file_change/script @@ -9,8 +9,11 @@ plan_no_changes() { } title "first deploy triggers a run" -trace $CLI bundle plan -o json > out.plan.create.json -$CLI bundle deploy $(readplanarg out.plan.create.json) +# Deploy reads the raw plan so it keeps the DMS version_id it needs; the committed +# golden is that plan with the DMS stamp stripped, so it matches across DMS variants. +trace $CLI bundle plan -o json > tmp.plan.create.json +nostamp < tmp.plan.create.json > out.plan.create.json +$CLI bundle deploy $(readplanarg tmp.plan.create.json) trace read_id.py my_job # Name the first run so the recreated one becomes [MY_RUN_ID_2]. read_id.py my_run > /dev/null From a55df3255a8e3463bb3dec78fd7b2b0c4af65b60 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 17:56:30 +0200 Subject: [PATCH 55/60] job_runs: drop stale AI-snapshot mention from the include comment sync force-includes only the internal dir; the AI Runtime sync overlay was replaced by a tgz artifact on main (#6494, #6532). --- bundle/config/mutator/resolve_job_run_file_triggers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index 0d3248c6e72..a0206aac69b 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -79,7 +79,7 @@ func (*resolveJobRunFileTriggers) Apply(ctx context.Context, b *bundle.Bundle) d func listSyncableRelPaths(ctx context.Context, b *bundle.Bundle) ([]string, error) { // Match sync's effective include set, not just Sync.Include, so a pattern can - // hash the internal and AI-snapshot dirs sync force-includes. + // hash the internal dir sync force-includes. // // This is the candidate set, not sync's post-snapshot upload set: NewSnapshotState // additionally drops files whose notebook type it cannot determine, such as a From 964ccddbff0312a6bc218228499f5f5938b51053 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 18:03:14 +0200 Subject: [PATCH 56/60] job_runs: drop the prevent_destroy trigger validation, rely on the generic check A fired trigger recreates the run, and checkForPreventDestroy already rejects a recreate on a prevent_destroy resource generically (bundle/phases/plan.go), so the special-case validation and its schema-doc note were redundant. Removing them lets the combo create once and error only when a re-fire actually plans a recreate, matching every other recreate field. --- .../mutator/validate_job_run_triggers.go | 22 ------------------- .../mutator/validate_job_run_triggers_test.go | 22 +++---------------- bundle/internal/schema/annotations.yml | 4 ++-- bundle/schema/jsonschema.json | 4 ++-- 4 files changed, 7 insertions(+), 45 deletions(-) diff --git a/bundle/config/mutator/validate_job_run_triggers.go b/bundle/config/mutator/validate_job_run_triggers.go index eff70b05e5f..d33a7263bde 100644 --- a/bundle/config/mutator/validate_job_run_triggers.go +++ b/bundle/config/mutator/validate_job_run_triggers.go @@ -26,15 +26,6 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D if jr == nil || jr.Lifecycle == nil { continue } - if jr.Lifecycle.PreventDestroy { - if summary := preventDestroyError(jr.HasOnBundleDeploy(), jr.HasOnFileChange()); summary != "" { - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: summary, - Locations: b.Config.GetLocations(fmt.Sprintf("resources.job_runs.%s.lifecycle", name)), - }) - } - } for i, t := range jr.Lifecycle.Triggers { path := fmt.Sprintf("resources.job_runs.%s.lifecycle.triggers[%d]", name, i) if t.OnBundleDeploy == nil && t.OnFileChange == nil { @@ -77,16 +68,3 @@ func (*validateJobRunTriggers) Apply(_ context.Context, b *bundle.Bundle) diag.D } return diags } - -func preventDestroyError(onBundleDeploy, onFileChange bool) string { - switch { - case onBundleDeploy && onFileChange: - return "lifecycle.triggers.on_bundle_deploy and on_file_change are incompatible with lifecycle.prevent_destroy" - case onBundleDeploy: - return "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy" - case onFileChange: - return "lifecycle.triggers.on_file_change is incompatible with lifecycle.prevent_destroy" - default: - return "" - } -} diff --git a/bundle/config/mutator/validate_job_run_triggers_test.go b/bundle/config/mutator/validate_job_run_triggers_test.go index 0b14e8cfbd0..cb5d13e8f2f 100644 --- a/bundle/config/mutator/validate_job_run_triggers_test.go +++ b/bundle/config/mutator/validate_job_run_triggers_test.go @@ -79,29 +79,13 @@ func TestValidateJobRunTriggers(t *testing.T) { summary: "lifecycle.triggers.on_file_change must be non-empty when set", }, { - name: "on_bundle_deploy with prevent_destroy", - triggers: []resources.JobRunTrigger{ - {OnBundleDeploy: &trueVal}, - }, - preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy is incompatible with lifecycle.prevent_destroy", - }, - { - name: "on_file_change with prevent_destroy", + // A trigger with prevent_destroy is valid at this stage; the recreate a + // fired trigger plans is rejected generically by checkForPreventDestroy. + name: "trigger with prevent_destroy is allowed here", triggers: []resources.JobRunTrigger{ {OnFileChange: &fileChange}, }, preventDestroy: true, - summary: "lifecycle.triggers.on_file_change is incompatible with lifecycle.prevent_destroy", - }, - { - name: "both triggers with prevent_destroy", - triggers: []resources.JobRunTrigger{ - {OnFileChange: &fileChange}, - {OnBundleDeploy: &trueVal}, - }, - preventDestroy: true, - summary: "lifecycle.triggers.on_bundle_deploy and on_file_change are incompatible with lifecycle.prevent_destroy", }, { name: "prevent_destroy alone", diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 05b9b8a5925..215a156b0f8 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1035,10 +1035,10 @@ resources: "$fields": "on_bundle_deploy": "description": |- - If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + If true, re-fire the run on every bundle deploy. "on_file_change": "description": |- - Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy. + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. "python_named_params": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index b438956e0fe..6d499819f61 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1533,11 +1533,11 @@ "type": "object", "properties": { "on_bundle_deploy": { - "description": "If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy.", + "description": "If true, re-fire the run on every bundle deploy.", "$ref": "#/$defs/bool" }, "on_file_change": { - "description": "Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy.", + "description": "Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported.", "$ref": "#/$defs/string" } }, From d2c544e39d97f263e01a93742e6c135bda03070d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 18:08:25 +0200 Subject: [PATCH 57/60] job_runs: drop stale generated-overlays mention from the fingerprint comment The AI Runtime sync overlay is gone (tgz artifact now); build and the predeploy script are the only steps that produce a watched file here. --- cmd/bundle/utils/process.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 810bd11ef5e..2dcd549eba2 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -444,10 +444,10 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } // Fingerprint on_file_change triggers once, after every step that can produce - // a watched file: build, generated overlays and the predeploy script. Reads - // the sync root that phases.Initialize resolves, so it is skipped along with - // it; `bundle deploy --plan` recomputes fingerprints that the loaded plan then - // overrides with the ones it recorded. + // a watched file: build and the predeploy script. Reads the sync root that + // phases.Initialize resolves, so it is skipped along with it; `bundle deploy + // --plan` recomputes fingerprints that the loaded plan then overrides with the + // ones it recorded. if !opts.SkipInitialize { bundle.ApplyContext(ctx, b, mutator.ResolveJobRunFileTriggers()) if logdiag.HasError(ctx) { From a32d777ce3823031e777b24754fb9f85b01dd812 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 9 Sep 2026 18:11:52 +0200 Subject: [PATCH 58/60] job_runs: drop vfs.Overlay from the file-trigger hash test Write the watched file to disk and hash it directly instead of layering an in-memory overlay over the sync root. --- .../config/mutator/resolve_job_run_file_triggers_test.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index d9770117ccc..15b84a8a789 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -18,7 +18,7 @@ import ( func TestResolveJobRunFileTriggersHashesThroughSyncRoot(t *testing.T) { dir := t.TempDir() - require.NoError(t, os.WriteFile(filepath.Join(dir, "watched.txt"), []byte("native"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "watched.txt"), []byte("watched"), 0o644)) root := vfs.MustNew(dir) pattern := "watched.txt" b := &bundle.Bundle{ @@ -42,14 +42,11 @@ func TestResolveJobRunFileTriggersHashesThroughSyncRoot(t *testing.T) { }, }, } - overlay, err := vfs.Overlay(b.SyncRoot, map[string][]byte{"watched.txt": []byte("overlay")}) - require.NoError(t, err) - b.SyncRoot = overlay diags := bundle.Apply(t.Context(), b, mutator.ResolveJobRunFileTriggers()) require.False(t, diags.HasError()) assert.Equal(t, - contentHash("watched.txt\x00"+contentHash("overlay")+"\x00"), + contentHash("watched.txt\x00"+contentHash("watched")+"\x00"), b.Config.Resources.JobRuns["my_run"].Lifecycle.TriggersState.OnFileChange["watched.txt"], ) } From 63913421d45744cd1c33caa9a1535cc22eba1007 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 10 Sep 2026 14:52:40 +0200 Subject: [PATCH 59/60] job_runs: regenerate pydabs docstrings after dropping the prevent_destroy note validate-generated failed because the pydabs job_run_trigger.py docstrings still carried "Incompatible with lifecycle.prevent_destroy"; only annotations.yml and jsonschema.json had been regenerated. --- .../bundles/job_runs/_models/job_run_trigger.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/databricks/bundles/job_runs/_models/job_run_trigger.py b/python/databricks/bundles/job_runs/_models/job_run_trigger.py index 9878235db6e..75bdf970a4e 100644 --- a/python/databricks/bundles/job_runs/_models/job_run_trigger.py +++ b/python/databricks/bundles/job_runs/_models/job_run_trigger.py @@ -17,12 +17,12 @@ class JobRunTrigger: on_bundle_deploy: VariableOrOptional[bool] = None """ - If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + If true, re-fire the run on every bundle deploy. """ on_file_change: VariableOrOptional[str] = None """ - Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy. + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. """ @classmethod @@ -38,12 +38,12 @@ class JobRunTriggerDict(TypedDict, total=False): on_bundle_deploy: VariableOrOptional[bool] """ - If true, re-fire the run on every bundle deploy. Incompatible with lifecycle.prevent_destroy. + If true, re-fire the run on every bundle deploy. """ on_file_change: VariableOrOptional[str] """ - Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. Incompatible with lifecycle.prevent_destroy. + Path or glob relative to the defining YAML file. It must resolve under the sync root. Re-fire the run when a matched file's content hash changes, or when the set of matches appears or disappears. Only files the bundle syncs are hashed, so .gitignore and sync.exclude apply. Use * to match a single directory level; ** is not supported. """ From f964f6fd00a6016539c04ffc8b34f178267a178e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 11 Sep 2026 09:45:24 +0200 Subject: [PATCH 60/60] job_runs: reject only genuinely absolute on_file_change patterns Drop the hand-rolled isWindowsAbs cross-OS check: it rejected "c:foo" and "\foo" which are valid relative filenames on POSIX, with a misleading "must be relative" error. filepath.IsAbs is already OS-aware, so a path rooted only on the other OS falls through to the sync-root containment check, which is what actually guards against escaping the tree. Reported by shreyas-goenka. --- .../mutator/resolve_job_run_file_triggers.go | 25 +++++-------------- .../resolve_job_run_file_triggers_test.go | 9 +++---- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/bundle/config/mutator/resolve_job_run_file_triggers.go b/bundle/config/mutator/resolve_job_run_file_triggers.go index a0206aac69b..35cffd0dcbe 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers.go @@ -161,11 +161,12 @@ func validateFileTriggerPattern(b *bundle.Bundle, loc, pattern string) (string, Locations: b.Config.GetLocations(loc), }) } - // filepath.IsAbs only recognises the host's flavour, so check both plus the - // Windows forms lexically. Otherwise "C:\watched.txt" is rejected on Windows - // but silently treated as a relative "C:" directory elsewhere, and the same - // bundle validates differently depending on where it is deployed from. - if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) || isWindowsAbs(pattern) { + // Reject a genuinely absolute path; Join would otherwise silently reinterpret it + // as relative to the bundle root. filepath.IsAbs is OS-aware and pathlib.IsAbs + // covers a POSIX path on Windows. A string that is rooted only on the other OS + // (e.g. "c:foo" or "\foo" on POSIX) is a valid relative name here and falls + // through to the sync-root containment check below. + if filepath.IsAbs(pattern) || pathlib.IsAbs(pattern) { return "", diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fileTriggerPrefix + fmt.Sprintf("pattern %q must be relative to the defining YAML file", pattern), @@ -196,20 +197,6 @@ func validateFileTriggerPattern(b *bundle.Bundle, loc, pattern string) (string, return relPattern, diags } -// isWindowsAbs reports whether pattern is rooted in Windows terms - a drive -// letter, a UNC share, or a leading separator - whatever the host OS is. -func isWindowsAbs(pattern string) bool { - if strings.HasPrefix(pattern, `\`) { - return true - } - // "C:", "C:/x" and "C:\x", plus the drive-relative "C:x". - if len(pattern) < 2 || pattern[1] != ':' { - return false - } - c := pattern[0] - return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' -} - func hashFile(root fs.FS, path string) (string, error) { f, err := root.Open(path) if err != nil { diff --git a/bundle/config/mutator/resolve_job_run_file_triggers_test.go b/bundle/config/mutator/resolve_job_run_file_triggers_test.go index 15b84a8a789..2af318b0178 100644 --- a/bundle/config/mutator/resolve_job_run_file_triggers_test.go +++ b/bundle/config/mutator/resolve_job_run_file_triggers_test.go @@ -55,13 +55,12 @@ func TestResolveJobRunFileTriggersHashesThroughSyncRoot(t *testing.T) { // alone only knows the local flavour, so a Windows-rooted pattern used to pass // validation on POSIX and fail on Windows. func TestResolveJobRunFileTriggersRejectsRootedPatterns(t *testing.T) { + // Only genuinely absolute paths on this OS are rejected. A string that is + // rooted only on Windows (e.g. "c:foo" or "\foo") is a valid relative name on + // POSIX and is left to the sync-root containment check, so it is not listed + // here; on Windows filepath.IsAbs classifies those forms itself. for _, pattern := range []string{ "/abs/watched.txt", - `C:\watched.txt`, - "C:/watched.txt", - "c:watched.txt", - `\\server\share\watched.txt`, - `\rooted.txt`, } { t.Run(pattern, func(t *testing.T) { dir := t.TempDir()