From a21f96caa80e6966c3c2215e771192b6d921c198 Mon Sep 17 00:00:00 2001 From: wtruongdata Date: Wed, 8 Jul 2026 01:33:26 -0500 Subject: [PATCH 1/4] fix: allow an explicit execution time to extend the plan end past the prod frontier Signed-off-by: wtruongdata --- sqlmesh/core/context.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/sqlmesh/core/context.py b/sqlmesh/core/context.py index 5902977331..5eba0d0677 100644 --- a/sqlmesh/core/context.py +++ b/sqlmesh/core/context.py @@ -1721,6 +1721,35 @@ def plan_builder( execution_time or now(), ) + execution_time_ts = to_timestamp(execution_time) if execution_time is not None else None + if ( + execution_time_ts is not None + and end is None + and default_end is not None + and execution_time_ts > default_end + ): + # An explicitly provided execution time acts as the plan's effective "now", so the + # default end is allowed to extend past the recorded prod frontier instead of being + # capped by it. This mirrors how an explicit `end` causes the per-model interval end + # caps to be dropped entirely in PlanBuilder.build() (see `self.override_end`, + # sqlmesh/core/plan/builder.py line 206). Raising (rather than dropping) the caps is + # safe here because `plan --run` already runs with no caps at all. + # + # Note this raises every entry already present in max_interval_end_per_model (i.e. + # every model in this dict, which _get_max_interval_end_per_model above has already + # scoped down to backfill_models/its ancestors when a selection is in effect) - not + # just modified or explicitly selected models within that scope. That's intentional, + # not an oversight. It's what makes a plain, unscoped `sqlmesh plan --execution-time X` + # in prod report the same missing intervals as `sqlmesh plan --run --execution-time X` + # would at the same simulated time. Narrowing this further would reintroduce a `plan` + # vs `plan --run` mismatch for models within that scope. + default_end = execution_time_ts + execution_time_dt = to_datetime(execution_time_ts) + max_interval_end_per_model = { + model_fqn: max(interval_end, execution_time_dt) + for model_fqn, interval_end in max_interval_end_per_model.items() + } + # Refresh snapshot intervals to ensure that they are up to date with values reflected in the max_interval_end_per_model. self.state_sync.refresh_snapshot_intervals(context_diff.snapshots.values()) From a6b9c63be924689fdd7be7bdd4d921d8eb493166 Mon Sep 17 00:00:00 2001 From: wtruongdata Date: Wed, 8 Jul 2026 01:33:26 -0500 Subject: [PATCH 2/4] test: cover plans with an execution time ahead of the prod frontier Signed-off-by: wtruongdata --- tests/core/test_context.py | 177 ++++++++++++++++++++++++++++++++++++- 1 file changed, 176 insertions(+), 1 deletion(-) diff --git a/tests/core/test_context.py b/tests/core/test_context.py index 365d31d3fd..74ce0d7916 100644 --- a/tests/core/test_context.py +++ b/tests/core/test_context.py @@ -1671,6 +1671,176 @@ def test_plan_start_ahead_of_end(copy_to_temp_path): context.close() +@pytest.mark.slow +def test_plan_execution_time_ahead_of_prod_frontier(copy_to_temp_path): + """An explicitly provided `execution_time` should be able to extend the plan's default end + past the recorded prod frontier, since it represents the plan's effective "now". Without + this, an explicit execution_time beyond the last applied interval is silently ignored and + the plan reports no changes/no backfill even though new intervals are due. + + See: https://github.com/SQLMesh/sqlmesh/issues/5640 + """ + path = copy_to_temp_path("examples/sushi") + with time_machine.travel("2024-01-02 00:00:00 UTC"): + context = Context(paths=path, gateway="duckdb_persistent") + context.plan("prod", no_prompts=True, auto_apply=True) + assert all( + i == to_timestamp("2024-01-02") + for i in context.state_sync.max_interval_end_per_model("prod").values() + ) + context.close() + + # No model changes, but a subsequent plan explicitly passes an execution_time that is ahead + # of the recorded prod frontier (2024-01-02). This mimics running + # `sqlmesh plan --execution-time '2024-01-05'` a few days later. + with time_machine.travel("2024-01-06 00:00:00 UTC"): + context = Context(paths=path, gateway="duckdb_persistent") + plan = context.plan_builder("prod", execution_time="2024-01-05").build() + assert plan.requires_backfill + assert to_timestamp(plan.end) == to_timestamp("2024-01-05") + context.apply(plan) + # The seed model isn't backfilled on a cron schedule like the other models, so its + # recorded interval end doesn't advance to the new execution time (same exclusion as + # test_plan_seed_model_excluded_from_default_end). + max_ends = context.state_sync.max_interval_end_per_model("prod") + assert all( + i == to_timestamp("2024-01-05") + for fqn, i in max_ends.items() + if "waiter_names" not in fqn + ) + context.close() + + # Sanity check that the downward clamp is unaffected: an explicit execution_time that is + # *behind* the recorded prod frontier should still clamp the default end down to it (this is + # already covered for the dev case by test_plan_execution_time_start_end). + with time_machine.travel("2024-01-07 00:00:00 UTC"): + context = Context(paths=path, gateway="duckdb_persistent") + plan = context.plan_builder("prod", execution_time="2024-01-04").build() + assert not plan.requires_backfill + assert to_timestamp(plan.end) == to_timestamp("2024-01-04") + context.close() + + +def _write_daily_and_weekly_model_project(tmp_path: Path) -> None: + """Minimal 2-model project used to exercise the interaction between execution_time and + multiple, differently-cadenced models' recorded prod frontiers. A daily and a weekly model + are used so that, after an initial backfill, they end up with different recorded interval + ends (the weekly model's frontier lags the daily model's), which is what the original bug + report's project topology looked like. + """ + (tmp_path / "models").mkdir() + (tmp_path / "config.yaml").write_text( + """ +model_defaults: + dialect: duckdb +""" + ) + (tmp_path / "models" / "daily_model.sql").write_text( + """ +MODEL ( + name daily_model, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column start_dt + ), + start '2024-01-01', + cron '@daily' +); + +select @start_ds as start_ds, @end_ds as end_ds, @start_dt as start_dt, @end_dt as end_dt; +""" + ) + (tmp_path / "models" / "weekly_model.sql").write_text( + """ +MODEL ( + name weekly_model, + kind INCREMENTAL_BY_TIME_RANGE ( + time_column start_dt + ), + start '2024-01-01', + cron '@weekly' +); + +select @start_ds as start_ds, @end_ds as end_ds, @start_dt as start_dt, @end_dt as end_dt; +""" + ) + + +def _missing_intervals_by_name(plan: Plan) -> t.Dict[str, t.Tuple[t.Tuple[int, int], ...]]: + return {si.snapshot_id.name: tuple(si.merged_intervals) for si in plan.missing_intervals} + + +def test_plan_execution_time_ahead_of_prod_frontier_matches_run_for_all_models(tmp_path: Path): + """Locks in that raising `max_interval_end_per_model` for an explicitly provided + `execution_time` sweeps in *every* model with a recorded prod frontier, not just + modified/selected ones. This is intentional, not an oversight: it's what makes a plain, + unscoped `sqlmesh plan --execution-time X` in prod report the exact same missing intervals + that `sqlmesh plan --run --execution-time X` would report at the same simulated time - the + parity the original bug report asks for (https://github.com/SQLMesh/sqlmesh/issues/5640, + which cites `--run` as already having the correct behavior). A future change that "scopes" + the raise down to fewer models would silently break this `plan`/`plan --run` parity and + should fail this test. + """ + _write_daily_and_weekly_model_project(tmp_path) + context = Context(paths=tmp_path) + + # Catch both models up, but to different frontiers: the weekly model's cadence means its + # last fully-elapsed interval (2024-01-14) is a day behind the daily model's (2024-01-15). + context.plan(auto_apply=True, no_prompts=True, execution_time="2024-01-15 00:00:01") + max_ends = context.state_sync.max_interval_end_per_model("prod") + assert max_ends['"daily_model"'] == to_timestamp("2024-01-15") + assert max_ends['"weekly_model"'] == to_timestamp("2024-01-14") + + # A plain, unscoped prod plan (no model changes, no --select-model/--backfill-model, no + # restatement) with execution_time set well ahead of both frontiers and no explicit end. + execution_time = "2024-01-25 00:00:01" + plan = context.plan_builder("prod", execution_time=execution_time).build() + assert plan.requires_backfill + assert to_timestamp(plan.end) == to_timestamp(execution_time) + + missing = _missing_intervals_by_name(plan) + # Both models show missing intervals, even though only the daily model's cadence would + # naturally put it "due" first - the weekly model is swept in too. + assert set(missing) == {'"daily_model"', '"weekly_model"'} + + # An equivalent `plan --run` at the same execution_time computes missing intervals with no + # caps at all. If it matches exactly, that confirms the plain-plan raise reproduces the + # `--run` result rather than under- or over-shooting it. + run_plan = context.plan_builder("prod", execution_time=execution_time, run=True).build() + assert run_plan.requires_backfill + assert _missing_intervals_by_name(run_plan) == missing + + +def test_plan_execution_time_ahead_of_prod_frontier_with_explicit_end(tmp_path: Path): + """When the user provides an explicit `end` alongside `execution_time`, the new + `end is None` guard means the per-model interval end caps are never raised towards + `execution_time` in the first place - the plan's end is exactly the explicit end, and + the pre-existing `PlanBuilder.override_end` behavior (which drops the per-model caps + entirely once `end` is explicit, see sqlmesh/core/plan/builder.py) takes over instead, same + as it did before this fix. This locks in that combining explicit `end` with a far-future + `execution_time` cannot make the backfill silently jump past the requested end. + """ + _write_daily_and_weekly_model_project(tmp_path) + context = Context(paths=tmp_path) + context.plan(auto_apply=True, no_prompts=True, execution_time="2024-01-15 00:00:01") + + # Explicit start/end is only allowed for dev plans (or prod plans with restatements), so use + # a dev plan here; the guard being exercised doesn't depend on which of those it is. + plan = context.plan_builder( + "dev", + execution_time="2024-01-30 00:00:01", + end="2024-01-21", + include_unmodified=True, + ).build() + assert plan.requires_backfill + # The plan's end matches the explicit end exactly - it is not raised towards execution_time. + assert to_timestamp(plan.end) == to_timestamp("2024-01-21") + + # Missing intervals for both models stop at the explicit end and never reach anywhere near + # the much-later execution_time. + for si in plan.missing_intervals: + assert si.merged_intervals[-1][1] <= to_timestamp("2024-01-22") + + @pytest.mark.slow def test_plan_seed_model_excluded_from_default_end(copy_to_temp_path: t.Callable): path = copy_to_temp_path("examples/sushi") @@ -3254,7 +3424,12 @@ def test_plan_min_intervals(tmp_path: Path): plan = context.plan(execution_time=current_time) assert to_datetime(plan.start) == to_datetime("2020-01-01 00:00:00") - assert to_datetime(plan.end) == to_datetime("2020-02-01 00:00:00") + # the explicitly provided execution_time is 1 second past the day-aligned frontier of the + # other, already-caught-up models, so the plan end now matches it exactly instead of being + # capped at that frontier (see https://github.com/SQLMesh/sqlmesh/issues/5640). This + # doesn't change which intervals are missing below since they're still bucketed by each + # model's own cron. + assert to_datetime(plan.end) == to_datetime("2020-02-01 00:00:01") assert to_datetime(plan.execution_time) == to_datetime("2020-02-01 00:00:01") def _get_missing_intervals(plan: Plan, name: str) -> t.List[t.Tuple[datetime, datetime]]: From 7376ec290f170be6dfcbfbe304576427a43d070a Mon Sep 17 00:00:00 2001 From: wtruongdata Date: Wed, 22 Jul 2026 18:16:19 -0500 Subject: [PATCH 3/4] docs: note that an explicit --execution-time extends a prod plan past the frontier Documents the behavior from #5640 in the plan 'Start and end dates' section: passing --execution-time later than the intervals already loaded in prod extends the plan end and backfills the missing intervals. Signed-off-by: wtruongdata --- docs/concepts/plans.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/concepts/plans.md b/docs/concepts/plans.md index defcd06c0d..c3e12652ee 100644 --- a/docs/concepts/plans.md +++ b/docs/concepts/plans.md @@ -109,6 +109,12 @@ For context, every model has a start date. The start can be specified in [the mo Because the prod environment supports business operations, prod plans ensure every model is backfilled from its start date until the most recent completed time interval. Due to that restriction, the `plan` command's `--start` and `--end` options are not supported for regular plans against prod. The options are supported for [restatement plans](#restatement-plans) against prod to allow re-processing a subset of existing data. +!!! note "Explicit execution time" + + "The most recent completed time interval" is measured relative to the plan's *execution time*, which defaults to now. If you pass an explicit `--execution-time` that is later than the intervals already loaded in prod, the plan extends its end date up to that time and backfills the intervals in between. + + For example, if a daily model in prod is loaded through 2025-12-25, running `sqlmesh plan --execution-time '2025-12-28'` backfills the missing 2025-12-26 and 2025-12-27 intervals. + Non-prod plans are typically used for development, so their models can optionally be backfilled for any date range with the `--start` and `--end` options. Limiting the date range makes backfills faster and development more efficient, especially for incremental models using large tables. #### Model kind limitations From 52880c71752f813939cb78efae64fe9522c315b3 Mon Sep 17 00:00:00 2001 From: wtruongdata Date: Wed, 22 Jul 2026 22:39:33 -0500 Subject: [PATCH 4/4] refactor: trim the plan_builder execution-time comment Shorten the inline comment to the rationale (an explicit execution time is the effective now; raise the per-model caps so plan matches plan --run) and drop the hard-coded builder.py line reference that would drift. No behavior change. Signed-off-by: wtruongdata --- sqlmesh/core/context.py | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/sqlmesh/core/context.py b/sqlmesh/core/context.py index 5eba0d0677..f30c401a36 100644 --- a/sqlmesh/core/context.py +++ b/sqlmesh/core/context.py @@ -1728,21 +1728,11 @@ def plan_builder( and default_end is not None and execution_time_ts > default_end ): - # An explicitly provided execution time acts as the plan's effective "now", so the - # default end is allowed to extend past the recorded prod frontier instead of being - # capped by it. This mirrors how an explicit `end` causes the per-model interval end - # caps to be dropped entirely in PlanBuilder.build() (see `self.override_end`, - # sqlmesh/core/plan/builder.py line 206). Raising (rather than dropping) the caps is - # safe here because `plan --run` already runs with no caps at all. - # - # Note this raises every entry already present in max_interval_end_per_model (i.e. - # every model in this dict, which _get_max_interval_end_per_model above has already - # scoped down to backfill_models/its ancestors when a selection is in effect) - not - # just modified or explicitly selected models within that scope. That's intentional, - # not an oversight. It's what makes a plain, unscoped `sqlmesh plan --execution-time X` - # in prod report the same missing intervals as `sqlmesh plan --run --execution-time X` - # would at the same simulated time. Narrowing this further would reintroduce a `plan` - # vs `plan --run` mismatch for models within that scope. + # An explicit execution time is the plan's effective "now", so the default end may + # extend past the recorded prod frontier (as an explicit `end` already does via + # PlanBuilder.override_end). Raising every per-model cap to it keeps a plain + # `plan --execution-time X` in step with `plan --run --execution-time X`, which + # already runs with no caps. default_end = execution_time_ts execution_time_dt = to_datetime(execution_time_ts) max_interval_end_per_model = {