fix(sdk,dashboard-agent): stop a single tool-using turn compacting the chat conversation

A `chat.agent` with compaction configured could summarise a conversation
after its very first question. The between-turns check was handed the
turn's token usage summed over every tool-calling step, so a five-step
turn reported its context five times over. It now receives the last
step's usage, which is the context the model held on its final call. The
summed figure is still on the event as `turnUsage`.

A head-start handover that completes its pending tool call under the
same message id now replaces the spliced partial in the model lane
directly. Before, the replacement never matched and the lane was rebuilt
from the transcript with a warning, which also dropped any pending lane
injections.

The dashboard agent decides compaction on the whole context the provider
billed for the last call, 100k tokens by default and configurable with
`DASHBOARD_AGENT_CONTEXT_TOKEN_BUDGET`. There is no longer a prefix
constant to keep in step with the model or the prompt.

Mono-RevId: 1ec83a82ec108975607b9aea4becd61deff1b6ee

Author: ericallam

.changeset/outer-compaction-last-step-usage.md

@@ -0,0 +1,5 @@
+---
+"@trigger.dev/sdk": patch
+---
+
+chat.agent: the between-turns compaction check now receives the last step's token usage (the context the model actually held) instead of the turn's sum over every tool-calling step, so a single tool-using turn no longer compacts a short conversation. The summed figure is still available as `turnUsage` on the event. A head-start handover whose pending tool call completes under the same message id now replaces its spliced partial in the model lane directly instead of falling back to a full reconversion.

internal-packages/dashboard-agent/src/compaction.test.ts

@@ -20,13 +20,13 @@ import {
   collectDurableState,
   COMPACTION_KEPT_TAIL,
   COMPACTION_KEPT_TAIL_CHARS,
-  CONVERSATION_TOKEN_BUDGET,
   describeDurableState,
   estimateConversationTokens,
   renderTranscriptForSummary,
   safeTail,
   shouldCompactConversation,
-  STATIC_PREFIX_TOKENS,
+  contextTokenBudget,
+  DEFAULT_CONTEXT_TOKEN_BUDGET,
   withDurableState,
 } from "./compaction";
 
@@ -146,24 +146,45 @@ describe("when the conversation is compacted", () => {
 
   it("compacts on our own estimate, with no usage reported at all", () => {
     // 4 chars ≈ 1 token, so this is comfortably past the budget.
-    const messages = bulk(40, (CONVERSATION_TOKEN_BUDGET * 4) / 20);
-    expect(estimateConversationTokens(messages)).toBeGreaterThan(CONVERSATION_TOKEN_BUDGET);
+    const messages = bulk(40, 12_000);
+    expect(estimateConversationTokens(messages)).toBeGreaterThan(DEFAULT_CONTEXT_TOKEN_BUDGET);
     expect(shouldCompactConversation({ messages })).toBe(true);
   });
 
-  it("compacts on the provider's input count, net of the static prefix", () => {
+  it("compacts on the context the provider billed for the last call", () => {
+    const messages = bulk(4, 100);
+    const budget = DEFAULT_CONTEXT_TOKEN_BUDGET;
+    expect(shouldCompactConversation({ messages, inputTokens: 39_500 }, budget)).toBe(false);
+    expect(shouldCompactConversation({ messages, inputTokens: budget }, budget)).toBe(false);
+    expect(shouldCompactConversation({ messages, inputTokens: budget + 1 }, budget)).toBe(true);
+  });
+
+  it("takes the context budget from the environment, falling back to the default", () => {
+    expect(contextTokenBudget(undefined)).toBe(DEFAULT_CONTEXT_TOKEN_BUDGET);
+    expect(contextTokenBudget(" 150000 ")).toBe(150_000);
+    expect(contextTokenBudget("0")).toBe(DEFAULT_CONTEXT_TOKEN_BUDGET);
+    expect(contextTokenBudget("lots")).toBe(DEFAULT_CONTEXT_TOKEN_BUDGET);
+    expect(contextTokenBudget("1e5")).toBe(100_000);
+    expect(contextTokenBudget("100k")).toBe(DEFAULT_CONTEXT_TOKEN_BUDGET);
+    expect(contextTokenBudget("1.5")).toBe(DEFAULT_CONTEXT_TOKEN_BUDGET);
     const messages = bulk(4, 100);
-    // The prefix alone must never trigger it.
-    expect(shouldCompactConversation({ messages, inputTokens: STATIC_PREFIX_TOKENS + 100 })).toBe(
-      false
-    );
     expect(
-      shouldCompactConversation({
-        messages,
-        inputTokens: STATIC_PREFIX_TOKENS + CONVERSATION_TOKEN_BUDGET + 1,
-      })
+      shouldCompactConversation({ messages, inputTokens: 50_001 }, contextTokenBudget("50000"))
     ).toBe(true);
   });
+
+  it("lets a raised budget stand when the provider count is below it", () => {
+    const messages = bulk(40, 12_000);
+    expect(estimateConversationTokens(messages)).toBeGreaterThan(DEFAULT_CONTEXT_TOKEN_BUDGET);
+    expect(estimateConversationTokens(messages)).toBeLessThan(150_000);
+    expect(shouldCompactConversation({ messages, inputTokens: 120_000 }, 150_000)).toBe(false);
+    expect(shouldCompactConversation({ messages, inputTokens: 150_001 }, 150_000)).toBe(true);
+  });
+
+  it("does not compact a short conversation on a tool-using turn's summed usage", () => {
+    const messages = bulk(12, 600);
+    expect(shouldCompactConversation({ messages, inputTokens: 39_557 })).toBe(false);
+  });
 });
 
 describe("the state a summary may not swallow", () => {

internal-packages/dashboard-agent/src/compaction.ts

@@ -28,20 +28,26 @@ import { summaryPrompt } from "./prompts";
  */
 
 /**
- * The static prefix (system prompt + tool schemas) as `prompt-prefix.test.ts`
- * measures it: ~20.9k estimated tokens. Subtracted so the budget below is about the
- * conversation rather than the total.
- */
-export const STATIC_PREFIX_TOKENS = 21_000;
-
-/**
- * How much conversation rides on top of that prefix before we summarise.
+ * How large the model's context may grow before we summarise, as the provider
+ * billed the LAST call of the turn: prefix, conversation and tool results together.
  *
- * 60k keeps a turn's input near 80k — well inside the 200k window even when a
- * 10-step turn's run traces and query rows add tens of thousands more — and it is
- * about where the uncached tail costs more per turn than one summary call does.
+ * Budgeting the whole context rather than "input minus a prefix constant" means no
+ * figure here has to track the model's tokenizer or the prompt's size (Sonnet 5
+ * bills the same prefix ~1.8x what a chars/4 estimate gives). 100k is roughly the
+ * old 60k of conversation on top of the measured ~38k prefix: well inside every
+ * model's window even when a 10-step turn's run traces and query rows add tens of
+ * thousands more, and about where the uncached tail costs more per turn than one
+ * summary call does. `DASHBOARD_AGENT_CONTEXT_TOKEN_BUDGET` overrides it per
+ * deployment.
  */
-export const CONVERSATION_TOKEN_BUDGET = 60_000;
+export const DEFAULT_CONTEXT_TOKEN_BUDGET = 100_000;
+
+export function contextTokenBudget(
+  envValue = process.env.DASHBOARD_AGENT_CONTEXT_TOKEN_BUDGET
+): number {
+  const parsed = Number(envValue?.trim() ?? "");
+  return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : DEFAULT_CONTEXT_TOKEN_BUDGET;
+}
 
 /** Messages kept verbatim after the summary, so the last exchange reads normally. */
 export const COMPACTION_KEPT_TAIL = 8;
@@ -87,19 +93,24 @@ export function estimateConversationTokens(messages: ModelMessage[]): number {
 }
 
 /**
- * Two signals, either of which fires: what the provider billed for input minus the
- * prefix, and our own estimate of the conversation. The estimate is what makes this
- * testable and what covers a call the provider reported no usage for.
+ * Two signals, either of which fires against the same budget: the context the
+ * provider billed on the last call, and our own chars/4 estimate of the conversation.
+ * The estimate leaves out the prefix (it is not in `messages`) and so can only fire
+ * later than the provider would; it covers a call with no reported usage and keeps
+ * the decision testable without a provider. `inputTokens` must be the last step's,
+ * not the turn's sum over steps, which `chat.agent` guarantees for both checks.
  */
-export function shouldCompactConversation(event: {
-  messages: ModelMessage[];
-  inputTokens?: number;
-  totalTokens?: number;
-}): boolean {
+export function shouldCompactConversation(
+  event: {
+    messages: ModelMessage[];
+    inputTokens?: number;
+    totalTokens?: number;
+  },
+  budget = contextTokenBudget()
+): boolean {
   const reported = typeof event.inputTokens === "number" ? event.inputTokens : event.totalTokens;
-  const fromProvider = typeof reported === "number" ? reported - STATIC_PREFIX_TOKENS : 0;
-  const estimated = estimateConversationTokens(event.messages);
-  return Math.max(fromProvider, estimated) > CONVERSATION_TOKEN_BUDGET;
+  if (typeof reported === "number" && reported > budget) return true;
+  return estimateConversationTokens(event.messages) > budget;
 }
 
 /* ------------------------------------------------------------------ *
@@ -304,6 +315,7 @@ export const dashboardAgentCompaction: ChatAgentCompactionOptions = {
         messageCount: event.messages.length,
         estimatedConversationTokens: estimateConversationTokens(event.messages),
         inputTokens: event.inputTokens ?? null,
+        contextTokenBudget: contextTokenBudget(),
       });
     }
     return compact;