fix(apple): bound the cold toolchain probes by the owning request budget

Three synchronous probes could each retry once at 30 s, so a wedged
toolchain host blocked a request for ~180 s with no deadline and no
cancellation check.

The runner cache decision now takes the owning request's budget
(remaining ms + abort signal) and builds one clock per fingerprint read:
every attempt runs at min(per-call ceiling, remaining), the retry is
skipped once the budget is spent, an exhausted budget fails the decision
without starting another probe, and an aborted signal surfaces the
cancellation instead of retrying. `ensureXctestrunArtifact` passes the
build budget and signal, session reuse passes the startup budget and the
request signal, and lease adoption passes the startup budget; a caller
with neither is still capped at 45 s total, so the worst case falls from
~180 s to 45 s. Error codes, texts, and the probe hint are unchanged.

Both retry classifiers now read the exec layer's structured timeout
detail instead of matching "timed out after Nms" in the message. The
predicate is exported once from host-kit's command surface and reaches
`runner-cache-metadata.ts` through the Apple runner host port, so the
file's eager closure is unchanged.

Tests use a fake clock that only advances when a probe actually blocks
for the timeout it was given, so the exhausted-budget and cancellation
cases have to spend the budget to pass; both consumers also pin that an
error saying "timed out after 10ms" without the structured detail is not
retried.

Refs #2422

Author: thymikee

packages/host-kit/src/command.ts

@@ -7,6 +7,7 @@ export {
   execFailureDetails,
   type ExecOptions,
   type ExecResult,
+  isCommandTimeoutError,
   isExecutablePath,
   requireExecSuccess,
   resolveExecutableOverridePath,

packages/host-kit/src/internal/exec.test.ts

@@ -5,6 +5,7 @@ import path from 'node:path';
 import { flushDiagnosticsToSessionFile, withDiagnosticsScope } from './diagnostics.ts';
 import {
   coerceExecResult,
+  isCommandTimeoutError,
   requireExecSuccess,
   runCmd,
   runCmdBackground,
@@ -424,3 +425,37 @@ test('coerceExecResult repairs loosely-typed provider results and keeps typed on
   } as unknown as ExecResult);
   assert.deepEqual(loose, { stdout: '', stderr: '42', exitCode: 1 });
 });
+
+test('isCommandTimeoutError reads the structured timeout, not the message text', async () => {
+  const killedAtTimeout = await runCmd(process.execPath, ['-e', 'setTimeout(() => {}, 10_000)'], {
+    timeoutMs: 50,
+  }).then(
+    () => null,
+    (error: unknown) => error,
+  );
+  assert.ok(isCommandTimeoutError(killedAtTimeout));
+
+  assert.ok(
+    isCommandTimeoutError(
+      (() => {
+        try {
+          runCmdSync(process.execPath, ['-e', 'setTimeout(() => {}, 10_000)'], { timeoutMs: 50 });
+          return null;
+        } catch (error) {
+          return error;
+        }
+      })(),
+    ),
+  );
+
+  // A tool that failed on its own and said "timed out" in its output: same
+  // code, same wording, no timeout we imposed.
+  assert.equal(
+    isCommandTimeoutError(
+      new AppError('COMMAND_FAILED', 'xcodebuild timed out after 10ms', { cmd: 'xcodebuild' }),
+    ),
+    false,
+  );
+  assert.equal(isCommandTimeoutError(new Error('timed out after 10ms')), false);
+  assert.equal(isCommandTimeoutError(undefined), false);
+});

packages/host-kit/src/internal/exec.ts

@@ -592,6 +592,22 @@ function createTimeoutError(
   });
 }
 
+/**
+ * True only for the COMMAND_FAILED error this module raises when it kills a
+ * command at its own `timeoutMs` — the structured signal both timeout sites
+ * above stamp on `details.timeoutMs`. Callers that retry a timed-out command
+ * classify with this instead of matching the message text, so a command whose
+ * own output happens to say "timed out after 10ms" is not mistaken for a
+ * timeout the exec layer imposed.
+ */
+export function isCommandTimeoutError(error: unknown): boolean {
+  return (
+    error instanceof AppError &&
+    error.code === 'COMMAND_FAILED' &&
+    typeof error.details?.timeoutMs === 'number'
+  );
+}
+
 function createExitError(
   executable: string,
   cmd: string,

packages/platform-apple/src/core/runner-host.ts

@@ -3,6 +3,7 @@ import { publishFileSync, acquireProcessLock } from '@agent-device/host-kit/file
 import { resolveIosSimulatorDeviceSetPath } from '@agent-device/kernel/device-isolation';
 
 import {
+  isCommandTimeoutError,
   requireExecSuccess,
   runCmdBackground,
   runCmdStreaming,
@@ -55,6 +56,7 @@ export const appleRunnerHost: AppleRunnerHost = {
   runCmdSync,
   runCmdBackground,
   requireExecSuccess,
+  isCommandTimeoutError,
   emitDiagnostic,
   withDiagnosticTimer,
   retryWithPolicy,

packages/platform-apple/src/runner/__tests__/runner-cache-metadata.test.ts

@@ -1,7 +1,10 @@
-import { expect, test } from 'vitest';
+import { beforeEach, describe, expect, test } from 'vitest';
 import assert from 'node:assert/strict';
-import { AppError } from '@agent-device/kernel/errors';
+import { AppError, isRequestCanceledError } from '@agent-device/kernel/errors';
+import { resetAllProcessMemosForTests } from '@agent-device/kernel/ttl-memo';
 import { IOS_DEVICE, IOS_SIMULATOR, MACOS_DEVICE } from './device-fixtures.ts';
+import { appleRunnerTestHost } from '../test-host.ts';
+import type { ExecOptions } from '../host.ts';
 import {
   COLD_TOOLCHAIN_PROBE_TIMEOUT_MS,
   diffComparableRunnerCacheMetadata,
@@ -253,58 +256,182 @@ test('a timed-out probe leaves the toolchain unavailable instead of a comparable
 
 // Apple's syspolicyd signature scan blocks the first xcodebuild/xcrun exec
 // after a fresh macOS host boots for roughly 18 to 19 seconds; the immediate
-// next exec of the same tool is instant (#2422). These two cases exercise
-// the resulting one-retry policy without waiting on a real cold-start stall.
-// They use device fixtures untouched by the tests above so the toolchain
-// fingerprint cache starts empty for each.
+// next exec of the same tool is instant (#2422). These cases exercise the
+// resulting one-retry policy, and the budget that bounds it, without waiting
+// on a real cold-start stall: the fake clock only moves when a probe actually
+// blocks for the timeout it was given, so a case that claims the budget was
+// spent had to spend it.
+describe('toolchain probe budget', () => {
+  // Failures are never memoized, but the recovery case below succeeds; each
+  // case starts from an empty toolchain fingerprint cache so none of them
+  // reads another's answer.
+  beforeEach(resetAllProcessMemosForTests);
+
+  test('a cold-start probe recovers on retry, and the stall it survived is charged to the budget', () => {
+    const clock = installFakeToolchainClock();
+    const xcodebuildTimeouts: number[] = [];
+    runCmdSync.mockImplementation((command: string, args: string[], options: ExecOptions) => {
+      if (command !== 'xcodebuild') return appleToolchainProbeResult(command, args);
+      xcodebuildTimeouts.push(options.timeoutMs ?? 0);
+      if (xcodebuildTimeouts.length > 1) return appleToolchainProbeResult(command, args);
+      throw blockForWholeTimeout(clock, command, args, options);
+    });
+    runCmdSync.mockClear();
+
+    const metadata = resolveExpectedRunnerCacheMetadata(IOS_DEVICE);
+
+    assert.equal(metadata.xcodeVersion, '26.2');
+    assert.equal(metadata.xcodeBuildVersion, '17C52');
+    // The retry runs on what the shared budget has left, not on a fresh
+    // per-call ceiling: 45 s total minus the 30 s the first attempt burned.
+    assert.deepEqual(xcodebuildTimeouts, [COLD_TOOLCHAIN_PROBE_TIMEOUT_MS, 15_000]);
+  });
 
-test('a cold-start toolchain probe recovers on retry: the first call exceeds the budget, the second returns immediately', () => {
-  let xcodebuildAttempts = 0;
-  runCmdSync.mockImplementation((command: string, args: readonly string[]) => {
-    if (command === 'xcodebuild') {
-      xcodebuildAttempts += 1;
-      if (xcodebuildAttempts === 1) {
-        throw new AppError('COMMAND_FAILED', 'xcodebuild timed out after 30000ms', {
-          timeoutMs: 30_000,
-        });
-      }
-    }
-    return appleToolchainProbeResult(command, args);
+  test('a toolchain host that never returns stops at the shared budget instead of once per probe', () => {
+    const clock = installFakeToolchainClock();
+    runCmdSync.mockImplementation((command: string, args: string[], options: ExecOptions) => {
+      throw blockForWholeTimeout(clock, command, args, options);
+    });
+    runCmdSync.mockClear();
+
+    assert.throws(
+      () => resolveExpectedRunnerCacheMetadata(MACOS_DEVICE),
+      (error: unknown) => {
+        assert.ok(error instanceof AppError);
+        assert.equal(error.code, 'COMMAND_FAILED');
+        assert.equal(error.details?.reason, 'apple_toolchain_probe_unavailable');
+        assert.equal(error.details?.retriable, true);
+        expect(error.details?.hint).toContain('xcode-select -p');
+        expect(error.message).toContain('xcodebuild -version');
+        // The reported failure is the last attempt: 30 s, then the 15 s the
+        // budget still had.
+        expect(error.message).toContain('xcodebuild timed out after 15000ms');
+        expect(error.message).toContain('request budget');
+        return true;
+      },
+    );
+    // 30 s + a 15 s retry spends the whole budget on the first probe; the two
+    // xcrun probes then fail on the budget instead of blocking for 30 s each.
+    assert.equal(runCmdSync.mock.calls.length, 2);
+    assert.equal(clock.nowMs, 45_000);
   });
-  runCmdSync.mockClear();
 
-  const metadata = resolveExpectedRunnerCacheMetadata(IOS_DEVICE);
+  test('an owning request with 4 s left gets one 4 s attempt and no retry', () => {
+    const clock = installFakeToolchainClock();
+    runCmdSync.mockImplementation((command: string, args: string[], options: ExecOptions) => {
+      throw blockForWholeTimeout(clock, command, args, options);
+    });
+    runCmdSync.mockClear();
+
+    assert.throws(
+      () => resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR, undefined, { timeoutMs: 4_000 }),
+      (error: unknown) => {
+        assert.ok(error instanceof AppError);
+        assert.equal(error.details?.reason, 'apple_toolchain_probe_unavailable');
+        expect(error.message).toContain('xcodebuild timed out after 4000ms');
+        expect(error.message).toContain('request budget');
+        return true;
+      },
+    );
+    assert.equal(runCmdSync.mock.calls.length, 1);
+    assert.equal(clock.nowMs, 4_000);
+  });
 
-  assert.equal(metadata.xcodeVersion, '26.2');
-  assert.equal(metadata.xcodeBuildVersion, '17C52');
-  expect(runCmdSync.mock.calls.filter(([command]) => command === 'xcodebuild')).toHaveLength(2);
-});
+  test('a request canceled while a probe blocked surfaces the cancellation instead of retrying', () => {
+    const clock = installFakeToolchainClock();
+    const request = new AbortController();
+    runCmdSync.mockImplementation((command: string, args: string[], options: ExecOptions) => {
+      const timeout = blockForWholeTimeout(clock, command, args, options);
+      request.abort();
+      throw timeout;
+    });
+    runCmdSync.mockClear();
+
+    assert.throws(
+      () =>
+        resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR, undefined, { signal: request.signal }),
+      (error: unknown) => isRequestCanceledError(error),
+    );
+    assert.equal(runCmdSync.mock.calls.length, 1);
+  });
 
-test('a toolchain host that never returns still fails at the deadline with the same timeout error', () => {
-  runCmdSync.mockImplementation((command: string, args: readonly string[]) => {
-    if (command === 'xcodebuild') {
-      throw new AppError('COMMAND_FAILED', 'xcodebuild timed out after 30000ms', {
-        timeoutMs: 30_000,
-      });
-    }
-    return appleToolchainProbeResult(command, args);
+  test('an already-canceled request runs no toolchain probe at all', () => {
+    installFakeToolchainClock();
+    runCmdSync.mockClear();
+
+    assert.throws(
+      () =>
+        resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR, undefined, {
+          signal: AbortSignal.abort(),
+        }),
+      (error: unknown) => isRequestCanceledError(error),
+    );
+    assert.equal(runCmdSync.mock.calls.length, 0);
   });
-  runCmdSync.mockClear();
 
-  try {
-    resolveExpectedRunnerCacheMetadata(MACOS_DEVICE);
-    assert.fail('expected an always-timing-out toolchain probe to fail the cache decision');
-  } catch (error) {
-    assert.ok(error instanceof AppError);
-    assert.equal(error.code, 'COMMAND_FAILED');
-    assert.equal(error.details?.retriable, true);
-    expect(error.message).toContain('xcodebuild -version');
-    expect(error.message).toContain('xcodebuild timed out after 30000ms');
-  }
-  // Exactly one retry, not an unbounded loop: the original attempt plus one retry.
-  expect(runCmdSync.mock.calls.filter(([command]) => command === 'xcodebuild')).toHaveLength(2);
+  test('a probe that failed on its own and merely says "timed out" in its message is not retried', () => {
+    installFakeToolchainClock();
+    runCmdSync.mockImplementation((command: string, args: string[]) => {
+      if (command !== 'xcodebuild') return appleToolchainProbeResult(command, args);
+      // No `timeoutMs` detail: this is the tool reporting its own failure, not
+      // the exec layer killing it at a timeout we asked for.
+      throw new AppError('COMMAND_FAILED', 'xcodebuild timed out after 10ms', {
+        cmd: command,
+        args,
+      });
+    });
+    runCmdSync.mockClear();
+
+    assert.throws(
+      () => resolveExpectedRunnerCacheMetadata(IOS_SIMULATOR),
+      (error: unknown) => {
+        assert.ok(error instanceof AppError);
+        expect(error.message).toContain('xcodebuild timed out after 10ms');
+        return true;
+      },
+    );
+    expect(runCmdSync.mock.calls.filter(([command]) => command === 'xcodebuild')).toHaveLength(1);
+  });
 });
 
+/**
+ * A clock the probes' own budget reads, advanced only by
+ * {@link blockForWholeTimeout}. Without it a mock that throws immediately
+ * proves nothing about a deadline: no time passes, so every budget looks
+ * untouched however many attempts run.
+ */
+function installFakeToolchainClock(): { nowMs: number } {
+  const clock = { nowMs: 0 };
+  appleRunnerTestHost.update({
+    deadlineFromTimeoutMs: (timeoutMs: number) => {
+      const startedAtMs = clock.nowMs;
+      const expiresAtMs = startedAtMs + Math.max(0, timeoutMs);
+      return {
+        remainingMs: () => Math.max(0, expiresAtMs - clock.nowMs),
+        elapsedMs: () => Math.max(0, clock.nowMs - startedAtMs),
+        isExpired: () => expiresAtMs - clock.nowMs <= 0,
+      };
+    },
+  });
+  return clock;
+}
+
+/** A probe that blocked for its whole timeout and was then killed, as the exec layer reports it. */
+function blockForWholeTimeout(
+  clock: { nowMs: number },
+  command: string,
+  args: string[],
+  options: ExecOptions,
+): AppError {
+  const timeoutMs = options.timeoutMs ?? 0;
+  clock.nowMs += timeoutMs;
+  return new AppError('COMMAND_FAILED', `${command} timed out after ${timeoutMs}ms`, {
+    cmd: command,
+    args,
+    timeoutMs,
+  });
+}
+
 test('a failing probe reports its exit status rather than a fabricated SDK version', () => {
   runCmdSync.mockImplementation((command: string, args: readonly string[]) =>
     command === 'xcrun'

packages/platform-apple/src/runner/host.ts

@@ -149,6 +149,12 @@ export type AppleRunnerHost = {
     message: string,
     extra?: Record<string, unknown> | ((result: ExecResult) => Record<string, unknown>),
   ): ExecResult;
+  /**
+   * True only for the error the exec layer raises when it killed a command at
+   * the `timeoutMs` the caller asked for, read from the structured detail it
+   * stamps rather than from the message text.
+   */
+  isCommandTimeoutError(error: unknown): boolean;
   // Diagnostics (@agent-device/host-kit/diagnostics)
   emitDiagnostic(event: DiagnosticEventInput): void;
   withDiagnosticTimer<T>(
@@ -277,6 +283,8 @@ export const runCmdBackground: AppleRunnerHost['runCmdBackground'] = (cmd, args,
   requireHost().runCmdBackground(cmd, args, options);
 export const requireExecSuccess: AppleRunnerHost['requireExecSuccess'] = (result, message, extra) =>
   requireHost().requireExecSuccess(result, message, extra);
+export const isCommandTimeoutError: AppleRunnerHost['isCommandTimeoutError'] = (error) =>
+  requireHost().isCommandTimeoutError(error);
 export const emitDiagnostic: AppleRunnerHost['emitDiagnostic'] = (event) =>
   requireHost().emitDiagnostic(event);
 export const withDiagnosticTimer = <T>(

packages/platform-apple/src/runner/runner-adoption.ts

@@ -19,6 +19,7 @@ import {
 import {
   resolveExpectedRunnerCacheMetadata,
   resolveRunnerDerivedPath,
+  type RunnerCacheProbeBudget,
   type RunnerXctestrunArtifact,
 } from './runner-xctestrun.ts';
 import {
@@ -86,7 +87,9 @@ export async function tryAdoptRunnerSessionFromLease(
   if (!verifyLeaseRunnerPidIdentity(lease, runnerPid)) {
     return skip('runner_pid_recycled');
   }
-  const expectedDerived = resolveExpectedDerivedPath(device);
+  const expectedDerived = resolveExpectedDerivedPath(device, {
+    timeoutMs: options.startupTimeoutMs,
+  });
   if (!expectedDerived) return skip('expected_derived_unresolved');
   if (!lease.xctestrunPath.startsWith(`${expectedDerived}${path.sep}`)) {
     return skip('artifact_fingerprint_mismatch');
@@ -134,9 +137,15 @@ async function probeRunnerAnswersUptime(device: DeviceInfo, port: number): Promi
   }
 }
 
-function resolveExpectedDerivedPath(device: DeviceInfo): string | null {
+function resolveExpectedDerivedPath(
+  device: DeviceInfo,
+  budget: RunnerCacheProbeBudget,
+): string | null {
   try {
-    return resolveRunnerDerivedPath(device, resolveExpectedRunnerCacheMetadata(device));
+    return resolveRunnerDerivedPath(
+      device,
+      resolveExpectedRunnerCacheMetadata(device, undefined, budget),
+    );
   } catch {
     return null;
   }