perf(ios): speed up deep snapshots and keep first taps reliable

.github/workflows/ios.yml

@@ -147,6 +147,7 @@ jobs:
 
       - name: Run targeted iOS runner XCTest regressions
         run: |
+          set -o pipefail
           XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)"
           test -n "$XCTESTRUN_PATH"
           xcodebuild test-without-building \
@@ -164,6 +165,12 @@ jobs:
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testEmptyReplacementWithoutResolvableTargetFailsClosed \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextEntryTapWitnessIsBoundToTargetIdentity \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTapTextInputProbeSkipsPenalizedXCTestChannel \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testFreshCoordinateTapContainsUnavailableTextInputProbe \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextInputProbeIssueScopeIsThreadBound \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextInputProbePreservesEnclosingRunnerWait \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testSuppressedAxIssueMakesTextInputProbeUnavailable \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHealthyCoordinateTapPreservesBareTypingWitness \
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testTextInputProbeContainmentExcludesRequiredReadsAndLaterIssues \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCoordinateTextInputCandidateMustBeEnabledAndContainTheTouchPoint \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testQuerySelectorPrefersHittableMatchOverNonHittableDuplicate \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActivateTargetSkipsForegroundAndActivatesNonForegroundApplication \
@@ -234,7 +241,14 @@ jobs:
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testCustomActionCoverageParsesOnlyCompletePairs \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testPartialCustomActionPassIsDisclosedAndCompleteOneIsNot \
             -only-testing:AgentDeviceRunnerUITests/RunnerTests/testActionNamesAreCappedPerElementAndReported \
-            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHungCustomActionReadIsContainedAndRecovers
+            -only-testing:AgentDeviceRunnerUITests/RunnerTests/testHungCustomActionReadIsContainedAndRecovers 2>&1 | tee /tmp/agent-device-runner-regressions.log
+          node --input-type=module -e '
+            import { readFileSync } from "node:fs";
+            const log = readFileSync("/tmp/agent-device-runner-regressions.log", "utf8");
+            if (!/\] AGENT_DEVICE_RUNNER_OPTIONAL_PROBE_WAIT_COMPLETED$/m.test(log)) {
+              throw new Error("Optional observation ended the runner test before its wait completed");
+            }
+          '
 
       - name: Preflight iOS runner through public CLI
         run: |

.github/workflows/size.yml

@@ -73,6 +73,7 @@ jobs:
       - name: Measure base size
         run: |
           git checkout --detach "${{ github.event.pull_request.base.sha }}"
+          cp scripts/size-report-package.mjs /tmp/agent-device-size-report/
           pnpm install --frozen-lockfile
           if [ "${{ steps.base-dist-cache.outputs.cache-hit }}" != "true" ]; then
             pnpm build

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -1795,7 +1795,7 @@ extension RunnerTests {
         )
         let textInput: XCUIElement?
         if !xCTestTextInputProbeSkipped {
-          textInput = textInputAt(app: activeApp, x: x, y: y)
+          textInput = coordinateTapTextInputAt(app: activeApp, x: x, y: y)
         } else {
           // A process-scoped tap cannot authorize later typing without concrete element identity.
           textInput = nil

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -318,37 +318,6 @@ extension RunnerTests {
     return nil
   }
 
-  func textInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? {
-    return textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y)).first
-  }
-
-  private func textInputCandidatesAt(app: XCUIApplication, point: CGPoint) -> [XCUIElement] {
-    safely("TEXT_INPUT_AT_POINT", []) {
-      // Query the text-input element types directly instead of enumerating the entire tree
-      // (app.descendants(.any).allElementsBoundByIndex snapshots every element and is ~10x
-      // slower — it dominated fill latency because resolveTextEntryElement re-runs this on
-      // each verify/repair poll once the focused field reference goes stale).
-      // Prefer the smallest matching field so nested editable controls win over large containers.
-      [
-        app.textFields,
-        app.secureTextFields,
-        app.searchFields,
-        app.textViews,
-      ]
-        .flatMap { $0.allElementsBoundByIndex }
-        .filter { element in
-          guard element.exists else { return false }
-          let frame = element.frame
-          return isCoordinateTextInputCandidate(
-            enabled: element.isEnabled,
-            frame: frame,
-            point: point
-          )
-        }
-        .sorted(by: smallestElementFirst)
-    }
-  }
-
   private func readableText(for element: XCUIElement) -> String? {
     let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
     let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift

@@ -0,0 +1,101 @@
+import XCTest
+
+final class TextInputProbeIssues {
+  let thread = Thread.current
+  var count = 0
+}
+
+enum TextInputProbeFailure: String {
+  case recordedIssue = "text_input_probe_recorded_issue"
+  case exception = "text_input_probe_exception"
+}
+
+enum TextInputProbeOutcome {
+  case matches([XCUIElement])
+  case absent
+  case unavailable(TextInputProbeFailure)
+}
+
+extension RunnerTests {
+  func textInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? {
+    textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y)).first
+  }
+
+  func textInputCandidatesAt(app: XCUIApplication, point: CGPoint) -> [XCUIElement] {
+    safely("TEXT_INPUT_AT_POINT", []) {
+      queryTextInputs(app: app, point: point, shouldStop: { false })
+    }
+  }
+
+  func coordinateTapTextInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? {
+    switch probeTextInputs(app: app, point: CGPoint(x: x, y: y)) {
+    case .matches(let elements):
+      return elements.first
+    case .absent:
+      return nil
+    case .unavailable:
+      return nil
+    }
+  }
+
+  func probeTextInputs(app: XCUIApplication, point: CGPoint) -> TextInputProbeOutcome {
+    precondition(Thread.isMainThread)
+    let issues = TextInputProbeIssues()
+    suppressedIssueLock.lock()
+    let previous = textInputProbeIssues
+    textInputProbeIssues = issues
+    suppressedIssueLock.unlock()
+    defer {
+      suppressedIssueLock.lock()
+      textInputProbeIssues = previous
+      suppressedIssueLock.unlock()
+    }
+    let (elements, exception) = catchingObjCException(fallback: []) {
+      queryTextInputs(app: app, point: point, shouldStop: { self.hasTextInputProbeIssues(issues) })
+    }
+    if hasTextInputProbeIssues(issues) { return .unavailable(.recordedIssue) }
+    if exception != nil { return .unavailable(.exception) }
+    return elements.isEmpty ? .absent : .matches(elements)
+  }
+
+  private func hasTextInputProbeIssues(_ scope: TextInputProbeIssues) -> Bool {
+    suppressedIssueLock.lock()
+    defer { suppressedIssueLock.unlock() }
+    return scope.count > 0
+  }
+
+  func containTextInputProbeIssue(_ issue: XCTIssue) -> Bool {
+    suppressedIssueLock.lock()
+    guard let scope = textInputProbeIssues, scope.thread === Thread.current else {
+      suppressedIssueLock.unlock()
+      return false
+    }
+    scope.count += 1
+    suppressedIssueLock.unlock()
+    NSLog("AGENT_DEVICE_RUNNER_TEXT_INPUT_PROBE_UNAVAILABLE issue=%@", issue.compactDescription)
+    return true
+  }
+
+  private func queryTextInputs(
+    app: XCUIApplication,
+    point: CGPoint,
+    shouldStop: () -> Bool
+  ) -> [XCUIElement] {
+    var candidates: [XCUIElement] = []
+    for query in [app.textFields, app.secureTextFields, app.searchFields, app.textViews] {
+      if shouldStop() { break }
+      candidates.append(contentsOf: query.allElementsBoundByIndex)
+#if AGENT_DEVICE_RUNNER_UNIT_TESTS
+      if let issue = textInputProbeIssueForTesting {
+        textInputProbeIssueForTesting = nil
+        record(issue)
+      }
+#endif
+    }
+    guard !shouldStop() else { return [] }
+    return candidates.filter { element in
+      guard !shouldStop(), element.exists else { return false }
+      return isCoordinateTextInputCandidate(enabled: element.isEnabled, frame: element.frame, point: point)
+    }.sorted(by: smallestElementFirst)
+  }
+}

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -143,6 +143,8 @@ final class RunnerTests: XCTestCase {
   // The injection records a real XCTIssue AFTER the real gesture, so
   // `xctestRecordedFailureResponse` and target invalidation fire byte-for-byte
   // like a field failure. Production builds compile none of this.
+  var textInputProbeIssueForTesting: XCTIssue?
+
   static let injectedTapFailureFlagPathForTesting =
     "/tmp/agent-device-inject-tap-recorded-failure-for-testing"
 
@@ -182,6 +184,7 @@ final class RunnerTests: XCTestCase {
   #endif
   // Observability for the record(_:) suppression below: how many AX-broken-screen snapshot
   // issues this session muted, so wedge investigations see the volume without grepping logs.
+  var textInputProbeIssues: TextInputProbeIssues?
   let suppressedIssueLock = NSLock()
   var suppressedAxSnapshotIssueCount = 0
   // Keep blocker actions narrow to avoid false positives from generic hittable containers.
@@ -221,6 +224,7 @@ final class RunnerTests: XCTestCase {
   /// outcomes stay honest through their own error paths — only this issue side-channel is
   /// muted. Everything else still records (and still drives XCTEST_RECORDED_FAILURE).
   override func record(_ issue: XCTIssue) {
+    if containTextInputProbeIssue(issue) { return }
     let description = issue.compactDescription
     if Self.isSuppressedAxSnapshotIssueDescription(description) {
       suppressedIssueLock.lock()

apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+TextInputProbeTests.swift

@@ -0,0 +1,177 @@
+import XCTest
+
+extension RunnerTests {
+#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS)
+  func testTextInputProbePreservesEnclosingRunnerWait() {
+    app.launchArguments = ["--agent-device-text-entry-regression"]
+    app.launch()
+    defer {
+      textInputProbeIssueForTesting = nil
+      app.terminate()
+    }
+    let field = app.textFields["agent-device-hardware-keyboard-input"]
+    XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout))
+    let point = CGPoint(x: field.frame.midX, y: field.frame.midY)
+    let completed = expectation(description: "optional probe completed inside runner wait")
+    DispatchQueue.main.async {
+      self.textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Optional probe issue during runner wait")
+      _ = self.probeTextInputs(app: self.app, point: point)
+      completed.fulfill()
+    }
+    guard XCTWaiter.wait(for: [completed], timeout: 5) == .completed else {
+      return XCTFail("Optional probe interrupted the runner wait")
+    }
+    XCTAssertNil(textInputProbeIssues)
+    NSLog("AGENT_DEVICE_RUNNER_OPTIONAL_PROBE_WAIT_COMPLETED")
+  }
+
+  func testTextInputProbeIssueScopeIsThreadBound() {
+    let issue = XCTIssue(type: .assertionFailure, compactDescription: "Issue scope thread check")
+    XCTAssertFalse(containTextInputProbeIssue(issue))
+    let scope = TextInputProbeIssues()
+    suppressedIssueLock.lock()
+    textInputProbeIssues = scope
+    suppressedIssueLock.unlock()
+    defer {
+      suppressedIssueLock.lock()
+      textInputProbeIssues = nil
+      suppressedIssueLock.unlock()
+    }
+    let finished = DispatchSemaphore(value: 0)
+    let result = ProbeThreadResult()
+    Thread.detachNewThread {
+      result.contained = self.containTextInputProbeIssue(issue)
+      finished.signal()
+    }
+    guard finished.wait(timeout: .now() + 2) == .success else {
+      return XCTFail("Background issue classification did not finish")
+    }
+    XCTAssertFalse(result.contained)
+    XCTAssertEqual(scope.count, 0)
+    XCTAssertTrue(containTextInputProbeIssue(issue))
+    XCTAssertEqual(scope.count, 1)
+  }
+
+  func testHealthyCoordinateTapPreservesBareTypingWitness() throws {
+    app.launchArguments = ["--agent-device-text-entry-regression"]
+    app.launch()
+    defer {
+      invalidateCachedTarget(reason: "unit_test_cleanup")
+      app.terminate()
+    }
+    let field = app.textFields["agent-device-hardware-keyboard-input"]
+    XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout))
+    let frame = field.frame
+    currentApp = app
+    currentBundleId = "com.callstack.agentdevice.runner"
+    currentAppProcessIdentifier = try XCTUnwrap(Self.processIdentifier(of: app))
+    clearSnapshotXCTestChannelPenalty(reason: "fresh-runner")
+    let failures = currentXCTestFailureCount()
+    let tap = try runnerCommandFixture(
+      #"{"appBundleId":"com.callstack.agentdevice.runner","command":"tap","commandId":"tap-healthy-probe","x":\#(frame.midX),"y":\#(frame.midY),"synthesized":true}"#
+    )
+    let tapped = try execute(command: tap)
+    XCTAssertTrue(tapped.ok, String(describing: tapped.error))
+    XCTAssertNotNil(textEntryTapWitness)
+    XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: currentBundleId))
+    try XCTSkipIf(isKeyboardVisible(app: app), "software keyboard is up; hidden-keyboard witness cannot be exercised")
+    let type = try runnerCommandFixture(#"{"appBundleId":"com.callstack.agentdevice.runner","command":"type","commandId":"type-healthy-probe","text":"probe-witness"}"#)
+    let typed = try execute(command: type)
+    XCTAssertTrue(typed.ok, String(describing: typed.error))
+    XCTAssertEqual(typed.data?.textEntryRoute, "synthesized-first-responder")
+    XCTAssertEqual(field.value as? String, "probe-witness")
+    XCTAssertFalse(didRecordXCTestFailure(since: failures))
+  }
+
+  func testTextInputProbeContainmentExcludesRequiredReadsAndLaterIssues() {
+    app.launchArguments = ["--agent-device-text-entry-regression"]
+    app.launch()
+    defer {
+      textInputProbeIssueForTesting = nil
+      app.terminate()
+    }
+    let field = app.textFields["agent-device-hardware-keyboard-input"]
+    XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout))
+    let point = CGPoint(x: field.frame.midX, y: field.frame.midY)
+    let expected = XCTIssue(type: .assertionFailure, compactDescription: "Required query failure must escape optional containment")
+    let options = XCTExpectedFailure.Options()
+    var observed = 0
+    options.issueMatcher = { issue in
+      guard issue.type == expected.type, issue.compactDescription == expected.compactDescription else { return false }
+      observed += 1
+      return true
+    }
+    XCTExpectFailure("Required read and later issue belong to their caller", options: options) {
+      textInputProbeIssueForTesting = expected
+      _ = textInputAt(app: app, x: point.x, y: point.y)
+      _ = probeTextInputs(app: app, point: point)
+      record(expected)
+    }
+    XCTAssertEqual(observed, 2)
+  }
+
+  func testSuppressedAxIssueMakesTextInputProbeUnavailable() throws {
+    app.launchArguments = ["--agent-device-text-entry-regression"]
+    app.launch()
+    defer {
+      textInputProbeIssueForTesting = nil
+      app.terminate()
+    }
+    let field = app.textFields["agent-device-hardware-keyboard-input"]
+    XCTAssertTrue(field.waitForExistence(timeout: appExistenceTimeout))
+    let frame = field.frame
+    textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Failed to get matching snapshot: kAXErrorIllegalArgument")
+    let outcome = probeTextInputs(app: app, point: CGPoint(x: frame.midX, y: frame.midY))
+    guard case .unavailable = outcome else {
+      return XCTFail("A suppressed AX issue must discard the matching candidate")
+    }
+  }
+
+  func testFreshCoordinateTapContainsUnavailableTextInputProbe() throws {
+    app.launchArguments = ["--agent-device-text-entry-regression"]
+    app.launch()
+    defer {
+      textInputProbeIssueForTesting = nil
+      clearSnapshotXCTestChannelPenalty(reason: "test-cleanup")
+      invalidateCachedTarget(reason: "unit_test_cleanup")
+      app.terminate()
+    }
+    let target = app.staticTexts["Agent Device Runner"]
+    XCTAssertTrue(target.waitForExistence(timeout: appExistenceTimeout))
+    let frame = target.frame
+    currentApp = app
+    currentBundleId = "com.callstack.agentdevice.runner"
+    currentAppProcessIdentifier = try XCTUnwrap(Self.processIdentifier(of: app))
+    clearSnapshotXCTestChannelPenalty(reason: "fresh-runner")
+    let failures = currentXCTestFailureCount()
+    textInputProbeIssueForTesting = XCTIssue(type: .assertionFailure, compactDescription: "Injected optional text input query failure")
+    let command = try runnerCommandFixture(
+      #"{"appBundleId":"com.callstack.agentdevice.runner","command":"tap","commandId":"tap-probe-unavailable","x":\#(frame.midX),"y":\#(frame.midY),"synthesized":true}"#
+    )
+    let response = try execute(command: command)
+    XCTAssertTrue(response.ok, String(describing: response.error))
+    XCTAssertFalse(didRecordXCTestFailure(since: failures))
+    XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: currentBundleId))
+    XCTAssertNil(textEntryTapWitness)
+    let type = try runnerCommandFixture(#"{"appBundleId":"com.callstack.agentdevice.runner","command":"type","commandId":"type-after-unavailable-probe","text":"must-not-type"}"#)
+    let typed = try execute(command: type)
+    XCTAssertFalse(typed.ok)
+    XCTAssertEqual(typed.error?.code, "TEXT_INPUT_NOT_FOCUSED")
+    let field = app.textFields["agent-device-hardware-keyboard-input"]
+    let fieldFrame = field.frame
+    let nextTap = try runnerCommandFixture(
+      #"{"appBundleId":"com.callstack.agentdevice.runner","command":"tap","commandId":"tap-after-probe-recovery","x":\#(fieldFrame.midX),"y":\#(fieldFrame.midY),"synthesized":true}"#
+    )
+    XCTAssertTrue(try execute(command: nextTap).ok)
+    XCTAssertNotNil(textEntryTapWitness)
+    XCTAssertTrue(try execute(command: type).ok)
+    XCTAssertEqual(field.value as? String, "must-not-type")
+  }
+#endif
+}
+
+#if AGENT_DEVICE_RUNNER_UNIT_TESTS && os(iOS)
+private final class ProbeThreadResult: @unchecked Sendable {
+  var contained = false
+}
+#endif