From 778171f2195d603c338d599e8dcdec4e5503709d Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Thu, 12 Nov 2026 03:33:26 +0000 Subject: [PATCH] fix(computer): verify macOS type-text and paste-text sent as keys by reading the field back When text can't be written through AXValue (Chromium/Electron fields, fields that refuse the write), type-text and paste-text send keys or Cmd+V and always reported unverified. Now the helper predicts the focused field's value from its value and selection before sending, then reads it back until it settles: verified on a match, unverified (text_mismatch) otherwise. Secure fields and fields without a readable value and selection stay unverified as before, and the result never repeats the typed text or the field's contents. --- .../Sources/OrcaComputerUseMacOS/main.swift | 52 ++++++++++++-- .../TextInsertionReadback.swift | 70 +++++++++++++++++++ .../TextInsertionReadbackTests.swift | 48 +++++++++++++ skill-guides/orca-computer-use.md | 3 +- src/cli/skills/bundled-skill-guides.ts | 3 +- src/shared/computer-use-runtime-types.ts | 2 + 6 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/TextInsertionReadback.swift create mode 100644 native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/TextInsertionReadbackTests.swift diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index 9a43b296403..21f63e543e9 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -985,14 +985,16 @@ final class Provider { private func typeText(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) let text = try requiredString(params, "text") - if let focused = focusedRecord(snapshot), let verification = try TextInput.replaceSelection(focused.element, with: text) { + let focused = focusedRecord(snapshot)?.element + if let focused, let verification = try TextInput.replaceSelection(focused, with: text) { return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification) } + let insertion = TextInput.Insertion(into: focused, text: text) try deliverKeys(try withTransportErrors { try BackgroundInputTransport.textEvents(text) }, snapshot: snapshot, params: params) return actionMetadata( path: "synthetic", actionName: "typeText", - verification: unverifiedAction(reason: "synthetic_input") + verification: insertion?.verification() ?? unverifiedAction(reason: "synthetic_input") ) } @@ -1027,17 +1029,19 @@ final class Provider { private func pasteText(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) let text = try requiredString(params, "text") - if let focused = focusedRecord(snapshot), let verification = try TextInput.replaceSelection(focused.element, with: text) { + let focused = focusedRecord(snapshot)?.element + if let focused, let verification = try TextInput.replaceSelection(focused, with: text) { return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification) } let pasteKeys = try Input.keyEvents("cmd+v") + let insertion = TextInput.Insertion(into: focused, text: text) try Input.withPasteboardText(text) { try deliverKeys(pasteKeys, snapshot: snapshot, params: params) } return actionMetadata( path: "clipboard", actionName: "paste", - verification: unverifiedAction(reason: "clipboard_paste") + verification: insertion?.verification() ?? unverifiedAction(reason: "clipboard_paste") ) } @@ -3219,6 +3223,46 @@ private enum TextInput { return verifiedAction(property: "focusedText") } + /// Text sent as keys or a paste: what the focused field should read once it lands. + struct Insertion { + let element: AXUIElement + let expected: String + + /// Nil when the field can't prove the result: none focused, a secure field, or no readable value and selection. + init?(into element: AXUIElement?, text: String) { + guard let element, + !AXSnapshotReader().isSecureTextElement(element), + let current = rawStringAttribute(element, kAXValueAttribute as String) + else { + return nil + } + let selection = TextInput.selectedTextRange(element).flatMap { range -> Range? in + range.location >= 0 && range.length >= 0 ? range.location..<(range.location + range.length) : nil + } + guard let expected = TextInsertionReadback.expectedValue(current: current, selection: selection, inserting: text) else { + return nil + } + self.element = element + self.expected = expected + } + + /// Nil when the field can no longer be read. + func verification() -> [String: Any]? { + let outcome = TextInsertionReadback.settle(expected: expected) { + rawStringAttribute(element, kAXValueAttribute as String) + } + switch outcome { + case .landed: + return verifiedAction(property: "focusedText") + case .mismatch: + // Why: the field may hold part of the text, so the result never shows what it reads. + return unverifiedAction(reason: "text_mismatch") + case .unreadable: + return nil + } + } + } + static func selectAll(_ element: AXUIElement) -> Bool { guard let current = rawStringAttribute(element, kAXValueAttribute as String) else { return false diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/TextInsertionReadback.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/TextInsertionReadback.swift new file mode 100644 index 00000000000..22d021a9191 --- /dev/null +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/TextInsertionReadback.swift @@ -0,0 +1,70 @@ +import Foundation + +/// Checks that text sent as keys or a paste landed: predict the field's value, then read it back. +public enum TextInsertionReadback { + public enum Outcome: Equatable { + case landed + case mismatch + case unreadable + } + + /// The value a field should hold once `text` replaces its selection. `selection` is in UTF-16 offsets, as + /// AXSelectedTextRange reports it. Nil means the result can't be predicted: a non-empty field with no + /// selection, or a range outside the value or inside a character. + public static func expectedValue(current: String, selection: Range?, inserting text: String) -> String? { + let units = Array(current.utf16) + let range: Range + if let selection { + guard selection.lowerBound >= 0, selection.upperBound <= units.count else { + return nil + } + range = selection + } else if units.isEmpty { + range = 0..<0 + } else { + return nil + } + let prefix = Array(units[.. Double = CFAbsoluteTimeGetCurrent, + sleep: (Double) -> Void = { Thread.sleep(forTimeInterval: $0) }, + read: () -> String? + ) -> Outcome { + let start = now() + var quietDeadline = start + quietSeconds + var readback = read() + while let current = readback, !sameText(current, expected) { + let time = now() + guard time < quietDeadline, time < start + capSeconds else { + return .mismatch + } + sleep(intervalSeconds) + readback = read() + if readback != current { + quietDeadline = now() + quietSeconds + } + } + return readback == nil ? .unreadable : .landed + } + + // Why: Chromium contenteditables store typed trailing or repeated spaces as no-break spaces. + private static func sameText(_ lhs: String, _ rhs: String) -> Bool { + lhs.replacingOccurrences(of: "\u{00A0}", with: " ") == rhs.replacingOccurrences(of: "\u{00A0}", with: " ") + } +} diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/TextInsertionReadbackTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/TextInsertionReadbackTests.swift new file mode 100644 index 00000000000..0fa97fbabf9 --- /dev/null +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/TextInsertionReadbackTests.swift @@ -0,0 +1,48 @@ +import XCTest +@testable import OrcaComputerUseMacOSCore + +final class TextInsertionReadbackTests: XCTestCase { + + func testInsertionReplacesTheSelectionInUTF16Offsets() { + XCTAssertEqual( + TextInsertionReadback.expectedValue(current: "a😀bcd", selection: 3..<5, inserting: "XY"), + "a😀XYd" + ) + XCTAssertEqual(TextInsertionReadback.expectedValue(current: "", selection: nil, inserting: "hi"), "hi") + } + + func testUnpredictableInsertionsReturnNil() { + XCTAssertNil(TextInsertionReadback.expectedValue(current: "abc", selection: nil, inserting: "d")) + XCTAssertNil(TextInsertionReadback.expectedValue(current: "abc", selection: 2..<4, inserting: "d")) + XCTAssertNil(TextInsertionReadback.expectedValue(current: "a😀", selection: 2..<2, inserting: "d")) + } + + func testFieldStillFillingPastTheQuietWindowLands() { + // One character every 0.4 s: each read is past the first 0.5 s window, but each change restarts it. + let reads = ["", "h", "he", "hel", "hell", "hello"] + XCTAssertEqual(settle(expected: "hello", reads: reads, secondsPerRead: 0.4), .landed) + } + + func testContentEditableNoBreakSpacesLand() { + XCTAssertEqual(settle(expected: "a b ", reads: ["a \u{00A0}b\u{00A0}"], secondsPerRead: 0.025), .landed) + } + + func testFieldThatDropsTheTextIsAMismatch() { + XCTAssertEqual(settle(expected: "hello", reads: Array(repeating: "", count: 100), secondsPerRead: 0.025), .mismatch) + } + + func testFieldThatStopsAnsweringIsUnreadable() { + XCTAssertEqual(settle(expected: "hello", reads: ["", "h"], secondsPerRead: 0.025), .unreadable) + } + + private func settle(expected: String, reads: [String], secondsPerRead: Double) -> TextInsertionReadback.Outcome { + var clock = 0.0 + var remaining = reads[...] + return TextInsertionReadback.settle( + expected: expected, + now: { clock }, + sleep: { _ in clock += secondsPerRead }, + read: { remaining.popFirst() } + ) + } +} diff --git a/skill-guides/orca-computer-use.md b/skill-guides/orca-computer-use.md index 4e1506d0e88..18125c7e0f8 100644 --- a/skill-guides/orca-computer-use.md +++ b/skill-guides/orca-computer-use.md @@ -116,11 +116,12 @@ printf '%s' "$TEXT" | ORCA computer set-value --app --element-index ` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held. - Coordinates are window-local; use coordinates from the latest screenshot/state for the same target window. diff --git a/src/cli/skills/bundled-skill-guides.ts b/src/cli/skills/bundled-skill-guides.ts index 33aeaf25c71..bcb17907daa 100644 --- a/src/cli/skills/bundled-skill-guides.ts +++ b/src/cli/skills/bundled-skill-guides.ts @@ -1135,11 +1135,12 @@ const ORCA_COMPUTER_USE_MARKDOWN = [ " - `unverified (accessibility action unasserted)` means the accessibility call succeeded but no post-state assertion was made.", " - `unverified (suspected noop)` means the accessibility call succeeded but the window's tree and the app's windows were unchanged about a second later. Web and Electron controls often ignore it. Do not repeat the same element click; if the effect you expect would show in the window, retry once with `click --x --y` at the control from a fresh screenshot. If the effect lives elsewhere (clipboard, another app), check there first.", " - `unverified (synthetic input)` means input was fired into the void and is unverifiable.", + " - `unverified (text mismatch)` means typed or pasted text was read back and the focused field does not hold what inserting it should produce: it may be missing, partly entered, reformatted or autocompleted. Do not type it again blindly, or it may land twice; run `get-app-state` and fix the field from what it actually holds.", " - Missing verification metadata is unverified, including responses from older runtimes.", " - Never report an unverified action as success. If it could have sent, submitted, bought, or deleted something, say the effect is unproven.", "- Prefer semantic actions: `set-value` for editable fields, `click` for controls, and `perform-secondary-action` only for listed action names.", "- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.", - "- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; synthetic keyboard delivery is reported as unverified, so inspect the returned state before assuming text landed.", + "- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; unless the result says `verified focusedText`, inspect the returned state before assuming text landed.", "- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.", "- Use `click --modifiers ` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.", "- Coordinates are window-local; use coordinates from the latest screenshot/state for the same target window.", diff --git a/src/shared/computer-use-runtime-types.ts b/src/shared/computer-use-runtime-types.ts index 42de55e7b3c..e2e1fff7181 100644 --- a/src/shared/computer-use-runtime-types.ts +++ b/src/shared/computer-use-runtime-types.ts @@ -121,6 +121,8 @@ export type ComputerActionVerification = | 'value_mismatch' // The accessibility call succeeded but the window and the app's windows looked the same about a second later. | 'suspected_noop' + // Typed or pasted text was read back and the focused field does not hold what inserting it should produce. + | 'text_mismatch' // Only on value_mismatch from a non-secure field; results never repeat the requested value. actualPreview?: string | null }