fix(computer): verify macOS type-text and paste-text sent as keys by reading the field back

When text can't be written through AXValue (Chromium/Electron fields, fields
that refuse the write), type-text and paste-text send keys or Cmd+V and always
reported unverified. Now the helper predicts the focused field's value from its
value and selection before sending, then reads it back until it settles:
verified on a match, unverified (text_mismatch) otherwise. Secure fields and
fields without a readable value and selection stay unverified as before, and
the result never repeats the typed text or the field's contents.
This commit is contained in:
Kelvin Amoaba
2026-11-12 21:50:11 +00:00
parent c55af116d1
commit 778171f219
6 changed files with 172 additions and 6 deletions
@@ -985,14 +985,16 @@ final class Provider {
private func typeText(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentKeyboardSnapshot(params: params)
let text = try requiredString(params, "text")
if let focused = focusedRecord(snapshot), let verification = try TextInput.replaceSelection(focused.element, with: text) {
let focused = focusedRecord(snapshot)?.element
if let focused, let verification = try TextInput.replaceSelection(focused, with: text) {
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
}
let insertion = TextInput.Insertion(into: focused, text: text)
try deliverKeys(try withTransportErrors { try BackgroundInputTransport.textEvents(text) }, snapshot: snapshot, params: params)
return actionMetadata(
path: "synthetic",
actionName: "typeText",
verification: unverifiedAction(reason: "synthetic_input")
verification: insertion?.verification() ?? unverifiedAction(reason: "synthetic_input")
)
}
@@ -1027,17 +1029,19 @@ final class Provider {
private func pasteText(params: [String: JSONValue]) throws -> [String: Any] {
let snapshot = try currentKeyboardSnapshot(params: params)
let text = try requiredString(params, "text")
if let focused = focusedRecord(snapshot), let verification = try TextInput.replaceSelection(focused.element, with: text) {
let focused = focusedRecord(snapshot)?.element
if let focused, let verification = try TextInput.replaceSelection(focused, with: text) {
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
}
let pasteKeys = try Input.keyEvents("cmd+v")
let insertion = TextInput.Insertion(into: focused, text: text)
try Input.withPasteboardText(text) {
try deliverKeys(pasteKeys, snapshot: snapshot, params: params)
}
return actionMetadata(
path: "clipboard",
actionName: "paste",
verification: unverifiedAction(reason: "clipboard_paste")
verification: insertion?.verification() ?? unverifiedAction(reason: "clipboard_paste")
)
}
@@ -3219,6 +3223,46 @@ private enum TextInput {
return verifiedAction(property: "focusedText")
}
/// Text sent as keys or a paste: what the focused field should read once it lands.
struct Insertion {
let element: AXUIElement
let expected: String
/// Nil when the field can't prove the result: none focused, a secure field, or no readable value and selection.
init?(into element: AXUIElement?, text: String) {
guard let element,
!AXSnapshotReader().isSecureTextElement(element),
let current = rawStringAttribute(element, kAXValueAttribute as String)
else {
return nil
}
let selection = TextInput.selectedTextRange(element).flatMap { range -> Range<Int>? in
range.location >= 0 && range.length >= 0 ? range.location..<(range.location + range.length) : nil
}
guard let expected = TextInsertionReadback.expectedValue(current: current, selection: selection, inserting: text) else {
return nil
}
self.element = element
self.expected = expected
}
/// Nil when the field can no longer be read.
func verification() -> [String: Any]? {
let outcome = TextInsertionReadback.settle(expected: expected) {
rawStringAttribute(element, kAXValueAttribute as String)
}
switch outcome {
case .landed:
return verifiedAction(property: "focusedText")
case .mismatch:
// Why: the field may hold part of the text, so the result never shows what it reads.
return unverifiedAction(reason: "text_mismatch")
case .unreadable:
return nil
}
}
}
static func selectAll(_ element: AXUIElement) -> Bool {
guard let current = rawStringAttribute(element, kAXValueAttribute as String) else {
return false
@@ -0,0 +1,70 @@
import Foundation
/// Checks that text sent as keys or a paste landed: predict the field's value, then read it back.
public enum TextInsertionReadback {
public enum Outcome: Equatable {
case landed
case mismatch
case unreadable
}
/// The value a field should hold once `text` replaces its selection. `selection` is in UTF-16 offsets, as
/// AXSelectedTextRange reports it. Nil means the result can't be predicted: a non-empty field with no
/// selection, or a range outside the value or inside a character.
public static func expectedValue(current: String, selection: Range<Int>?, inserting text: String) -> String? {
let units = Array(current.utf16)
let range: Range<Int>
if let selection {
guard selection.lowerBound >= 0, selection.upperBound <= units.count else {
return nil
}
range = selection
} else if units.isEmpty {
range = 0..<0
} else {
return nil
}
let prefix = Array(units[..<range.lowerBound])
let suffix = Array(units[range.upperBound...])
let before = String(decoding: prefix, as: UTF16.self)
let after = String(decoding: suffix, as: UTF16.self)
// Why: a boundary inside a surrogate pair decodes to U+FFFD, which no real field would read back.
guard Array(before.utf16) == prefix, Array(after.utf16) == suffix else {
return nil
}
return before + text + after
}
/// Reads the field until it holds `expected`. Keys reach it through the app's event queue, so it can still be
/// filling after the last key is posted: each change restarts the quiet window, up to `capSeconds` in all.
public static func settle(
expected: String,
quietSeconds: Double = 0.5,
capSeconds: Double = 2,
intervalSeconds: Double = 0.025,
now: () -> Double = CFAbsoluteTimeGetCurrent,
sleep: (Double) -> Void = { Thread.sleep(forTimeInterval: $0) },
read: () -> String?
) -> Outcome {
let start = now()
var quietDeadline = start + quietSeconds
var readback = read()
while let current = readback, !sameText(current, expected) {
let time = now()
guard time < quietDeadline, time < start + capSeconds else {
return .mismatch
}
sleep(intervalSeconds)
readback = read()
if readback != current {
quietDeadline = now() + quietSeconds
}
}
return readback == nil ? .unreadable : .landed
}
// Why: Chromium contenteditables store typed trailing or repeated spaces as no-break spaces.
private static func sameText(_ lhs: String, _ rhs: String) -> Bool {
lhs.replacingOccurrences(of: "\u{00A0}", with: " ") == rhs.replacingOccurrences(of: "\u{00A0}", with: " ")
}
}
@@ -0,0 +1,48 @@
import XCTest
@testable import OrcaComputerUseMacOSCore
final class TextInsertionReadbackTests: XCTestCase {
func testInsertionReplacesTheSelectionInUTF16Offsets() {
XCTAssertEqual(
TextInsertionReadback.expectedValue(current: "a😀bcd", selection: 3..<5, inserting: "XY"),
"a😀XYd"
)
XCTAssertEqual(TextInsertionReadback.expectedValue(current: "", selection: nil, inserting: "hi"), "hi")
}
func testUnpredictableInsertionsReturnNil() {
XCTAssertNil(TextInsertionReadback.expectedValue(current: "abc", selection: nil, inserting: "d"))
XCTAssertNil(TextInsertionReadback.expectedValue(current: "abc", selection: 2..<4, inserting: "d"))
XCTAssertNil(TextInsertionReadback.expectedValue(current: "a😀", selection: 2..<2, inserting: "d"))
}
func testFieldStillFillingPastTheQuietWindowLands() {
// One character every 0.4 s: each read is past the first 0.5 s window, but each change restarts it.
let reads = ["", "h", "he", "hel", "hell", "hello"]
XCTAssertEqual(settle(expected: "hello", reads: reads, secondsPerRead: 0.4), .landed)
}
func testContentEditableNoBreakSpacesLand() {
XCTAssertEqual(settle(expected: "a b ", reads: ["a \u{00A0}b\u{00A0}"], secondsPerRead: 0.025), .landed)
}
func testFieldThatDropsTheTextIsAMismatch() {
XCTAssertEqual(settle(expected: "hello", reads: Array(repeating: "", count: 100), secondsPerRead: 0.025), .mismatch)
}
func testFieldThatStopsAnsweringIsUnreadable() {
XCTAssertEqual(settle(expected: "hello", reads: ["", "h"], secondsPerRead: 0.025), .unreadable)
}
private func settle(expected: String, reads: [String], secondsPerRead: Double) -> TextInsertionReadback.Outcome {
var clock = 0.0
var remaining = reads[...]
return TextInsertionReadback.settle(
expected: expected,
now: { clock },
sleep: { _ in clock += secondsPerRead },
read: { remaining.popFirst() }
)
}
}
+2 -1
View File
@@ -116,11 +116,12 @@ printf '%s' "$TEXT" | ORCA computer set-value --app <app> --element-index <index
- `unverified (accessibility action unasserted)` means the accessibility call succeeded but no post-state assertion was made.
- `unverified (suspected noop)` means the accessibility call succeeded but the window's tree and the app's windows were unchanged about a second later. Web and Electron controls often ignore it. Do not repeat the same element click; if the effect you expect would show in the window, retry once with `click --x --y` at the control from a fresh screenshot. If the effect lives elsewhere (clipboard, another app), check there first.
- `unverified (synthetic input)` means input was fired into the void and is unverifiable.
- `unverified (text mismatch)` means typed or pasted text was read back and the focused field does not hold what inserting it should produce: it may be missing, partly entered, reformatted or autocompleted. Do not type it again blindly, or it may land twice; run `get-app-state` and fix the field from what it actually holds.
- Missing verification metadata is unverified, including responses from older runtimes.
- Never report an unverified action as success. If it could have sent, submitted, bought, or deleted something, say the effect is unproven.
- Prefer semantic actions: `set-value` for editable fields, `click` for controls, and `perform-secondary-action` only for listed action names.
- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.
- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; synthetic keyboard delivery is reported as unverified, so inspect the returned state before assuming text landed.
- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; unless the result says `verified focusedText`, inspect the returned state before assuming text landed.
- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.
- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.
- Coordinates are window-local; use coordinates from the latest screenshot/state for the same target window.
+2 -1
View File
@@ -1135,11 +1135,12 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
" - `unverified (accessibility action unasserted)` means the accessibility call succeeded but no post-state assertion was made.",
" - `unverified (suspected noop)` means the accessibility call succeeded but the window's tree and the app's windows were unchanged about a second later. Web and Electron controls often ignore it. Do not repeat the same element click; if the effect you expect would show in the window, retry once with `click --x --y` at the control from a fresh screenshot. If the effect lives elsewhere (clipboard, another app), check there first.",
" - `unverified (synthetic input)` means input was fired into the void and is unverifiable.",
" - `unverified (text mismatch)` means typed or pasted text was read back and the focused field does not hold what inserting it should produce: it may be missing, partly entered, reformatted or autocompleted. Do not type it again blindly, or it may land twice; run `get-app-state` and fix the field from what it actually holds.",
" - Missing verification metadata is unverified, including responses from older runtimes.",
" - Never report an unverified action as success. If it could have sent, submitted, bought, or deleted something, say the effect is unproven.",
"- Prefer semantic actions: `set-value` for editable fields, `click` for controls, and `perform-secondary-action` only for listed action names.",
"- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.",
"- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; synthetic keyboard delivery is reported as unverified, so inspect the returned state before assuming text landed.",
"- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; unless the result says `verified focusedText`, inspect the returned state before assuming text landed.",
"- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.",
"- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.",
"- Coordinates are window-local; use coordinates from the latest screenshot/state for the same target window.",
+2
View File
@@ -121,6 +121,8 @@ export type ComputerActionVerification =
| 'value_mismatch'
// The accessibility call succeeded but the window and the app's windows looked the same about a second later.
| 'suspected_noop'
// Typed or pasted text was read back and the focused field does not hold what inserting it should produce.
| 'text_mismatch'
// Only on value_mismatch from a non-secure field; results never repeat the requested value.
actualPreview?: string | null
}