mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 19:34:29 +00:00
fix(computer): verify macOS type-text and paste-text sent as keys by reading the field back
When text can't be written through AXValue (Chromium/Electron fields, fields that refuse the write), type-text and paste-text send keys or Cmd+V and always reported unverified. Now the helper predicts the focused field's value from its value and selection before sending, then reads it back until it settles: verified on a match, unverified (text_mismatch) otherwise. Secure fields and fields without a readable value and selection stay unverified as before, and the result never repeats the typed text or the field's contents.
This commit is contained in:
@@ -985,14 +985,16 @@ final class Provider {
|
||||
private func typeText(params: [String: JSONValue]) throws -> [String: Any] {
|
||||
let snapshot = try currentKeyboardSnapshot(params: params)
|
||||
let text = try requiredString(params, "text")
|
||||
if let focused = focusedRecord(snapshot), let verification = try TextInput.replaceSelection(focused.element, with: text) {
|
||||
let focused = focusedRecord(snapshot)?.element
|
||||
if let focused, let verification = try TextInput.replaceSelection(focused, with: text) {
|
||||
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
|
||||
}
|
||||
let insertion = TextInput.Insertion(into: focused, text: text)
|
||||
try deliverKeys(try withTransportErrors { try BackgroundInputTransport.textEvents(text) }, snapshot: snapshot, params: params)
|
||||
return actionMetadata(
|
||||
path: "synthetic",
|
||||
actionName: "typeText",
|
||||
verification: unverifiedAction(reason: "synthetic_input")
|
||||
verification: insertion?.verification() ?? unverifiedAction(reason: "synthetic_input")
|
||||
)
|
||||
}
|
||||
|
||||
@@ -1027,17 +1029,19 @@ final class Provider {
|
||||
private func pasteText(params: [String: JSONValue]) throws -> [String: Any] {
|
||||
let snapshot = try currentKeyboardSnapshot(params: params)
|
||||
let text = try requiredString(params, "text")
|
||||
if let focused = focusedRecord(snapshot), let verification = try TextInput.replaceSelection(focused.element, with: text) {
|
||||
let focused = focusedRecord(snapshot)?.element
|
||||
if let focused, let verification = try TextInput.replaceSelection(focused, with: text) {
|
||||
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
|
||||
}
|
||||
let pasteKeys = try Input.keyEvents("cmd+v")
|
||||
let insertion = TextInput.Insertion(into: focused, text: text)
|
||||
try Input.withPasteboardText(text) {
|
||||
try deliverKeys(pasteKeys, snapshot: snapshot, params: params)
|
||||
}
|
||||
return actionMetadata(
|
||||
path: "clipboard",
|
||||
actionName: "paste",
|
||||
verification: unverifiedAction(reason: "clipboard_paste")
|
||||
verification: insertion?.verification() ?? unverifiedAction(reason: "clipboard_paste")
|
||||
)
|
||||
}
|
||||
|
||||
@@ -3219,6 +3223,46 @@ private enum TextInput {
|
||||
return verifiedAction(property: "focusedText")
|
||||
}
|
||||
|
||||
/// Text sent as keys or a paste: what the focused field should read once it lands.
|
||||
struct Insertion {
|
||||
let element: AXUIElement
|
||||
let expected: String
|
||||
|
||||
/// Nil when the field can't prove the result: none focused, a secure field, or no readable value and selection.
|
||||
init?(into element: AXUIElement?, text: String) {
|
||||
guard let element,
|
||||
!AXSnapshotReader().isSecureTextElement(element),
|
||||
let current = rawStringAttribute(element, kAXValueAttribute as String)
|
||||
else {
|
||||
return nil
|
||||
}
|
||||
let selection = TextInput.selectedTextRange(element).flatMap { range -> Range<Int>? in
|
||||
range.location >= 0 && range.length >= 0 ? range.location..<(range.location + range.length) : nil
|
||||
}
|
||||
guard let expected = TextInsertionReadback.expectedValue(current: current, selection: selection, inserting: text) else {
|
||||
return nil
|
||||
}
|
||||
self.element = element
|
||||
self.expected = expected
|
||||
}
|
||||
|
||||
/// Nil when the field can no longer be read.
|
||||
func verification() -> [String: Any]? {
|
||||
let outcome = TextInsertionReadback.settle(expected: expected) {
|
||||
rawStringAttribute(element, kAXValueAttribute as String)
|
||||
}
|
||||
switch outcome {
|
||||
case .landed:
|
||||
return verifiedAction(property: "focusedText")
|
||||
case .mismatch:
|
||||
// Why: the field may hold part of the text, so the result never shows what it reads.
|
||||
return unverifiedAction(reason: "text_mismatch")
|
||||
case .unreadable:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static func selectAll(_ element: AXUIElement) -> Bool {
|
||||
guard let current = rawStringAttribute(element, kAXValueAttribute as String) else {
|
||||
return false
|
||||
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
import Foundation
|
||||
|
||||
/// Checks that text sent as keys or a paste landed: predict the field's value, then read it back.
|
||||
public enum TextInsertionReadback {
|
||||
public enum Outcome: Equatable {
|
||||
case landed
|
||||
case mismatch
|
||||
case unreadable
|
||||
}
|
||||
|
||||
/// The value a field should hold once `text` replaces its selection. `selection` is in UTF-16 offsets, as
|
||||
/// AXSelectedTextRange reports it. Nil means the result can't be predicted: a non-empty field with no
|
||||
/// selection, or a range outside the value or inside a character.
|
||||
public static func expectedValue(current: String, selection: Range<Int>?, inserting text: String) -> String? {
|
||||
let units = Array(current.utf16)
|
||||
let range: Range<Int>
|
||||
if let selection {
|
||||
guard selection.lowerBound >= 0, selection.upperBound <= units.count else {
|
||||
return nil
|
||||
}
|
||||
range = selection
|
||||
} else if units.isEmpty {
|
||||
range = 0..<0
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
let prefix = Array(units[..<range.lowerBound])
|
||||
let suffix = Array(units[range.upperBound...])
|
||||
let before = String(decoding: prefix, as: UTF16.self)
|
||||
let after = String(decoding: suffix, as: UTF16.self)
|
||||
// Why: a boundary inside a surrogate pair decodes to U+FFFD, which no real field would read back.
|
||||
guard Array(before.utf16) == prefix, Array(after.utf16) == suffix else {
|
||||
return nil
|
||||
}
|
||||
return before + text + after
|
||||
}
|
||||
|
||||
/// Reads the field until it holds `expected`. Keys reach it through the app's event queue, so it can still be
|
||||
/// filling after the last key is posted: each change restarts the quiet window, up to `capSeconds` in all.
|
||||
public static func settle(
|
||||
expected: String,
|
||||
quietSeconds: Double = 0.5,
|
||||
capSeconds: Double = 2,
|
||||
intervalSeconds: Double = 0.025,
|
||||
now: () -> Double = CFAbsoluteTimeGetCurrent,
|
||||
sleep: (Double) -> Void = { Thread.sleep(forTimeInterval: $0) },
|
||||
read: () -> String?
|
||||
) -> Outcome {
|
||||
let start = now()
|
||||
var quietDeadline = start + quietSeconds
|
||||
var readback = read()
|
||||
while let current = readback, !sameText(current, expected) {
|
||||
let time = now()
|
||||
guard time < quietDeadline, time < start + capSeconds else {
|
||||
return .mismatch
|
||||
}
|
||||
sleep(intervalSeconds)
|
||||
readback = read()
|
||||
if readback != current {
|
||||
quietDeadline = now() + quietSeconds
|
||||
}
|
||||
}
|
||||
return readback == nil ? .unreadable : .landed
|
||||
}
|
||||
|
||||
// Why: Chromium contenteditables store typed trailing or repeated spaces as no-break spaces.
|
||||
private static func sameText(_ lhs: String, _ rhs: String) -> Bool {
|
||||
lhs.replacingOccurrences(of: "\u{00A0}", with: " ") == rhs.replacingOccurrences(of: "\u{00A0}", with: " ")
|
||||
}
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
import XCTest
|
||||
@testable import OrcaComputerUseMacOSCore
|
||||
|
||||
final class TextInsertionReadbackTests: XCTestCase {
|
||||
|
||||
func testInsertionReplacesTheSelectionInUTF16Offsets() {
|
||||
XCTAssertEqual(
|
||||
TextInsertionReadback.expectedValue(current: "a😀bcd", selection: 3..<5, inserting: "XY"),
|
||||
"a😀XYd"
|
||||
)
|
||||
XCTAssertEqual(TextInsertionReadback.expectedValue(current: "", selection: nil, inserting: "hi"), "hi")
|
||||
}
|
||||
|
||||
func testUnpredictableInsertionsReturnNil() {
|
||||
XCTAssertNil(TextInsertionReadback.expectedValue(current: "abc", selection: nil, inserting: "d"))
|
||||
XCTAssertNil(TextInsertionReadback.expectedValue(current: "abc", selection: 2..<4, inserting: "d"))
|
||||
XCTAssertNil(TextInsertionReadback.expectedValue(current: "a😀", selection: 2..<2, inserting: "d"))
|
||||
}
|
||||
|
||||
func testFieldStillFillingPastTheQuietWindowLands() {
|
||||
// One character every 0.4 s: each read is past the first 0.5 s window, but each change restarts it.
|
||||
let reads = ["", "h", "he", "hel", "hell", "hello"]
|
||||
XCTAssertEqual(settle(expected: "hello", reads: reads, secondsPerRead: 0.4), .landed)
|
||||
}
|
||||
|
||||
func testContentEditableNoBreakSpacesLand() {
|
||||
XCTAssertEqual(settle(expected: "a b ", reads: ["a \u{00A0}b\u{00A0}"], secondsPerRead: 0.025), .landed)
|
||||
}
|
||||
|
||||
func testFieldThatDropsTheTextIsAMismatch() {
|
||||
XCTAssertEqual(settle(expected: "hello", reads: Array(repeating: "", count: 100), secondsPerRead: 0.025), .mismatch)
|
||||
}
|
||||
|
||||
func testFieldThatStopsAnsweringIsUnreadable() {
|
||||
XCTAssertEqual(settle(expected: "hello", reads: ["", "h"], secondsPerRead: 0.025), .unreadable)
|
||||
}
|
||||
|
||||
private func settle(expected: String, reads: [String], secondsPerRead: Double) -> TextInsertionReadback.Outcome {
|
||||
var clock = 0.0
|
||||
var remaining = reads[...]
|
||||
return TextInsertionReadback.settle(
|
||||
expected: expected,
|
||||
now: { clock },
|
||||
sleep: { _ in clock += secondsPerRead },
|
||||
read: { remaining.popFirst() }
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -116,11 +116,12 @@ printf '%s' "$TEXT" | ORCA computer set-value --app <app> --element-index <index
|
||||
- `unverified (accessibility action unasserted)` means the accessibility call succeeded but no post-state assertion was made.
|
||||
- `unverified (suspected noop)` means the accessibility call succeeded but the window's tree and the app's windows were unchanged about a second later. Web and Electron controls often ignore it. Do not repeat the same element click; if the effect you expect would show in the window, retry once with `click --x --y` at the control from a fresh screenshot. If the effect lives elsewhere (clipboard, another app), check there first.
|
||||
- `unverified (synthetic input)` means input was fired into the void and is unverifiable.
|
||||
- `unverified (text mismatch)` means typed or pasted text was read back and the focused field does not hold what inserting it should produce: it may be missing, partly entered, reformatted or autocompleted. Do not type it again blindly, or it may land twice; run `get-app-state` and fix the field from what it actually holds.
|
||||
- Missing verification metadata is unverified, including responses from older runtimes.
|
||||
- Never report an unverified action as success. If it could have sent, submitted, bought, or deleted something, say the effect is unproven.
|
||||
- Prefer semantic actions: `set-value` for editable fields, `click` for controls, and `perform-secondary-action` only for listed action names.
|
||||
- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.
|
||||
- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; synthetic keyboard delivery is reported as unverified, so inspect the returned state before assuming text landed.
|
||||
- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; unless the result says `verified focusedText`, inspect the returned state before assuming text landed.
|
||||
- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.
|
||||
- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.
|
||||
- Coordinates are window-local; use coordinates from the latest screenshot/state for the same target window.
|
||||
|
||||
@@ -1135,11 +1135,12 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
|
||||
" - `unverified (accessibility action unasserted)` means the accessibility call succeeded but no post-state assertion was made.",
|
||||
" - `unverified (suspected noop)` means the accessibility call succeeded but the window's tree and the app's windows were unchanged about a second later. Web and Electron controls often ignore it. Do not repeat the same element click; if the effect you expect would show in the window, retry once with `click --x --y` at the control from a fresh screenshot. If the effect lives elsewhere (clipboard, another app), check there first.",
|
||||
" - `unverified (synthetic input)` means input was fired into the void and is unverifiable.",
|
||||
" - `unverified (text mismatch)` means typed or pasted text was read back and the focused field does not hold what inserting it should produce: it may be missing, partly entered, reformatted or autocompleted. Do not type it again blindly, or it may land twice; run `get-app-state` and fix the field from what it actually holds.",
|
||||
" - Missing verification metadata is unverified, including responses from older runtimes.",
|
||||
" - Never report an unverified action as success. If it could have sent, submitted, bought, or deleted something, say the effect is unproven.",
|
||||
"- Prefer semantic actions: `set-value` for editable fields, `click` for controls, and `perform-secondary-action` only for listed action names.",
|
||||
"- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.",
|
||||
"- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; synthetic keyboard delivery is reported as unverified, so inspect the returned state before assuming text landed.",
|
||||
"- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; unless the result says `verified focusedText`, inspect the returned state before assuming text landed.",
|
||||
"- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.",
|
||||
"- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.",
|
||||
"- Coordinates are window-local; use coordinates from the latest screenshot/state for the same target window.",
|
||||
|
||||
@@ -121,6 +121,8 @@ export type ComputerActionVerification =
|
||||
| 'value_mismatch'
|
||||
// The accessibility call succeeded but the window and the app's windows looked the same about a second later.
|
||||
| 'suspected_noop'
|
||||
// Typed or pasted text was read back and the focused field does not hold what inserting it should produce.
|
||||
| 'text_mismatch'
|
||||
// Only on value_mismatch from a non-secure field; results never repeat the requested value.
|
||||
actualPreview?: string | null
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user