Files
orca/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/TextInsertionReadbackTests.swift
Kelvin Amoaba 792d4e8028 fix(computer): verify macOS type-text and paste-text by reading the field back (#41402)
* fix(computer): verify macOS type-text and paste-text sent as keys by reading the field back

When text can't be written through AXValue (Chromium/Electron fields, fields
that refuse the write), type-text and paste-text send keys or Cmd+V and always
reported unverified. Now the helper predicts the focused field's value from its
value and selection before sending, then reads it back until it settles:
verified on a match, unverified (text_mismatch) otherwise. Secure fields and
fields without a readable value and selection stay unverified as before, and
the result never repeats the typed text or the field's contents.

* fix(computer): keep the caret put after a refused macOS text write and skip unprovable readbacks

A refused AX value write no longer moves the caret before falling back to keys, so the keys and the prediction both use the user's selection. Text with control characters, a value the field already holds, and a field still filling at the 2 s cap stay unverified rather than verified or text_mismatch. The quiet window starts after the first read answers.

* fix(computer): bound macOS text readback size and say a mismatch can mean keys went elsewhere

* fix(computer): treat a macOS text readback past the cap as undetermined
2026-11-12 22:45:11 +00:00

87 lines
3.5 KiB
Swift

import XCTest
@testable import OrcaComputerUseMacOSCore
final class TextInsertionReadbackTests: XCTestCase {
func testInsertionReplacesTheSelectionInUTF16Offsets() {
XCTAssertEqual(
TextInsertionReadback.expectedValue(current: "a😀bcd", selection: 3..<5, inserting: "XY"),
"a😀XYd"
)
XCTAssertEqual(TextInsertionReadback.expectedValue(current: "", selection: nil, inserting: "hi"), "hi")
}
func testUnpredictableInsertionsReturnNil() {
XCTAssertNil(TextInsertionReadback.expectedValue(current: "abc", selection: nil, inserting: "d"))
XCTAssertNil(TextInsertionReadback.expectedValue(current: "abc", selection: 2..<4, inserting: "d"))
XCTAssertNil(TextInsertionReadback.expectedValue(current: "a😀", selection: 2..<2, inserting: "d"))
XCTAssertNil(TextInsertionReadback.expectedValue(current: "", selection: 0..<0, inserting: "send\n"))
XCTAssertNil(TextInsertionReadback.expectedValue(current: "", selection: 0..<0, inserting: "a\tb"))
XCTAssertNil(TextInsertionReadback.expectedValue(current: "hello", selection: 0..<5, inserting: "hello"))
}
func testFieldStillFillingPastTheQuietWindowLands() {
// One character every 0.4 s: each read is past the first 0.5 s window, but each change restarts it.
let reads = ["", "h", "he", "hel", "hell", "hello"]
XCTAssertEqual(settle(expected: "hello", reads: reads, secondsPerRead: 0.4), .landed)
}
func testSlowFirstReadStillGetsAQuietWindow() {
var clock = 0.0
var reads = ["h", "hello"][...]
let outcome = TextInsertionReadback.settle(
expected: "hello",
now: { clock },
sleep: { clock += $0 },
read: {
if reads.count == 2 { clock += 0.6 }
return reads.popFirst()
}
)
XCTAssertEqual(outcome, .landed)
}
func testReadBlockedPastTheCapIsUndetermined() {
var clock = 0.0
var reads = ["h", "h"][...]
let outcome = TextInsertionReadback.settle(
expected: "hello",
now: { clock },
sleep: { clock += $0 },
read: {
if reads.count == 1 { clock = 2.2 }
return reads.popFirst()
}
)
XCTAssertEqual(outcome, .undetermined)
}
func testContentEditableNoBreakSpacesLand() {
XCTAssertEqual(settle(expected: "a b ", reads: ["a \u{00A0}b\u{00A0}"], secondsPerRead: 0.025), .landed)
}
func testFieldThatDropsTheTextIsAMismatch() {
XCTAssertEqual(settle(expected: "hello", reads: Array(repeating: "", count: 100), secondsPerRead: 0.025), .mismatch)
}
func testFieldThatStopsAnsweringIsUndetermined() {
XCTAssertEqual(settle(expected: "hello", reads: ["", "h"], secondsPerRead: 0.025), .undetermined)
}
func testFieldStillFillingAtTheCapIsUndetermined() {
let reads = (0..<100).map { String(repeating: "a", count: $0) }
XCTAssertEqual(settle(expected: String(repeating: "a", count: 200), reads: reads, secondsPerRead: 0.1), .undetermined)
}
private func settle(expected: String, reads: [String], secondsPerRead: Double) -> TextInsertionReadback.Outcome {
var clock = 0.0
var remaining = reads[...]
return TextInsertionReadback.settle(
expected: expected,
now: { clock },
sleep: { _ in clock += secondsPerRead },
read: { remaining.popFirst() }
)
}
}