mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 19:07:41 +00:00
fix(computer): select list and table rows when a macOS click leaves them unselected
A plain click on an AXRow/AXCell answered success (AXPress) or unverified synthetic input while nothing got selected: AppKit rows have no AXPress, and background pointer input does not select them. After the click, read the selection back; if the row is still unselected, write it through the owning table/outline/list (AXSelectedCells/AXSelectedRows/AXSelectedChildren), falling back to the row's own AXSelected, and report verified only when the selection reads back on.
This commit is contained in:
@@ -816,18 +816,35 @@ final class Provider {
|
||||
if !foreground {
|
||||
try requireBackgroundClickAllowed(backgroundClickHit(record.element, snapshot: snapshot))
|
||||
}
|
||||
let rowSelection = AccessibilityRowSelection()
|
||||
let unselectedRow = modifiers.isEmpty && count <= 1 && button == .left
|
||||
&& RowSelectionClick.isUnselectedRow(record.element, ax: rowSelection)
|
||||
if modifiers.isEmpty,
|
||||
count <= 1,
|
||||
button.hasAccessibilityAction,
|
||||
let actionName = try performClickAction(record: record, mouseButton: button) {
|
||||
return actionMetadata(path: "accessibility", actionName: actionName)
|
||||
guard unselectedRow else {
|
||||
return actionMetadata(path: "accessibility", actionName: actionName)
|
||||
}
|
||||
return rowClickMetadata(
|
||||
try RowSelectionClick.confirm(record.element, ax: rowSelection),
|
||||
path: "accessibility",
|
||||
actionName: actionName
|
||||
)
|
||||
}
|
||||
if let point = center(record.localFrame, in: snapshot.windowBounds) {
|
||||
try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground, element: record.element)
|
||||
return actionMetadata(
|
||||
guard unselectedRow else {
|
||||
return actionMetadata(
|
||||
path: "synthetic",
|
||||
fallbackReason: "actionUnsupported",
|
||||
verification: unverifiedAction(reason: "synthetic_input")
|
||||
)
|
||||
}
|
||||
return rowClickMetadata(
|
||||
try RowSelectionClick.confirm(record.element, ax: rowSelection),
|
||||
path: "synthetic",
|
||||
fallbackReason: "actionUnsupported",
|
||||
verification: unverifiedAction(reason: "synthetic_input")
|
||||
actionName: nil
|
||||
)
|
||||
}
|
||||
throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame")
|
||||
@@ -840,6 +857,22 @@ final class Provider {
|
||||
)
|
||||
}
|
||||
|
||||
/// The selection reading back on verifies a row click; a row it never selected stays unverified.
|
||||
private func rowClickMetadata(_ outcome: RowSelectionClick.Outcome, path: String, actionName: String?) -> [String: Any] {
|
||||
let fallbackReason = path == "synthetic" ? "actionUnsupported" : nil
|
||||
switch outcome {
|
||||
case .selectedByClick:
|
||||
return actionMetadata(path: path, actionName: actionName, fallbackReason: fallbackReason, verification: verifiedAction(property: "selection"))
|
||||
case let .selectedByWrite(attribute):
|
||||
return actionMetadata(path: "accessibility", actionName: attribute, fallbackReason: fallbackReason, verification: verifiedAction(property: "selection"))
|
||||
case .unselected:
|
||||
// Why: an accessibility click leaves verification to the tree diff, which reports suspected_noop if nothing changed.
|
||||
return path == "synthetic"
|
||||
? actionMetadata(path: path, fallbackReason: fallbackReason, verification: unverifiedAction(reason: "synthetic_input"))
|
||||
: actionMetadata(path: path, actionName: actionName)
|
||||
}
|
||||
}
|
||||
|
||||
private func performClickAction(record: ElementRecord, mouseButton: MouseButtonSelection) throws -> String? {
|
||||
if mouseButton == .right {
|
||||
return try performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil
|
||||
@@ -1875,6 +1908,75 @@ private func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool {
|
||||
return AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) == .success && settable.boolValue
|
||||
}
|
||||
|
||||
/// Once the app stops answering, later reads return nothing at once so a hung app costs one timeout, not dozens.
|
||||
private final class AccessibilityRowSelection: RowSelectionAccessibility {
|
||||
private var unanswered = false
|
||||
|
||||
func role(of element: AXUIElement) -> String? {
|
||||
read(element, kAXRoleAttribute as String) as? String
|
||||
}
|
||||
|
||||
func isSelected(_ element: AXUIElement) -> Bool? {
|
||||
read(element, kAXSelectedAttribute as String) as? Bool
|
||||
}
|
||||
|
||||
func parent(of element: AXUIElement) -> AXUIElement? {
|
||||
guard let value = read(element, kAXParentAttribute as String),
|
||||
CFGetTypeID(value) == AXUIElementGetTypeID()
|
||||
else {
|
||||
return nil
|
||||
}
|
||||
return (value as! AXUIElement)
|
||||
}
|
||||
|
||||
func isSettable(_ attribute: String, on element: AXUIElement) -> Bool {
|
||||
guard !unanswered else { return false }
|
||||
var settable = DarwinBoolean(false)
|
||||
let start = CFAbsoluteTimeGetCurrent()
|
||||
let result = AXUIElementIsAttributeSettable(element, attribute as CFString, &settable)
|
||||
unanswered = AccessibilityMessaging.timedOut(result, since: start)
|
||||
return result == .success && settable.boolValue
|
||||
}
|
||||
|
||||
func setSelected(_ element: AXUIElement) throws -> Bool {
|
||||
try write(kAXSelectedAttribute as String, on: element, value: kCFBooleanTrue)
|
||||
}
|
||||
|
||||
func setSelection(_ attribute: String, of container: AXUIElement, to member: AXUIElement) throws -> Bool {
|
||||
try write(attribute, on: container, value: [member] as CFArray)
|
||||
}
|
||||
|
||||
func settles(_ check: () -> Bool) -> Bool {
|
||||
// Why: settle counts only its sleeps, and one slow AX read can take the full 3 s timeout.
|
||||
let deadline = CFAbsoluteTimeGetCurrent() + 0.3
|
||||
var shown = false
|
||||
_ = PermissionTrustSettling.settle(timeoutMs: 300, intervalMs: 25) {
|
||||
guard !unanswered, CFAbsoluteTimeGetCurrent() < deadline else { return true }
|
||||
shown = check()
|
||||
return shown
|
||||
}
|
||||
return shown
|
||||
}
|
||||
|
||||
private func read(_ element: AXUIElement, _ attribute: String) -> CFTypeRef? {
|
||||
guard !unanswered else { return nil }
|
||||
var value: CFTypeRef?
|
||||
let start = CFAbsoluteTimeGetCurrent()
|
||||
let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value)
|
||||
unanswered = AccessibilityMessaging.timedOut(result, since: start)
|
||||
return result == .success ? value : nil
|
||||
}
|
||||
|
||||
private func write(_ attribute: String, on element: AXUIElement, value: CFTypeRef) throws -> Bool {
|
||||
let start = CFAbsoluteTimeGetCurrent()
|
||||
let result = AXUIElementSetAttributeValue(element, attribute as CFString, value)
|
||||
if AccessibilityMessaging.timedOut(result, since: start) {
|
||||
throw AccessibilityMessaging.unanswered("the \(attribute) write")
|
||||
}
|
||||
return result == .success
|
||||
}
|
||||
}
|
||||
|
||||
private func frame(_ element: AXUIElement, windowBounds: CGRect) -> CGRect? {
|
||||
var positionValue: CFTypeRef?
|
||||
var sizeValue: CFTypeRef?
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
/// The accessibility reads and writes a row click needs, so the click policy runs against a fake tree in tests.
|
||||
public protocol RowSelectionAccessibility {
|
||||
associatedtype Element
|
||||
func role(of element: Element) -> String?
|
||||
func isSelected(_ element: Element) -> Bool?
|
||||
func parent(of element: Element) -> Element?
|
||||
func isSettable(_ attribute: String, on element: Element) -> Bool
|
||||
/// Sets AXSelected to true; false when the app refused the write.
|
||||
func setSelected(_ element: Element) throws -> Bool
|
||||
/// Replaces the container's selection with the member; false when the app refused the write.
|
||||
func setSelection(_ attribute: String, of container: Element, to member: Element) throws -> Bool
|
||||
/// Re-checks for a moment, since apps apply a selection on a later run-loop turn.
|
||||
func settles(_ check: () -> Bool) -> Bool
|
||||
}
|
||||
|
||||
/// A plain click on a list, table or outline row means "select it", yet AXPress on a row often answers
|
||||
/// success and selects nothing. For a row that read unselected, the selection reading back on is what
|
||||
/// makes the click count.
|
||||
public enum RowSelectionClick {
|
||||
public static let rowRoles: Set<String> = ["AXRow", "AXCell", "AXOutlineRow"]
|
||||
static let containerRoles: Set<String> = ["AXTable", "AXOutline", "AXList", "AXGrid"]
|
||||
static let maxContainerDepth = 6
|
||||
|
||||
public enum Outcome: Equatable {
|
||||
case selectedByClick
|
||||
case selectedByWrite(attribute: String)
|
||||
case unselected
|
||||
}
|
||||
|
||||
/// Rows that read selected already, or publish no selection, prove nothing by reading selected afterwards.
|
||||
public static func isUnselectedRow<AX: RowSelectionAccessibility>(_ element: AX.Element, ax: AX) -> Bool {
|
||||
guard let role = ax.role(of: element), rowRoles.contains(role) else { return false }
|
||||
return selectionShows(element, ax: ax) == false
|
||||
}
|
||||
|
||||
/// Call after a click reached the app. A selection write is idempotent, so it is safe even when the
|
||||
/// click did something else; it still comes second because it skips the app's own click handling.
|
||||
public static func confirm<AX: RowSelectionAccessibility>(_ element: AX.Element, ax: AX) throws -> Outcome {
|
||||
if ax.settles({ selectionShows(element, ax: ax) == true }) {
|
||||
return .selectedByClick
|
||||
}
|
||||
guard let attribute = try writeSelection(element, ax: ax),
|
||||
ax.settles({ selectionShows(element, ax: ax) == true })
|
||||
else {
|
||||
return .unselected
|
||||
}
|
||||
return .selectedByWrite(attribute: attribute)
|
||||
}
|
||||
|
||||
/// A cell in a table that selects whole rows reads selected through its row.
|
||||
static func selectionShows<AX: RowSelectionAccessibility>(_ element: AX.Element, ax: AX) -> Bool? {
|
||||
let own = ax.isSelected(element)
|
||||
guard own != true, let row = enclosingRow(of: element, ax: ax) else { return own }
|
||||
let rowSelected = ax.isSelected(row)
|
||||
return rowSelected == true ? true : own ?? rowSelected
|
||||
}
|
||||
|
||||
/// Returns the first write the app accepted. It stops there: a selection that has not shown yet may
|
||||
/// still land, and a second write could then select something else.
|
||||
private static func writeSelection<AX: RowSelectionAccessibility>(_ element: AX.Element, ax: AX) throws -> String? {
|
||||
let row = enclosingRow(of: element, ax: ax)
|
||||
let isCell = ax.role(of: element) == "AXCell"
|
||||
var child = element
|
||||
for _ in 0..<maxContainerDepth {
|
||||
guard let parent = ax.parent(of: child) else { break }
|
||||
guard let role = ax.role(of: parent), containerRoles.contains(role) else {
|
||||
child = parent
|
||||
continue
|
||||
}
|
||||
// Why: a spreadsheet owns cell selection on AXSelectedCells; a table that selects rows takes the
|
||||
// cell's row; AXSelectedChildren only takes the container's own children.
|
||||
let writes: [(String, AX.Element)] = (isCell ? [("AXSelectedCells", element)] : []) + [("AXSelectedRows", row ?? element), ("AXSelectedChildren", child)]
|
||||
for (attribute, member) in writes where ax.isSettable(attribute, on: parent) {
|
||||
if try ax.setSelection(attribute, of: parent, to: member) {
|
||||
return attribute
|
||||
}
|
||||
}
|
||||
// Why: an outer list's selection is not this row's; writing it would change something else.
|
||||
break
|
||||
}
|
||||
// Why: last, because in a multi-select list AXSelected may add the row to the selection instead of replacing it.
|
||||
for target in [element, row].compactMap({ $0 }) where ax.isSettable("AXSelected", on: target) {
|
||||
if try ax.setSelected(target) {
|
||||
return "AXSelected"
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
private static func enclosingRow<AX: RowSelectionAccessibility>(of element: AX.Element, ax: AX) -> AX.Element? {
|
||||
guard ax.role(of: element) == "AXCell",
|
||||
let parent = ax.parent(of: element),
|
||||
let parentRole = ax.role(of: parent),
|
||||
parentRole != "AXCell",
|
||||
rowRoles.contains(parentRole)
|
||||
else {
|
||||
return nil
|
||||
}
|
||||
return parent
|
||||
}
|
||||
}
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
import OrcaComputerUseMacOSCore
|
||||
import XCTest
|
||||
|
||||
final class RowSelectionClickTests: XCTestCase {
|
||||
private final class FakeTree: RowSelectionAccessibility {
|
||||
struct Node {
|
||||
var role: String
|
||||
var selected: Bool?
|
||||
var parent: Int?
|
||||
var settable: Set<String> = []
|
||||
}
|
||||
|
||||
var nodes: [Int: Node]
|
||||
var acceptedWritesTakeEffect = true
|
||||
private(set) var writes: [String] = []
|
||||
|
||||
init(_ nodes: [Int: Node]) {
|
||||
self.nodes = nodes
|
||||
}
|
||||
|
||||
func role(of element: Int) -> String? { nodes[element]?.role }
|
||||
func isSelected(_ element: Int) -> Bool? { nodes[element]?.selected }
|
||||
func parent(of element: Int) -> Int? { nodes[element]?.parent }
|
||||
func isSettable(_ attribute: String, on element: Int) -> Bool { nodes[element]?.settable.contains(attribute) == true }
|
||||
|
||||
func setSelected(_ element: Int) throws -> Bool {
|
||||
writes.append("AXSelected:\(element)")
|
||||
if acceptedWritesTakeEffect { select(element) }
|
||||
return true
|
||||
}
|
||||
|
||||
func setSelection(_ attribute: String, of container: Int, to member: Int) throws -> Bool {
|
||||
writes.append("\(attribute):\(container):\(member)")
|
||||
if acceptedWritesTakeEffect { select(member) }
|
||||
return true
|
||||
}
|
||||
|
||||
func settles(_ check: () -> Bool) -> Bool { check() }
|
||||
|
||||
private func select(_ member: Int) {
|
||||
for id in nodes.keys where nodes[id]?.selected != nil {
|
||||
nodes[id]?.selected = id == member
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func outline(containerSettable: Set<String> = ["AXSelectedRows"], rowSettable: Set<String> = []) -> FakeTree {
|
||||
FakeTree([
|
||||
1: .init(role: "AXOutline", parent: nil, settable: containerSettable),
|
||||
2: .init(role: "AXRow", selected: true, parent: 1),
|
||||
3: .init(role: "AXRow", selected: false, parent: 1, settable: rowSettable),
|
||||
4: .init(role: "AXCell", parent: 3),
|
||||
])
|
||||
}
|
||||
|
||||
func testOnlyARowThatReadsUnselectedQualifies() {
|
||||
let tree = outline()
|
||||
tree.nodes[5] = .init(role: "AXButton", selected: false, parent: 1)
|
||||
tree.nodes[6] = .init(role: "AXRow", parent: 1)
|
||||
|
||||
XCTAssertTrue(RowSelectionClick.isUnselectedRow(3, ax: tree))
|
||||
XCTAssertTrue(RowSelectionClick.isUnselectedRow(4, ax: tree))
|
||||
XCTAssertFalse(RowSelectionClick.isUnselectedRow(2, ax: tree))
|
||||
XCTAssertFalse(RowSelectionClick.isUnselectedRow(5, ax: tree))
|
||||
XCTAssertFalse(RowSelectionClick.isUnselectedRow(6, ax: tree))
|
||||
}
|
||||
|
||||
func testAClickThatSelectedTheRowNeedsNoWrite() throws {
|
||||
let tree = outline(rowSettable: ["AXSelected"])
|
||||
tree.nodes[3]?.selected = true
|
||||
|
||||
XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .selectedByClick)
|
||||
XCTAssertEqual(tree.writes, [])
|
||||
}
|
||||
|
||||
func testAClickThatSelectedNothingReplacesTheContainerSelection() throws {
|
||||
let tree = outline(rowSettable: ["AXSelected"])
|
||||
|
||||
XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .selectedByWrite(attribute: "AXSelectedRows"))
|
||||
XCTAssertEqual(tree.writes, ["AXSelectedRows:1:3"])
|
||||
}
|
||||
|
||||
func testARowWhoseContainerTakesNoWriteSelectsItself() throws {
|
||||
let tree = outline(containerSettable: [], rowSettable: ["AXSelected"])
|
||||
|
||||
XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .selectedByWrite(attribute: "AXSelected"))
|
||||
XCTAssertEqual(tree.writes, ["AXSelected:3"])
|
||||
}
|
||||
|
||||
func testACellIsSelectedThroughItsRowInTheOwningContainer() throws {
|
||||
let tree = outline()
|
||||
|
||||
XCTAssertEqual(try RowSelectionClick.confirm(4, ax: tree), .selectedByWrite(attribute: "AXSelectedRows"))
|
||||
XCTAssertEqual(tree.writes, ["AXSelectedRows:1:3"])
|
||||
}
|
||||
|
||||
func testSelectedChildrenTakesTheContainersOwnChild() throws {
|
||||
let tree = outline(containerSettable: ["AXSelectedChildren"])
|
||||
|
||||
XCTAssertEqual(try RowSelectionClick.confirm(4, ax: tree), .selectedByWrite(attribute: "AXSelectedChildren"))
|
||||
XCTAssertEqual(tree.writes, ["AXSelectedChildren:1:3"])
|
||||
}
|
||||
|
||||
func testAnOuterListThatTakesWritesIsLeftAlone() throws {
|
||||
let tree = outline(containerSettable: [], rowSettable: ["AXSelected"])
|
||||
tree.nodes[10] = .init(role: "AXList", parent: nil, settable: ["AXSelectedChildren"])
|
||||
tree.nodes[11] = .init(role: "AXGroup", selected: false, parent: 10)
|
||||
tree.nodes[1]?.parent = 11
|
||||
|
||||
XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .selectedByWrite(attribute: "AXSelected"))
|
||||
XCTAssertEqual(tree.writes, ["AXSelected:3"])
|
||||
}
|
||||
|
||||
func testAnAcceptedWriteThatNeverShowsIsNotFollowedByAnotherWrite() throws {
|
||||
let tree = outline(rowSettable: ["AXSelected"])
|
||||
tree.acceptedWritesTakeEffect = false
|
||||
|
||||
XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .unselected)
|
||||
XCTAssertEqual(tree.writes, ["AXSelectedRows:1:3"])
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user