diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index f25b5e8b7c9..3f3b40b9fbb 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -816,18 +816,35 @@ final class Provider { if !foreground { try requireBackgroundClickAllowed(backgroundClickHit(record.element, snapshot: snapshot)) } + let rowSelection = AccessibilityRowSelection() + let unselectedRow = modifiers.isEmpty && count <= 1 && button == .left + && RowSelectionClick.isUnselectedRow(record.element, ax: rowSelection) if modifiers.isEmpty, count <= 1, button.hasAccessibilityAction, let actionName = try performClickAction(record: record, mouseButton: button) { - return actionMetadata(path: "accessibility", actionName: actionName) + guard unselectedRow else { + return actionMetadata(path: "accessibility", actionName: actionName) + } + return rowClickMetadata( + try RowSelectionClick.confirm(record.element, ax: rowSelection), + path: "accessibility", + actionName: actionName + ) } if let point = center(record.localFrame, in: snapshot.windowBounds) { try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground, element: record.element) - return actionMetadata( + guard unselectedRow else { + return actionMetadata( + path: "synthetic", + fallbackReason: "actionUnsupported", + verification: unverifiedAction(reason: "synthetic_input") + ) + } + return rowClickMetadata( + try RowSelectionClick.confirm(record.element, ax: rowSelection), path: "synthetic", - fallbackReason: "actionUnsupported", - verification: unverifiedAction(reason: "synthetic_input") + actionName: nil ) } throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame") @@ -840,6 +857,22 @@ final class Provider { ) } + /// The selection reading back on verifies a row click; a row it never selected stays unverified. + private func rowClickMetadata(_ outcome: RowSelectionClick.Outcome, path: String, actionName: String?) -> [String: Any] { + let fallbackReason = path == "synthetic" ? "actionUnsupported" : nil + switch outcome { + case .selectedByClick: + return actionMetadata(path: path, actionName: actionName, fallbackReason: fallbackReason, verification: verifiedAction(property: "selection")) + case let .selectedByWrite(attribute): + return actionMetadata(path: "accessibility", actionName: attribute, fallbackReason: fallbackReason, verification: verifiedAction(property: "selection")) + case .unselected: + // Why: an accessibility click leaves verification to the tree diff, which reports suspected_noop if nothing changed. + return path == "synthetic" + ? actionMetadata(path: path, fallbackReason: fallbackReason, verification: unverifiedAction(reason: "synthetic_input")) + : actionMetadata(path: path, actionName: actionName) + } + } + private func performClickAction(record: ElementRecord, mouseButton: MouseButtonSelection) throws -> String? { if mouseButton == .right { return try performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil @@ -1875,6 +1908,75 @@ private func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool { return AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) == .success && settable.boolValue } +/// Once the app stops answering, later reads return nothing at once so a hung app costs one timeout, not dozens. +private final class AccessibilityRowSelection: RowSelectionAccessibility { + private var unanswered = false + + func role(of element: AXUIElement) -> String? { + read(element, kAXRoleAttribute as String) as? String + } + + func isSelected(_ element: AXUIElement) -> Bool? { + read(element, kAXSelectedAttribute as String) as? Bool + } + + func parent(of element: AXUIElement) -> AXUIElement? { + guard let value = read(element, kAXParentAttribute as String), + CFGetTypeID(value) == AXUIElementGetTypeID() + else { + return nil + } + return (value as! AXUIElement) + } + + func isSettable(_ attribute: String, on element: AXUIElement) -> Bool { + guard !unanswered else { return false } + var settable = DarwinBoolean(false) + let start = CFAbsoluteTimeGetCurrent() + let result = AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) + unanswered = AccessibilityMessaging.timedOut(result, since: start) + return result == .success && settable.boolValue + } + + func setSelected(_ element: AXUIElement) throws -> Bool { + try write(kAXSelectedAttribute as String, on: element, value: kCFBooleanTrue) + } + + func setSelection(_ attribute: String, of container: AXUIElement, to member: AXUIElement) throws -> Bool { + try write(attribute, on: container, value: [member] as CFArray) + } + + func settles(_ check: () -> Bool) -> Bool { + // Why: settle counts only its sleeps, and one slow AX read can take the full 3 s timeout. + let deadline = CFAbsoluteTimeGetCurrent() + 0.3 + var shown = false + _ = PermissionTrustSettling.settle(timeoutMs: 300, intervalMs: 25) { + guard !unanswered, CFAbsoluteTimeGetCurrent() < deadline else { return true } + shown = check() + return shown + } + return shown + } + + private func read(_ element: AXUIElement, _ attribute: String) -> CFTypeRef? { + guard !unanswered else { return nil } + var value: CFTypeRef? + let start = CFAbsoluteTimeGetCurrent() + let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value) + unanswered = AccessibilityMessaging.timedOut(result, since: start) + return result == .success ? value : nil + } + + private func write(_ attribute: String, on element: AXUIElement, value: CFTypeRef) throws -> Bool { + let start = CFAbsoluteTimeGetCurrent() + let result = AXUIElementSetAttributeValue(element, attribute as CFString, value) + if AccessibilityMessaging.timedOut(result, since: start) { + throw AccessibilityMessaging.unanswered("the \(attribute) write") + } + return result == .success + } +} + private func frame(_ element: AXUIElement, windowBounds: CGRect) -> CGRect? { var positionValue: CFTypeRef? var sizeValue: CFTypeRef? diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/RowSelectionClick.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/RowSelectionClick.swift new file mode 100644 index 00000000000..50f8b0fd52b --- /dev/null +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/RowSelectionClick.swift @@ -0,0 +1,101 @@ +/// The accessibility reads and writes a row click needs, so the click policy runs against a fake tree in tests. +public protocol RowSelectionAccessibility { + associatedtype Element + func role(of element: Element) -> String? + func isSelected(_ element: Element) -> Bool? + func parent(of element: Element) -> Element? + func isSettable(_ attribute: String, on element: Element) -> Bool + /// Sets AXSelected to true; false when the app refused the write. + func setSelected(_ element: Element) throws -> Bool + /// Replaces the container's selection with the member; false when the app refused the write. + func setSelection(_ attribute: String, of container: Element, to member: Element) throws -> Bool + /// Re-checks for a moment, since apps apply a selection on a later run-loop turn. + func settles(_ check: () -> Bool) -> Bool +} + +/// A plain click on a list, table or outline row means "select it", yet AXPress on a row often answers +/// success and selects nothing. For a row that read unselected, the selection reading back on is what +/// makes the click count. +public enum RowSelectionClick { + public static let rowRoles: Set = ["AXRow", "AXCell", "AXOutlineRow"] + static let containerRoles: Set = ["AXTable", "AXOutline", "AXList", "AXGrid"] + static let maxContainerDepth = 6 + + public enum Outcome: Equatable { + case selectedByClick + case selectedByWrite(attribute: String) + case unselected + } + + /// Rows that read selected already, or publish no selection, prove nothing by reading selected afterwards. + public static func isUnselectedRow(_ element: AX.Element, ax: AX) -> Bool { + guard let role = ax.role(of: element), rowRoles.contains(role) else { return false } + return selectionShows(element, ax: ax) == false + } + + /// Call after a click reached the app. A selection write is idempotent, so it is safe even when the + /// click did something else; it still comes second because it skips the app's own click handling. + public static func confirm(_ element: AX.Element, ax: AX) throws -> Outcome { + if ax.settles({ selectionShows(element, ax: ax) == true }) { + return .selectedByClick + } + guard let attribute = try writeSelection(element, ax: ax), + ax.settles({ selectionShows(element, ax: ax) == true }) + else { + return .unselected + } + return .selectedByWrite(attribute: attribute) + } + + /// A cell in a table that selects whole rows reads selected through its row. + static func selectionShows(_ element: AX.Element, ax: AX) -> Bool? { + let own = ax.isSelected(element) + guard own != true, let row = enclosingRow(of: element, ax: ax) else { return own } + let rowSelected = ax.isSelected(row) + return rowSelected == true ? true : own ?? rowSelected + } + + /// Returns the first write the app accepted. It stops there: a selection that has not shown yet may + /// still land, and a second write could then select something else. + private static func writeSelection(_ element: AX.Element, ax: AX) throws -> String? { + let row = enclosingRow(of: element, ax: ax) + let isCell = ax.role(of: element) == "AXCell" + var child = element + for _ in 0..(of element: AX.Element, ax: AX) -> AX.Element? { + guard ax.role(of: element) == "AXCell", + let parent = ax.parent(of: element), + let parentRole = ax.role(of: parent), + parentRole != "AXCell", + rowRoles.contains(parentRole) + else { + return nil + } + return parent + } +} diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/RowSelectionClickTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/RowSelectionClickTests.swift new file mode 100644 index 00000000000..a754e63e7de --- /dev/null +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/RowSelectionClickTests.swift @@ -0,0 +1,121 @@ +import OrcaComputerUseMacOSCore +import XCTest + +final class RowSelectionClickTests: XCTestCase { + private final class FakeTree: RowSelectionAccessibility { + struct Node { + var role: String + var selected: Bool? + var parent: Int? + var settable: Set = [] + } + + var nodes: [Int: Node] + var acceptedWritesTakeEffect = true + private(set) var writes: [String] = [] + + init(_ nodes: [Int: Node]) { + self.nodes = nodes + } + + func role(of element: Int) -> String? { nodes[element]?.role } + func isSelected(_ element: Int) -> Bool? { nodes[element]?.selected } + func parent(of element: Int) -> Int? { nodes[element]?.parent } + func isSettable(_ attribute: String, on element: Int) -> Bool { nodes[element]?.settable.contains(attribute) == true } + + func setSelected(_ element: Int) throws -> Bool { + writes.append("AXSelected:\(element)") + if acceptedWritesTakeEffect { select(element) } + return true + } + + func setSelection(_ attribute: String, of container: Int, to member: Int) throws -> Bool { + writes.append("\(attribute):\(container):\(member)") + if acceptedWritesTakeEffect { select(member) } + return true + } + + func settles(_ check: () -> Bool) -> Bool { check() } + + private func select(_ member: Int) { + for id in nodes.keys where nodes[id]?.selected != nil { + nodes[id]?.selected = id == member + } + } + } + + private func outline(containerSettable: Set = ["AXSelectedRows"], rowSettable: Set = []) -> FakeTree { + FakeTree([ + 1: .init(role: "AXOutline", parent: nil, settable: containerSettable), + 2: .init(role: "AXRow", selected: true, parent: 1), + 3: .init(role: "AXRow", selected: false, parent: 1, settable: rowSettable), + 4: .init(role: "AXCell", parent: 3), + ]) + } + + func testOnlyARowThatReadsUnselectedQualifies() { + let tree = outline() + tree.nodes[5] = .init(role: "AXButton", selected: false, parent: 1) + tree.nodes[6] = .init(role: "AXRow", parent: 1) + + XCTAssertTrue(RowSelectionClick.isUnselectedRow(3, ax: tree)) + XCTAssertTrue(RowSelectionClick.isUnselectedRow(4, ax: tree)) + XCTAssertFalse(RowSelectionClick.isUnselectedRow(2, ax: tree)) + XCTAssertFalse(RowSelectionClick.isUnselectedRow(5, ax: tree)) + XCTAssertFalse(RowSelectionClick.isUnselectedRow(6, ax: tree)) + } + + func testAClickThatSelectedTheRowNeedsNoWrite() throws { + let tree = outline(rowSettable: ["AXSelected"]) + tree.nodes[3]?.selected = true + + XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .selectedByClick) + XCTAssertEqual(tree.writes, []) + } + + func testAClickThatSelectedNothingReplacesTheContainerSelection() throws { + let tree = outline(rowSettable: ["AXSelected"]) + + XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .selectedByWrite(attribute: "AXSelectedRows")) + XCTAssertEqual(tree.writes, ["AXSelectedRows:1:3"]) + } + + func testARowWhoseContainerTakesNoWriteSelectsItself() throws { + let tree = outline(containerSettable: [], rowSettable: ["AXSelected"]) + + XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .selectedByWrite(attribute: "AXSelected")) + XCTAssertEqual(tree.writes, ["AXSelected:3"]) + } + + func testACellIsSelectedThroughItsRowInTheOwningContainer() throws { + let tree = outline() + + XCTAssertEqual(try RowSelectionClick.confirm(4, ax: tree), .selectedByWrite(attribute: "AXSelectedRows")) + XCTAssertEqual(tree.writes, ["AXSelectedRows:1:3"]) + } + + func testSelectedChildrenTakesTheContainersOwnChild() throws { + let tree = outline(containerSettable: ["AXSelectedChildren"]) + + XCTAssertEqual(try RowSelectionClick.confirm(4, ax: tree), .selectedByWrite(attribute: "AXSelectedChildren")) + XCTAssertEqual(tree.writes, ["AXSelectedChildren:1:3"]) + } + + func testAnOuterListThatTakesWritesIsLeftAlone() throws { + let tree = outline(containerSettable: [], rowSettable: ["AXSelected"]) + tree.nodes[10] = .init(role: "AXList", parent: nil, settable: ["AXSelectedChildren"]) + tree.nodes[11] = .init(role: "AXGroup", selected: false, parent: 10) + tree.nodes[1]?.parent = 11 + + XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .selectedByWrite(attribute: "AXSelected")) + XCTAssertEqual(tree.writes, ["AXSelected:3"]) + } + + func testAnAcceptedWriteThatNeverShowsIsNotFollowedByAnotherWrite() throws { + let tree = outline(rowSettable: ["AXSelected"]) + tree.acceptedWritesTakeEffect = false + + XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .unselected) + XCTAssertEqual(tree.writes, ["AXSelectedRows:1:3"]) + } +}