fix(computer): select list and table rows when a macOS click leaves them unselected

A plain click on an AXRow/AXCell answered success (AXPress) or unverified synthetic input while nothing got selected: AppKit rows have no AXPress, and background pointer input does not select them. After the click, read the selection back; if the row is still unselected, write it through the owning table/outline/list (AXSelectedCells/AXSelectedRows/AXSelectedChildren), falling back to the row's own AXSelected, and report verified only when the selection reads back on.
This commit is contained in:
Kelvin Amoaba
2026-11-12 07:58:00 +00:00
parent 34bde5edd4
commit f7f080c123
3 changed files with 328 additions and 4 deletions
@@ -816,18 +816,35 @@ final class Provider {
if !foreground {
try requireBackgroundClickAllowed(backgroundClickHit(record.element, snapshot: snapshot))
}
let rowSelection = AccessibilityRowSelection()
let unselectedRow = modifiers.isEmpty && count <= 1 && button == .left
&& RowSelectionClick.isUnselectedRow(record.element, ax: rowSelection)
if modifiers.isEmpty,
count <= 1,
button.hasAccessibilityAction,
let actionName = try performClickAction(record: record, mouseButton: button) {
return actionMetadata(path: "accessibility", actionName: actionName)
guard unselectedRow else {
return actionMetadata(path: "accessibility", actionName: actionName)
}
return rowClickMetadata(
try RowSelectionClick.confirm(record.element, ax: rowSelection),
path: "accessibility",
actionName: actionName
)
}
if let point = center(record.localFrame, in: snapshot.windowBounds) {
try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground, element: record.element)
return actionMetadata(
guard unselectedRow else {
return actionMetadata(
path: "synthetic",
fallbackReason: "actionUnsupported",
verification: unverifiedAction(reason: "synthetic_input")
)
}
return rowClickMetadata(
try RowSelectionClick.confirm(record.element, ax: rowSelection),
path: "synthetic",
fallbackReason: "actionUnsupported",
verification: unverifiedAction(reason: "synthetic_input")
actionName: nil
)
}
throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame")
@@ -840,6 +857,22 @@ final class Provider {
)
}
/// The selection reading back on verifies a row click; a row it never selected stays unverified.
private func rowClickMetadata(_ outcome: RowSelectionClick.Outcome, path: String, actionName: String?) -> [String: Any] {
let fallbackReason = path == "synthetic" ? "actionUnsupported" : nil
switch outcome {
case .selectedByClick:
return actionMetadata(path: path, actionName: actionName, fallbackReason: fallbackReason, verification: verifiedAction(property: "selection"))
case let .selectedByWrite(attribute):
return actionMetadata(path: "accessibility", actionName: attribute, fallbackReason: fallbackReason, verification: verifiedAction(property: "selection"))
case .unselected:
// Why: an accessibility click leaves verification to the tree diff, which reports suspected_noop if nothing changed.
return path == "synthetic"
? actionMetadata(path: path, fallbackReason: fallbackReason, verification: unverifiedAction(reason: "synthetic_input"))
: actionMetadata(path: path, actionName: actionName)
}
}
private func performClickAction(record: ElementRecord, mouseButton: MouseButtonSelection) throws -> String? {
if mouseButton == .right {
return try performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil
@@ -1875,6 +1908,75 @@ private func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool {
return AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) == .success && settable.boolValue
}
/// Once the app stops answering, later reads return nothing at once so a hung app costs one timeout, not dozens.
private final class AccessibilityRowSelection: RowSelectionAccessibility {
private var unanswered = false
func role(of element: AXUIElement) -> String? {
read(element, kAXRoleAttribute as String) as? String
}
func isSelected(_ element: AXUIElement) -> Bool? {
read(element, kAXSelectedAttribute as String) as? Bool
}
func parent(of element: AXUIElement) -> AXUIElement? {
guard let value = read(element, kAXParentAttribute as String),
CFGetTypeID(value) == AXUIElementGetTypeID()
else {
return nil
}
return (value as! AXUIElement)
}
func isSettable(_ attribute: String, on element: AXUIElement) -> Bool {
guard !unanswered else { return false }
var settable = DarwinBoolean(false)
let start = CFAbsoluteTimeGetCurrent()
let result = AXUIElementIsAttributeSettable(element, attribute as CFString, &settable)
unanswered = AccessibilityMessaging.timedOut(result, since: start)
return result == .success && settable.boolValue
}
func setSelected(_ element: AXUIElement) throws -> Bool {
try write(kAXSelectedAttribute as String, on: element, value: kCFBooleanTrue)
}
func setSelection(_ attribute: String, of container: AXUIElement, to member: AXUIElement) throws -> Bool {
try write(attribute, on: container, value: [member] as CFArray)
}
func settles(_ check: () -> Bool) -> Bool {
// Why: settle counts only its sleeps, and one slow AX read can take the full 3 s timeout.
let deadline = CFAbsoluteTimeGetCurrent() + 0.3
var shown = false
_ = PermissionTrustSettling.settle(timeoutMs: 300, intervalMs: 25) {
guard !unanswered, CFAbsoluteTimeGetCurrent() < deadline else { return true }
shown = check()
return shown
}
return shown
}
private func read(_ element: AXUIElement, _ attribute: String) -> CFTypeRef? {
guard !unanswered else { return nil }
var value: CFTypeRef?
let start = CFAbsoluteTimeGetCurrent()
let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value)
unanswered = AccessibilityMessaging.timedOut(result, since: start)
return result == .success ? value : nil
}
private func write(_ attribute: String, on element: AXUIElement, value: CFTypeRef) throws -> Bool {
let start = CFAbsoluteTimeGetCurrent()
let result = AXUIElementSetAttributeValue(element, attribute as CFString, value)
if AccessibilityMessaging.timedOut(result, since: start) {
throw AccessibilityMessaging.unanswered("the \(attribute) write")
}
return result == .success
}
}
private func frame(_ element: AXUIElement, windowBounds: CGRect) -> CGRect? {
var positionValue: CFTypeRef?
var sizeValue: CFTypeRef?
@@ -0,0 +1,101 @@
/// The accessibility reads and writes a row click needs, so the click policy runs against a fake tree in tests.
public protocol RowSelectionAccessibility {
associatedtype Element
func role(of element: Element) -> String?
func isSelected(_ element: Element) -> Bool?
func parent(of element: Element) -> Element?
func isSettable(_ attribute: String, on element: Element) -> Bool
/// Sets AXSelected to true; false when the app refused the write.
func setSelected(_ element: Element) throws -> Bool
/// Replaces the container's selection with the member; false when the app refused the write.
func setSelection(_ attribute: String, of container: Element, to member: Element) throws -> Bool
/// Re-checks for a moment, since apps apply a selection on a later run-loop turn.
func settles(_ check: () -> Bool) -> Bool
}
/// A plain click on a list, table or outline row means "select it", yet AXPress on a row often answers
/// success and selects nothing. For a row that read unselected, the selection reading back on is what
/// makes the click count.
public enum RowSelectionClick {
public static let rowRoles: Set<String> = ["AXRow", "AXCell", "AXOutlineRow"]
static let containerRoles: Set<String> = ["AXTable", "AXOutline", "AXList", "AXGrid"]
static let maxContainerDepth = 6
public enum Outcome: Equatable {
case selectedByClick
case selectedByWrite(attribute: String)
case unselected
}
/// Rows that read selected already, or publish no selection, prove nothing by reading selected afterwards.
public static func isUnselectedRow<AX: RowSelectionAccessibility>(_ element: AX.Element, ax: AX) -> Bool {
guard let role = ax.role(of: element), rowRoles.contains(role) else { return false }
return selectionShows(element, ax: ax) == false
}
/// Call after a click reached the app. A selection write is idempotent, so it is safe even when the
/// click did something else; it still comes second because it skips the app's own click handling.
public static func confirm<AX: RowSelectionAccessibility>(_ element: AX.Element, ax: AX) throws -> Outcome {
if ax.settles({ selectionShows(element, ax: ax) == true }) {
return .selectedByClick
}
guard let attribute = try writeSelection(element, ax: ax),
ax.settles({ selectionShows(element, ax: ax) == true })
else {
return .unselected
}
return .selectedByWrite(attribute: attribute)
}
/// A cell in a table that selects whole rows reads selected through its row.
static func selectionShows<AX: RowSelectionAccessibility>(_ element: AX.Element, ax: AX) -> Bool? {
let own = ax.isSelected(element)
guard own != true, let row = enclosingRow(of: element, ax: ax) else { return own }
let rowSelected = ax.isSelected(row)
return rowSelected == true ? true : own ?? rowSelected
}
/// Returns the first write the app accepted. It stops there: a selection that has not shown yet may
/// still land, and a second write could then select something else.
private static func writeSelection<AX: RowSelectionAccessibility>(_ element: AX.Element, ax: AX) throws -> String? {
let row = enclosingRow(of: element, ax: ax)
let isCell = ax.role(of: element) == "AXCell"
var child = element
for _ in 0..<maxContainerDepth {
guard let parent = ax.parent(of: child) else { break }
guard let role = ax.role(of: parent), containerRoles.contains(role) else {
child = parent
continue
}
// Why: a spreadsheet owns cell selection on AXSelectedCells; a table that selects rows takes the
// cell's row; AXSelectedChildren only takes the container's own children.
let writes: [(String, AX.Element)] = (isCell ? [("AXSelectedCells", element)] : []) + [("AXSelectedRows", row ?? element), ("AXSelectedChildren", child)]
for (attribute, member) in writes where ax.isSettable(attribute, on: parent) {
if try ax.setSelection(attribute, of: parent, to: member) {
return attribute
}
}
// Why: an outer list's selection is not this row's; writing it would change something else.
break
}
// Why: last, because in a multi-select list AXSelected may add the row to the selection instead of replacing it.
for target in [element, row].compactMap({ $0 }) where ax.isSettable("AXSelected", on: target) {
if try ax.setSelected(target) {
return "AXSelected"
}
}
return nil
}
private static func enclosingRow<AX: RowSelectionAccessibility>(of element: AX.Element, ax: AX) -> AX.Element? {
guard ax.role(of: element) == "AXCell",
let parent = ax.parent(of: element),
let parentRole = ax.role(of: parent),
parentRole != "AXCell",
rowRoles.contains(parentRole)
else {
return nil
}
return parent
}
}
@@ -0,0 +1,121 @@
import OrcaComputerUseMacOSCore
import XCTest
final class RowSelectionClickTests: XCTestCase {
private final class FakeTree: RowSelectionAccessibility {
struct Node {
var role: String
var selected: Bool?
var parent: Int?
var settable: Set<String> = []
}
var nodes: [Int: Node]
var acceptedWritesTakeEffect = true
private(set) var writes: [String] = []
init(_ nodes: [Int: Node]) {
self.nodes = nodes
}
func role(of element: Int) -> String? { nodes[element]?.role }
func isSelected(_ element: Int) -> Bool? { nodes[element]?.selected }
func parent(of element: Int) -> Int? { nodes[element]?.parent }
func isSettable(_ attribute: String, on element: Int) -> Bool { nodes[element]?.settable.contains(attribute) == true }
func setSelected(_ element: Int) throws -> Bool {
writes.append("AXSelected:\(element)")
if acceptedWritesTakeEffect { select(element) }
return true
}
func setSelection(_ attribute: String, of container: Int, to member: Int) throws -> Bool {
writes.append("\(attribute):\(container):\(member)")
if acceptedWritesTakeEffect { select(member) }
return true
}
func settles(_ check: () -> Bool) -> Bool { check() }
private func select(_ member: Int) {
for id in nodes.keys where nodes[id]?.selected != nil {
nodes[id]?.selected = id == member
}
}
}
private func outline(containerSettable: Set<String> = ["AXSelectedRows"], rowSettable: Set<String> = []) -> FakeTree {
FakeTree([
1: .init(role: "AXOutline", parent: nil, settable: containerSettable),
2: .init(role: "AXRow", selected: true, parent: 1),
3: .init(role: "AXRow", selected: false, parent: 1, settable: rowSettable),
4: .init(role: "AXCell", parent: 3),
])
}
func testOnlyARowThatReadsUnselectedQualifies() {
let tree = outline()
tree.nodes[5] = .init(role: "AXButton", selected: false, parent: 1)
tree.nodes[6] = .init(role: "AXRow", parent: 1)
XCTAssertTrue(RowSelectionClick.isUnselectedRow(3, ax: tree))
XCTAssertTrue(RowSelectionClick.isUnselectedRow(4, ax: tree))
XCTAssertFalse(RowSelectionClick.isUnselectedRow(2, ax: tree))
XCTAssertFalse(RowSelectionClick.isUnselectedRow(5, ax: tree))
XCTAssertFalse(RowSelectionClick.isUnselectedRow(6, ax: tree))
}
func testAClickThatSelectedTheRowNeedsNoWrite() throws {
let tree = outline(rowSettable: ["AXSelected"])
tree.nodes[3]?.selected = true
XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .selectedByClick)
XCTAssertEqual(tree.writes, [])
}
func testAClickThatSelectedNothingReplacesTheContainerSelection() throws {
let tree = outline(rowSettable: ["AXSelected"])
XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .selectedByWrite(attribute: "AXSelectedRows"))
XCTAssertEqual(tree.writes, ["AXSelectedRows:1:3"])
}
func testARowWhoseContainerTakesNoWriteSelectsItself() throws {
let tree = outline(containerSettable: [], rowSettable: ["AXSelected"])
XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .selectedByWrite(attribute: "AXSelected"))
XCTAssertEqual(tree.writes, ["AXSelected:3"])
}
func testACellIsSelectedThroughItsRowInTheOwningContainer() throws {
let tree = outline()
XCTAssertEqual(try RowSelectionClick.confirm(4, ax: tree), .selectedByWrite(attribute: "AXSelectedRows"))
XCTAssertEqual(tree.writes, ["AXSelectedRows:1:3"])
}
func testSelectedChildrenTakesTheContainersOwnChild() throws {
let tree = outline(containerSettable: ["AXSelectedChildren"])
XCTAssertEqual(try RowSelectionClick.confirm(4, ax: tree), .selectedByWrite(attribute: "AXSelectedChildren"))
XCTAssertEqual(tree.writes, ["AXSelectedChildren:1:3"])
}
func testAnOuterListThatTakesWritesIsLeftAlone() throws {
let tree = outline(containerSettable: [], rowSettable: ["AXSelected"])
tree.nodes[10] = .init(role: "AXList", parent: nil, settable: ["AXSelectedChildren"])
tree.nodes[11] = .init(role: "AXGroup", selected: false, parent: 10)
tree.nodes[1]?.parent = 11
XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .selectedByWrite(attribute: "AXSelected"))
XCTAssertEqual(tree.writes, ["AXSelected:3"])
}
func testAnAcceptedWriteThatNeverShowsIsNotFollowedByAnotherWrite() throws {
let tree = outline(rowSettable: ["AXSelected"])
tree.acceptedWritesTakeEffect = false
XCTAssertEqual(try RowSelectionClick.confirm(3, ax: tree), .unselected)
XCTAssertEqual(tree.writes, ["AXSelectedRows:1:3"])
}
}