fix(computer): check macOS click targets are enabled, in view and uncovered before clicking (#41519)

* fix(computer): check macOS click targets are enabled, in view and on top before sending input

* fix(computer): scope macOS click cover checks to the target window

* fix(computer): scroll before trusting container hits and stop hit walks at the shared container

* refactor(computer): share the one-scroll rule in the macOS click check

native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift

@@ -813,6 +813,7 @@ final class Provider {
         }
         if let elementIndex = try optionalInteger(params, "elementIndex") {
             let record = try element(snapshot, elementIndex)
+            try requireEnabled(record, snapshot: snapshot)
             if !foreground {
                 try requireBackgroundClickAllowed(backgroundClickHit(record.element, snapshot: snapshot))
             }
@@ -832,22 +833,20 @@ final class Provider {
                     actionName: actionName
                 )
             }
-            if let point = center(record.localFrame, in: snapshot.windowBounds) {
-                try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground, element: record.element)
-                guard unselectedRow else {
-                    return actionMetadata(
-                        path: "synthetic",
-                        fallbackReason: "actionUnsupported",
-                        verification: unverifiedAction(reason: "synthetic_input")
-                    )
-                }
-                return rowClickMetadata(
-                    try RowSelectionClick.confirm(record.element, ax: rowSelection),
+            let point = try pointerTarget(record, snapshot: snapshot)
+            try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground, element: record.element)
+            guard unselectedRow else {
+                return actionMetadata(
                     path: "synthetic",
-                    actionName: nil
+                    fallbackReason: "actionUnsupported",
+                    verification: unverifiedAction(reason: "synthetic_input")
                 )
             }
-            throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame")
+            return rowClickMetadata(
+                try RowSelectionClick.confirm(record.element, ax: rowSelection),
+                path: "synthetic",
+                actionName: nil
+            )
         }
         let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot)
         try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground)
@@ -873,6 +872,38 @@ final class Provider {
         }
     }
 
+    private func requireEnabled(_ record: ElementRecord, snapshot: Snapshot) throws {
+        let ax = AccessibilityActionability(pid: snapshot.app.pid)
+        let enabled = ElementActionability.waitUntilEnabled(record.element, ax: ax)
+        if ax.unanswered { throw AccessibilityMessaging.appNotResponding(snapshot.app.name) }
+        guard enabled else { throw notEnabledError(record) }
+    }
+
+    /// The screen point a pointer click on the element should use, once it is enabled, in view and on top there.
+    private func pointerTarget(_ record: ElementRecord, snapshot: Snapshot) throws -> CGPoint {
+        // Why: its own wait budget, so time spent enabling or trying AXPress cannot cut short the settle after a scroll.
+        let ax = AccessibilityActionability(pid: snapshot.app.pid)
+        let target = try ElementActionability.pointerTarget(record.element, windowFrame: snapshot.windowBounds, ax: ax)
+        if ax.unanswered { throw AccessibilityMessaging.appNotResponding(snapshot.app.name) }
+        switch target {
+        case let .ready(point):
+            return point
+        case .notEnabled:
+            throw notEnabledError(record)
+        case .noFrame:
+            throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame; no click was sent")
+        case let .outOfView(scrolled):
+            let reason = scrolled ? "is still out of view after scrolling it into view" : "is out of view and cannot be scrolled into view"
+            throw ProviderError.coded("element_not_clickable", "element \(record.index) \(reason), so no click was sent; scroll its container, run get-app-state again and retry")
+        case let .covered(by: cover):
+            throw ProviderError.coded("element_not_clickable", "element \(record.index) is covered by \(describeElement(cover)), so no click was sent; close or finish what covers it, run get-app-state again and retry")
+        }
+    }
+
+    private func notEnabledError(_ record: ElementRecord) -> ProviderError {
+        .coded("element_not_enabled", "element \(record.index) is disabled, so no input was sent; do what enables it first, then run get-app-state and retry once its line no longer lists disabled")
+    }
+
     private func performClickAction(record: ElementRecord, mouseButton: MouseButtonSelection) throws -> String? {
         if mouseButton == .right {
             return try performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil
@@ -893,6 +924,10 @@ final class Provider {
         guard let action else {
             throw ProviderError.coded("action_not_supported", "'\(requested)' is not a valid secondary action for element \(record.index)")
         }
+        // Why: scrolling a control into view does not operate it.
+        if action != "AXScrollToVisible" {
+            try requireEnabled(record, snapshot: snapshot)
+        }
         guard try performAction(record.element, action) else {
             throw ProviderError.coded("accessibility_error", "AXUIElementPerformAction(\(action)) failed")
         }
@@ -903,6 +938,7 @@ final class Provider {
         let snapshot = try currentSnapshot(params: params)
         let record = try element(snapshot, try requiredInteger(params, "elementIndex"))
         let expected = try requiredStringAllowingEmpty(params, "value")
+        try requireEnabled(record, snapshot: snapshot)
         guard isSettable(record.element, kAXValueAttribute as String) else {
             throw ProviderError.coded("value_not_settable", "element \(record.index) is not settable")
         }
@@ -1980,6 +2016,94 @@ private final class AccessibilityRowSelection: RowSelectionAccessibility {
     }
 }
 
+/// Once the app stops answering, later reads return nothing at once so a hung app costs one timeout, not dozens.
+private final class AccessibilityActionability: ElementActionabilityAccessibility {
+    private let appElement: AXUIElement
+    // Why: wall time, since one slow AX read can take the full 3 s messaging timeout.
+    private let deadline = CFAbsoluteTimeGetCurrent() + 0.5
+    private(set) var unanswered = false
+
+    init(pid: pid_t) {
+        appElement = AXUIElementCreateApplication(pid)
+    }
+
+    func isEnabled(_ element: AXUIElement) -> Bool? {
+        read(element, kAXEnabledAttribute as String) as? Bool
+    }
+
+    func frame(of element: AXUIElement) -> CGRect? {
+        var point = CGPoint.zero
+        var size = CGSize.zero
+        guard let position = read(element, kAXPositionAttribute as String),
+              let extent = read(element, kAXSizeAttribute as String),
+              CFGetTypeID(position) == AXValueGetTypeID(),
+              CFGetTypeID(extent) == AXValueGetTypeID(),
+              AXValueGetValue(position as! AXValue, .cgPoint, &point),
+              AXValueGetValue(extent as! AXValue, .cgSize, &size)
+        else {
+            return nil
+        }
+        return CGRect(origin: point, size: size)
+    }
+
+    func role(of element: AXUIElement) -> String? {
+        read(element, kAXRoleAttribute as String) as? String
+    }
+
+    func parent(of element: AXUIElement) -> AXUIElement? {
+        guard let value = read(element, kAXParentAttribute as String), CFGetTypeID(value) == AXUIElementGetTypeID() else {
+            return nil
+        }
+        return (value as! AXUIElement)
+    }
+
+    func isSame(_ lhs: AXUIElement, _ rhs: AXUIElement) -> Bool {
+        CFEqual(lhs, rhs)
+    }
+
+    func isModal(_ window: AXUIElement) -> Bool {
+        read(window, kAXModalAttribute as String) as? Bool == true
+    }
+
+    /// Asks the target app itself: background input reaches it whatever other apps' windows are on top.
+    func element(at point: CGPoint) -> AXUIElement? {
+        guard !unanswered, Float(point.x).isFinite, Float(point.y).isFinite else { return nil }
+        var found: AXUIElement?
+        let start = CFAbsoluteTimeGetCurrent()
+        let result = AXUIElementCopyElementAtPosition(appElement, Float(point.x), Float(point.y), &found)
+        unanswered = AccessibilityMessaging.timedOut(result, since: start)
+        return result == .success ? found : nil
+    }
+
+    func scrollToVisible(_ element: AXUIElement) throws -> Bool {
+        try performAction(element, "AXScrollToVisible")
+    }
+
+    func waitBeforeRetry() -> Bool {
+        let interval = 0.05
+        guard !unanswered, CFAbsoluteTimeGetCurrent() + interval < deadline else { return false }
+        Thread.sleep(forTimeInterval: interval)
+        return true
+    }
+
+    private func read(_ element: AXUIElement, _ attribute: String) -> CFTypeRef? {
+        guard !unanswered else { return nil }
+        var value: CFTypeRef?
+        let start = CFAbsoluteTimeGetCurrent()
+        let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value)
+        unanswered = AccessibilityMessaging.timedOut(result, since: start)
+        return result == .success ? value : nil
+    }
+}
+
+private func describeElement(_ element: AXUIElement) -> String {
+    let role = stringAttribute(element, kAXRoleAttribute as String) ?? "an element"
+    guard let label = stringAttribute(element, kAXTitleAttribute as String) ?? stringAttribute(element, kAXDescriptionAttribute as String) else {
+        return role
+    }
+    return "\(role) \"\(preview(label, maxLength: 60))\""
+}
+
 private func frame(_ element: AXUIElement, windowBounds: CGRect) -> CGRect? {
     var positionValue: CFTypeRef?
     var sizeValue: CFTypeRef?

native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ElementActionability.swift

@@ -0,0 +1,209 @@
+import CoreGraphics
+
+/// The accessibility reads an element action needs before it sends input, so the policy runs against a fake tree in tests.
+public protocol ElementActionabilityAccessibility {
+    associatedtype Element
+    /// AXEnabled; nil when the element does not publish it.
+    func isEnabled(_ element: Element) -> Bool?
+    /// Frame in global top-left screen points.
+    func frame(of element: Element) -> CGRect?
+    func role(of element: Element) -> String?
+    func parent(of element: Element) -> Element?
+    func isSame(_ lhs: Element, _ rhs: Element) -> Bool
+    /// AXModal: an app-modal window blocks input to the app's other windows.
+    func isModal(_ window: Element) -> Bool
+    /// The target app's own topmost element at a screen point; nil when the app cannot say.
+    func element(at point: CGPoint) -> Element?
+    /// Asks the app to scroll the element into view; false when the element does not offer that.
+    func scrollToVisible(_ element: Element) throws -> Bool
+    /// Pauses before the next check; false once the wait budget is spent.
+    func waitBeforeRetry() -> Bool
+}
+
+/// Checks that an element can take an action before any input is sent: it is enabled and, for pointer
+/// input, it is in view (scrolled there if needed) and is what the app hit-tests under the click point.
+public enum ElementActionability {
+    public enum PointerTarget<Element> {
+        case ready(CGPoint)
+        case notEnabled
+        case noFrame
+        /// Nothing of the element shows inside its window and scroll areas.
+        case outOfView(scrolled: Bool)
+        /// Another element is on top at every point tried.
+        case covered(by: Element)
+    }
+
+    static let maxAncestorDepth = 64
+
+    /// Waits briefly for a disabled element to enable; an element that publishes no AXEnabled counts as enabled.
+    public static func waitUntilEnabled<AX: ElementActionabilityAccessibility>(_ element: AX.Element, ax: AX) -> Bool {
+        while ax.isEnabled(element) == false {
+            guard ax.waitBeforeRetry() else { return false }
+        }
+        return true
+    }
+
+    /// Where a pointer click on the element lands, re-checked until it holds or the wait budget runs out.
+    public static func pointerTarget<AX: ElementActionabilityAccessibility>(
+        _ element: AX.Element,
+        windowFrame: CGRect,
+        ax: AX
+    ) throws -> PointerTarget<AX.Element> {
+        var scrolled = false
+        var scrollTried = false
+        var settledFrame: CGRect?
+        var last: PointerTarget<AX.Element> = .noFrame
+        // Why: one scroll at most, so a target that keeps failing cannot scroll the user's view around.
+        func scrollOnce() throws -> Bool {
+            guard !scrollTried else { return false }
+            scrollTried = true
+            scrolled = try ax.scrollToVisible(element)
+            return scrolled
+        }
+        repeat {
+            guard ax.isEnabled(element) != false else {
+                last = .notEnabled
+                continue
+            }
+            guard let frame = ax.frame(of: element), frame.width > 0, frame.height > 0 else {
+                return .noFrame
+            }
+            // Why: a scroll can animate, and a click mid-animation lands where the element used to be.
+            if scrolled, settledFrame.map({ !framesMatch($0, frame) }) ?? true {
+                settledFrame = frame
+                last = .outOfView(scrolled: true)
+                continue
+            }
+            let (clip, ancestors) = clipRect(of: element, windowFrame: windowFrame, ax: ax)
+            let visible = frame.intersection(clip)
+            // Why: an element bigger than its viewport cannot be brought fully into view, so it is clicked where it shows.
+            let fits = frame.width <= clip.width && frame.height <= clip.height
+            if visible.isEmpty || (fits && !visible.contains(CGPoint(x: frame.midX, y: frame.midY))), try scrollOnce() {
+                last = .outOfView(scrolled: true)
+                continue
+            }
+            guard !visible.isEmpty else {
+                last = .outOfView(scrolled: scrolled)
+                if scrolled { continue }
+                return last
+            }
+            switch hitTarget(element, visible: visible, ancestors: ancestors, ax: ax) {
+            case let .direct(point):
+                return .ready(point)
+            case let .ancestorOnly(point):
+                // Why: a scroller not exposed as AXScrollArea (a web overflow div) can hide the element behind its own container.
+                guard try scrollOnce() else { return .ready(point) }
+                last = .outOfView(scrolled: true)
+            case let .covered(by: cover):
+                last = .covered(by: cover)
+                // Why: a table or sticky header inside the scroll area can cover a row that scrolling would clear.
+                _ = try scrollOnce()
+            }
+        } while ax.waitBeforeRetry()
+        return last
+    }
+
+    /// The part of the screen the element can show in (its window and every scroll area it sits in), plus its ancestors.
+    static func clipRect<AX: ElementActionabilityAccessibility>(
+        of element: AX.Element,
+        windowFrame: CGRect,
+        ax: AX
+    ) -> (CGRect, [AX.Element]) {
+        var visible = windowFrame
+        var ancestors: [AX.Element] = []
+        var current = element
+        for _ in 0..<maxAncestorDepth {
+            guard let parent = ax.parent(of: current) else { break }
+            ancestors.append(parent)
+            let role = ax.role(of: parent)
+            if role == "AXScrollArea", let clip = ax.frame(of: parent) {
+                visible = visible.intersection(clip)
+            }
+            if role == "AXWindow" { break }
+            current = parent
+        }
+        return (visible, ancestors)
+    }
+
+    enum Hit<Element> {
+        /// The element, inside it, or a hit that shows nothing covers it.
+        case direct(CGPoint)
+        /// Only the element's own ancestors answered, as when it is see-through to hit tests.
+        case ancestorOnly(CGPoint)
+        case covered(by: Element)
+    }
+
+    static func hitTarget<AX: ElementActionabilityAccessibility>(
+        _ element: AX.Element,
+        visible: CGRect,
+        ancestors: [AX.Element],
+        ax: AX
+    ) -> Hit<AX.Element> {
+        var ancestorPoint: CGPoint?
+        var coveredBy: AX.Element?
+        for point in candidatePoints(in: visible) {
+            guard let hit = ax.element(at: point) else { return .direct(point) }
+            switch owner(of: hit, element, ancestors: ancestors, ax: ax) {
+            case .clear:
+                return .direct(point)
+            case .ancestor:
+                ancestorPoint = ancestorPoint ?? point
+            case .cover:
+                coveredBy = coveredBy ?? hit
+            }
+        }
+        if let ancestorPoint { return .ancestorOnly(ancestorPoint) }
+        // SAFETY: candidatePoints is never empty, so a loop without a direct or ancestor hit recorded a cover.
+        return .covered(by: coveredBy!)
+    }
+
+    private enum HitOwner {
+        /// The element, inside it, or a hit that shows nothing of the target window is on top.
+        case clear
+        case ancestor
+        case cover
+    }
+
+    private static func owner<AX: ElementActionabilityAccessibility>(
+        of hit: AX.Element,
+        _ element: AX.Element,
+        ancestors: [AX.Element],
+        ax: AX
+    ) -> HitOwner {
+        if ancestors.contains(where: { ax.isSame($0, hit) }) { return .ancestor }
+        var current = hit
+        for _ in 0..<maxAncestorDepth {
+            if ax.isSame(current, element) { return .clear }
+            // Why: reaching the element's own container first means the hit sits beside the element, on top of it.
+            if ancestors.contains(where: { ax.isSame($0, current) }) { return .cover }
+            switch ax.role(of: current) {
+            case "AXWindow":
+                // Why: the app hit-tests its topmost window, but background input is stamped with the target window,
+                // so only a modal window on top blocks it; the foreground fence checks other windows on top.
+                return ax.isModal(current) ? .cover : .clear
+            case "AXApplication":
+                // Why: the app found none of its windows there (hidden, minimized, another Space), which says nothing.
+                return .clear
+            default:
+                break
+            }
+            guard let parent = ax.parent(of: current) else { return .cover }
+            current = parent
+        }
+        return .cover
+    }
+
+    /// The centre first, then the middle of each quadrant, so a partly covered element is still reachable.
+    static func candidatePoints(in rect: CGRect) -> [CGPoint] {
+        let center = CGPoint(x: rect.midX, y: rect.midY)
+        let quarters = [(0.25, 0.25), (0.75, 0.25), (0.25, 0.75), (0.75, 0.75)].map {
+            CGPoint(x: rect.minX + rect.width * $0.0, y: rect.minY + rect.height * $0.1)
+        }
+        return [center] + quarters
+    }
+
+    private static func framesMatch(_ lhs: CGRect, _ rhs: CGRect) -> Bool {
+        abs(lhs.minX - rhs.minX) < 0.5 && abs(lhs.minY - rhs.minY) < 0.5
+            && abs(lhs.width - rhs.width) < 0.5 && abs(lhs.height - rhs.height) < 0.5
+    }
+}

native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ElementActionabilityTests.swift

@@ -0,0 +1,221 @@
+import CoreGraphics
+import OrcaComputerUseMacOSCore
+import XCTest
+
+final class ElementActionabilityTests: XCTestCase {
+    private final class FakeTree: ElementActionabilityAccessibility {
+        struct Node {
+            var role: String
+            var parent: Int?
+            var frame: CGRect
+            var enabled: Bool? = true
+            var modal = false
+            /// Hides the parts of descendants outside its frame from hit tests, as a scroller does.
+            var clips = false
+            /// Where the node sits after AXScrollToVisible; nil when it offers no scroll.
+            var scrollsTo: CGRect?
+        }
+
+        var nodes: [Int: Node]
+        var retriesLeft = 5
+        var onRetry: (FakeTree) -> Void = { _ in }
+        private(set) var scrollRequests = 0
+
+        init(_ nodes: [Int: Node]) {
+            self.nodes = nodes
+        }
+
+        func isEnabled(_ element: Int) -> Bool? { nodes[element]?.enabled }
+        func frame(of element: Int) -> CGRect? { nodes[element]?.frame }
+        func role(of element: Int) -> String? { nodes[element]?.role }
+        func parent(of element: Int) -> Int? { nodes[element]?.parent }
+        func isSame(_ lhs: Int, _ rhs: Int) -> Bool { lhs == rhs }
+        func isModal(_ window: Int) -> Bool { nodes[window]?.modal == true }
+
+        /// The highest-numbered node under the point is on top.
+        func element(at point: CGPoint) -> Int? {
+            nodes.keys.filter { shows($0, at: point) }.max()
+        }
+
+        private func shows(_ id: Int, at point: CGPoint) -> Bool {
+            guard let node = nodes[id], node.frame.contains(point) else { return false }
+            var ancestor = node.parent
+            while let current = ancestor.flatMap({ nodes[$0] }) {
+                if current.clips, !current.frame.contains(point) { return false }
+                ancestor = current.parent
+            }
+            return true
+        }
+
+        func scrollToVisible(_ element: Int) throws -> Bool {
+            guard let target = nodes[element]?.scrollsTo else { return false }
+            scrollRequests += 1
+            nodes[element]?.frame = target
+            return true
+        }
+
+        func waitBeforeRetry() -> Bool {
+            guard retriesLeft > 0 else { return false }
+            retriesLeft -= 1
+            onRetry(self)
+            return true
+        }
+    }
+
+    private let window = CGRect(x: 0, y: 0, width: 400, height: 300)
+
+    /// A window with a scroll area showing y 0-200 and a button inside it.
+    private func scrollingWindow(button: FakeTree.Node) -> FakeTree {
+        FakeTree([
+            1: .init(role: "AXWindow", parent: nil, frame: window),
+            2: .init(role: "AXScrollArea", parent: 1, frame: CGRect(x: 0, y: 0, width: 400, height: 200), clips: true),
+            3: button,
+        ])
+    }
+
+    func testADisabledElementIsRefusedWithoutAPoint() throws {
+        let tree = scrollingWindow(button: .init(role: "AXButton", parent: 2, frame: CGRect(x: 10, y: 10, width: 80, height: 20), enabled: false))
+
+        guard case .notEnabled = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else {
+            return XCTFail("a disabled button must not get a click point")
+        }
+        XCTAssertFalse(ElementActionability.waitUntilEnabled(3, ax: tree))
+    }
+
+    func testAnElementThatEnablesWithinTheWaitIsClicked() {
+        let tree = scrollingWindow(button: .init(role: "AXButton", parent: 2, frame: CGRect(x: 10, y: 10, width: 80, height: 20), enabled: false))
+        tree.onRetry = { $0.nodes[3]?.enabled = true }
+
+        XCTAssertTrue(ElementActionability.waitUntilEnabled(3, ax: tree))
+    }
+
+    func testAnElementBelowTheScrolledAreaIsScrolledIntoViewThenClicked() throws {
+        let tree = scrollingWindow(button: .init(
+            role: "AXButton",
+            parent: 2,
+            frame: CGRect(x: 10, y: 250, width: 80, height: 20),
+            scrollsTo: CGRect(x: 10, y: 150, width: 80, height: 20)
+        ))
+
+        guard case let .ready(point) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else {
+            return XCTFail("the button should be clickable once scrolled into view")
+        }
+        XCTAssertEqual(tree.scrollRequests, 1)
+        XCTAssertEqual(point, CGPoint(x: 50, y: 160))
+    }
+
+    func testAnElementThatCannotBeScrolledIntoViewIsRefused() throws {
+        let tree = scrollingWindow(button: .init(role: "AXButton", parent: 2, frame: CGRect(x: 10, y: 250, width: 80, height: 20)))
+
+        guard case .outOfView(scrolled: false) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else {
+            return XCTFail("a button clipped by its scroll area must not be clicked")
+        }
+    }
+
+    func testAnElementBiggerThanItsScrollAreaIsClickedWhereItShowsWithoutScrolling() throws {
+        let tree = scrollingWindow(button: .init(
+            role: "AXGroup",
+            parent: 2,
+            frame: CGRect(x: 0, y: 100, width: 400, height: 1000),
+            scrollsTo: CGRect(x: 0, y: 0, width: 400, height: 1000)
+        ))
+
+        guard case let .ready(point) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else {
+            return XCTFail("the visible part of the group should take the click")
+        }
+        XCTAssertEqual(tree.scrollRequests, 0)
+        XCTAssertEqual(point, CGPoint(x: 200, y: 150))
+    }
+
+    func testACoveredElementIsRefusedAndNamesWhatCoversIt() throws {
+        let tree = scrollingWindow(button: .init(role: "AXButton", parent: 2, frame: CGRect(x: 10, y: 10, width: 80, height: 20)))
+        tree.nodes[4] = .init(role: "AXSheet", parent: 1, frame: CGRect(x: 0, y: 0, width: 400, height: 100))
+
+        guard case let .covered(by: cover) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else {
+            return XCTFail("a button under a sheet must not be clicked")
+        }
+        XCTAssertEqual(cover, 4)
+    }
+
+    func testARowUnderAStickyHeaderIsScrolledClearThenClicked() throws {
+        let tree = scrollingWindow(button: .init(
+            role: "AXRow",
+            parent: 2,
+            frame: CGRect(x: 0, y: 0, width: 100, height: 20),
+            scrollsTo: CGRect(x: 0, y: 40, width: 100, height: 20)
+        ))
+        tree.nodes[4] = .init(role: "AXGroup", parent: 2, frame: CGRect(x: 0, y: 0, width: 400, height: 30))
+
+        guard case let .ready(point) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else {
+            return XCTFail("the row should be clicked once scrolled out from under the header")
+        }
+        XCTAssertEqual(tree.scrollRequests, 1)
+        XCTAssertEqual(point, CGPoint(x: 50, y: 50))
+    }
+
+    func testAnotherWindowOfTheAppOnTopDoesNotCoverTheTarget() throws {
+        let tree = scrollingWindow(button: .init(role: "AXButton", parent: 2, frame: CGRect(x: 10, y: 10, width: 80, height: 20)))
+        tree.nodes[5] = .init(role: "AXWindow", parent: nil, frame: CGRect(x: 0, y: 0, width: 200, height: 200))
+
+        guard case .ready = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else {
+            return XCTFail("background input goes to the target window even under another window of the app")
+        }
+
+        tree.nodes[5]?.modal = true
+        guard case .covered(by: 5) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else {
+            return XCTFail("an app-modal window on top blocks input to the target window")
+        }
+    }
+
+    func testAPartlyCoveredElementIsClickedWhereItShows() throws {
+        let tree = scrollingWindow(button: .init(role: "AXButton", parent: 2, frame: CGRect(x: 0, y: 0, width: 100, height: 100)))
+        tree.nodes[4] = .init(role: "AXGroup", parent: 2, frame: CGRect(x: 0, y: 0, width: 60, height: 100))
+
+        guard case let .ready(point) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else {
+            return XCTFail("the uncovered right half should take the click")
+        }
+        XCTAssertEqual(point, CGPoint(x: 75, y: 25))
+    }
+
+    func testAHitOnTheElementsOwnChildCountsAsTheElement() throws {
+        let tree = scrollingWindow(button: .init(role: "AXRow", parent: 2, frame: CGRect(x: 0, y: 0, width: 100, height: 20)))
+        tree.nodes[4] = .init(role: "AXStaticText", parent: 3, frame: CGRect(x: 0, y: 0, width: 100, height: 20))
+
+        guard case .ready = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else {
+            return XCTFail("a hit on the row's own text is a hit on the row")
+        }
+    }
+
+    func testAnElementHiddenByAScrollerThatIsNotAScrollAreaIsScrolledIntoViewFirst() throws {
+        // A web overflow div shows as AXGroup (2), so only the window (1) answers below it.
+        let tree = FakeTree([
+            1: .init(role: "AXWindow", parent: nil, frame: window),
+            2: .init(role: "AXGroup", parent: 1, frame: CGRect(x: 0, y: 0, width: 400, height: 100), clips: true),
+            3: .init(
+                role: "AXLink",
+                parent: 2,
+                frame: CGRect(x: 10, y: 150, width: 80, height: 20),
+                scrollsTo: CGRect(x: 10, y: 50, width: 80, height: 20)
+            ),
+        ])
+
+        guard case let .ready(point) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else {
+            return XCTFail("the link should be clicked once scrolled into its container's view")
+        }
+        XCTAssertEqual(tree.scrollRequests, 1)
+        XCTAssertEqual(point, CGPoint(x: 50, y: 60))
+    }
+
+    func testAHitOnTheElementsAncestorMeansNothingCoversIt() throws {
+        // The group (9) is above the image (3) in hit order, as when the image ignores hit tests.
+        let tree = FakeTree([
+            1: .init(role: "AXWindow", parent: nil, frame: window),
+            3: .init(role: "AXImage", parent: 9, frame: CGRect(x: 10, y: 10, width: 20, height: 20)),
+            9: .init(role: "AXGroup", parent: 1, frame: window),
+        ])
+
+        guard case .ready = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else {
+            return XCTFail("a hit on the element's own ancestor is not a covering element")
+        }
+    }
+}

skill-guides/orca-computer-use.md

@@ -172,7 +172,8 @@ Slack: the accessibility tree may be shallow while the screenshot contains usefu
 - `unsupported_capability`: the provider or desktop environment cannot do that action; use a semantic alternative or install the missing dependency if the message names one.
 - `action_not_supported`: inspect the element's listed actions and retry with one of those names, or use click/set-value when appropriate.
 - `value_not_settable`: the element cannot accept direct value writes; focus it and use keyboard input only when the returned state can be inspected.
-- `element_not_clickable`: the element has no actionable frame; use a parent/child element with a frame or choose window-local coordinates from the latest screenshot.
+- `element_not_clickable`: no click was sent. If the message says the element is covered, deal with what covers it (a dialog, sheet or popover), then click again from a fresh `get-app-state`; never click by coordinates to get past the cover. If it is out of view, scroll its container first. Otherwise the element has no actionable frame; use a parent/child element with a frame or choose window-local coordinates from the latest screenshot.
+- `element_not_enabled`: the element is disabled (its line lists `disabled`), so nothing was sent. Do what enables it first, then act once a fresh `get-app-state` no longer lists it as disabled.
 - `invalid_argument`: fix the command flags; do not retry the same command unchanged.
 - `action_timeout`: inspect current state before retrying, then use a simpler semantic action or `--no-screenshot` if observation is slow.
 - `verification_failed`: the write was read back and the element holds a different value, so the action failed. Run `get-app-state` to see which field changed. For editable text, click the intended field and confirm focus, select all (`cmd+a` on macOS, `ctrl+a` on Windows/Linux), then use `type-text` and inspect the resulting value instead of repeating `set-value`.

src/cli/skills/bundled-skill-guides.ts

@@ -1191,7 +1191,8 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
   "- `unsupported_capability`: the provider or desktop environment cannot do that action; use a semantic alternative or install the missing dependency if the message names one.",
   "- `action_not_supported`: inspect the element's listed actions and retry with one of those names, or use click/set-value when appropriate.",
   "- `value_not_settable`: the element cannot accept direct value writes; focus it and use keyboard input only when the returned state can be inspected.",
-  "- `element_not_clickable`: the element has no actionable frame; use a parent/child element with a frame or choose window-local coordinates from the latest screenshot.",
+  "- `element_not_clickable`: no click was sent. If the message says the element is covered, deal with what covers it (a dialog, sheet or popover), then click again from a fresh `get-app-state`; never click by coordinates to get past the cover. If it is out of view, scroll its container first. Otherwise the element has no actionable frame; use a parent/child element with a frame or choose window-local coordinates from the latest screenshot.",
+  "- `element_not_enabled`: the element is disabled (its line lists `disabled`), so nothing was sent. Do what enables it first, then act once a fresh `get-app-state` no longer lists it as disabled.",
   "- `invalid_argument`: fix the command flags; do not retry the same command unchanged.",
   "- `action_timeout`: inspect current state before retrying, then use a simpler semantic action or `--no-screenshot` if observation is slow.",
   "- `verification_failed`: the write was read back and the element holds a different value, so the action failed. Run `get-app-state` to see which field changed. For editable text, click the intended field and confirm focus, select all (`cmd+a` on macOS, `ctrl+a` on Windows/Linux), then use `type-text` and inspect the resulting value instead of repeating `set-value`.",

src/shared/computer-use-error-recovery.ts

@@ -72,10 +72,26 @@ export function computerUseErrorRecoveryData(
         'Do not infer valid indexes from `elementCount` or reuse indexes after navigation, scrolling, focus changes, or delays.'
       )
     case 'element_not_clickable':
+      if (message?.includes(' is covered by ')) {
+        return recoverWith(
+          'Run `orca computer get-app-state --app <app> --json` and deal with what covers the element first (answer or close the dialog, sheet or popover).',
+          'Then click the element again using an index from the fresh tree; do not click by coordinates to get past the cover.'
+        )
+      }
+      if (message?.includes(' out of view')) {
+        return recoverWith(
+          "Scroll the element's container with `orca computer scroll`, then run `orca computer get-app-state --app <app> --json` and click the element using an index from the fresh tree."
+        )
+      }
       return recoverWith(
         'Choose a nearby parent or child element that has an actionable frame.',
         'If using coordinates, derive window-local coordinates from the latest screenshot/state for the same target window.'
       )
+    case 'element_not_enabled':
+      return recoverWith(
+        'The element is disabled, so nothing was sent. Do what enables it first (for example fill required fields), then run `orca computer get-app-state --app <app> --json` and act once its line no longer lists `disabled`.',
+        'Do not retry the same action unchanged or click it by coordinates instead.'
+      )
     case 'action_not_supported':
       return recoverWith(
         'Inspect the element in a fresh `get-app-state` result and use one of its advertised secondary actions.',

src/shared/computer-use-runtime-types.ts

@@ -10,6 +10,7 @@ export const COMPUTER_ERROR_CODES = {
   permission_denied: 'permission_denied',
   element_not_found: 'element_not_found',
   element_not_clickable: 'element_not_clickable',
+  element_not_enabled: 'element_not_enabled',
   action_not_supported: 'action_not_supported',
   value_not_settable: 'value_not_settable',
   invalid_argument: 'invalid_argument',