diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index 4ca1844479e..43ab1f782f3 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -813,6 +813,7 @@ final class Provider { } if let elementIndex = try optionalInteger(params, "elementIndex") { let record = try element(snapshot, elementIndex) + try requireEnabled(record, snapshot: snapshot) if !foreground { try requireBackgroundClickAllowed(backgroundClickHit(record.element, snapshot: snapshot)) } @@ -832,22 +833,20 @@ final class Provider { actionName: actionName ) } - if let point = center(record.localFrame, in: snapshot.windowBounds) { - try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground, element: record.element) - guard unselectedRow else { - return actionMetadata( - path: "synthetic", - fallbackReason: "actionUnsupported", - verification: unverifiedAction(reason: "synthetic_input") - ) - } - return rowClickMetadata( - try RowSelectionClick.confirm(record.element, ax: rowSelection), + let point = try pointerTarget(record, snapshot: snapshot) + try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground, element: record.element) + guard unselectedRow else { + return actionMetadata( path: "synthetic", - actionName: nil + fallbackReason: "actionUnsupported", + verification: unverifiedAction(reason: "synthetic_input") ) } - throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame") + return rowClickMetadata( + try RowSelectionClick.confirm(record.element, ax: rowSelection), + path: "synthetic", + actionName: nil + ) } let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot) try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground) @@ -873,6 +872,38 @@ final class Provider { } } + private func requireEnabled(_ record: ElementRecord, snapshot: Snapshot) throws { + let ax = AccessibilityActionability(pid: snapshot.app.pid) + let enabled = ElementActionability.waitUntilEnabled(record.element, ax: ax) + if ax.unanswered { throw AccessibilityMessaging.appNotResponding(snapshot.app.name) } + guard enabled else { throw notEnabledError(record) } + } + + /// The screen point a pointer click on the element should use, once it is enabled, in view and on top there. + private func pointerTarget(_ record: ElementRecord, snapshot: Snapshot) throws -> CGPoint { + // Why: its own wait budget, so time spent enabling or trying AXPress cannot cut short the settle after a scroll. + let ax = AccessibilityActionability(pid: snapshot.app.pid) + let target = try ElementActionability.pointerTarget(record.element, windowFrame: snapshot.windowBounds, ax: ax) + if ax.unanswered { throw AccessibilityMessaging.appNotResponding(snapshot.app.name) } + switch target { + case let .ready(point): + return point + case .notEnabled: + throw notEnabledError(record) + case .noFrame: + throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame; no click was sent") + case let .outOfView(scrolled): + let reason = scrolled ? "is still out of view after scrolling it into view" : "is out of view and cannot be scrolled into view" + throw ProviderError.coded("element_not_clickable", "element \(record.index) \(reason), so no click was sent; scroll its container, run get-app-state again and retry") + case let .covered(by: cover): + throw ProviderError.coded("element_not_clickable", "element \(record.index) is covered by \(describeElement(cover)), so no click was sent; close or finish what covers it, run get-app-state again and retry") + } + } + + private func notEnabledError(_ record: ElementRecord) -> ProviderError { + .coded("element_not_enabled", "element \(record.index) is disabled, so no input was sent; do what enables it first, then run get-app-state and retry once its line no longer lists disabled") + } + private func performClickAction(record: ElementRecord, mouseButton: MouseButtonSelection) throws -> String? { if mouseButton == .right { return try performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil @@ -893,6 +924,10 @@ final class Provider { guard let action else { throw ProviderError.coded("action_not_supported", "'\(requested)' is not a valid secondary action for element \(record.index)") } + // Why: scrolling a control into view does not operate it. + if action != "AXScrollToVisible" { + try requireEnabled(record, snapshot: snapshot) + } guard try performAction(record.element, action) else { throw ProviderError.coded("accessibility_error", "AXUIElementPerformAction(\(action)) failed") } @@ -903,6 +938,7 @@ final class Provider { let snapshot = try currentSnapshot(params: params) let record = try element(snapshot, try requiredInteger(params, "elementIndex")) let expected = try requiredStringAllowingEmpty(params, "value") + try requireEnabled(record, snapshot: snapshot) guard isSettable(record.element, kAXValueAttribute as String) else { throw ProviderError.coded("value_not_settable", "element \(record.index) is not settable") } @@ -1980,6 +2016,94 @@ private final class AccessibilityRowSelection: RowSelectionAccessibility { } } +/// Once the app stops answering, later reads return nothing at once so a hung app costs one timeout, not dozens. +private final class AccessibilityActionability: ElementActionabilityAccessibility { + private let appElement: AXUIElement + // Why: wall time, since one slow AX read can take the full 3 s messaging timeout. + private let deadline = CFAbsoluteTimeGetCurrent() + 0.5 + private(set) var unanswered = false + + init(pid: pid_t) { + appElement = AXUIElementCreateApplication(pid) + } + + func isEnabled(_ element: AXUIElement) -> Bool? { + read(element, kAXEnabledAttribute as String) as? Bool + } + + func frame(of element: AXUIElement) -> CGRect? { + var point = CGPoint.zero + var size = CGSize.zero + guard let position = read(element, kAXPositionAttribute as String), + let extent = read(element, kAXSizeAttribute as String), + CFGetTypeID(position) == AXValueGetTypeID(), + CFGetTypeID(extent) == AXValueGetTypeID(), + AXValueGetValue(position as! AXValue, .cgPoint, &point), + AXValueGetValue(extent as! AXValue, .cgSize, &size) + else { + return nil + } + return CGRect(origin: point, size: size) + } + + func role(of element: AXUIElement) -> String? { + read(element, kAXRoleAttribute as String) as? String + } + + func parent(of element: AXUIElement) -> AXUIElement? { + guard let value = read(element, kAXParentAttribute as String), CFGetTypeID(value) == AXUIElementGetTypeID() else { + return nil + } + return (value as! AXUIElement) + } + + func isSame(_ lhs: AXUIElement, _ rhs: AXUIElement) -> Bool { + CFEqual(lhs, rhs) + } + + func isModal(_ window: AXUIElement) -> Bool { + read(window, kAXModalAttribute as String) as? Bool == true + } + + /// Asks the target app itself: background input reaches it whatever other apps' windows are on top. + func element(at point: CGPoint) -> AXUIElement? { + guard !unanswered, Float(point.x).isFinite, Float(point.y).isFinite else { return nil } + var found: AXUIElement? + let start = CFAbsoluteTimeGetCurrent() + let result = AXUIElementCopyElementAtPosition(appElement, Float(point.x), Float(point.y), &found) + unanswered = AccessibilityMessaging.timedOut(result, since: start) + return result == .success ? found : nil + } + + func scrollToVisible(_ element: AXUIElement) throws -> Bool { + try performAction(element, "AXScrollToVisible") + } + + func waitBeforeRetry() -> Bool { + let interval = 0.05 + guard !unanswered, CFAbsoluteTimeGetCurrent() + interval < deadline else { return false } + Thread.sleep(forTimeInterval: interval) + return true + } + + private func read(_ element: AXUIElement, _ attribute: String) -> CFTypeRef? { + guard !unanswered else { return nil } + var value: CFTypeRef? + let start = CFAbsoluteTimeGetCurrent() + let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value) + unanswered = AccessibilityMessaging.timedOut(result, since: start) + return result == .success ? value : nil + } +} + +private func describeElement(_ element: AXUIElement) -> String { + let role = stringAttribute(element, kAXRoleAttribute as String) ?? "an element" + guard let label = stringAttribute(element, kAXTitleAttribute as String) ?? stringAttribute(element, kAXDescriptionAttribute as String) else { + return role + } + return "\(role) \"\(preview(label, maxLength: 60))\"" +} + private func frame(_ element: AXUIElement, windowBounds: CGRect) -> CGRect? { var positionValue: CFTypeRef? var sizeValue: CFTypeRef? diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ElementActionability.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ElementActionability.swift new file mode 100644 index 00000000000..0360ebf6919 --- /dev/null +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ElementActionability.swift @@ -0,0 +1,209 @@ +import CoreGraphics + +/// The accessibility reads an element action needs before it sends input, so the policy runs against a fake tree in tests. +public protocol ElementActionabilityAccessibility { + associatedtype Element + /// AXEnabled; nil when the element does not publish it. + func isEnabled(_ element: Element) -> Bool? + /// Frame in global top-left screen points. + func frame(of element: Element) -> CGRect? + func role(of element: Element) -> String? + func parent(of element: Element) -> Element? + func isSame(_ lhs: Element, _ rhs: Element) -> Bool + /// AXModal: an app-modal window blocks input to the app's other windows. + func isModal(_ window: Element) -> Bool + /// The target app's own topmost element at a screen point; nil when the app cannot say. + func element(at point: CGPoint) -> Element? + /// Asks the app to scroll the element into view; false when the element does not offer that. + func scrollToVisible(_ element: Element) throws -> Bool + /// Pauses before the next check; false once the wait budget is spent. + func waitBeforeRetry() -> Bool +} + +/// Checks that an element can take an action before any input is sent: it is enabled and, for pointer +/// input, it is in view (scrolled there if needed) and is what the app hit-tests under the click point. +public enum ElementActionability { + public enum PointerTarget { + case ready(CGPoint) + case notEnabled + case noFrame + /// Nothing of the element shows inside its window and scroll areas. + case outOfView(scrolled: Bool) + /// Another element is on top at every point tried. + case covered(by: Element) + } + + static let maxAncestorDepth = 64 + + /// Waits briefly for a disabled element to enable; an element that publishes no AXEnabled counts as enabled. + public static func waitUntilEnabled(_ element: AX.Element, ax: AX) -> Bool { + while ax.isEnabled(element) == false { + guard ax.waitBeforeRetry() else { return false } + } + return true + } + + /// Where a pointer click on the element lands, re-checked until it holds or the wait budget runs out. + public static func pointerTarget( + _ element: AX.Element, + windowFrame: CGRect, + ax: AX + ) throws -> PointerTarget { + var scrolled = false + var scrollTried = false + var settledFrame: CGRect? + var last: PointerTarget = .noFrame + // Why: one scroll at most, so a target that keeps failing cannot scroll the user's view around. + func scrollOnce() throws -> Bool { + guard !scrollTried else { return false } + scrollTried = true + scrolled = try ax.scrollToVisible(element) + return scrolled + } + repeat { + guard ax.isEnabled(element) != false else { + last = .notEnabled + continue + } + guard let frame = ax.frame(of: element), frame.width > 0, frame.height > 0 else { + return .noFrame + } + // Why: a scroll can animate, and a click mid-animation lands where the element used to be. + if scrolled, settledFrame.map({ !framesMatch($0, frame) }) ?? true { + settledFrame = frame + last = .outOfView(scrolled: true) + continue + } + let (clip, ancestors) = clipRect(of: element, windowFrame: windowFrame, ax: ax) + let visible = frame.intersection(clip) + // Why: an element bigger than its viewport cannot be brought fully into view, so it is clicked where it shows. + let fits = frame.width <= clip.width && frame.height <= clip.height + if visible.isEmpty || (fits && !visible.contains(CGPoint(x: frame.midX, y: frame.midY))), try scrollOnce() { + last = .outOfView(scrolled: true) + continue + } + guard !visible.isEmpty else { + last = .outOfView(scrolled: scrolled) + if scrolled { continue } + return last + } + switch hitTarget(element, visible: visible, ancestors: ancestors, ax: ax) { + case let .direct(point): + return .ready(point) + case let .ancestorOnly(point): + // Why: a scroller not exposed as AXScrollArea (a web overflow div) can hide the element behind its own container. + guard try scrollOnce() else { return .ready(point) } + last = .outOfView(scrolled: true) + case let .covered(by: cover): + last = .covered(by: cover) + // Why: a table or sticky header inside the scroll area can cover a row that scrolling would clear. + _ = try scrollOnce() + } + } while ax.waitBeforeRetry() + return last + } + + /// The part of the screen the element can show in (its window and every scroll area it sits in), plus its ancestors. + static func clipRect( + of element: AX.Element, + windowFrame: CGRect, + ax: AX + ) -> (CGRect, [AX.Element]) { + var visible = windowFrame + var ancestors: [AX.Element] = [] + var current = element + for _ in 0.. { + /// The element, inside it, or a hit that shows nothing covers it. + case direct(CGPoint) + /// Only the element's own ancestors answered, as when it is see-through to hit tests. + case ancestorOnly(CGPoint) + case covered(by: Element) + } + + static func hitTarget( + _ element: AX.Element, + visible: CGRect, + ancestors: [AX.Element], + ax: AX + ) -> Hit { + var ancestorPoint: CGPoint? + var coveredBy: AX.Element? + for point in candidatePoints(in: visible) { + guard let hit = ax.element(at: point) else { return .direct(point) } + switch owner(of: hit, element, ancestors: ancestors, ax: ax) { + case .clear: + return .direct(point) + case .ancestor: + ancestorPoint = ancestorPoint ?? point + case .cover: + coveredBy = coveredBy ?? hit + } + } + if let ancestorPoint { return .ancestorOnly(ancestorPoint) } + // SAFETY: candidatePoints is never empty, so a loop without a direct or ancestor hit recorded a cover. + return .covered(by: coveredBy!) + } + + private enum HitOwner { + /// The element, inside it, or a hit that shows nothing of the target window is on top. + case clear + case ancestor + case cover + } + + private static func owner( + of hit: AX.Element, + _ element: AX.Element, + ancestors: [AX.Element], + ax: AX + ) -> HitOwner { + if ancestors.contains(where: { ax.isSame($0, hit) }) { return .ancestor } + var current = hit + for _ in 0.. [CGPoint] { + let center = CGPoint(x: rect.midX, y: rect.midY) + let quarters = [(0.25, 0.25), (0.75, 0.25), (0.25, 0.75), (0.75, 0.75)].map { + CGPoint(x: rect.minX + rect.width * $0.0, y: rect.minY + rect.height * $0.1) + } + return [center] + quarters + } + + private static func framesMatch(_ lhs: CGRect, _ rhs: CGRect) -> Bool { + abs(lhs.minX - rhs.minX) < 0.5 && abs(lhs.minY - rhs.minY) < 0.5 + && abs(lhs.width - rhs.width) < 0.5 && abs(lhs.height - rhs.height) < 0.5 + } +} diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ElementActionabilityTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ElementActionabilityTests.swift new file mode 100644 index 00000000000..92a411fa7b2 --- /dev/null +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ElementActionabilityTests.swift @@ -0,0 +1,221 @@ +import CoreGraphics +import OrcaComputerUseMacOSCore +import XCTest + +final class ElementActionabilityTests: XCTestCase { + private final class FakeTree: ElementActionabilityAccessibility { + struct Node { + var role: String + var parent: Int? + var frame: CGRect + var enabled: Bool? = true + var modal = false + /// Hides the parts of descendants outside its frame from hit tests, as a scroller does. + var clips = false + /// Where the node sits after AXScrollToVisible; nil when it offers no scroll. + var scrollsTo: CGRect? + } + + var nodes: [Int: Node] + var retriesLeft = 5 + var onRetry: (FakeTree) -> Void = { _ in } + private(set) var scrollRequests = 0 + + init(_ nodes: [Int: Node]) { + self.nodes = nodes + } + + func isEnabled(_ element: Int) -> Bool? { nodes[element]?.enabled } + func frame(of element: Int) -> CGRect? { nodes[element]?.frame } + func role(of element: Int) -> String? { nodes[element]?.role } + func parent(of element: Int) -> Int? { nodes[element]?.parent } + func isSame(_ lhs: Int, _ rhs: Int) -> Bool { lhs == rhs } + func isModal(_ window: Int) -> Bool { nodes[window]?.modal == true } + + /// The highest-numbered node under the point is on top. + func element(at point: CGPoint) -> Int? { + nodes.keys.filter { shows($0, at: point) }.max() + } + + private func shows(_ id: Int, at point: CGPoint) -> Bool { + guard let node = nodes[id], node.frame.contains(point) else { return false } + var ancestor = node.parent + while let current = ancestor.flatMap({ nodes[$0] }) { + if current.clips, !current.frame.contains(point) { return false } + ancestor = current.parent + } + return true + } + + func scrollToVisible(_ element: Int) throws -> Bool { + guard let target = nodes[element]?.scrollsTo else { return false } + scrollRequests += 1 + nodes[element]?.frame = target + return true + } + + func waitBeforeRetry() -> Bool { + guard retriesLeft > 0 else { return false } + retriesLeft -= 1 + onRetry(self) + return true + } + } + + private let window = CGRect(x: 0, y: 0, width: 400, height: 300) + + /// A window with a scroll area showing y 0-200 and a button inside it. + private func scrollingWindow(button: FakeTree.Node) -> FakeTree { + FakeTree([ + 1: .init(role: "AXWindow", parent: nil, frame: window), + 2: .init(role: "AXScrollArea", parent: 1, frame: CGRect(x: 0, y: 0, width: 400, height: 200), clips: true), + 3: button, + ]) + } + + func testADisabledElementIsRefusedWithoutAPoint() throws { + let tree = scrollingWindow(button: .init(role: "AXButton", parent: 2, frame: CGRect(x: 10, y: 10, width: 80, height: 20), enabled: false)) + + guard case .notEnabled = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else { + return XCTFail("a disabled button must not get a click point") + } + XCTAssertFalse(ElementActionability.waitUntilEnabled(3, ax: tree)) + } + + func testAnElementThatEnablesWithinTheWaitIsClicked() { + let tree = scrollingWindow(button: .init(role: "AXButton", parent: 2, frame: CGRect(x: 10, y: 10, width: 80, height: 20), enabled: false)) + tree.onRetry = { $0.nodes[3]?.enabled = true } + + XCTAssertTrue(ElementActionability.waitUntilEnabled(3, ax: tree)) + } + + func testAnElementBelowTheScrolledAreaIsScrolledIntoViewThenClicked() throws { + let tree = scrollingWindow(button: .init( + role: "AXButton", + parent: 2, + frame: CGRect(x: 10, y: 250, width: 80, height: 20), + scrollsTo: CGRect(x: 10, y: 150, width: 80, height: 20) + )) + + guard case let .ready(point) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else { + return XCTFail("the button should be clickable once scrolled into view") + } + XCTAssertEqual(tree.scrollRequests, 1) + XCTAssertEqual(point, CGPoint(x: 50, y: 160)) + } + + func testAnElementThatCannotBeScrolledIntoViewIsRefused() throws { + let tree = scrollingWindow(button: .init(role: "AXButton", parent: 2, frame: CGRect(x: 10, y: 250, width: 80, height: 20))) + + guard case .outOfView(scrolled: false) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else { + return XCTFail("a button clipped by its scroll area must not be clicked") + } + } + + func testAnElementBiggerThanItsScrollAreaIsClickedWhereItShowsWithoutScrolling() throws { + let tree = scrollingWindow(button: .init( + role: "AXGroup", + parent: 2, + frame: CGRect(x: 0, y: 100, width: 400, height: 1000), + scrollsTo: CGRect(x: 0, y: 0, width: 400, height: 1000) + )) + + guard case let .ready(point) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else { + return XCTFail("the visible part of the group should take the click") + } + XCTAssertEqual(tree.scrollRequests, 0) + XCTAssertEqual(point, CGPoint(x: 200, y: 150)) + } + + func testACoveredElementIsRefusedAndNamesWhatCoversIt() throws { + let tree = scrollingWindow(button: .init(role: "AXButton", parent: 2, frame: CGRect(x: 10, y: 10, width: 80, height: 20))) + tree.nodes[4] = .init(role: "AXSheet", parent: 1, frame: CGRect(x: 0, y: 0, width: 400, height: 100)) + + guard case let .covered(by: cover) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else { + return XCTFail("a button under a sheet must not be clicked") + } + XCTAssertEqual(cover, 4) + } + + func testARowUnderAStickyHeaderIsScrolledClearThenClicked() throws { + let tree = scrollingWindow(button: .init( + role: "AXRow", + parent: 2, + frame: CGRect(x: 0, y: 0, width: 100, height: 20), + scrollsTo: CGRect(x: 0, y: 40, width: 100, height: 20) + )) + tree.nodes[4] = .init(role: "AXGroup", parent: 2, frame: CGRect(x: 0, y: 0, width: 400, height: 30)) + + guard case let .ready(point) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else { + return XCTFail("the row should be clicked once scrolled out from under the header") + } + XCTAssertEqual(tree.scrollRequests, 1) + XCTAssertEqual(point, CGPoint(x: 50, y: 50)) + } + + func testAnotherWindowOfTheAppOnTopDoesNotCoverTheTarget() throws { + let tree = scrollingWindow(button: .init(role: "AXButton", parent: 2, frame: CGRect(x: 10, y: 10, width: 80, height: 20))) + tree.nodes[5] = .init(role: "AXWindow", parent: nil, frame: CGRect(x: 0, y: 0, width: 200, height: 200)) + + guard case .ready = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else { + return XCTFail("background input goes to the target window even under another window of the app") + } + + tree.nodes[5]?.modal = true + guard case .covered(by: 5) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else { + return XCTFail("an app-modal window on top blocks input to the target window") + } + } + + func testAPartlyCoveredElementIsClickedWhereItShows() throws { + let tree = scrollingWindow(button: .init(role: "AXButton", parent: 2, frame: CGRect(x: 0, y: 0, width: 100, height: 100))) + tree.nodes[4] = .init(role: "AXGroup", parent: 2, frame: CGRect(x: 0, y: 0, width: 60, height: 100)) + + guard case let .ready(point) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else { + return XCTFail("the uncovered right half should take the click") + } + XCTAssertEqual(point, CGPoint(x: 75, y: 25)) + } + + func testAHitOnTheElementsOwnChildCountsAsTheElement() throws { + let tree = scrollingWindow(button: .init(role: "AXRow", parent: 2, frame: CGRect(x: 0, y: 0, width: 100, height: 20))) + tree.nodes[4] = .init(role: "AXStaticText", parent: 3, frame: CGRect(x: 0, y: 0, width: 100, height: 20)) + + guard case .ready = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else { + return XCTFail("a hit on the row's own text is a hit on the row") + } + } + + func testAnElementHiddenByAScrollerThatIsNotAScrollAreaIsScrolledIntoViewFirst() throws { + // A web overflow div shows as AXGroup (2), so only the window (1) answers below it. + let tree = FakeTree([ + 1: .init(role: "AXWindow", parent: nil, frame: window), + 2: .init(role: "AXGroup", parent: 1, frame: CGRect(x: 0, y: 0, width: 400, height: 100), clips: true), + 3: .init( + role: "AXLink", + parent: 2, + frame: CGRect(x: 10, y: 150, width: 80, height: 20), + scrollsTo: CGRect(x: 10, y: 50, width: 80, height: 20) + ), + ]) + + guard case let .ready(point) = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else { + return XCTFail("the link should be clicked once scrolled into its container's view") + } + XCTAssertEqual(tree.scrollRequests, 1) + XCTAssertEqual(point, CGPoint(x: 50, y: 60)) + } + + func testAHitOnTheElementsAncestorMeansNothingCoversIt() throws { + // The group (9) is above the image (3) in hit order, as when the image ignores hit tests. + let tree = FakeTree([ + 1: .init(role: "AXWindow", parent: nil, frame: window), + 3: .init(role: "AXImage", parent: 9, frame: CGRect(x: 10, y: 10, width: 20, height: 20)), + 9: .init(role: "AXGroup", parent: 1, frame: window), + ]) + + guard case .ready = try ElementActionability.pointerTarget(3, windowFrame: window, ax: tree) else { + return XCTFail("a hit on the element's own ancestor is not a covering element") + } + } +} diff --git a/skill-guides/orca-computer-use.md b/skill-guides/orca-computer-use.md index 62718ae0921..8cc9609a614 100644 --- a/skill-guides/orca-computer-use.md +++ b/skill-guides/orca-computer-use.md @@ -172,7 +172,8 @@ Slack: the accessibility tree may be shallow while the screenshot contains usefu - `unsupported_capability`: the provider or desktop environment cannot do that action; use a semantic alternative or install the missing dependency if the message names one. - `action_not_supported`: inspect the element's listed actions and retry with one of those names, or use click/set-value when appropriate. - `value_not_settable`: the element cannot accept direct value writes; focus it and use keyboard input only when the returned state can be inspected. -- `element_not_clickable`: the element has no actionable frame; use a parent/child element with a frame or choose window-local coordinates from the latest screenshot. +- `element_not_clickable`: no click was sent. If the message says the element is covered, deal with what covers it (a dialog, sheet or popover), then click again from a fresh `get-app-state`; never click by coordinates to get past the cover. If it is out of view, scroll its container first. Otherwise the element has no actionable frame; use a parent/child element with a frame or choose window-local coordinates from the latest screenshot. +- `element_not_enabled`: the element is disabled (its line lists `disabled`), so nothing was sent. Do what enables it first, then act once a fresh `get-app-state` no longer lists it as disabled. - `invalid_argument`: fix the command flags; do not retry the same command unchanged. - `action_timeout`: inspect current state before retrying, then use a simpler semantic action or `--no-screenshot` if observation is slow. - `verification_failed`: the write was read back and the element holds a different value, so the action failed. Run `get-app-state` to see which field changed. For editable text, click the intended field and confirm focus, select all (`cmd+a` on macOS, `ctrl+a` on Windows/Linux), then use `type-text` and inspect the resulting value instead of repeating `set-value`. diff --git a/src/cli/skills/bundled-skill-guides.ts b/src/cli/skills/bundled-skill-guides.ts index 692505b818a..8f7e90423c3 100644 --- a/src/cli/skills/bundled-skill-guides.ts +++ b/src/cli/skills/bundled-skill-guides.ts @@ -1191,7 +1191,8 @@ const ORCA_COMPUTER_USE_MARKDOWN = [ "- `unsupported_capability`: the provider or desktop environment cannot do that action; use a semantic alternative or install the missing dependency if the message names one.", "- `action_not_supported`: inspect the element's listed actions and retry with one of those names, or use click/set-value when appropriate.", "- `value_not_settable`: the element cannot accept direct value writes; focus it and use keyboard input only when the returned state can be inspected.", - "- `element_not_clickable`: the element has no actionable frame; use a parent/child element with a frame or choose window-local coordinates from the latest screenshot.", + "- `element_not_clickable`: no click was sent. If the message says the element is covered, deal with what covers it (a dialog, sheet or popover), then click again from a fresh `get-app-state`; never click by coordinates to get past the cover. If it is out of view, scroll its container first. Otherwise the element has no actionable frame; use a parent/child element with a frame or choose window-local coordinates from the latest screenshot.", + "- `element_not_enabled`: the element is disabled (its line lists `disabled`), so nothing was sent. Do what enables it first, then act once a fresh `get-app-state` no longer lists it as disabled.", "- `invalid_argument`: fix the command flags; do not retry the same command unchanged.", "- `action_timeout`: inspect current state before retrying, then use a simpler semantic action or `--no-screenshot` if observation is slow.", "- `verification_failed`: the write was read back and the element holds a different value, so the action failed. Run `get-app-state` to see which field changed. For editable text, click the intended field and confirm focus, select all (`cmd+a` on macOS, `ctrl+a` on Windows/Linux), then use `type-text` and inspect the resulting value instead of repeating `set-value`.", diff --git a/src/shared/computer-use-error-recovery.ts b/src/shared/computer-use-error-recovery.ts index ac43c974643..ee858c96dc6 100644 --- a/src/shared/computer-use-error-recovery.ts +++ b/src/shared/computer-use-error-recovery.ts @@ -72,10 +72,26 @@ export function computerUseErrorRecoveryData( 'Do not infer valid indexes from `elementCount` or reuse indexes after navigation, scrolling, focus changes, or delays.' ) case 'element_not_clickable': + if (message?.includes(' is covered by ')) { + return recoverWith( + 'Run `orca computer get-app-state --app --json` and deal with what covers the element first (answer or close the dialog, sheet or popover).', + 'Then click the element again using an index from the fresh tree; do not click by coordinates to get past the cover.' + ) + } + if (message?.includes(' out of view')) { + return recoverWith( + "Scroll the element's container with `orca computer scroll`, then run `orca computer get-app-state --app --json` and click the element using an index from the fresh tree." + ) + } return recoverWith( 'Choose a nearby parent or child element that has an actionable frame.', 'If using coordinates, derive window-local coordinates from the latest screenshot/state for the same target window.' ) + case 'element_not_enabled': + return recoverWith( + 'The element is disabled, so nothing was sent. Do what enables it first (for example fill required fields), then run `orca computer get-app-state --app --json` and act once its line no longer lists `disabled`.', + 'Do not retry the same action unchanged or click it by coordinates instead.' + ) case 'action_not_supported': return recoverWith( 'Inspect the element in a fresh `get-app-state` result and use one of its advertised secondary actions.', diff --git a/src/shared/computer-use-runtime-types.ts b/src/shared/computer-use-runtime-types.ts index ee3a38e260a..1b853a115dd 100644 --- a/src/shared/computer-use-runtime-types.ts +++ b/src/shared/computer-use-runtime-types.ts @@ -10,6 +10,7 @@ export const COMPUTER_ERROR_CODES = { permission_denied: 'permission_denied', element_not_found: 'element_not_found', element_not_clickable: 'element_not_clickable', + element_not_enabled: 'element_not_enabled', action_not_supported: 'action_not_supported', value_not_settable: 'value_not_settable', invalid_argument: 'invalid_argument',