From c845274d24e01176a6baaeb6bdea45f2491e5638 Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Fri, 13 Nov 2026 21:44:14 +0000 Subject: [PATCH] fix(computer-use): let web comboboxes click in the background and stop probing a hung app --- .../Sources/OrcaComputerUseMacOS/main.swift | 7 ++- .../BackgroundInputTransport.swift | 57 ++++++++++++------- .../ForegroundEffect.swift | 31 +++++++--- .../ForegroundEffectTests.swift | 29 ++++++---- skill-guides/orca-computer-use.md | 2 +- src/cli/skills/bundled-skill-guides.ts | 2 +- 6 files changed, 85 insertions(+), 43 deletions(-) diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index 1099c4365c5..fea70a792c9 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -1667,7 +1667,10 @@ private func clickPointer( } hitElement = found } - try requireNoForegroundEffect(clickEffect(button: button, modifiers: modifiers, on: hitElement)) + // Why: an element click already checked its target before trying AXPress. + if element == nil { + try requireNoForegroundEffect(clickEffect(button: button, modifiers: modifiers, on: hitElement)) + } let hit = try backgroundClickHit(hitElement, snapshot: snapshot) try withTransportErrors { try BackgroundInputTransport.click( @@ -1705,7 +1708,7 @@ private func requireNoForegroundEffect(_ effect: ForegroundEffect?) throws { case nil: return case .showsMenu: - throw ProviderError.coded("background_unavailable", "this would open a menu over the user's screen and take their keyboard, so nothing was sent; for a popup menu, combo box or web , set the option with set-value on that element. Ask the user before retrying with --restore-window.") case .opensDialog: throw ProviderError.coded("background_unavailable", "this button's title ends in an ellipsis, so it most likely opens a dialog or sheet that brings the app to the front; nothing was sent. Ask the user before retrying with --restore-window.") } diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/BackgroundInputTransport.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/BackgroundInputTransport.swift index 67bf60a27c4..46e41bd5ede 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/BackgroundInputTransport.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/BackgroundInputTransport.swift @@ -137,37 +137,56 @@ public enum BackgroundInputTransport { return isChromiumNode(current) ? .other : .webKitContent } if role == kAXWindowRole as String { return .other } - var parent: CFTypeRef? - guard AXUIElementCopyAttributeValue(current, kAXParentAttribute as CFString, &parent) == .success, - let parent, - CFGetTypeID(parent) == AXUIElementGetTypeID() - else { - return .other - } - // SAFETY: the CFTypeID check above proves this is an AXUIElement. - current = parent as! AXUIElement + guard let next = parent(of: current) else { return .other } + current = next } return .other } /// The element and its parent as `ForegroundEffect` reads them. - public static func pressTarget(_ element: AXUIElement) -> [(role: String?, labels: [String?])] { - var chain: [(role: String?, labels: [String?])] = [] + public static func pressTarget(_ element: AXUIElement) -> [ForegroundEffect.Node] { + var chain: [ForegroundEffect.Node] = [] var current: AXUIElement? = element while let node = current, chain.count < 2 { - chain.append(( - role: stringAttribute(node, kAXRoleAttribute as String), - labels: [stringAttribute(node, kAXTitleAttribute as String), stringAttribute(node, kAXDescriptionAttribute as String)] + var roleValue: CFTypeRef? + // Why: a hung app answers each read only after the messaging timeout; the action that follows reports it. + guard AXUIElementCopyAttributeValue(node, kAXRoleAttribute as CFString, &roleValue) != .cannotComplete else { break } + let role = roleValue as? String + chain.append(ForegroundEffect.Node( + role: role, + labels: [stringAttribute(node, kAXTitleAttribute as String), stringAttribute(node, kAXDescriptionAttribute as String)], + // Why: only a combobox needs the walk up to its page, so other presses cost no extra reads. + inWebContent: role == kAXComboBoxRole as String && isInWebArea(node) )) - var parent: CFTypeRef? - current = AXUIElementCopyAttributeValue(node, kAXParentAttribute as CFString, &parent) == .success - && parent.map { CFGetTypeID($0) == AXUIElementGetTypeID() } == true - // SAFETY: the CFTypeID check above proves parent is an AXUIElement. - ? (parent as! AXUIElement) : nil + current = parent(of: node) } return chain } + private static func isInWebArea(_ element: AXUIElement) -> Bool { + var current: AXUIElement? = element + for _ in 0..<48 { + guard let node = current else { return false } + let role = stringAttribute(node, kAXRoleAttribute as String) + if role == "AXWebArea" { return true } + if role == kAXWindowRole as String { return false } + current = parent(of: node) + } + return false + } + + private static func parent(of element: AXUIElement) -> AXUIElement? { + var parent: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, kAXParentAttribute as CFString, &parent) == .success, + let parent, + CFGetTypeID(parent) == AXUIElementGetTypeID() + else { + return nil + } + // SAFETY: the CFTypeID check above proves this is an AXUIElement. + return (parent as! AXUIElement) + } + // MARK: Keys /// Keys of a chord without a Unicode string: AppKit only matches menu key equivalents (⌘A/C/V/Z) diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ForegroundEffect.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ForegroundEffect.swift index 15ac5346249..49fbb57e4ce 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ForegroundEffect.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ForegroundEffect.swift @@ -9,15 +9,30 @@ public enum ForegroundEffect: Equatable, Sendable { /// which can activate the app. A predictor, not a proof: no accessibility attribute says so. case opensDialog + /// One element a press would land on, as the classifier reads it. + public struct Node: Equatable, Sendable { + public let role: String? + /// Title and description. + public let labels: [String?] + /// Inside a web page, where a combobox is a text input whose suggestions the page draws itself. + public let inWebContent: Bool + + public init(role: String?, labels: [String?] = [], inWebContent: Bool = false) { + self.role = role + self.labels = labels + self.inWebContent = inWebContent + } + } + static let menuOwnerRoles: Set = ["AXPopUpButton", "AXMenuButton", "AXComboBox"] static let ellipses = ["\u{2026}", "..."] - /// The effect of a press (AXPress or a left click) on an element, given its role and title or - /// description, then its parent's; a hit on a popup's label lands one level below the popup. - public static func ofPress(_ chain: [(role: String?, labels: [String?])]) -> ForegroundEffect? { + /// The effect of a press (AXPress or a click) on an element, then its parent; a hit on a popup's + /// label lands one level below the popup. + public static func ofPress(_ chain: [Node]) -> ForegroundEffect? { for node in chain.prefix(2) { guard let role = node.role else { continue } - if menuOwnerRoles.contains(role) { return .showsMenu } + if menuOwnerRoles.contains(role), !(role == "AXComboBox" && node.inWebContent) { return .showsMenu } // Why: only buttons, so a truncated "Loading..." text or a "Read more..." link still clicks. if role == "AXButton", node.labels.contains(where: endsInEllipsis) { return .opensDialog } } @@ -25,15 +40,15 @@ public enum ForegroundEffect: Equatable, Sendable { } /// A right-click, or a Control-click, opens the context menu; any other click presses the target. - public static func ofClick(button: MouseButtonSelection, control: Bool, target chain: [(role: String?, labels: [String?])]) -> ForegroundEffect? { - button == .right || (button == .left && control) ? .showsMenu : ofPress(chain) + public static func ofClick(button: MouseButtonSelection, control: Bool, target chain: @autoclosure () -> [Node]) -> ForegroundEffect? { + button == .right || (button == .left && control) ? .showsMenu : ofPress(chain()) } /// The effect of an accessibility action run on its own, as perform-secondary-action does. - public static func ofAction(_ action: String, target chain: [(role: String?, labels: [String?])]) -> ForegroundEffect? { + public static func ofAction(_ action: String, target chain: @autoclosure () -> [Node]) -> ForegroundEffect? { switch action { case "AXShowMenu": return .showsMenu - case "AXPress", "AXPick": return ofPress(chain) + case "AXPress", "AXPick": return ofPress(chain()) default: return nil } } diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ForegroundEffectTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ForegroundEffectTests.swift index b9a7c0ee007..fb3cdc4bddd 100644 --- a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ForegroundEffectTests.swift +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ForegroundEffectTests.swift @@ -2,30 +2,35 @@ import OrcaComputerUseMacOSCore import XCTest final class ForegroundEffectTests: XCTestCase { + private typealias Node = ForegroundEffect.Node + func testPressingAMenuOwnerOrItsLabelShowsAMenu() { for role in ["AXPopUpButton", "AXMenuButton", "AXComboBox"] { - XCTAssertEqual(ForegroundEffect.ofPress([(role: role, labels: [])]), .showsMenu, role) - XCTAssertEqual(ForegroundEffect.ofPress([(role: "AXStaticText", labels: ["Small"]), (role: role, labels: [])]), .showsMenu, role) + XCTAssertEqual(ForegroundEffect.ofPress([Node(role: role)]), .showsMenu, role) + XCTAssertEqual(ForegroundEffect.ofPress([Node(role: "AXStaticText", labels: ["Small"]), Node(role: role)]), .showsMenu, role) } - XCTAssertNil(ForegroundEffect.ofPress([ - (role: "AXStaticText", labels: []), (role: "AXGroup", labels: []), (role: "AXPopUpButton", labels: []), - ])) + XCTAssertNil(ForegroundEffect.ofPress([Node(role: "AXStaticText"), Node(role: "AXGroup"), Node(role: "AXPopUpButton")])) + } + + func testAWebComboboxIsATextInputNotAMenu() { + XCTAssertNil(ForegroundEffect.ofPress([Node(role: "AXComboBox", inWebContent: true)])) + XCTAssertEqual(ForegroundEffect.ofPress([Node(role: "AXPopUpButton", inWebContent: true)]), .showsMenu) } func testOnlyAButtonWhoseTitleEndsInAnEllipsisOpensADialog() { - XCTAssertEqual(ForegroundEffect.ofPress([(role: "AXButton", labels: ["Save\u{2026}", nil])]), .opensDialog) - XCTAssertEqual(ForegroundEffect.ofPress([(role: "AXButton", labels: [nil, "Export... "])]), .opensDialog) - XCTAssertNil(ForegroundEffect.ofPress([(role: "AXButton", labels: ["Save", "Saves the file\u{2026} now"])])) - XCTAssertNil(ForegroundEffect.ofPress([(role: "AXLink", labels: ["Read more..."])])) - XCTAssertNil(ForegroundEffect.ofPress([(role: "AXStaticText", labels: ["Loading..."]), (role: "AXGroup", labels: [])])) + XCTAssertEqual(ForegroundEffect.ofPress([Node(role: "AXButton", labels: ["Save\u{2026}", nil])]), .opensDialog) + XCTAssertEqual(ForegroundEffect.ofPress([Node(role: "AXButton", labels: [nil, "Export... "])]), .opensDialog) + XCTAssertNil(ForegroundEffect.ofPress([Node(role: "AXButton", labels: ["Save", "Saves the file\u{2026} now"])])) + XCTAssertNil(ForegroundEffect.ofPress([Node(role: "AXLink", labels: ["Read more..."])])) + XCTAssertNil(ForegroundEffect.ofPress([Node(role: "AXStaticText", labels: ["Loading..."]), Node(role: "AXGroup")])) } func testContextMenuClicksShowAMenuOnAnyTarget() { - let plainButton: [(role: String?, labels: [String?])] = [(role: "AXButton", labels: ["OK"])] + let plainButton = [Node(role: "AXButton", labels: ["OK"])] XCTAssertEqual(ForegroundEffect.ofClick(button: .right, control: false, target: plainButton), .showsMenu) XCTAssertEqual(ForegroundEffect.ofClick(button: .left, control: true, target: plainButton), .showsMenu) XCTAssertNil(ForegroundEffect.ofClick(button: .left, control: false, target: plainButton)) XCTAssertEqual(ForegroundEffect.ofAction("AXShowMenu", target: plainButton), .showsMenu) - XCTAssertNil(ForegroundEffect.ofAction("AXScrollToVisible", target: [(role: "AXPopUpButton", labels: [])])) + XCTAssertNil(ForegroundEffect.ofAction("AXScrollToVisible", target: [Node(role: "AXPopUpButton")])) } } diff --git a/skill-guides/orca-computer-use.md b/skill-guides/orca-computer-use.md index c42a42fe98e..227bc39f8d0 100644 --- a/skill-guides/orca-computer-use.md +++ b/skill-guides/orca-computer-use.md @@ -148,7 +148,7 @@ Do not pass `--restore-window` by default. Use it only when foreground input is - Drags in Chromium or Electron apps, and apps that read the hardware pointer (games, OpenGL/Metal views, Java/Tk), which ignore background pointer events. - The window is minimized or hidden. Accessibility actions still work there; pointer and keyboard events do not. -On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks on popup menus, menu buttons, combo boxes and web ``, set the option with `set-value` on that element. +On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks on popup menus, menu buttons, native combo boxes and web ``, set the option with `set-value` on that element. On Linux, clicks, scrolling, drags and keyboard input still bring the target window forward. On Windows, these go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), element scrolls by whole `--pages`, and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Everything else brings the window forward and reports `foreground`. diff --git a/src/cli/skills/bundled-skill-guides.ts b/src/cli/skills/bundled-skill-guides.ts index 34c23623fa0..30534e9ffa5 100644 --- a/src/cli/skills/bundled-skill-guides.ts +++ b/src/cli/skills/bundled-skill-guides.ts @@ -1169,7 +1169,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [ "- Drags in Chromium or Electron apps, and apps that read the hardware pointer (games, OpenGL/Metal views, Java/Tk), which ignore background pointer events.", "- The window is minimized or hidden. Accessibility actions still work there; pointer and keyboard events do not.", "", - "On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks on popup menus, menu buttons, combo boxes and web ``, set the option with `set-value` on that element.", + "On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks on popup menus, menu buttons, native combo boxes and web ``, set the option with `set-value` on that element.", "", "On Linux, clicks, scrolling, drags and keyboard input still bring the target window forward. On Windows, these go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), element scrolls by whole `--pages`, and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Everything else brings the window forward and reports `foreground`.", "",