mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
fix(computer-use): let web comboboxes click in the background and stop probing a hung app
This commit is contained in:
@@ -1667,7 +1667,10 @@ private func clickPointer(
|
||||
}
|
||||
hitElement = found
|
||||
}
|
||||
try requireNoForegroundEffect(clickEffect(button: button, modifiers: modifiers, on: hitElement))
|
||||
// Why: an element click already checked its target before trying AXPress.
|
||||
if element == nil {
|
||||
try requireNoForegroundEffect(clickEffect(button: button, modifiers: modifiers, on: hitElement))
|
||||
}
|
||||
let hit = try backgroundClickHit(hitElement, snapshot: snapshot)
|
||||
try withTransportErrors {
|
||||
try BackgroundInputTransport.click(
|
||||
@@ -1705,7 +1708,7 @@ private func requireNoForegroundEffect(_ effect: ForegroundEffect?) throws {
|
||||
case nil:
|
||||
return
|
||||
case .showsMenu:
|
||||
throw ProviderError.coded("background_unavailable", "this would open a menu over the user's screen and take their keyboard, so nothing was sent; for a popup menu, combo box or web <select>, set the option with set-value on that element. Ask the user before retrying with --restore-window.")
|
||||
throw ProviderError.coded("background_unavailable", "this would open a menu over the user's screen and take their keyboard, so nothing was sent; for a popup menu, native combo box or web <select>, set the option with set-value on that element. Ask the user before retrying with --restore-window.")
|
||||
case .opensDialog:
|
||||
throw ProviderError.coded("background_unavailable", "this button's title ends in an ellipsis, so it most likely opens a dialog or sheet that brings the app to the front; nothing was sent. Ask the user before retrying with --restore-window.")
|
||||
}
|
||||
|
||||
+38
-19
@@ -137,37 +137,56 @@ public enum BackgroundInputTransport {
|
||||
return isChromiumNode(current) ? .other : .webKitContent
|
||||
}
|
||||
if role == kAXWindowRole as String { return .other }
|
||||
var parent: CFTypeRef?
|
||||
guard AXUIElementCopyAttributeValue(current, kAXParentAttribute as CFString, &parent) == .success,
|
||||
let parent,
|
||||
CFGetTypeID(parent) == AXUIElementGetTypeID()
|
||||
else {
|
||||
return .other
|
||||
}
|
||||
// SAFETY: the CFTypeID check above proves this is an AXUIElement.
|
||||
current = parent as! AXUIElement
|
||||
guard let next = parent(of: current) else { return .other }
|
||||
current = next
|
||||
}
|
||||
return .other
|
||||
}
|
||||
|
||||
/// The element and its parent as `ForegroundEffect` reads them.
|
||||
public static func pressTarget(_ element: AXUIElement) -> [(role: String?, labels: [String?])] {
|
||||
var chain: [(role: String?, labels: [String?])] = []
|
||||
public static func pressTarget(_ element: AXUIElement) -> [ForegroundEffect.Node] {
|
||||
var chain: [ForegroundEffect.Node] = []
|
||||
var current: AXUIElement? = element
|
||||
while let node = current, chain.count < 2 {
|
||||
chain.append((
|
||||
role: stringAttribute(node, kAXRoleAttribute as String),
|
||||
labels: [stringAttribute(node, kAXTitleAttribute as String), stringAttribute(node, kAXDescriptionAttribute as String)]
|
||||
var roleValue: CFTypeRef?
|
||||
// Why: a hung app answers each read only after the messaging timeout; the action that follows reports it.
|
||||
guard AXUIElementCopyAttributeValue(node, kAXRoleAttribute as CFString, &roleValue) != .cannotComplete else { break }
|
||||
let role = roleValue as? String
|
||||
chain.append(ForegroundEffect.Node(
|
||||
role: role,
|
||||
labels: [stringAttribute(node, kAXTitleAttribute as String), stringAttribute(node, kAXDescriptionAttribute as String)],
|
||||
// Why: only a combobox needs the walk up to its page, so other presses cost no extra reads.
|
||||
inWebContent: role == kAXComboBoxRole as String && isInWebArea(node)
|
||||
))
|
||||
var parent: CFTypeRef?
|
||||
current = AXUIElementCopyAttributeValue(node, kAXParentAttribute as CFString, &parent) == .success
|
||||
&& parent.map { CFGetTypeID($0) == AXUIElementGetTypeID() } == true
|
||||
// SAFETY: the CFTypeID check above proves parent is an AXUIElement.
|
||||
? (parent as! AXUIElement) : nil
|
||||
current = parent(of: node)
|
||||
}
|
||||
return chain
|
||||
}
|
||||
|
||||
private static func isInWebArea(_ element: AXUIElement) -> Bool {
|
||||
var current: AXUIElement? = element
|
||||
for _ in 0..<48 {
|
||||
guard let node = current else { return false }
|
||||
let role = stringAttribute(node, kAXRoleAttribute as String)
|
||||
if role == "AXWebArea" { return true }
|
||||
if role == kAXWindowRole as String { return false }
|
||||
current = parent(of: node)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
private static func parent(of element: AXUIElement) -> AXUIElement? {
|
||||
var parent: CFTypeRef?
|
||||
guard AXUIElementCopyAttributeValue(element, kAXParentAttribute as CFString, &parent) == .success,
|
||||
let parent,
|
||||
CFGetTypeID(parent) == AXUIElementGetTypeID()
|
||||
else {
|
||||
return nil
|
||||
}
|
||||
// SAFETY: the CFTypeID check above proves this is an AXUIElement.
|
||||
return (parent as! AXUIElement)
|
||||
}
|
||||
|
||||
// MARK: Keys
|
||||
|
||||
/// Keys of a chord without a Unicode string: AppKit only matches menu key equivalents (⌘A/C/V/Z)
|
||||
|
||||
@@ -9,15 +9,30 @@ public enum ForegroundEffect: Equatable, Sendable {
|
||||
/// which can activate the app. A predictor, not a proof: no accessibility attribute says so.
|
||||
case opensDialog
|
||||
|
||||
/// One element a press would land on, as the classifier reads it.
|
||||
public struct Node: Equatable, Sendable {
|
||||
public let role: String?
|
||||
/// Title and description.
|
||||
public let labels: [String?]
|
||||
/// Inside a web page, where a combobox is a text input whose suggestions the page draws itself.
|
||||
public let inWebContent: Bool
|
||||
|
||||
public init(role: String?, labels: [String?] = [], inWebContent: Bool = false) {
|
||||
self.role = role
|
||||
self.labels = labels
|
||||
self.inWebContent = inWebContent
|
||||
}
|
||||
}
|
||||
|
||||
static let menuOwnerRoles: Set<String> = ["AXPopUpButton", "AXMenuButton", "AXComboBox"]
|
||||
static let ellipses = ["\u{2026}", "..."]
|
||||
|
||||
/// The effect of a press (AXPress or a left click) on an element, given its role and title or
|
||||
/// description, then its parent's; a hit on a popup's label lands one level below the popup.
|
||||
public static func ofPress(_ chain: [(role: String?, labels: [String?])]) -> ForegroundEffect? {
|
||||
/// The effect of a press (AXPress or a click) on an element, then its parent; a hit on a popup's
|
||||
/// label lands one level below the popup.
|
||||
public static func ofPress(_ chain: [Node]) -> ForegroundEffect? {
|
||||
for node in chain.prefix(2) {
|
||||
guard let role = node.role else { continue }
|
||||
if menuOwnerRoles.contains(role) { return .showsMenu }
|
||||
if menuOwnerRoles.contains(role), !(role == "AXComboBox" && node.inWebContent) { return .showsMenu }
|
||||
// Why: only buttons, so a truncated "Loading..." text or a "Read more..." link still clicks.
|
||||
if role == "AXButton", node.labels.contains(where: endsInEllipsis) { return .opensDialog }
|
||||
}
|
||||
@@ -25,15 +40,15 @@ public enum ForegroundEffect: Equatable, Sendable {
|
||||
}
|
||||
|
||||
/// A right-click, or a Control-click, opens the context menu; any other click presses the target.
|
||||
public static func ofClick(button: MouseButtonSelection, control: Bool, target chain: [(role: String?, labels: [String?])]) -> ForegroundEffect? {
|
||||
button == .right || (button == .left && control) ? .showsMenu : ofPress(chain)
|
||||
public static func ofClick(button: MouseButtonSelection, control: Bool, target chain: @autoclosure () -> [Node]) -> ForegroundEffect? {
|
||||
button == .right || (button == .left && control) ? .showsMenu : ofPress(chain())
|
||||
}
|
||||
|
||||
/// The effect of an accessibility action run on its own, as perform-secondary-action does.
|
||||
public static func ofAction(_ action: String, target chain: [(role: String?, labels: [String?])]) -> ForegroundEffect? {
|
||||
public static func ofAction(_ action: String, target chain: @autoclosure () -> [Node]) -> ForegroundEffect? {
|
||||
switch action {
|
||||
case "AXShowMenu": return .showsMenu
|
||||
case "AXPress", "AXPick": return ofPress(chain)
|
||||
case "AXPress", "AXPick": return ofPress(chain())
|
||||
default: return nil
|
||||
}
|
||||
}
|
||||
|
||||
+17
-12
@@ -2,30 +2,35 @@ import OrcaComputerUseMacOSCore
|
||||
import XCTest
|
||||
|
||||
final class ForegroundEffectTests: XCTestCase {
|
||||
private typealias Node = ForegroundEffect.Node
|
||||
|
||||
func testPressingAMenuOwnerOrItsLabelShowsAMenu() {
|
||||
for role in ["AXPopUpButton", "AXMenuButton", "AXComboBox"] {
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([(role: role, labels: [])]), .showsMenu, role)
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([(role: "AXStaticText", labels: ["Small"]), (role: role, labels: [])]), .showsMenu, role)
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([Node(role: role)]), .showsMenu, role)
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([Node(role: "AXStaticText", labels: ["Small"]), Node(role: role)]), .showsMenu, role)
|
||||
}
|
||||
XCTAssertNil(ForegroundEffect.ofPress([
|
||||
(role: "AXStaticText", labels: []), (role: "AXGroup", labels: []), (role: "AXPopUpButton", labels: []),
|
||||
]))
|
||||
XCTAssertNil(ForegroundEffect.ofPress([Node(role: "AXStaticText"), Node(role: "AXGroup"), Node(role: "AXPopUpButton")]))
|
||||
}
|
||||
|
||||
func testAWebComboboxIsATextInputNotAMenu() {
|
||||
XCTAssertNil(ForegroundEffect.ofPress([Node(role: "AXComboBox", inWebContent: true)]))
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([Node(role: "AXPopUpButton", inWebContent: true)]), .showsMenu)
|
||||
}
|
||||
|
||||
func testOnlyAButtonWhoseTitleEndsInAnEllipsisOpensADialog() {
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([(role: "AXButton", labels: ["Save\u{2026}", nil])]), .opensDialog)
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([(role: "AXButton", labels: [nil, "Export... "])]), .opensDialog)
|
||||
XCTAssertNil(ForegroundEffect.ofPress([(role: "AXButton", labels: ["Save", "Saves the file\u{2026} now"])]))
|
||||
XCTAssertNil(ForegroundEffect.ofPress([(role: "AXLink", labels: ["Read more..."])]))
|
||||
XCTAssertNil(ForegroundEffect.ofPress([(role: "AXStaticText", labels: ["Loading..."]), (role: "AXGroup", labels: [])]))
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([Node(role: "AXButton", labels: ["Save\u{2026}", nil])]), .opensDialog)
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([Node(role: "AXButton", labels: [nil, "Export... "])]), .opensDialog)
|
||||
XCTAssertNil(ForegroundEffect.ofPress([Node(role: "AXButton", labels: ["Save", "Saves the file\u{2026} now"])]))
|
||||
XCTAssertNil(ForegroundEffect.ofPress([Node(role: "AXLink", labels: ["Read more..."])]))
|
||||
XCTAssertNil(ForegroundEffect.ofPress([Node(role: "AXStaticText", labels: ["Loading..."]), Node(role: "AXGroup")]))
|
||||
}
|
||||
|
||||
func testContextMenuClicksShowAMenuOnAnyTarget() {
|
||||
let plainButton: [(role: String?, labels: [String?])] = [(role: "AXButton", labels: ["OK"])]
|
||||
let plainButton = [Node(role: "AXButton", labels: ["OK"])]
|
||||
XCTAssertEqual(ForegroundEffect.ofClick(button: .right, control: false, target: plainButton), .showsMenu)
|
||||
XCTAssertEqual(ForegroundEffect.ofClick(button: .left, control: true, target: plainButton), .showsMenu)
|
||||
XCTAssertNil(ForegroundEffect.ofClick(button: .left, control: false, target: plainButton))
|
||||
XCTAssertEqual(ForegroundEffect.ofAction("AXShowMenu", target: plainButton), .showsMenu)
|
||||
XCTAssertNil(ForegroundEffect.ofAction("AXScrollToVisible", target: [(role: "AXPopUpButton", labels: [])]))
|
||||
XCTAssertNil(ForegroundEffect.ofAction("AXScrollToVisible", target: [Node(role: "AXPopUpButton")]))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,7 +148,7 @@ Do not pass `--restore-window` by default. Use it only when foreground input is
|
||||
- Drags in Chromium or Electron apps, and apps that read the hardware pointer (games, OpenGL/Metal views, Java/Tk), which ignore background pointer events.
|
||||
- The window is minimized or hidden. Accessibility actions still work there; pointer and keyboard events do not.
|
||||
|
||||
On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks on popup menus, menu buttons, combo boxes and web `<select>` controls, and buttons whose title ends in an ellipsis (`Save…`), which usually open a dialog. For a popup, combo box or `<select>`, set the option with `set-value` on that element.
|
||||
On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks on popup menus, menu buttons, native combo boxes and web `<select>` controls, and buttons whose title ends in an ellipsis (`Save…`), which usually open a dialog. For a popup, native combo box or `<select>`, set the option with `set-value` on that element.
|
||||
|
||||
On Linux, clicks, scrolling, drags and keyboard input still bring the target window forward. On Windows, these go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), element scrolls by whole `--pages`, and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Everything else brings the window forward and reports `foreground`.
|
||||
|
||||
|
||||
@@ -1169,7 +1169,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
|
||||
"- Drags in Chromium or Electron apps, and apps that read the hardware pointer (games, OpenGL/Metal views, Java/Tk), which ignore background pointer events.",
|
||||
"- The window is minimized or hidden. Accessibility actions still work there; pointer and keyboard events do not.",
|
||||
"",
|
||||
"On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks on popup menus, menu buttons, combo boxes and web `<select>` controls, and buttons whose title ends in an ellipsis (`Save…`), which usually open a dialog. For a popup, combo box or `<select>`, set the option with `set-value` on that element.",
|
||||
"On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks on popup menus, menu buttons, native combo boxes and web `<select>` controls, and buttons whose title ends in an ellipsis (`Save…`), which usually open a dialog. For a popup, native combo box or `<select>`, set the option with `set-value` on that element.",
|
||||
"",
|
||||
"On Linux, clicks, scrolling, drags and keyboard input still bring the target window forward. On Windows, these go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), element scrolls by whole `--pages`, and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Everything else brings the window forward and reports `foreground`.",
|
||||
"",
|
||||
|
||||
Reference in New Issue
Block a user