mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
fix(computer-use): refuse macOS background actions that open a menu or dialog over the user
This commit is contained in:
@@ -879,7 +879,7 @@ final class Provider {
|
||||
let record = try element(snapshot, elementIndex)
|
||||
try requireEnabled(record, snapshot: snapshot)
|
||||
if !foreground {
|
||||
try requireBackgroundClickAllowed(backgroundClickHit(record.element, snapshot: snapshot))
|
||||
try requireNoForegroundEffect(clickEffect(button: button, modifiers: modifiers, on: record.element))
|
||||
}
|
||||
let rowSelection = AccessibilityRowSelection()
|
||||
let unselectedRow = modifiers.isEmpty && count <= 1 && button == .left
|
||||
@@ -992,6 +992,9 @@ final class Provider {
|
||||
if action != "AXScrollToVisible" {
|
||||
try requireEnabled(record, snapshot: snapshot)
|
||||
}
|
||||
if !isForegroundRequest(params) {
|
||||
try requireNoForegroundEffect(ForegroundEffect.ofAction(action, target: BackgroundInputTransport.pressTarget(record.element)))
|
||||
}
|
||||
guard try performAction(record.element, action) else {
|
||||
throw ProviderError.coded("accessibility_error", "AXUIElementPerformAction(\(action)) failed")
|
||||
}
|
||||
@@ -1003,6 +1006,7 @@ final class Provider {
|
||||
let record = try element(snapshot, try requiredInteger(params, "elementIndex"))
|
||||
let expected = try requiredStringAllowingEmpty(params, "value")
|
||||
try requireEnabled(record, snapshot: snapshot)
|
||||
try requireNotFileNameField(record)
|
||||
guard isSettable(record.element, kAXValueAttribute as String) else {
|
||||
throw ProviderError.coded("value_not_settable", "element \(record.index) is not settable")
|
||||
}
|
||||
@@ -1065,6 +1069,7 @@ final class Provider {
|
||||
}
|
||||
|
||||
private func typeTextIntoElement(_ record: ElementRecord, text: String) throws -> [String: Any] {
|
||||
try requireNotFileNameField(record)
|
||||
// Why: typing at focus instead would put the text in whatever field has focus, not the requested one.
|
||||
if BackgroundInputTransport.isChromiumNode(record.element) {
|
||||
throw ProviderError.coded("value_not_settable", "element \(record.index) is web content, which ignores accessibility text insertion; click it, then run type-text without --element-index")
|
||||
@@ -1335,8 +1340,14 @@ private func enableManualAccessibilityIfNeeded(_ appElement: AXUIElement, app: A
|
||||
guard app.needsManualAccessibilityMode else {
|
||||
return
|
||||
}
|
||||
_ = AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue)
|
||||
_ = AXUIElementSetAttributeValue(appElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue)
|
||||
if isSettable(appElement, "AXManualAccessibility") {
|
||||
_ = AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue)
|
||||
return
|
||||
}
|
||||
// Why: the older, broader switch, so only for apps that lack AXManualAccessibility and only when it is off.
|
||||
if boolAttribute(appElement, "AXEnhancedUserInterface") != true {
|
||||
_ = AXUIElementSetAttributeValue(appElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue)
|
||||
}
|
||||
}
|
||||
|
||||
private func focusedWindow(appElement: AXUIElement, app: AppDescriptor, visibleWindowCount: Int, allowRecovery: Bool) throws -> AXUIElement {
|
||||
@@ -1623,7 +1634,7 @@ private func withTransportErrors<T>(_ body: () throws -> T) throws -> T {
|
||||
case .eventCreationFailed:
|
||||
throw ProviderError.coded("accessibility_error", "failed to create input event")
|
||||
case .targetWindowNotKey:
|
||||
throw ProviderError.coded("background_unavailable", "background input cannot change the key window of the active app; retry with --restore-window to target this window explicitly")
|
||||
throw ProviderError.coded("background_unavailable", "the user is working in another window of this app, and input to this window would move their typing here, so nothing was sent; use an element action (click --element-index, set-value) or wait until the user leaves the app. Ask the user before retrying with --restore-window.")
|
||||
case let .focusRecordFailed(status):
|
||||
throw ProviderError.coded("window_not_focused", "background key-window focus record failed (status \(status)); input may already have been delivered, so inspect current state before retrying")
|
||||
}
|
||||
@@ -1656,8 +1667,8 @@ private func clickPointer(
|
||||
}
|
||||
hitElement = found
|
||||
}
|
||||
try requireNoForegroundEffect(clickEffect(button: button, modifiers: modifiers, on: hitElement))
|
||||
let hit = try backgroundClickHit(hitElement, snapshot: snapshot)
|
||||
try requireBackgroundClickAllowed(hit)
|
||||
try withTransportErrors {
|
||||
try BackgroundInputTransport.click(
|
||||
backgroundTarget(snapshot),
|
||||
@@ -1680,9 +1691,23 @@ private func backgroundClickHit(_ element: AXUIElement, snapshot: Snapshot) thro
|
||||
return BackgroundInputTransport.classifyPointerHit(element)
|
||||
}
|
||||
|
||||
private func requireBackgroundClickAllowed(_ hit: BackgroundInputTransport.PointerHit) throws {
|
||||
guard hit != .popupMenu else {
|
||||
throw ProviderError.coded("background_unavailable", "a background click on a popup menu or web <select> opens the menu off screen or wedges the app's input; set the option with set-value on the popup element, or retry with --restore-window for a foreground click")
|
||||
private func clickEffect(button: MouseButtonSelection, modifiers: [KeyModifier], on element: AXUIElement) -> ForegroundEffect? {
|
||||
ForegroundEffect.ofClick(
|
||||
button: button,
|
||||
control: modifiers.contains { $0.flag == .maskControl },
|
||||
target: BackgroundInputTransport.pressTarget(element)
|
||||
)
|
||||
}
|
||||
|
||||
/// Refuses, before anything is sent, an action that would put a menu or dialog in front of the user.
|
||||
private func requireNoForegroundEffect(_ effect: ForegroundEffect?) throws {
|
||||
switch effect {
|
||||
case nil:
|
||||
return
|
||||
case .showsMenu:
|
||||
throw ProviderError.coded("background_unavailable", "this would open a menu over the user's screen and take their keyboard, so nothing was sent; for a popup menu, combo box or web <select>, set the option with set-value on that element. Ask the user before retrying with --restore-window.")
|
||||
case .opensDialog:
|
||||
throw ProviderError.coded("background_unavailable", "this button's title ends in an ellipsis, so it most likely opens a dialog or sheet that brings the app to the front; nothing was sent. Ask the user before retrying with --restore-window.")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2045,6 +2070,16 @@ private func performAction(_ element: AXUIElement, _ action: String) throws -> B
|
||||
return result == .success
|
||||
}
|
||||
|
||||
/// A file-name field (a Finder name cell) shows and reads back an AX write, but the file keeps its old name.
|
||||
private func isFileNameField(_ element: AXUIElement) -> Bool {
|
||||
rawStringAttribute(element, "AXFilename") != nil
|
||||
}
|
||||
|
||||
private func requireNotFileNameField(_ record: ElementRecord) throws {
|
||||
guard isFileNameField(record.element) else { return }
|
||||
throw ProviderError.coded("value_not_settable", "element \(record.index) shows a file's name; an accessibility write changes what it shows but does not rename the file, so nothing was written; with the field in edit mode, type the name with type-text without --element-index and press-key Return, or rename the file on disk")
|
||||
}
|
||||
|
||||
private func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool {
|
||||
var settable = DarwinBoolean(false)
|
||||
return AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) == .success && settable.boolValue
|
||||
@@ -3290,6 +3325,7 @@ private enum TextInput {
|
||||
static func replaceSelection(_ element: AXUIElement, with text: String, property: String = "focusedText") throws -> [String: Any]? {
|
||||
// Why: Chromium applies AX value writes without the input events pages listen to; a contenteditable gets none, so editors like Discord's never see the text.
|
||||
guard !BackgroundInputTransport.isChromiumNode(element),
|
||||
!isFileNameField(element),
|
||||
isSettable(element, kAXValueAttribute as String),
|
||||
let current = rawStringAttribute(element, kAXValueAttribute as String)
|
||||
else {
|
||||
|
||||
+19
-7
@@ -31,8 +31,6 @@ public enum BackgroundInputTransport {
|
||||
public enum PointerHit: Equatable, Sendable {
|
||||
/// WebKit content (Safari, WKWebView): drops clicks into a window that is not key.
|
||||
case webKitContent
|
||||
/// A popup menu or web `<select>`: a background click opens menu tracking off screen or wedges Chromium input.
|
||||
case popupMenu
|
||||
case other
|
||||
}
|
||||
|
||||
@@ -131,12 +129,8 @@ public enum BackgroundInputTransport {
|
||||
|
||||
public static func classifyPointerHit(_ element: AXUIElement) -> PointerHit {
|
||||
var current = element
|
||||
for depth in 0..<48 {
|
||||
for _ in 0..<48 {
|
||||
let role = stringAttribute(current, kAXRoleAttribute as String)
|
||||
// Why: the hit lands on a popup's label or the popup itself, never deeper than its parent.
|
||||
if depth < 2, role == kAXPopUpButtonRole as String || role == kAXMenuButtonRole as String {
|
||||
return .popupMenu
|
||||
}
|
||||
if role == "AXWebArea" {
|
||||
// Why: Chromium accepts window-stamped clicks as is; only WebKit needs the key-window scope,
|
||||
// which fires the app's own did-become-active code (Slack, for one, marks messages read).
|
||||
@@ -156,6 +150,24 @@ public enum BackgroundInputTransport {
|
||||
return .other
|
||||
}
|
||||
|
||||
/// The element and its parent as `ForegroundEffect` reads them.
|
||||
public static func pressTarget(_ element: AXUIElement) -> [(role: String?, labels: [String?])] {
|
||||
var chain: [(role: String?, labels: [String?])] = []
|
||||
var current: AXUIElement? = element
|
||||
while let node = current, chain.count < 2 {
|
||||
chain.append((
|
||||
role: stringAttribute(node, kAXRoleAttribute as String),
|
||||
labels: [stringAttribute(node, kAXTitleAttribute as String), stringAttribute(node, kAXDescriptionAttribute as String)]
|
||||
))
|
||||
var parent: CFTypeRef?
|
||||
current = AXUIElementCopyAttributeValue(node, kAXParentAttribute as CFString, &parent) == .success
|
||||
&& parent.map { CFGetTypeID($0) == AXUIElementGetTypeID() } == true
|
||||
// SAFETY: the CFTypeID check above proves parent is an AXUIElement.
|
||||
? (parent as! AXUIElement) : nil
|
||||
}
|
||||
return chain
|
||||
}
|
||||
|
||||
// MARK: Keys
|
||||
|
||||
/// Keys of a chord without a Unicode string: AppKit only matches menu key equivalents (⌘A/C/V/Z)
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import Foundation
|
||||
|
||||
/// What an action would put in front of the user. Without --restore-window an action with an effect
|
||||
/// is refused before anything is sent. Design follows agent-desktop's `background_effect.rs` (Apache-2.0).
|
||||
public enum ForegroundEffect: Equatable, Sendable {
|
||||
/// A menu drawn over the user's window that takes their keyboard; AppKit has no background route to its items.
|
||||
case showsMenu
|
||||
/// A button whose title ends in an ellipsis, the platform convention for "opens a dialog or sheet",
|
||||
/// which can activate the app. A predictor, not a proof: no accessibility attribute says so.
|
||||
case opensDialog
|
||||
|
||||
static let menuOwnerRoles: Set<String> = ["AXPopUpButton", "AXMenuButton", "AXComboBox"]
|
||||
static let ellipses = ["\u{2026}", "..."]
|
||||
|
||||
/// The effect of a press (AXPress or a left click) on an element, given its role and title or
|
||||
/// description, then its parent's; a hit on a popup's label lands one level below the popup.
|
||||
public static func ofPress(_ chain: [(role: String?, labels: [String?])]) -> ForegroundEffect? {
|
||||
for node in chain.prefix(2) {
|
||||
guard let role = node.role else { continue }
|
||||
if menuOwnerRoles.contains(role) { return .showsMenu }
|
||||
// Why: only buttons, so a truncated "Loading..." text or a "Read more..." link still clicks.
|
||||
if role == "AXButton", node.labels.contains(where: endsInEllipsis) { return .opensDialog }
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/// A right-click, or a Control-click, opens the context menu; any other click presses the target.
|
||||
public static func ofClick(button: MouseButtonSelection, control: Bool, target chain: [(role: String?, labels: [String?])]) -> ForegroundEffect? {
|
||||
button == .right || (button == .left && control) ? .showsMenu : ofPress(chain)
|
||||
}
|
||||
|
||||
/// The effect of an accessibility action run on its own, as perform-secondary-action does.
|
||||
public static func ofAction(_ action: String, target chain: [(role: String?, labels: [String?])]) -> ForegroundEffect? {
|
||||
switch action {
|
||||
case "AXShowMenu": return .showsMenu
|
||||
case "AXPress", "AXPick": return ofPress(chain)
|
||||
default: return nil
|
||||
}
|
||||
}
|
||||
|
||||
static func endsInEllipsis(_ label: String?) -> Bool {
|
||||
guard let label else { return false }
|
||||
let trimmed = label.trimmingCharacters(in: .whitespacesAndNewlines)
|
||||
return ellipses.contains { trimmed.hasSuffix($0) }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import OrcaComputerUseMacOSCore
|
||||
import XCTest
|
||||
|
||||
final class ForegroundEffectTests: XCTestCase {
|
||||
func testPressingAMenuOwnerOrItsLabelShowsAMenu() {
|
||||
for role in ["AXPopUpButton", "AXMenuButton", "AXComboBox"] {
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([(role: role, labels: [])]), .showsMenu, role)
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([(role: "AXStaticText", labels: ["Small"]), (role: role, labels: [])]), .showsMenu, role)
|
||||
}
|
||||
XCTAssertNil(ForegroundEffect.ofPress([
|
||||
(role: "AXStaticText", labels: []), (role: "AXGroup", labels: []), (role: "AXPopUpButton", labels: []),
|
||||
]))
|
||||
}
|
||||
|
||||
func testOnlyAButtonWhoseTitleEndsInAnEllipsisOpensADialog() {
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([(role: "AXButton", labels: ["Save\u{2026}", nil])]), .opensDialog)
|
||||
XCTAssertEqual(ForegroundEffect.ofPress([(role: "AXButton", labels: [nil, "Export... "])]), .opensDialog)
|
||||
XCTAssertNil(ForegroundEffect.ofPress([(role: "AXButton", labels: ["Save", "Saves the file\u{2026} now"])]))
|
||||
XCTAssertNil(ForegroundEffect.ofPress([(role: "AXLink", labels: ["Read more..."])]))
|
||||
XCTAssertNil(ForegroundEffect.ofPress([(role: "AXStaticText", labels: ["Loading..."]), (role: "AXGroup", labels: [])]))
|
||||
}
|
||||
|
||||
func testContextMenuClicksShowAMenuOnAnyTarget() {
|
||||
let plainButton: [(role: String?, labels: [String?])] = [(role: "AXButton", labels: ["OK"])]
|
||||
XCTAssertEqual(ForegroundEffect.ofClick(button: .right, control: false, target: plainButton), .showsMenu)
|
||||
XCTAssertEqual(ForegroundEffect.ofClick(button: .left, control: true, target: plainButton), .showsMenu)
|
||||
XCTAssertNil(ForegroundEffect.ofClick(button: .left, control: false, target: plainButton))
|
||||
XCTAssertEqual(ForegroundEffect.ofAction("AXShowMenu", target: plainButton), .showsMenu)
|
||||
XCTAssertNil(ForegroundEffect.ofAction("AXScrollToVisible", target: [(role: "AXPopUpButton", labels: [])]))
|
||||
}
|
||||
}
|
||||
@@ -148,7 +148,7 @@ Do not pass `--restore-window` by default. Use it only when foreground input is
|
||||
- Drags in Chromium or Electron apps, and apps that read the hardware pointer (games, OpenGL/Metal views, Java/Tk), which ignore background pointer events.
|
||||
- The window is minimized or hidden. Accessibility actions still work there; pointer and keyboard events do not.
|
||||
|
||||
Popup menus and web `<select>` controls: set the option with `set-value` on the popup element. Background clicks on them are refused because they open the menu off screen or wedge the app's input.
|
||||
On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks on popup menus, menu buttons, combo boxes and web `<select>` controls, and buttons whose title ends in an ellipsis (`Save…`), which usually open a dialog. For a popup, combo box or `<select>`, set the option with `set-value` on that element.
|
||||
|
||||
On Linux, clicks, scrolling, drags and keyboard input still bring the target window forward. On Windows, these go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), element scrolls by whole `--pages`, and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Everything else brings the window forward and reports `foreground`.
|
||||
|
||||
|
||||
@@ -1169,7 +1169,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
|
||||
"- Drags in Chromium or Electron apps, and apps that read the hardware pointer (games, OpenGL/Metal views, Java/Tk), which ignore background pointer events.",
|
||||
"- The window is minimized or hidden. Accessibility actions still work there; pointer and keyboard events do not.",
|
||||
"",
|
||||
"Popup menus and web `<select>` controls: set the option with `set-value` on the popup element. Background clicks on them are refused because they open the menu off screen or wedge the app's input.",
|
||||
"On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks on popup menus, menu buttons, combo boxes and web `<select>` controls, and buttons whose title ends in an ellipsis (`Save…`), which usually open a dialog. For a popup, combo box or `<select>`, set the option with `set-value` on that element.",
|
||||
"",
|
||||
"On Linux, clicks, scrolling, drags and keyboard input still bring the target window forward. On Windows, these go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), element scrolls by whole `--pages`, and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Everything else brings the window forward and reports `foreground`.",
|
||||
"",
|
||||
|
||||
Reference in New Issue
Block a user