fix(computer-use): refuse macOS background actions that open a menu or dialog over the user

This commit is contained in:
Kelvin Amoaba
2026-11-13 21:20:36 +00:00
parent 3cfdf07402
commit 70cfdb53da
6 changed files with 142 additions and 17 deletions
@@ -879,7 +879,7 @@ final class Provider {
let record = try element(snapshot, elementIndex)
try requireEnabled(record, snapshot: snapshot)
if !foreground {
try requireBackgroundClickAllowed(backgroundClickHit(record.element, snapshot: snapshot))
try requireNoForegroundEffect(clickEffect(button: button, modifiers: modifiers, on: record.element))
}
let rowSelection = AccessibilityRowSelection()
let unselectedRow = modifiers.isEmpty && count <= 1 && button == .left
@@ -992,6 +992,9 @@ final class Provider {
if action != "AXScrollToVisible" {
try requireEnabled(record, snapshot: snapshot)
}
if !isForegroundRequest(params) {
try requireNoForegroundEffect(ForegroundEffect.ofAction(action, target: BackgroundInputTransport.pressTarget(record.element)))
}
guard try performAction(record.element, action) else {
throw ProviderError.coded("accessibility_error", "AXUIElementPerformAction(\(action)) failed")
}
@@ -1003,6 +1006,7 @@ final class Provider {
let record = try element(snapshot, try requiredInteger(params, "elementIndex"))
let expected = try requiredStringAllowingEmpty(params, "value")
try requireEnabled(record, snapshot: snapshot)
try requireNotFileNameField(record)
guard isSettable(record.element, kAXValueAttribute as String) else {
throw ProviderError.coded("value_not_settable", "element \(record.index) is not settable")
}
@@ -1065,6 +1069,7 @@ final class Provider {
}
private func typeTextIntoElement(_ record: ElementRecord, text: String) throws -> [String: Any] {
try requireNotFileNameField(record)
// Why: typing at focus instead would put the text in whatever field has focus, not the requested one.
if BackgroundInputTransport.isChromiumNode(record.element) {
throw ProviderError.coded("value_not_settable", "element \(record.index) is web content, which ignores accessibility text insertion; click it, then run type-text without --element-index")
@@ -1335,8 +1340,14 @@ private func enableManualAccessibilityIfNeeded(_ appElement: AXUIElement, app: A
guard app.needsManualAccessibilityMode else {
return
}
_ = AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue)
_ = AXUIElementSetAttributeValue(appElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue)
if isSettable(appElement, "AXManualAccessibility") {
_ = AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue)
return
}
// Why: the older, broader switch, so only for apps that lack AXManualAccessibility and only when it is off.
if boolAttribute(appElement, "AXEnhancedUserInterface") != true {
_ = AXUIElementSetAttributeValue(appElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue)
}
}
private func focusedWindow(appElement: AXUIElement, app: AppDescriptor, visibleWindowCount: Int, allowRecovery: Bool) throws -> AXUIElement {
@@ -1623,7 +1634,7 @@ private func withTransportErrors<T>(_ body: () throws -> T) throws -> T {
case .eventCreationFailed:
throw ProviderError.coded("accessibility_error", "failed to create input event")
case .targetWindowNotKey:
throw ProviderError.coded("background_unavailable", "background input cannot change the key window of the active app; retry with --restore-window to target this window explicitly")
throw ProviderError.coded("background_unavailable", "the user is working in another window of this app, and input to this window would move their typing here, so nothing was sent; use an element action (click --element-index, set-value) or wait until the user leaves the app. Ask the user before retrying with --restore-window.")
case let .focusRecordFailed(status):
throw ProviderError.coded("window_not_focused", "background key-window focus record failed (status \(status)); input may already have been delivered, so inspect current state before retrying")
}
@@ -1656,8 +1667,8 @@ private func clickPointer(
}
hitElement = found
}
try requireNoForegroundEffect(clickEffect(button: button, modifiers: modifiers, on: hitElement))
let hit = try backgroundClickHit(hitElement, snapshot: snapshot)
try requireBackgroundClickAllowed(hit)
try withTransportErrors {
try BackgroundInputTransport.click(
backgroundTarget(snapshot),
@@ -1680,9 +1691,23 @@ private func backgroundClickHit(_ element: AXUIElement, snapshot: Snapshot) thro
return BackgroundInputTransport.classifyPointerHit(element)
}
private func requireBackgroundClickAllowed(_ hit: BackgroundInputTransport.PointerHit) throws {
guard hit != .popupMenu else {
throw ProviderError.coded("background_unavailable", "a background click on a popup menu or web <select> opens the menu off screen or wedges the app's input; set the option with set-value on the popup element, or retry with --restore-window for a foreground click")
private func clickEffect(button: MouseButtonSelection, modifiers: [KeyModifier], on element: AXUIElement) -> ForegroundEffect? {
ForegroundEffect.ofClick(
button: button,
control: modifiers.contains { $0.flag == .maskControl },
target: BackgroundInputTransport.pressTarget(element)
)
}
/// Refuses, before anything is sent, an action that would put a menu or dialog in front of the user.
private func requireNoForegroundEffect(_ effect: ForegroundEffect?) throws {
switch effect {
case nil:
return
case .showsMenu:
throw ProviderError.coded("background_unavailable", "this would open a menu over the user's screen and take their keyboard, so nothing was sent; for a popup menu, combo box or web <select>, set the option with set-value on that element. Ask the user before retrying with --restore-window.")
case .opensDialog:
throw ProviderError.coded("background_unavailable", "this button's title ends in an ellipsis, so it most likely opens a dialog or sheet that brings the app to the front; nothing was sent. Ask the user before retrying with --restore-window.")
}
}
@@ -2045,6 +2070,16 @@ private func performAction(_ element: AXUIElement, _ action: String) throws -> B
return result == .success
}
/// A file-name field (a Finder name cell) shows and reads back an AX write, but the file keeps its old name.
private func isFileNameField(_ element: AXUIElement) -> Bool {
rawStringAttribute(element, "AXFilename") != nil
}
private func requireNotFileNameField(_ record: ElementRecord) throws {
guard isFileNameField(record.element) else { return }
throw ProviderError.coded("value_not_settable", "element \(record.index) shows a file's name; an accessibility write changes what it shows but does not rename the file, so nothing was written; with the field in edit mode, type the name with type-text without --element-index and press-key Return, or rename the file on disk")
}
private func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool {
var settable = DarwinBoolean(false)
return AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) == .success && settable.boolValue
@@ -3290,6 +3325,7 @@ private enum TextInput {
static func replaceSelection(_ element: AXUIElement, with text: String, property: String = "focusedText") throws -> [String: Any]? {
// Why: Chromium applies AX value writes without the input events pages listen to; a contenteditable gets none, so editors like Discord's never see the text.
guard !BackgroundInputTransport.isChromiumNode(element),
!isFileNameField(element),
isSettable(element, kAXValueAttribute as String),
let current = rawStringAttribute(element, kAXValueAttribute as String)
else {
@@ -31,8 +31,6 @@ public enum BackgroundInputTransport {
public enum PointerHit: Equatable, Sendable {
/// WebKit content (Safari, WKWebView): drops clicks into a window that is not key.
case webKitContent
/// A popup menu or web `<select>`: a background click opens menu tracking off screen or wedges Chromium input.
case popupMenu
case other
}
@@ -131,12 +129,8 @@ public enum BackgroundInputTransport {
public static func classifyPointerHit(_ element: AXUIElement) -> PointerHit {
var current = element
for depth in 0..<48 {
for _ in 0..<48 {
let role = stringAttribute(current, kAXRoleAttribute as String)
// Why: the hit lands on a popup's label or the popup itself, never deeper than its parent.
if depth < 2, role == kAXPopUpButtonRole as String || role == kAXMenuButtonRole as String {
return .popupMenu
}
if role == "AXWebArea" {
// Why: Chromium accepts window-stamped clicks as is; only WebKit needs the key-window scope,
// which fires the app's own did-become-active code (Slack, for one, marks messages read).
@@ -156,6 +150,24 @@ public enum BackgroundInputTransport {
return .other
}
/// The element and its parent as `ForegroundEffect` reads them.
public static func pressTarget(_ element: AXUIElement) -> [(role: String?, labels: [String?])] {
var chain: [(role: String?, labels: [String?])] = []
var current: AXUIElement? = element
while let node = current, chain.count < 2 {
chain.append((
role: stringAttribute(node, kAXRoleAttribute as String),
labels: [stringAttribute(node, kAXTitleAttribute as String), stringAttribute(node, kAXDescriptionAttribute as String)]
))
var parent: CFTypeRef?
current = AXUIElementCopyAttributeValue(node, kAXParentAttribute as CFString, &parent) == .success
&& parent.map { CFGetTypeID($0) == AXUIElementGetTypeID() } == true
// SAFETY: the CFTypeID check above proves parent is an AXUIElement.
? (parent as! AXUIElement) : nil
}
return chain
}
// MARK: Keys
/// Keys of a chord without a Unicode string: AppKit only matches menu key equivalents (⌘A/C/V/Z)
@@ -0,0 +1,46 @@
import Foundation
/// What an action would put in front of the user. Without --restore-window an action with an effect
/// is refused before anything is sent. Design follows agent-desktop's `background_effect.rs` (Apache-2.0).
public enum ForegroundEffect: Equatable, Sendable {
/// A menu drawn over the user's window that takes their keyboard; AppKit has no background route to its items.
case showsMenu
/// A button whose title ends in an ellipsis, the platform convention for "opens a dialog or sheet",
/// which can activate the app. A predictor, not a proof: no accessibility attribute says so.
case opensDialog
static let menuOwnerRoles: Set<String> = ["AXPopUpButton", "AXMenuButton", "AXComboBox"]
static let ellipses = ["\u{2026}", "..."]
/// The effect of a press (AXPress or a left click) on an element, given its role and title or
/// description, then its parent's; a hit on a popup's label lands one level below the popup.
public static func ofPress(_ chain: [(role: String?, labels: [String?])]) -> ForegroundEffect? {
for node in chain.prefix(2) {
guard let role = node.role else { continue }
if menuOwnerRoles.contains(role) { return .showsMenu }
// Why: only buttons, so a truncated "Loading..." text or a "Read more..." link still clicks.
if role == "AXButton", node.labels.contains(where: endsInEllipsis) { return .opensDialog }
}
return nil
}
/// A right-click, or a Control-click, opens the context menu; any other click presses the target.
public static func ofClick(button: MouseButtonSelection, control: Bool, target chain: [(role: String?, labels: [String?])]) -> ForegroundEffect? {
button == .right || (button == .left && control) ? .showsMenu : ofPress(chain)
}
/// The effect of an accessibility action run on its own, as perform-secondary-action does.
public static func ofAction(_ action: String, target chain: [(role: String?, labels: [String?])]) -> ForegroundEffect? {
switch action {
case "AXShowMenu": return .showsMenu
case "AXPress", "AXPick": return ofPress(chain)
default: return nil
}
}
static func endsInEllipsis(_ label: String?) -> Bool {
guard let label else { return false }
let trimmed = label.trimmingCharacters(in: .whitespacesAndNewlines)
return ellipses.contains { trimmed.hasSuffix($0) }
}
}
@@ -0,0 +1,31 @@
import OrcaComputerUseMacOSCore
import XCTest
final class ForegroundEffectTests: XCTestCase {
func testPressingAMenuOwnerOrItsLabelShowsAMenu() {
for role in ["AXPopUpButton", "AXMenuButton", "AXComboBox"] {
XCTAssertEqual(ForegroundEffect.ofPress([(role: role, labels: [])]), .showsMenu, role)
XCTAssertEqual(ForegroundEffect.ofPress([(role: "AXStaticText", labels: ["Small"]), (role: role, labels: [])]), .showsMenu, role)
}
XCTAssertNil(ForegroundEffect.ofPress([
(role: "AXStaticText", labels: []), (role: "AXGroup", labels: []), (role: "AXPopUpButton", labels: []),
]))
}
func testOnlyAButtonWhoseTitleEndsInAnEllipsisOpensADialog() {
XCTAssertEqual(ForegroundEffect.ofPress([(role: "AXButton", labels: ["Save\u{2026}", nil])]), .opensDialog)
XCTAssertEqual(ForegroundEffect.ofPress([(role: "AXButton", labels: [nil, "Export... "])]), .opensDialog)
XCTAssertNil(ForegroundEffect.ofPress([(role: "AXButton", labels: ["Save", "Saves the file\u{2026} now"])]))
XCTAssertNil(ForegroundEffect.ofPress([(role: "AXLink", labels: ["Read more..."])]))
XCTAssertNil(ForegroundEffect.ofPress([(role: "AXStaticText", labels: ["Loading..."]), (role: "AXGroup", labels: [])]))
}
func testContextMenuClicksShowAMenuOnAnyTarget() {
let plainButton: [(role: String?, labels: [String?])] = [(role: "AXButton", labels: ["OK"])]
XCTAssertEqual(ForegroundEffect.ofClick(button: .right, control: false, target: plainButton), .showsMenu)
XCTAssertEqual(ForegroundEffect.ofClick(button: .left, control: true, target: plainButton), .showsMenu)
XCTAssertNil(ForegroundEffect.ofClick(button: .left, control: false, target: plainButton))
XCTAssertEqual(ForegroundEffect.ofAction("AXShowMenu", target: plainButton), .showsMenu)
XCTAssertNil(ForegroundEffect.ofAction("AXScrollToVisible", target: [(role: "AXPopUpButton", labels: [])]))
}
}
+1 -1
View File
@@ -148,7 +148,7 @@ Do not pass `--restore-window` by default. Use it only when foreground input is
- Drags in Chromium or Electron apps, and apps that read the hardware pointer (games, OpenGL/Metal views, Java/Tk), which ignore background pointer events.
- The window is minimized or hidden. Accessibility actions still work there; pointer and keyboard events do not.
Popup menus and web `<select>` controls: set the option with `set-value` on the popup element. Background clicks on them are refused because they open the menu off screen or wedge the app's input.
On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks on popup menus, menu buttons, combo boxes and web `<select>` controls, and buttons whose title ends in an ellipsis (`Save…`), which usually open a dialog. For a popup, combo box or `<select>`, set the option with `set-value` on that element.
On Linux, clicks, scrolling, drags and keyboard input still bring the target window forward. On Windows, these go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), element scrolls by whole `--pages`, and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Everything else brings the window forward and reports `foreground`.
+1 -1
View File
@@ -1169,7 +1169,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
"- Drags in Chromium or Electron apps, and apps that read the hardware pointer (games, OpenGL/Metal views, Java/Tk), which ignore background pointer events.",
"- The window is minimized or hidden. Accessibility actions still work there; pointer and keyboard events do not.",
"",
"Popup menus and web `<select>` controls: set the option with `set-value` on the popup element. Background clicks on them are refused because they open the menu off screen or wedge the app's input.",
"On macOS, background actions that would open a menu or dialog over the user are refused before anything is sent: right-clicks and Control-clicks, clicks on popup menus, menu buttons, combo boxes and web `<select>` controls, and buttons whose title ends in an ellipsis (`Save…`), which usually open a dialog. For a popup, combo box or `<select>`, set the option with `set-value` on that element.",
"",
"On Linux, clicks, scrolling, drags and keyboard input still bring the target window forward. On Windows, these go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), element scrolls by whole `--pages`, and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Everything else brings the window forward and reports `foreground`.",
"",