diff --git a/config/scripts/verify-computer-native.mjs b/config/scripts/verify-computer-native.mjs index 750d425ef14..896558aa919 100644 --- a/config/scripts/verify-computer-native.mjs +++ b/config/scripts/verify-computer-native.mjs @@ -344,11 +344,6 @@ function verifyNativeArgumentGuardrails() { 'macOS typeText must try verified focused AX text replacement before synthetic keyboard fallback' ) } - if (!macos.includes('even when the click uses an AX action path')) { - failures.push( - 'macOS clicks must activate the target window before accessibility or pointer input' - ) - } if ( !macos.includes('if visibleWindowCount > 0') || !macos.includes('ProviderError.coded("permission_denied", "app') || diff --git a/docs/site/content/docs/cli/computer-use.mdx b/docs/site/content/docs/cli/computer-use.mdx index 089600aae0a..edf911b9df4 100644 --- a/docs/site/content/docs/cli/computer-use.mdx +++ b/docs/site/content/docs/cli/computer-use.mdx @@ -73,6 +73,8 @@ orca computer perform-secondary-action --app --element-index --action Prefer semantic actions (`click`, `set-value`, `perform-secondary-action`) over raw `type-text` or `press-key` — they target accessibility elements directly and survive focus changes that keyboard input doesn't. +On macOS, actions run in the background: the target app isn't brought forward, your frontmost app and key window stay put, and the pointer doesn't move, so you can keep working while an agent drives another window. Each action result reports `action.delivery` (`background` or `foreground`). Pass `--restore-window` only when an app ignores background input, such as drags in Chromium or Electron apps or games; it brings the app forward and uses the real pointer and keyboard. Popup menus and web ` opens the menu off screen or wedges the app's input; set the option with set-value on the popup element, or retry with --restore-window for a foreground click") + } +} + +private func scrollPointer(at point: CGPoint, direction: String, pages: Double, snapshot: Snapshot, params: [String: JSONValue]) throws { + guard let delta = boundedInteger(max(1, (12 * pages).rounded()), as: Int32.self) else { + throw ProviderError.coded("invalid_argument", "pages is out of range") + } + let wheel1: Int32 = direction == "up" ? delta : direction == "down" ? -delta : 0 + let wheel2: Int32 = direction == "left" ? delta : direction == "right" ? -delta : 0 + if isForegroundRequest(params) { + try requireTargetWindowFocused(snapshot, restoreWindowRequested: true) + try Input.scroll(at: point, wheel1: wheel1, wheel2: wheel2, targetWindow: snapshot) + return + } + try withTransportErrors { + try BackgroundInputTransport.scroll(backgroundTarget(snapshot), at: point, wheel1: wheel1, wheel2: wheel2) + } +} + +private func deliverKeys(_ events: [CGEvent], snapshot: Snapshot, params: [String: JSONValue]) throws { + if isForegroundRequest(params) { + try requireTargetWindowFocused(snapshot, restoreWindowRequested: true) + for event in events { + event.post(tap: .cghidEventTap) + } + return + } + try withTransportErrors { + try BackgroundInputTransport.sendKeys(events, to: backgroundTarget(snapshot)) + } +} + +private func eventFlags(_ modifiers: [KeyModifier]) -> CGEventFlags { + modifiers.reduce(into: CGEventFlags()) { result, modifier in + result.insert(modifier.flag) + } +} + private func matchingWindow(appElement: AXUIElement, capture: WindowCapture, focused: AXUIElement, explicitTarget: Bool) -> AXUIElement? { guard let windows = copyArray(appElement, kAXWindowsAttribute as String) else { return nil @@ -1700,43 +1838,6 @@ private func screenshotScale(screenshot: ScreenshotPayload?, bounds: CGRect) -> ) } -extension MouseButtonSelection { - // Why: macOS has no dedicated middle-button event family; it rides `otherMouse*` - // with the button number carried by `mouseButton:` on the event constructor. - var cgButton: CGMouseButton { - switch self { - case .left: - return .left - case .right: - return .right - case .middle: - return .center - } - } - - var downEvent: CGEventType { - switch self { - case .left: - return .leftMouseDown - case .right: - return .rightMouseDown - case .middle: - return .otherMouseDown - } - } - - var upEvent: CGEventType { - switch self { - case .left: - return .leftMouseUp - case .right: - return .rightMouseUp - case .middle: - return .otherMouseUp - } - } -} - private func mouseButton(_ raw: String?) throws -> MouseButtonSelection { switch ActionArgumentValidation.mouseButton(raw) { case let .success(button): @@ -2567,11 +2668,9 @@ private enum Input { guard let source = CGEventSource(stateID: .combinedSessionState) else { throw ProviderError.coded("accessibility_error", "failed to create event source") } - let flags = modifiers.reduce(into: CGEventFlags()) { result, modifier in - result.insert(modifier.flag) - } + let flags = eventFlags(modifiers) let target = syntheticClickRecipient(pid: targetWindow.app.pid, windowId: targetWindow.windowId) - do { + try withRecipientErrors { try SyntheticMouseClickDelivery.deliver( clickCount: count, target: target, @@ -2606,6 +2705,12 @@ private enum Input { post: { $0.post(tap: .cghidEventTap) }, pause: { _ = usleep($0) } ) + } + } + + private static func withRecipientErrors(_ body: () throws -> Void) throws { + do { + try body() } catch let failure as SyntheticMouseClickDelivery.FenceFailure { switch failure { case let .recipientChanged(expected, actual, deliveredPresses): @@ -2617,79 +2722,59 @@ private enum Input { : "\(deliveredPresses) press(es) may already have been delivered; run get-app-state and verify state before retrying" throw ProviderError.coded( "window_not_focused", - "coordinate click aborted because target pid \(expected.ownerPID) window \(expected.windowID) is no longer the focused topmost recipient (current: \(actualDescription)); \(recovery)" + "pointer input aborted because target pid \(expected.ownerPID) window \(expected.windowID) is no longer the focused topmost recipient (current: \(actualDescription)); \(recovery)" ) } } } - static func scroll(pid: pid_t, at point: CGPoint, direction: String, pages: Double) throws { - guard let delta = boundedInteger(max(1, (12 * pages).rounded()), as: Int32.self) else { - throw ProviderError.coded("invalid_argument", "pages is out of range") - } - let wheel1: Int32 = direction == "up" ? delta : direction == "down" ? -delta : 0 - let wheel2: Int32 = direction == "left" ? delta : direction == "right" ? -delta : 0 - guard let event = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 2, wheel1: wheel1, wheel2: wheel2, wheel3: 0) else { + static func scroll(at point: CGPoint, wheel1: Int32, wheel2: Int32, targetWindow: Snapshot) throws { + let move = try mouseEvent(.mouseMoved, at: point) + guard let wheel = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 2, wheel1: wheel1, wheel2: wheel2, wheel3: 0) else { throw ProviderError.coded("accessibility_error", "failed to create scroll event") } - event.location = point - event.postToPid(pid) - } - - static func drag(pid: pid_t, from start: CGPoint, to end: CGPoint) throws { - guard let source = CGEventSource(stateID: .combinedSessionState) else { - throw ProviderError.coded("accessibility_error", "failed to create event source") - } - try mouse(.mouseMoved, source: source, point: start, button: .left, pid: pid) - try mouse(.leftMouseDown, source: source, point: start, button: .left, pid: pid) - for step in 1...10 { - let progress = CGFloat(step) / 10 - try mouse( - .leftMouseDragged, - source: source, - point: CGPoint(x: start.x + (end.x - start.x) * progress, y: start.y + (end.y - start.y) * progress), - button: .left, - pid: pid + wheel.location = point + try withRecipientErrors { + try SyntheticMouseClickDelivery.deliverScroll( + move: move, wheel: wheel, + target: syntheticClickRecipient(pid: targetWindow.app.pid, windowId: targetWindow.windowId), + currentObservation: { currentSyntheticClickRecipient(snapshot: targetWindow, point: point) }, + post: { $0.post(tap: .cghidEventTap) }, + pause: { _ = usleep($0) } ) } - try mouse(.leftMouseUp, source: source, point: end, button: .left, pid: pid) } - static func typeText(_ text: String, pid: pid_t) throws { - for unit in text.utf16 { - var char = unit - guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true), - let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) - else { - throw ProviderError.coded("accessibility_error", "failed to create keyboard event") - } - down.keyboardSetUnicodeString(stringLength: 1, unicodeString: &char) - up.keyboardSetUnicodeString(stringLength: 1, unicodeString: &char) - down.post(tap: .cghidEventTap) - up.post(tap: .cghidEventTap) + static func drag(from start: CGPoint, to end: CGPoint, targetWindow: Snapshot) throws { + let move = try mouseEvent(.mouseMoved, at: start) + let down = try mouseEvent(.leftMouseDown, at: start) + let up = try mouseEvent(.leftMouseUp, at: end) + let dragged = try (1...10).map { step in + let progress = CGFloat(step) / 10 + return try mouseEvent(.leftMouseDragged, at: CGPoint(x: start.x + (end.x - start.x) * progress, y: start.y + (end.y - start.y) * progress)) + } + try withRecipientErrors { + try SyntheticMouseClickDelivery.deliverDrag( + move: move, down: down, dragged: dragged, up: up, + target: syntheticClickRecipient(pid: targetWindow.app.pid, windowId: targetWindow.windowId), + currentObservation: { currentSyntheticClickRecipient(snapshot: targetWindow, point: start) }, + post: { $0.post(tap: .cghidEventTap) }, + pause: { _ = usleep($0) } + ) } } - static func pressKey(_ key: String, pid: pid_t) throws { + static func keyEvents(_ key: String) throws -> [CGEvent] { let parsed = try KeyMap.parse(key) - var flags = CGEventFlags() - var pressedModifiers: [KeyModifier] = [] - defer { - for modifier in pressedModifiers.reversed() { - flags.remove(modifier.flag) - try? keyEvent(modifier.keyCode, down: false, flags: flags, pid: pid) - } + return try withTransportErrors { + try BackgroundInputTransport.chordEvents( + keyCode: parsed.keyCode, + modifiers: parsed.modifiers.map { (keyCode: $0.keyCode, flag: $0.flag) } + ) } - for modifier in parsed.modifiers { - flags.insert(modifier.flag) - try keyEvent(modifier.keyCode, down: true, flags: flags, pid: pid) - pressedModifiers.append(modifier) - } - try keyEvent(parsed.keyCode, down: true, flags: flags, pid: pid) - try keyEvent(parsed.keyCode, down: false, flags: flags, pid: pid) } - static func pasteText(_ text: String, pid: pid_t) throws { + static func withPasteboardText(_ text: String, _ body: () throws -> Void) throws { let pasteboard = NSPasteboard.general let previousItems: [NSPasteboardItem] = pasteboard.pasteboardItems?.map { item in let copy = NSPasteboardItem() @@ -2702,36 +2787,28 @@ private enum Input { } ?? [] pasteboard.clearContents() pasteboard.setString(text, forType: .string) + let changeCount = pasteboard.changeCount defer { - pasteboard.clearContents() - if !previousItems.isEmpty { - pasteboard.writeObjects(previousItems) + // Why: posting Cmd+V returns before the receiver reads the pasteboard. + usleep(200_000) + if pasteboard.changeCount == changeCount { + pasteboard.clearContents() + if !previousItems.isEmpty { + pasteboard.writeObjects(previousItems) + } } } - try pressKey("cmd+v", pid: pid) + try body() } - private static func mouse( - _ type: CGEventType, - source: CGEventSource, - point: CGPoint, - button: CGMouseButton, - flags: CGEventFlags = [], - pid: pid_t - ) throws { - guard let event = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: point, mouseButton: button) else { + private static func mouseEvent(_ type: CGEventType, at point: CGPoint) throws -> CGEvent { + guard point.x.isFinite, point.y.isFinite else { + throw ProviderError.coded("invalid_argument", "pointer coordinates are out of range") + } + guard let event = CGEvent(mouseEventSource: nil, mouseType: type, mouseCursorPosition: point, mouseButton: .left) else { throw ProviderError.coded("accessibility_error", "failed to create mouse event") } - event.flags = flags - event.postToPid(pid) - } - - private static func keyEvent(_ keyCode: CGKeyCode, down: Bool, flags: CGEventFlags, pid: pid_t) throws { - guard let event = CGEvent(keyboardEventSource: nil, virtualKey: keyCode, keyDown: down) else { - throw ProviderError.coded("accessibility_error", "failed to create key event") - } - event.flags = flags - event.post(tap: .cghidEventTap) + return event } } diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/BackgroundInputTransport.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/BackgroundInputTransport.swift new file mode 100644 index 00000000000..5c49f131599 --- /dev/null +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/BackgroundInputTransport.swift @@ -0,0 +1,421 @@ +import AppKit +import ApplicationServices +import CoreGraphics +import Darwin + +/// Delivers pointer and key input to one window of an app without activating it, raising it, +/// changing the user's key window or moving the real pointer. It owns every private symbol it needs. +public enum BackgroundInputTransport { + public struct Target: Equatable, Sendable { + public let pid: pid_t + public let windowID: CGWindowID + /// Window frame in global top-left screen points, as CGWindowList reports it. + public let windowFrame: CGRect + + public init(pid: pid_t, windowID: CGWindowID, windowFrame: CGRect) { + self.pid = pid + self.windowID = windowID + self.windowFrame = windowFrame + } + } + + public enum Failure: Error, Equatable { + case unavailable(missingSymbols: [String]) + case processNotFound + case eventCreationFailed + case targetWindowNotKey + case focusRecordFailed(status: Int32) + } + + /// What the app's accessibility tree has under a point, which decides how a pointer event may be sent. + public enum PointerHit: Equatable, Sendable { + /// WebKit content (Safari, WKWebView): drops clicks into a window that is not key. + case webKitContent + /// A popup menu or web `` controls: set the option with `set-value` on the popup element. Background clicks on them are refused because they open the menu off screen or wedge the app's input. + +On Linux and Windows, clicks, scrolling, drags and keyboard input still bring the target window forward. + ## Screenshots `get-app-state` and actions request screenshots by default unless `--no-screenshot` is @@ -120,12 +136,12 @@ On Linux and Windows, screenshots may come from the visible desktop region for t ## App Notes -Browsers: for Edge, Chrome, Safari, and similar browser windows, set the address/search field directly, then press Return. Do not assume raw typing went to the address bar. Use `--restore-window` when the browser is not already frontmost. Large tab strips may show only the active tab plus an "inactive browser tabs omitted" marker; treat that as intentional noise reduction and operate on the current page/address bar unless the user asked to manage tabs. +Browsers: for Edge, Chrome, Safari, and similar browser windows, set the address/search field directly, then press Return. Do not assume raw typing went to the address bar. On Linux and Windows, use `--restore-window` when the browser is not already frontmost; on macOS this works in the background. Large tab strips may show only the active tab plus an "inactive browser tabs omitted" marker; treat that as intentional noise reduction and operate on the current page/address bar unless the user asked to manage tabs. For browser-hosted forms such as Gmail compose, verify the focused UI element after each field action. Page text fields can expose accessibility actions without moving DOM focus; if a click or `set-value` does not change the focused receiver, use `Tab` / `Shift+Tab` from a known focused field or window-local coordinates from a fresh screenshot. Prefer `paste-text` into the verified focused field for draft bodies, then inspect the returned state before continuing. ```text -ORCA computer get-app-state --app com.microsoft.edgemac --restore-window --json +ORCA computer get-app-state --app com.microsoft.edgemac --json ORCA computer set-value --app com.microsoft.edgemac --element-index --value "test123" --json ORCA computer press-key --app com.microsoft.edgemac --key Return --json ``` @@ -139,6 +155,7 @@ Slack: the accessibility tree may be shallow while the screenshot contains usefu - `app_not_found`: run `list-apps` and retry with the bundle ID. If the target is a web app such as Gmail, choose the desktop browser app/window that contains it; do not retry `ORCA computer ... --app Gmail` unchanged because `orca computer` app selectors refer to desktop apps, not website names. - `app_blocked`: stop; the target is intentionally blocked from computer-use. - `window_not_found` / `window_stale`: run `list-windows`, choose a current selector, then rerun `get-app-state`. +- `background_unavailable`: for a popup or `` controls: set the option with `set-value` on the popup element. Background clicks on them are refused because they open the menu off screen or wedge the app's input.", + "", + "On Linux and Windows, clicks, scrolling, drags and keyboard input still bring the target window forward.", + "", "## Screenshots", "", "`get-app-state` and actions request screenshots by default unless `--no-screenshot` is", @@ -137,12 +153,12 @@ const COMPUTER_USE_MARKDOWN = [ "", "## App Notes", "", - "Browsers: for Edge, Chrome, Safari, and similar browser windows, set the address/search field directly, then press Return. Do not assume raw typing went to the address bar. Use `--restore-window` when the browser is not already frontmost. Large tab strips may show only the active tab plus an \"inactive browser tabs omitted\" marker; treat that as intentional noise reduction and operate on the current page/address bar unless the user asked to manage tabs.", + "Browsers: for Edge, Chrome, Safari, and similar browser windows, set the address/search field directly, then press Return. Do not assume raw typing went to the address bar. On Linux and Windows, use `--restore-window` when the browser is not already frontmost; on macOS this works in the background. Large tab strips may show only the active tab plus an \"inactive browser tabs omitted\" marker; treat that as intentional noise reduction and operate on the current page/address bar unless the user asked to manage tabs.", "", "For browser-hosted forms such as Gmail compose, verify the focused UI element after each field action. Page text fields can expose accessibility actions without moving DOM focus; if a click or `set-value` does not change the focused receiver, use `Tab` / `Shift+Tab` from a known focused field or window-local coordinates from a fresh screenshot. Prefer `paste-text` into the verified focused field for draft bodies, then inspect the returned state before continuing.", "", "```text", - "ORCA computer get-app-state --app com.microsoft.edgemac --restore-window --json", + "ORCA computer get-app-state --app com.microsoft.edgemac --json", "ORCA computer set-value --app com.microsoft.edgemac --element-index --value \"test123\" --json", "ORCA computer press-key --app com.microsoft.edgemac --key Return --json", "```", @@ -156,6 +172,7 @@ const COMPUTER_USE_MARKDOWN = [ "- `app_not_found`: run `list-apps` and retry with the bundle ID. If the target is a web app such as Gmail, choose the desktop browser app/window that contains it; do not retry `ORCA computer ... --app Gmail` unchanged because `orca computer` app selectors refer to desktop apps, not website names.", "- `app_blocked`: stop; the target is intentionally blocked from computer-use.", "- `window_not_found` / `window_stale`: run `list-windows`, choose a current selector, then rerun `get-app-state`.", + "- `background_unavailable`: for a popup or ``, set its option with `orca computer set-value` on that element instead of clicking.', + 'Otherwise retry once with `--restore-window`; that brings the app forward and uses foreground input, so tell the user it will take over their screen.' + ) case 'window_stale': return recoverWith( 'Run `orca computer list-windows --app --json` and choose a current window selector.', diff --git a/src/shared/computer-use-runtime-types.ts b/src/shared/computer-use-runtime-types.ts index 8bbd0592351..e2e7cbebc4a 100644 --- a/src/shared/computer-use-runtime-types.ts +++ b/src/shared/computer-use-runtime-types.ts @@ -3,6 +3,7 @@ export const COMPUTER_ERROR_CODES = { app_blocked: 'app_blocked', window_not_found: 'window_not_found', window_not_focused: 'window_not_focused', + background_unavailable: 'background_unavailable', window_stale: 'window_stale', provider_incompatible: 'provider_incompatible', unsupported_capability: 'unsupported_capability', @@ -89,6 +90,8 @@ export type ComputerActionMetadata = { fallbackReason?: string | null targetWindowId?: number | null targetWindowIndex?: number | null + // Absent from providers that predate background input (Linux, Windows, older macOS helpers). + delivery?: 'background' | 'foreground' verification?: ComputerActionVerification } @@ -164,6 +167,10 @@ export type ComputerProviderCapabilities = { dock: boolean menubar: boolean } + // Absent from providers that predate background input. + input?: { + background: boolean + } } }