import AppKit import ApplicationServices import CoreGraphics import Darwin import Foundation import ImageIO import OrcaComputerUseMacOSCore import ScreenCaptureKit private let providerName = "orca-computer-use-macos" private let providerVersion = "1.0.0" private let providerProtocolVersion = 1 struct Request: Decodable { let id: Int let method: String let params: [String: JSONValue]? let token: String? } enum JSONValue: Decodable { case string(String) case number(Double) case bool(Bool) case object([String: JSONValue]) case array([JSONValue]) case null init(from decoder: Decoder) throws { let container = try decoder.singleValueContainer() if container.decodeNil() { self = .null } else if let value = try? container.decode(Bool.self) { self = .bool(value) } else if let value = try? container.decode(Double.self) { self = .number(value) } else if let value = try? container.decode(String.self) { self = .string(value) } else if let value = try? container.decode([JSONValue].self) { self = .array(value) } else { self = .object(try container.decode([String: JSONValue].self)) } } var string: String? { if case let .string(value) = self { return value } return nil } var number: Double? { if case let .number(value) = self { return value } return nil } var bool: Bool? { if case let .bool(value) = self { return value } return nil } } enum ProviderError: Error { case coded(String, String) var code: String { switch self { case let .coded(code, _): return code } } var message: String { switch self { case let .coded(_, message): return message } } } struct AppDescriptor { let name: String let bundleId: String? let pid: pid_t let app: NSRunningApplication var needsManualAccessibilityMode: Bool { // Chromium/Electron apps often need this private AX mode, but applying it // broadly can corrupt native Cocoa app trees into app-root-only nodes. guard let bundleId = bundleId?.lowercased() else { return false } return bundleId.hasPrefix("com.google.chrome") || bundleId.hasPrefix("com.microsoft.edgemac") || bundleId.hasPrefix("com.brave.browser") || bundleId.hasPrefix("com.operasoftware.opera") || bundleId.hasPrefix("com.vivaldi.vivaldi") || bundleId == "com.github.electron" || bundleId == "com.tinyspeck.slackmacgap" || bundleId == "com.spotify.client" || bundleId == "com.hnc.discord" || bundleId == "com.microsoft.teams2" || bundleId == "notion.id" } var isKnownBrowser: Bool { let bundle = bundleId?.lowercased() ?? "" let appName = name.lowercased() return bundle == "com.apple.safari" || bundle == "org.mozilla.firefox" || bundle == "company.thebrowser.browser" || bundle == "app.zen-browser.zen" || bundle.hasPrefix("com.google.chrome") || bundle.hasPrefix("com.microsoft.edgemac") || bundle.hasPrefix("com.brave.browser") || bundle.hasPrefix("com.operasoftware.opera") || bundle.hasPrefix("com.vivaldi.vivaldi") || appName == "safari" || appName == "firefox" || appName == "arc" || appName == "zen" || appName.contains("chrome") || appName.contains("chromium") || appName.contains("edge") || appName.contains("brave") || appName.contains("opera") || appName.contains("vivaldi") } } final class ElementRecord { let index: Int let element: AXUIElement let localFrame: CGRect? let actions: [String] let signature: String init(index: Int, element: AXUIElement, localFrame: CGRect?, actions: [String], signature: String) { self.index = index self.element = element self.localFrame = localFrame self.actions = actions self.signature = signature } } struct Snapshot { let id: String let app: AppDescriptor let windowTitle: String let windowBounds: CGRect let windowId: CGWindowID let windowLayer: Int let treeText: String let focusedElementId: Int? let screenshot: ScreenshotPayload? let screenshotStatus: ScreenshotStatus let screenshotScale: CGSize let screenshotEngine: String? let elements: [Int: ElementRecord] let elementOrder: [Int] let nextFreshElementIndex: Int let truncated: Bool let maxDepthReached: Bool /// On-screen windows of the app (menus and sheets included), so a new or closed window counts as a change. let appWindowIds: [CGWindowID] func showsSameState(as other: Snapshot) -> Bool { treeText == other.treeText && windowBounds == other.windowBounds && appWindowIds == other.appWindowIds } func withoutScreenshotPayload() -> Snapshot { Snapshot( id: id, app: app, windowTitle: windowTitle, windowBounds: windowBounds, windowId: windowId, windowLayer: windowLayer, treeText: treeText, focusedElementId: focusedElementId, screenshot: nil, screenshotStatus: .skipped, screenshotScale: CGSize(width: 1, height: 1), screenshotEngine: nil, elements: elements, elementOrder: elementOrder, nextFreshElementIndex: nextFreshElementIndex, truncated: truncated, maxDepthReached: maxDepthReached, appWindowIds: appWindowIds ) } } struct ScreenshotPayload { let data: String let width: Int let height: Int let scale: Double } struct CapturedImage { let image: CGImage let engine: String } enum ScreenshotStatus { case captured case skipped case failed(String) } private struct CachedSnapshotEntry { let snapshotId: String let keys: [String] let createdAt: Date } final class Provider { private var snapshots: [String: Snapshot] = [:] private var snapshotEntries: [CachedSnapshotEntry] = [] init() { // Why: the system-wide element sets the wait for every AX call in this process, so a hung app costs one bounded timeout whatever the macOS default. _ = AXUIElementSetMessagingTimeout(AXUIElementCreateSystemWide(), AccessibilityMessaging.timeoutSeconds) } func handle(method: String, params: [String: JSONValue]) throws -> Any { switch method { case "handshake": return providerHandshake() case "listApps": return ["apps": listApps().map(renderListedApp)] case "listWindows": return try listWindows(params: params) case "getAppState": return try renderSnapshot(observe(params: params)) case "click": return try actionResult(params: params) { try click(params: params) } case "performSecondaryAction": return try actionResult(params: params) { try performSecondaryAction(params: params) } case "setValue": return try actionResult(params: params) { try setValue(params: params) } case "typeText": return try actionResult(params: params) { try typeText(params: params) } case "pressKey": return try actionResult(params: params) { try pressKey(params: params) } case "hotkey": return try actionResult(params: params) { try hotkey(params: params) } case "pasteText": return try actionResult(params: params) { try pasteText(params: params) } case "scroll": return try actionResult(params: params) { try scroll(params: params) } case "drag": return try actionResult(params: params) { try drag(params: params) } default: throw ProviderError.coded("invalid_argument", "unknown method '\(method)'") } } private func actionResult(params: [String: JSONValue], action runAction: () throws -> [String: Any]) throws -> [String: Any] { var action = try runAction() // Why: only --restore-window activates the app or posts to the HID tap; every other route leaves the user's desktop alone. action["delivery"] = isForegroundRequest(params) ? "foreground" : "background" // Why: the action's currentSnapshot just cached this target's tree from before the action. let before = action["path"] as? String == "accessibility" && action["verification"] == nil ? try? cachedSnapshot(params: params) : nil do { let after = try observe(params: params) guard let before, after.showsSameState(as: before) else { return renderActionResult(action: action, snapshot: after) } if let changed = try observeLateChange(before: before, params: params) { return renderActionResult(action: action, snapshot: changed) } // Why: an AX call can succeed and do nothing, as AXPress does on many web controls (#31758). action["verification"] = unverifiedAction(reason: "suspected_noop") return renderActionResult(action: action, snapshot: after) } catch let error as ProviderError where (error.code == "window_not_found" || error.code == "window_stale") && hasRequestedWindowSelector(params) { var fallbackParams = params fallbackParams.removeValue(forKey: "windowId") fallbackParams.removeValue(forKey: "windowIndex") if action["verification"] == nil { action["verification"] = ["state": "unverified", "reason": "window_changed"] } return try renderActionResult(action: action, snapshot: observe(params: fallbackParams)) } } /// Re-reads the target for about a second, since apps often apply an action on a later run-loop turn. /// Returns the observation to report once something changed, or nil if nothing did. private func observeLateChange(before: Snapshot, params: [String: JSONValue]) throws -> Snapshot? { let settleParams = params.merging(["restoreWindow": .bool(false)]) { _, replacement in replacement } let quietParams = settleParams.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement } let deadline = Date().addingTimeInterval(1) while Date() < deadline { Thread.sleep(forTimeInterval: 0.2) let polled = try observe(params: quietParams) if !polled.showsSameState(as: before) { if params["noScreenshot"]?.bool == true { return polled } return try observe(params: settleParams) } } return nil } private func observe(params: [String: JSONValue]) throws -> Snapshot { let query = try requiredString(params, "app") let windowId = try requestedWindowId(params) let windowIndex = try requestedWindowIndex(params) let app = try resolveApp(query) if params["restoreWindow"]?.bool == true { recoverWindow(app) } let namespace = snapshotNamespace(params) pruneSnapshotCache() let snapshot = try buildSnapshot( app: app, includeScreenshot: params["noScreenshot"]?.bool != true, windowId: windowId, windowIndex: windowIndex, restoreWindow: params["restoreWindow"]?.bool == true, previousSnapshot: { [snapshots] resolvedWindowId in // Why: indexes carry over from this caller's own last view of exactly this window. snapshots[namespacedSnapshotKey(namespace, snapshotCanonicalWindowIdKey(resolvedWindowId))] .flatMap { $0.app.pid == app.pid ? $0 : nil } } ) // Why: cached snapshots only validate element identity for follow-up // actions; retaining MB-scale screenshot base64 in the long-lived agent grows memory. rememberSnapshot( query: query, app: app, snapshot: snapshot.withoutScreenshotPayload(), params: params, windowIndex: windowIndex ) return snapshot } private func rememberSnapshot( query: String, app: AppDescriptor, snapshot cachedSnapshot: Snapshot, params: [String: JSONValue], windowIndex: Int? ) { let keys = [query, app.name, app.bundleId ?? "", "pid:\(app.pid)"] .filter { !$0.isEmpty } .map { $0.lowercased() } let namespace = snapshotNamespace(params) var storedKeys: [String] = [] let canonicalWindowKey = snapshotCanonicalWindowIdKey(cachedSnapshot.windowId) if !isExplicitSnapshotNamespace(namespace) { snapshots[canonicalWindowKey.lowercased()] = cachedSnapshot storedKeys.append(canonicalWindowKey.lowercased()) } snapshots[namespacedSnapshotKey(namespace, canonicalWindowKey)] = cachedSnapshot storedKeys.append(namespacedSnapshotKey(namespace, canonicalWindowKey)) if let windowIndex { let canonicalWindowIndexKey = snapshotCanonicalWindowIndexKey(windowIndex) if !isExplicitSnapshotNamespace(namespace) { snapshots[canonicalWindowIndexKey.lowercased()] = cachedSnapshot storedKeys.append(canonicalWindowIndexKey.lowercased()) } snapshots[namespacedSnapshotKey(namespace, canonicalWindowIndexKey)] = cachedSnapshot storedKeys.append(namespacedSnapshotKey(namespace, canonicalWindowIndexKey)) } for key in keys { if !isExplicitSnapshotNamespace(namespace) { snapshots[key] = cachedSnapshot storedKeys.append(key) snapshots[snapshotWindowKey(key, cachedSnapshot.windowId)] = cachedSnapshot storedKeys.append(snapshotWindowKey(key, cachedSnapshot.windowId)) if let windowIndex { snapshots[snapshotWindowIndexKey(key, windowIndex)] = cachedSnapshot storedKeys.append(snapshotWindowIndexKey(key, windowIndex)) } } snapshots[namespacedSnapshotKey(namespace, key)] = cachedSnapshot storedKeys.append(namespacedSnapshotKey(namespace, key)) let namespacedWindowKey = namespacedSnapshotKey( namespace, snapshotWindowKey(key, cachedSnapshot.windowId) ) snapshots[namespacedWindowKey] = cachedSnapshot storedKeys.append(namespacedWindowKey) if let windowIndex { let namespacedWindowIndexKey = namespacedSnapshotKey( namespace, snapshotWindowIndexKey(key, windowIndex) ) snapshots[namespacedWindowIndexKey] = cachedSnapshot storedKeys.append(namespacedWindowIndexKey) } } snapshotEntries.append( CachedSnapshotEntry(snapshotId: cachedSnapshot.id, keys: storedKeys, createdAt: Date()) ) pruneSnapshotCache() } private func pruneSnapshotCache() { let now = Date() while let oldest = snapshotEntries.first, ComputerSnapshotCachePolicy.shouldPrune( entryCount: snapshotEntries.count, createdAt: oldest.createdAt, now: now ) { let expired = snapshotEntries.removeFirst() for key in expired.keys where snapshots[key]?.id == expired.snapshotId { snapshots.removeValue(forKey: key) } } } private func currentSnapshot(params: [String: JSONValue]) throws -> Snapshot { pruneSnapshotCache() let cached = try cachedSnapshot(params: params) // Why: cached AX frames can be stale after a window move or resize, and // stale geometry can turn an intended action into a misclick. let snapshot = try observe(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }) try validateRequestedElements(cached: cached, current: snapshot, params: params) return snapshot } private func currentKeyboardSnapshot(params: [String: JSONValue]) throws -> Snapshot { // Why: keyboard actions need fresh focused-element state, never pixels. try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }) } private func cachedSnapshot(params: [String: JSONValue]) throws -> Snapshot? { guard let query = params["app"]?.string, !query.isEmpty else { return nil } let namespace = snapshotNamespace(params) if let targetWindowId = try requestedWindowId(params) { let canonicalKey = snapshotCanonicalWindowIdKey(targetWindowId) if let cached = snapshots[namespacedSnapshotKey(namespace, canonicalKey)] { return cached } if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[canonicalKey.lowercased()] { return cached } let windowKey = snapshotWindowKey(query.lowercased(), targetWindowId) if let cached = snapshots[namespacedSnapshotKey(namespace, windowKey)] { return cached } if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[windowKey] { return cached } return nil } if let targetWindowIndex = try requestedWindowIndex(params) { let canonicalKey = snapshotCanonicalWindowIndexKey(targetWindowIndex) if let cached = snapshots[namespacedSnapshotKey(namespace, canonicalKey)] { return cached } if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[canonicalKey.lowercased()] { return cached } let windowKey = snapshotWindowIndexKey(query.lowercased(), targetWindowIndex) if let cached = snapshots[namespacedSnapshotKey(namespace, windowKey)] { return cached } if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[windowKey] { return cached } return nil } let key = query.lowercased() return snapshots[namespacedSnapshotKey(namespace, key)] ?? (isExplicitSnapshotNamespace(namespace) ? nil : snapshots[key]) } private func validateRequestedElements(cached: Snapshot?, current: Snapshot, params: [String: JSONValue]) throws { let requestedIndexes = try ["elementIndex", "fromElementIndex", "toElementIndex"].compactMap { key -> Int? in guard params[key]?.number != nil else { return nil } return try optionalInteger(params, key) } guard !requestedIndexes.isEmpty else { return } guard let cached else { throw ProviderError.coded("element_not_found", "element indexes require a fresh get-app-state snapshot for this app/window") } for index in requestedIndexes { guard let expected = cached.elements[index], let actual = current.elements[index] else { throw ProviderError.coded("element_not_found", "element \(index) is stale; run get-app-state again and use a fresh element index") } guard expected.signature == actual.signature else { throw ProviderError.coded("element_not_found", "element \(index) changed since the last snapshot; run get-app-state again and use a fresh element index") } } } private func ensureWindowStillAvailable(_ snapshot: Snapshot) throws { guard WindowCapture.candidates(pid: snapshot.app.pid).contains(where: { $0.windowId == snapshot.windowId }) else { throw ProviderError.coded("window_stale", "window \(Int(snapshot.windowId)) is no longer available; run get-app-state again to refresh the target window") } } private func listApps() -> [AppDescriptor] { var seen = Set() return NSWorkspace.shared.runningApplications .filter { !$0.isTerminated && $0.activationPolicy == .regular } .compactMap { app in guard let name = app.localizedName, !name.isEmpty else { return nil } let pid = app.processIdentifier guard pid > 0, pidIsLive(pid) else { return nil } let key = (app.bundleIdentifier ?? "pid:\(pid)").lowercased() guard seen.insert(key).inserted else { return nil } return AppDescriptor(name: name, bundleId: app.bundleIdentifier, pid: pid, app: app) } .sorted { lhs, rhs in if lhs.app.isActive != rhs.app.isActive { return lhs.app.isActive && !rhs.app.isActive } return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending } } private func renderListedApp(_ app: AppDescriptor) -> [String: Any] { [ "name": app.name, "bundleId": jsonNullable(app.bundleId), "pid": Int(app.pid), "isRunning": true, "lastUsedAt": NSNull(), "useCount": NSNull(), ] } private func providerHandshake() -> [String: Any] { [ "platform": "darwin", "provider": providerName, "providerVersion": providerVersion, "protocolVersion": providerProtocolVersion, "supports": [ "apps": [ "list": true, "bundleIds": true, "pids": true, ], "windows": [ "list": true, "targetById": true, "targetByIndex": true, "focus": false, "moveResize": false, ], "observation": [ "screenshot": true, "annotatedScreenshot": false, "elementFrames": true, "ocr": false, ], "actions": [ "click": true, "typeText": true, "pressKey": true, "hotkey": true, "pasteText": true, "scroll": true, "drag": true, "setValue": true, "performAction": true, ], "surfaces": [ "menus": false, "dialogs": false, "dock": false, "menubar": false, ], "input": [ "background": BackgroundInputTransport.isAvailable, ], ], ] } private func resolveApp(_ query: String) throws -> AppDescriptor { let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { throw ProviderError.coded("invalid_argument", "app query must not be empty") } if let pid = parsePid(trimmed) { if let app = appByPid(pid) { try rejectBlockedApp(app) return app } throw ProviderError.coded("app_not_found", "app '\(trimmed)' not found") } if blockedBundleIds.contains(trimmed) { throw ProviderError.coded("app_blocked", "app '\(trimmed)' is blocked for safety") } if let app = listApps().first(where: { matches($0, query: trimmed) }) { try rejectBlockedApp(app) return app } throw ProviderError.coded("app_not_found", "app '\(trimmed)' not found") } private func rejectBlockedApp(_ app: AppDescriptor) throws { if let bundle = app.bundleId, blockedBundleIds.contains(bundle) { throw ProviderError.coded("app_blocked", "app '\(bundle)' is blocked for safety") } } private func listWindows(params: [String: JSONValue]) throws -> [String: Any] { let app = try resolveApp(try requiredString(params, "app")) let windows = WindowCapture.candidates(pid: app.pid) .filter { $0.layer == 0 } .enumerated() .map { index, candidate -> [String: Any] in [ "index": index, "app": [ "name": app.name, "bundleId": jsonNullable(app.bundleId), "pid": Int(app.pid), ], "id": Int(candidate.windowId), "title": candidate.title ?? "", "x": Int(candidate.bounds.origin.x.rounded()), "y": Int(candidate.bounds.origin.y.rounded()), "width": Int(candidate.bounds.width.rounded()), "height": Int(candidate.bounds.height.rounded()), "isMinimized": false, "isOffscreen": !candidate.isOnScreen, "screenIndex": jsonNullable(screenIndex(for: candidate.bounds)), "isMain": NSNull(), "platform": [ "layer": candidate.layer, "alpha": candidate.alpha, ], ] } return [ "app": renderListedApp(app), "windows": windows, ] } private func appByPid(_ pid: pid_t) -> AppDescriptor? { guard let app = NSRunningApplication(processIdentifier: pid), !app.isTerminated, let name = app.localizedName else { return nil } return AppDescriptor(name: name, bundleId: app.bundleIdentifier, pid: pid, app: app) } private func buildSnapshot( app: AppDescriptor, includeScreenshot: Bool, windowId: CGWindowID?, windowIndex: Int?, restoreWindow: Bool, previousSnapshot: (CGWindowID) -> Snapshot? ) throws -> Snapshot { guard accessibilityTrustedSettled() else { // Why: agents retry failed observations. Only the explicit setup flow // should open macOS privacy prompts/settings; runtime calls stay quiet. throw ProviderError.coded( "permission_denied", "Accessibility permission is required for Orca Computer Use. Run `orca computer permissions` or open Settings > Computer Use, grant Accessibility to Orca Computer Use, then retry." ) } let appElement = AXUIElementCreateApplication(app.pid) var role: CFTypeRef? let probeStart = CFAbsoluteTimeGetCurrent() // Why: a hung app costs one timeout per AX read; walking it anyway held the request lock for every agent until the 60 s RPC timeout. if AccessibilityMessaging.timedOut(AXUIElementCopyAttributeValue(appElement, kAXRoleAttribute as CFString, &role), since: probeStart) { throw AccessibilityMessaging.appNotResponding(app.name) } enableManualAccessibilityIfNeeded(appElement, app: app) let windowCandidates = WindowCapture.candidates(pid: app.pid) let focused = try focusedWindow( appElement: appElement, app: app, visibleWindowCount: windowCandidates.count, allowRecovery: restoreWindow ) let focusedTitle = stringAttribute(focused, kAXTitleAttribute as String) ?? app.name let canCaptureScreenshot = includeScreenshot && screenCaptureTrustedSettled() guard let capture = WindowCapture.resolve( candidates: windowCandidates, titleHint: focusedTitle, windowId: windowId, windowIndex: windowIndex, captureImage: canCaptureScreenshot ) else { throw ProviderError.coded("window_not_found", "app '\(app.name)' has no on-screen window") } guard let window = matchingWindow(appElement: appElement, capture: capture, focused: focused, explicitTarget: windowId != nil || windowIndex != nil) else { throw ProviderError.coded("window_not_found", "could not match accessibility window to requested window; run get-app-state again or retry without a window selector") } let title = stringAttribute(window, kAXTitleAttribute as String) ?? capture.title ?? app.name let renderer = TreeRenderer( windowBounds: capture.bounds, focused: focusedElement(appElement: appElement), compactBrowserTabs: app.isKnownBrowser ) renderer.render(window) if renderer.appStoppedResponding { throw AccessibilityMessaging.appNotResponding(app.name) } let indexes = renderer.carryOverIndexes(from: previousSnapshot(capture.windowId)) let screenshot = includeScreenshot ? capture.screenshotPayload() : nil let screenshotStatus: ScreenshotStatus = if screenshot != nil { .captured } else if includeScreenshot && !canCaptureScreenshot { .failed("Screen Recording permission is required for Orca Computer Use; grant permission or pass --no-screenshot to inspect accessibility state only.") } else if includeScreenshot { .failed("window screenshot capture returned no image; retry with --no-screenshot if accessibility state is sufficient.") } else { .skipped } return Snapshot( id: UUID().uuidString, app: app, windowTitle: title, windowBounds: capture.bounds, windowId: capture.windowId, windowLayer: capture.layer, treeText: renderTreeText(app: app, title: title, bounds: capture.bounds, lines: renderer.lines, focused: renderer.focusedSummary), focusedElementId: renderer.focusedElementId, screenshot: screenshot, screenshotStatus: screenshotStatus, screenshotScale: screenshotScale(screenshot: screenshot, bounds: capture.bounds), screenshotEngine: capture.image?.engine, elements: renderer.records, elementOrder: indexes.order, nextFreshElementIndex: indexes.nextFreshIndex, truncated: renderer.truncated, maxDepthReached: renderer.maxDepthReached, appWindowIds: windowCandidates.map(\.windowId).sorted() ) } private func renderSnapshot(_ snapshot: Snapshot) -> [String: Any] { var screenshot: Any = NSNull() if let payload = snapshot.screenshot { screenshot = [ "data": payload.data, "format": "png", "width": payload.width, "height": payload.height, "scale": payload.scale, ] } return [ "snapshot": [ "id": snapshot.id, "app": [ "name": snapshot.app.name, "bundleId": jsonNullable(snapshot.app.bundleId), "pid": Int(snapshot.app.pid), ], "window": [ "id": Int(snapshot.windowId), "title": snapshot.windowTitle, "x": Int(snapshot.windowBounds.origin.x.rounded()), "y": Int(snapshot.windowBounds.origin.y.rounded()), "width": Int(snapshot.windowBounds.width.rounded()), "height": Int(snapshot.windowBounds.height.rounded()), "isMinimized": false, "isOffscreen": false, "screenIndex": jsonNullable(screenIndex(for: snapshot.windowBounds)), "platform": [ "layer": snapshot.windowLayer, ], ], "coordinateSpace": "window", "treeText": snapshot.treeText, "elementCount": snapshot.elements.count, "focusedElementId": snapshot.focusedElementId as Any, "truncation": [ "truncated": snapshot.truncated, "maxNodes": TreeRenderer.maxNodes, "maxDepth": TreeRenderer.maxDepth, "maxDepthReached": snapshot.maxDepthReached, ], ], "screenshot": screenshot, "screenshotStatus": renderScreenshotStatus(snapshot.screenshotStatus, snapshot: snapshot), ] } private func renderActionResult(action: [String: Any], snapshot: Snapshot) -> [String: Any] { var result = renderSnapshot(snapshot) var metadata = action metadata["targetWindowId"] = Int(snapshot.windowId) result["action"] = metadata return result } private func click(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentSnapshot(params: params) let button = try mouseButton(params["mouseButton"]?.string) let count = try positiveInteger(params["clickCount"]?.number, defaultValue: 1, name: "clickCount") guard count <= SyntheticMouseClickDelivery.maxClickCount else { throw ProviderError.coded( "invalid_argument", "clickCount must be at most \(SyntheticMouseClickDelivery.maxClickCount)" ) } let modifiers = try KeyMap.parseModifiers(params["modifiers"]?.string) let foreground = isForegroundRequest(params) if foreground { recoverWindow(snapshot.app, windowId: snapshot.windowId, windowBounds: snapshot.windowBounds) } if let elementIndex = try optionalInteger(params, "elementIndex") { let record = try element(snapshot, elementIndex) if !foreground { try requireBackgroundClickAllowed(backgroundClickHit(record.element, snapshot: snapshot)) } if modifiers.isEmpty, count <= 1, button.hasAccessibilityAction, let actionName = try performClickAction(record: record, mouseButton: button) { return actionMetadata(path: "accessibility", actionName: actionName) } if let point = center(record.localFrame, in: snapshot.windowBounds) { try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground, element: record.element) return actionMetadata( path: "synthetic", fallbackReason: "actionUnsupported", verification: unverifiedAction(reason: "synthetic_input") ) } throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame") } let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot) try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground) return actionMetadata( path: "synthetic", verification: unverifiedAction(reason: "synthetic_input") ) } private func performClickAction(record: ElementRecord, mouseButton: MouseButtonSelection) throws -> String? { if mouseButton == .right { return try performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil } for action in ["AXPress", "AXConfirm", "AXOpen"] { if try performAction(record.element, action) { return action } } return nil } private func performSecondaryAction(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentSnapshot(params: params) let record = try element(snapshot, try requiredInteger(params, "elementIndex")) let requested = try requiredString(params, "action") let action = record.actions.first { SnapshotRenderHeuristics.prettyAction($0).caseInsensitiveCompare(requested) == .orderedSame || $0.caseInsensitiveCompare(requested) == .orderedSame } guard let action else { throw ProviderError.coded("action_not_supported", "'\(requested)' is not a valid secondary action for element \(record.index)") } guard try performAction(record.element, action) else { throw ProviderError.coded("accessibility_error", "AXUIElementPerformAction(\(action)) failed") } return actionMetadata(path: "accessibility", actionName: action) } private func setValue(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentSnapshot(params: params) let record = try element(snapshot, try requiredInteger(params, "elementIndex")) let expected = try requiredStringAllowingEmpty(params, "value") guard isSettable(record.element, kAXValueAttribute as String) else { throw ProviderError.coded("value_not_settable", "element \(record.index) is not settable") } let current = rawAttributeValue(record.element, kAXValueAttribute as String) let coercion = AttributeValueCoercion(existingValue: current, requested: expected) let result: AXError let writeStart = CFAbsoluteTimeGetCurrent() switch coercion.writeValue { case .string: result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, expected as CFString) case let .integer(value): result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, NSNumber(value: value)) case let .double(value): result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, NSNumber(value: value)) case let .boolean(value): result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, value ? kCFBooleanTrue : kCFBooleanFalse) } if AccessibilityMessaging.timedOut(result, since: writeStart) { throw AccessibilityMessaging.unanswered("the value write") } guard result == .success else { throw ProviderError.coded("accessibility_error", "AXUIElementSetAttributeValue failed with \(result.rawValue)") } let verification: [String: Any] switch coercion.compare(readback: rawAttributeValue(record.element, kAXValueAttribute as String)) { case let .match(actualPreview): verification = verifiedAction(property: "value", expected: expected, actualPreview: actualPreview) case let .mismatch(actualPreview): verification = unverifiedAction(reason: "value_mismatch", expected: expected, actualPreview: actualPreview) case .unsupported: verification = unverifiedAction(reason: "readback_unsupported", expected: expected) } return actionMetadata(path: "accessibility", actionName: "AXSetValue", verification: verification) } private func typeText(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) let text = try requiredString(params, "text") if let focused = focusedRecord(snapshot), let verification = try TextInput.replaceSelection(focused.element, with: text) { return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification) } try deliverKeys(try withTransportErrors { try BackgroundInputTransport.textEvents(text) }, snapshot: snapshot, params: params) return actionMetadata( path: "synthetic", actionName: "typeText", verification: unverifiedAction(reason: "synthetic_input") ) } private func pressKey(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) try deliverKeys(try Input.keyEvents(try requiredString(params, "key")), snapshot: snapshot, params: params) return actionMetadata( path: "synthetic", actionName: "pressKey", verification: unverifiedAction(reason: "synthetic_input") ) } private func hotkey(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) let key = try requiredString(params, "key") if isSelectAllHotkey(key), let focused = focusedRecord(snapshot), TextInput.selectAll(focused.element) { return actionMetadata( path: "accessibility", actionName: "AXSelectAll", verification: TextInput.selectionVerification(focused.element) ) } try deliverKeys(try Input.keyEvents(key), snapshot: snapshot, params: params) return actionMetadata( path: "synthetic", actionName: "hotkey", verification: unverifiedAction(reason: "synthetic_input") ) } private func pasteText(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) let text = try requiredString(params, "text") if let focused = focusedRecord(snapshot), let verification = try TextInput.replaceSelection(focused.element, with: text) { return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification) } let pasteKeys = try Input.keyEvents("cmd+v") try Input.withPasteboardText(text) { try deliverKeys(pasteKeys, snapshot: snapshot, params: params) } return actionMetadata( path: "clipboard", actionName: "paste", verification: unverifiedAction(reason: "clipboard_paste") ) } private func scroll(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentSnapshot(params: params) let direction = try scrollDirection(try requiredString(params, "direction")) let pages = try positiveNumber(params["pages"]?.number, defaultValue: 1, name: "pages") if let elementIndex = try optionalInteger(params, "elementIndex") { let record = try element(snapshot, elementIndex) let action = "AXScroll\(direction.capitalized)ByPage" if pages.rounded() == pages, let pageCount = boundedInteger(pages, as: Int.self), record.actions.contains(action) { for _ in 0.. [String: Any] { let snapshot = try currentSnapshot(params: params) let start: CGPoint let end: CGPoint if let fromIndex = try optionalInteger(params, "fromElementIndex"), let toIndex = try optionalInteger(params, "toElementIndex") { let from = try element(snapshot, fromIndex) let to = try element(snapshot, toIndex) guard let fromPoint = center(from.localFrame, in: snapshot.windowBounds), let toPoint = center(to.localFrame, in: snapshot.windowBounds) else { throw ProviderError.coded("element_not_found", "drag element has no frame") } start = fromPoint end = toPoint } else { start = try coordinatePoint(params: params, xKey: "fromX", yKey: "fromY", snapshot: snapshot) end = try coordinatePoint(params: params, xKey: "toX", yKey: "toY", snapshot: snapshot) } if isForegroundRequest(params) { try requireTargetWindowFocused(snapshot, restoreWindowRequested: true) try Input.drag(from: start, to: end, targetWindow: snapshot) } else { try withTransportErrors { try BackgroundInputTransport.drag(backgroundTarget(snapshot), from: start, to: end) } } return actionMetadata(path: "synthetic") } private func element(_ snapshot: Snapshot, _ index: Int) throws -> ElementRecord { guard let record = snapshot.elements[index] else { throw ProviderError.coded("element_not_found", "element \(index) is not in the current cached snapshot for \(snapshot.app.name); run get-app-state again and use a fresh element index") } return record } private func focusedRecord(_ snapshot: Snapshot) -> ElementRecord? { guard let focusedElementId = snapshot.focusedElementId else { return nil } return snapshot.elements[focusedElementId] } } private let blockedBundleIds: Set = [ "com.1password.1password", "com.1password.safari", "com.bitwarden.desktop", "com.dashlane.dashlanephonefinal", "com.lastpass.LastPass", "com.nordsec.nordpass", "me.proton.pass.electron", "me.proton.pass.catalyst", ] private func requiredString(_ params: [String: JSONValue], _ key: String) throws -> String { guard let value = params[key]?.string, !value.isEmpty else { throw ProviderError.coded("invalid_argument", "missing \(key)") } return value } private func requiredStringAllowingEmpty(_ params: [String: JSONValue], _ key: String) throws -> String { guard let value = params[key]?.string else { throw ProviderError.coded("invalid_argument", "missing \(key)") } return value } private func requiredNumber(_ params: [String: JSONValue], _ key: String) throws -> Double { guard let value = params[key]?.number, value.isFinite else { throw ProviderError.coded("invalid_argument", "missing \(key)") } return value } private func requiredInteger(_ params: [String: JSONValue], _ key: String) throws -> Int { guard let value = boundedInteger(try requiredNumber(params, key), as: Int.self) else { throw ProviderError.coded("invalid_argument", "\(key) is out of range") } return value } private func optionalInteger(_ params: [String: JSONValue], _ key: String) throws -> Int? { guard let raw = params[key]?.number else { return nil } guard let value = boundedInteger(raw, as: Int.self) else { throw ProviderError.coded("invalid_argument", "\(key) is out of range") } return value } private func positiveInteger(_ value: Double?, defaultValue: Int, name: String) throws -> Int { switch ActionArgumentValidation.positiveInteger(value, defaultValue: defaultValue, name: name) { case let .success(value): return value case let .failure(error): throw ProviderError.coded("invalid_argument", error.message) } } private func positiveNumber(_ value: Double?, defaultValue: Double, name: String) throws -> Double { switch ActionArgumentValidation.positiveNumber(value, defaultValue: defaultValue, name: name) { case let .success(value): return value case let .failure(error): throw ProviderError.coded("invalid_argument", error.message) } } private func scrollDirection(_ value: String) throws -> String { switch ActionArgumentValidation.scrollDirection(value) { case let .success(value): return value case let .failure(error): throw ProviderError.coded("invalid_argument", error.message) } } private func parsePid(_ query: String) -> pid_t? { guard query.hasPrefix("pid:") else { return nil } guard let pid = Int32(query.dropFirst(4)), pid > 0 else { return nil } return pid } private func matches(_ app: AppDescriptor, query: String) -> Bool { app.name.caseInsensitiveCompare(query) == .orderedSame || app.bundleId?.caseInsensitiveCompare(query) == .orderedSame } private func pidIsLive(_ pid: pid_t) -> Bool { kill(pid, 0) == 0 } private func accessibilityTrusted() -> Bool { AXIsProcessTrusted() } private func accessibilityTrustedSettled() -> Bool { // Fresh helper processes can receive transient TCC preflight denials before the real grant settles. PermissionTrustSettling.settle(probe: accessibilityTrusted).settled } private func screenCaptureTrusted() -> Bool { CGPreflightScreenCaptureAccess() } private func screenCaptureTrustedSettled() -> Bool { PermissionTrustSettling.settle(timeoutMs: 2_000, probe: screenCaptureTrusted).settled } private func permissionStatusSnapshotSettled() -> PermissionStatusSnapshot { PermissionStatusSnapshotProbe.capture( accessibilityProbe: accessibilityTrustedSettled, screenshotsProbe: screenCaptureTrustedSettled ) } private func requestScreenCaptureAccess() -> Bool { CGRequestScreenCaptureAccess() } private func openAccessibilitySettings() { openSystemSettings("x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility") } private func openScreenRecordingSettings() { openSystemSettings("x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture") } private func openSystemSettings(_ value: String) { guard let url = URL(string: value) else { return } NSWorkspace.shared.open(url) } private func enableManualAccessibilityIfNeeded(_ appElement: AXUIElement, app: AppDescriptor) { guard app.needsManualAccessibilityMode else { return } _ = AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue) _ = AXUIElementSetAttributeValue(appElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) } private func focusedWindow(appElement: AXUIElement, app: AppDescriptor, visibleWindowCount: Int, allowRecovery: Bool) throws -> AXUIElement { let systemWide = AXUIElementCreateSystemWide() if let window = lookupUsableWindow(systemWide: systemWide, appElement: appElement, app: app) { return window } if allowRecovery { recoverWindow(app) if let window = lookupUsableWindow(systemWide: systemWide, appElement: appElement, app: app) { return window } } if visibleWindowCount > 0 { var settledWindow: AXUIElement? let outcome = PermissionTrustSettling.settle { settledWindow = lookupUsableWindow( systemWide: systemWide, appElement: appElement, app: app ) return settledWindow != nil } if let window = settledWindow, outcome.settled { return window } throw ProviderError.coded("permission_denied", "app '\(app.name)' has visible windows but no accessibility window (AX reads stayed blocked for \(outcome.waitedMs)ms after retries). macOS Accessibility may need Orca Computer Use toggled off and on again in System Settings.") } throw ProviderError.coded("window_not_found", "app '\(app.name)' has no accessibility window; make sure the app has a visible window, then retry with --restore-window.") } private func lookupUsableWindow( systemWide: AXUIElement, appElement: AXUIElement, app: AppDescriptor ) -> AXUIElement? { if let window = focusedSystemWindow(systemWide: systemWide, app: app) { return window } if let window = copyElement(appElement, kAXFocusedWindowAttribute as String), usableWindow(window) { return window } if let windows = copyArray(appElement, kAXWindowsAttribute as String) { return windows.first(where: usableWindow) } return nil } private func focusedSystemWindow(systemWide: AXUIElement, app: AppDescriptor) -> AXUIElement? { guard let focusedApp = copyElement(systemWide, kAXFocusedApplicationAttribute as String), pidAttribute(focusedApp) == app.pid else { return nil } if let window = copyElement(systemWide, kAXFocusedWindowAttribute as String), usableWindow(window) { return window } if let window = copyElement(focusedApp, kAXFocusedWindowAttribute as String), usableWindow(window) { return window } if let windows = copyArray(focusedApp, kAXWindowsAttribute as String) { return windows.first(where: usableWindow) } return nil } private func isTargetWindowFocused(_ snapshot: Snapshot) -> Bool { focusedSyntheticClickRecipient( targetPID: snapshot.app.pid, targetCandidates: { WindowCapture.candidates(pid: snapshot.app.pid) } ) == .focused(syntheticClickRecipient(pid: snapshot.app.pid, windowId: snapshot.windowId)) } private enum AXElementProbe { case value(AXUIElement) case absent case unavailable } private func copyElementProbe(_ element: AXUIElement, _ attribute: String) -> AXElementProbe { var value: CFTypeRef? switch AXUIElementCopyAttributeValue(element, attribute as CFString, &value) { case .success: guard let value else { return .unavailable } return .value(value as! AXUIElement) case .noValue: return .absent default: return .unavailable } } private func currentSyntheticClickRecipient( snapshot: Snapshot, point: CGPoint ) -> SyntheticMouseClickDelivery.RecipientObservation { let target = syntheticClickRecipient(pid: snapshot.app.pid, windowId: snapshot.windowId) var cachedTargetCandidates: [WindowCandidate]? func targetCandidates() -> [WindowCandidate] { if let cachedTargetCandidates { return cachedTargetCandidates } let candidates = WindowCapture.candidates(pid: snapshot.app.pid) cachedTargetCandidates = candidates return candidates } switch focusedSyntheticClickRecipient( targetPID: snapshot.app.pid, targetCandidates: targetCandidates ) { case let .focused(focused): guard focused == target else { return .focused(focused) } switch hitTestSyntheticClickRecipient( at: point, targetPID: snapshot.app.pid, targetCandidates: targetCandidates ) { case let .focused(recipient): return .focused(recipient) case .dismissed, .unavailable: return .unavailable } case .dismissed: return .dismissed case .unavailable: return .unavailable } } private func focusedSyntheticClickRecipient( targetPID: pid_t, targetCandidates: () -> [WindowCandidate] ) -> SyntheticMouseClickDelivery.RecipientObservation { let systemWide = AXUIElementCreateSystemWide() let focusedApp: AXUIElement switch copyElementProbe(systemWide, kAXFocusedApplicationAttribute as String) { case let .value(value): focusedApp = value case .absent: return .dismissed case .unavailable: return .unavailable } guard let ownerPID = pidAttribute(focusedApp) else { return .unavailable } let focusedWindow: AXUIElement switch copyElementProbe(focusedApp, kAXFocusedWindowAttribute as String) { case let .value(value): focusedWindow = value case .absent: guard ownerPID == targetPID else { return .unavailable } return .dismissed case .unavailable: return .unavailable } guard pidAttribute(focusedWindow) == ownerPID else { return .unavailable } return pointerRecipient(of: focusedWindow, targetPID: targetPID, targetCandidates: targetCandidates) } private func hitTestSyntheticClickRecipient( at point: CGPoint, targetPID: pid_t, targetCandidates: () -> [WindowCandidate] ) -> SyntheticMouseClickDelivery.RecipientObservation { guard Float(point.x).isFinite, Float(point.y).isFinite else { return .unavailable } let systemWide = AXUIElementCreateSystemWide() var hitElement: AXUIElement? switch AXUIElementCopyElementAtPosition( systemWide, Float(point.x), Float(point.y), &hitElement ) { case .success: break case .noValue: return .dismissed default: return .unavailable } guard let hitElement else { return .unavailable } return pointerRecipient(of: hitElement, targetPID: targetPID, targetCandidates: targetCandidates) } private func pointerRecipient( of element: AXUIElement, targetPID: pid_t, targetCandidates: () -> [WindowCandidate] ) -> SyntheticMouseClickDelivery.RecipientObservation { guard let ownerPID = pidAttribute(element), let window = containingWindow(element) else { return .unavailable } do { if let windowID = try AccessibilityWindowIdentity.windowID(of: window) { return .focused(syntheticClickRecipient(pid: ownerPID, windowId: windowID)) } } catch { return .unavailable } guard ownerPID == targetPID, let frame = absoluteFrame(window), let windows = copyArray(AXUIElementCreateApplication(ownerPID), kAXWindowsAttribute as String) else { return .unavailable } var matches: [AXUIElement] = [] for candidate in windows { guard let candidateFrame = absoluteFrame(candidate) else { return .unavailable } if windowFramesMatch(candidateFrame, frame) { matches.append(candidate) } } guard matches.count == 1, CFEqual(matches[0], window), let candidate = SyntheticMouseClickDelivery.uniqueWindowCandidate( from: targetCandidates(), matching: { windowFramesMatch($0.bounds, frame) } ) else { return .unavailable } return .focused(syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId)) } private func containingWindow(_ element: AXUIElement) -> AXUIElement? { var current = element for _ in 0..<64 { if stringAttribute(current, kAXRoleAttribute as String) == kAXWindowRole as String { return current } if let window = copyElement(current, kAXWindowAttribute as String) { return window } guard let parent = copyElement(current, kAXParentAttribute as String) else { return nil } current = parent } return nil } private func syntheticClickRecipient( pid: pid_t, windowId: CGWindowID ) -> SyntheticMouseClickDelivery.Recipient { SyntheticMouseClickDelivery.Recipient(ownerPID: pid, windowID: windowId) } private func requireTargetWindowFocused(_ snapshot: Snapshot, restoreWindowRequested: Bool) throws { guard let failure = KeyboardInputSafety.syntheticInputFocusFailure( targetWindowFocused: isTargetWindowFocused(snapshot), restoreWindowRequested: restoreWindowRequested ) else { return } switch failure { case .targetNotFocused: throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; retry with --restore-window or use set-value for editable elements") case .targetNotFocusedAfterRestore: throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; --restore-window was requested but the target is still not focused; bring it forward manually or check Accessibility permissions") } } private func isForegroundRequest(_ params: [String: JSONValue]) -> Bool { params["restoreWindow"]?.bool == true } private func backgroundTarget(_ snapshot: Snapshot) -> BackgroundInputTransport.Target { BackgroundInputTransport.Target(pid: snapshot.app.pid, windowID: snapshot.windowId, windowFrame: snapshot.windowBounds) } private func withTransportErrors(_ body: () throws -> T) throws -> T { do { return try body() } catch let failure as BackgroundInputTransport.Failure { switch failure { case let .unavailable(missingSymbols): throw ProviderError.coded("background_unavailable", "background input is unavailable on this macOS version (missing \(missingSymbols.joined(separator: ", "))); retry with --restore-window to bring the app forward and use foreground input") case .processNotFound: throw ProviderError.coded("app_not_found", "the target app quit before input was delivered") case .eventCreationFailed: throw ProviderError.coded("accessibility_error", "failed to create input event") case .targetWindowNotKey: throw ProviderError.coded("background_unavailable", "background input cannot change the key window of the active app; retry with --restore-window to target this window explicitly") case let .focusRecordFailed(status): throw ProviderError.coded("window_not_focused", "background key-window focus record failed (status \(status)); input may already have been delivered, so inspect current state before retrying") } } } private func clickPointer( at point: CGPoint, button: MouseButtonSelection, count: Int, modifiers: [KeyModifier], snapshot: Snapshot, foreground: Bool, element: AXUIElement? = nil ) throws { if foreground { try Input.click(at: point, button: button, count: count, modifiers: modifiers, targetWindow: snapshot) return } let hitElement: AXUIElement if let element { hitElement = element } else { var found: AXUIElement? guard Float(point.x).isFinite, Float(point.y).isFinite, AXUIElementCopyElementAtPosition(AXUIElementCreateApplication(snapshot.app.pid), Float(point.x), Float(point.y), &found) == .success, let found else { throw ProviderError.coded("background_unavailable", "cannot identify the background click target; run get-app-state again or retry with --restore-window") } hitElement = found } let hit = try backgroundClickHit(hitElement, snapshot: snapshot) try requireBackgroundClickAllowed(hit) try withTransportErrors { try BackgroundInputTransport.click( backgroundTarget(snapshot), at: point, button: button, clickCount: count, flags: eventFlags(modifiers), hit: hit ) } } private func backgroundClickHit(_ element: AXUIElement, snapshot: Snapshot) throws -> BackgroundInputTransport.PointerHit { let recipient = pointerRecipient(of: element, targetPID: snapshot.app.pid) { WindowCapture.candidates(pid: snapshot.app.pid) } guard recipient == .focused(syntheticClickRecipient(pid: snapshot.app.pid, windowId: snapshot.windowId)) else { throw ProviderError.coded("background_unavailable", "the background click target does not identify the requested window; run get-app-state again or retry with --restore-window") } return BackgroundInputTransport.classifyPointerHit(element) } private func requireBackgroundClickAllowed(_ hit: BackgroundInputTransport.PointerHit) throws { guard hit != .popupMenu else { throw ProviderError.coded("background_unavailable", "a background click on a popup menu or web