import AppKit import ApplicationServices import CoreGraphics import Darwin import Foundation import ImageIO import OrcaComputerUseMacOSCore import ScreenCaptureKit private let providerName = "orca-computer-use-macos" private let providerVersion = "1.0.0" private let providerProtocolVersion = 1 struct Request: Decodable { let id: Int let method: String let params: [String: JSONValue]? let token: String? } enum JSONValue: Decodable { case string(String) case number(Double) case bool(Bool) case object([String: JSONValue]) case array([JSONValue]) case null init(from decoder: Decoder) throws { let container = try decoder.singleValueContainer() if container.decodeNil() { self = .null } else if let value = try? container.decode(Bool.self) { self = .bool(value) } else if let value = try? container.decode(Double.self) { self = .number(value) } else if let value = try? container.decode(String.self) { self = .string(value) } else if let value = try? container.decode([JSONValue].self) { self = .array(value) } else { self = .object(try container.decode([String: JSONValue].self)) } } var string: String? { if case let .string(value) = self { return value } return nil } var number: Double? { if case let .number(value) = self { return value } return nil } var bool: Bool? { if case let .bool(value) = self { return value } return nil } } enum ProviderError: Error { case coded(String, String) var code: String { switch self { case let .coded(code, _): return code } } var message: String { switch self { case let .coded(_, message): return message } } } struct AppDescriptor { let name: String let bundleId: String? let pid: pid_t let app: NSRunningApplication var needsManualAccessibilityMode: Bool { // Chromium/Electron apps often need this private AX mode, but applying it // broadly can corrupt native Cocoa app trees into app-root-only nodes. guard let bundleId = bundleId?.lowercased() else { return false } return bundleId.hasPrefix("com.google.chrome") || bundleId.hasPrefix("com.microsoft.edgemac") || bundleId.hasPrefix("com.brave.browser") || bundleId.hasPrefix("com.operasoftware.opera") || bundleId.hasPrefix("com.vivaldi.vivaldi") || bundleId == "com.github.electron" || bundleId == "com.tinyspeck.slackmacgap" || bundleId == "com.spotify.client" || bundleId == "com.hnc.discord" || bundleId == "com.microsoft.teams2" || bundleId == "notion.id" } var isKnownBrowser: Bool { let bundle = bundleId?.lowercased() ?? "" let appName = name.lowercased() return bundle == "com.apple.safari" || bundle == "org.mozilla.firefox" || bundle == "company.thebrowser.browser" || bundle == "app.zen-browser.zen" || bundle.hasPrefix("com.google.chrome") || bundle.hasPrefix("com.microsoft.edgemac") || bundle.hasPrefix("com.brave.browser") || bundle.hasPrefix("com.operasoftware.opera") || bundle.hasPrefix("com.vivaldi.vivaldi") || appName == "safari" || appName == "firefox" || appName == "arc" || appName == "zen" || appName.contains("chrome") || appName.contains("chromium") || appName.contains("edge") || appName.contains("brave") || appName.contains("opera") || appName.contains("vivaldi") } } final class ElementRecord { let index: Int let element: AXUIElement let localFrame: CGRect? let actions: [String] let signature: String init(index: Int, element: AXUIElement, localFrame: CGRect?, actions: [String], signature: String) { self.index = index self.element = element self.localFrame = localFrame self.actions = actions self.signature = signature } } /// A snapshot that read one element's subtree; its elements also keep the window's earlier ones. struct SnapshotScope { let rootIndex: Int let walkedIndexes: Set } struct Snapshot { let id: String let app: AppDescriptor let windowTitle: String let windowBounds: CGRect let windowId: CGWindowID let windowLayer: Int let windowVisibility: WindowVisibility let treeText: String let focusedElementId: Int? let screenshot: ScreenshotPayload? let screenshotStatus: ScreenshotStatus let screenshotScale: CGSize let screenshotEngine: String? let elements: [Int: ElementRecord] let elementOrder: [Int] let nextFreshElementIndex: Int let truncated: Bool let maxDepthReached: Bool let scope: SnapshotScope? /// On-screen windows of the app (menus and sheets included), so a new or closed window counts as a change. let appWindowIds: [CGWindowID] func showsSameState(as other: Snapshot) -> Bool { treeText == other.treeText && windowBounds == other.windowBounds && appWindowIds == other.appWindowIds && windowVisibility == other.windowVisibility } func withoutScreenshotPayload() -> Snapshot { Snapshot( id: id, app: app, windowTitle: windowTitle, windowBounds: windowBounds, windowId: windowId, windowLayer: windowLayer, windowVisibility: windowVisibility, treeText: treeText, focusedElementId: focusedElementId, screenshot: nil, screenshotStatus: .skipped, screenshotScale: CGSize(width: 1, height: 1), screenshotEngine: nil, elements: elements, elementOrder: elementOrder, nextFreshElementIndex: nextFreshElementIndex, truncated: truncated, maxDepthReached: maxDepthReached, scope: scope, appWindowIds: appWindowIds ) } } struct ScreenshotPayload { let data: String let width: Int let height: Int let scale: Double } struct CapturedImage { let image: CGImage let engine: String } /// Where an observed window is; only an on-screen one can take background pointer or key input. enum WindowVisibility { case onScreen case minimized case appHidden case otherSpace var reason: String { switch self { case .onScreen: "is on screen" case .minimized: "is minimized" case .appHidden: "belongs to a hidden app" case .otherSpace: "is on another Space" } } } /// A window an agent can observe: on screen, or off screen with an accessibility element it can still read. private struct ObservableWindow { let candidate: WindowCandidate /// Off-screen windows carry the element found for them; on-screen ones are matched later. let element: AXUIElement? let visibility: WindowVisibility } enum ScreenshotStatus { case captured case skipped case failed(String) } private struct CachedSnapshotEntry { let snapshotId: String let keys: [String] let createdAt: Date } private struct OtherSpaceLookups { var elements: [CGWindowID: AXUIElement] = [:] var misses: [CGWindowID: Date] = [:] } /// The app's observable windows, plus other-Space windows that exist but accessibility can't read yet. private struct WindowInventory { let windows: [ObservableWindow] let unreadableOtherSpace: Set /// Every window the app has, closed-but-kept ones included. let windowIds: Set } final class Provider { private var snapshots: [String: Snapshot] = [:] private var snapshotEntries: [CachedSnapshotEntry] = [] /// Per app, window elements already read (valid while the window lives) for when a window moves to another Space, /// and when a remote-token lookup last missed, since a window becomes findable only once some client reads it. private var otherSpaceLookups: [pid_t: OtherSpaceLookups] = [:] init() { // Why: the system-wide element sets the wait for every AX call in this process, so a hung app costs one bounded timeout whatever the macOS default. _ = AXUIElementSetMessagingTimeout(AXUIElementCreateSystemWide(), AccessibilityMessaging.timeoutSeconds) } func handle(method: String, params: [String: JSONValue]) throws -> Any { switch method { case "handshake": return providerHandshake() case "listApps": return ["apps": listApps().map(renderListedApp)] case "launchApp": return try launchApp(params: params) case "listWindows": return try listWindows(params: params) case "getAppState": let params = isForegroundRequest(params) ? try recoverRequestedWindow(params: params) : params return try renderSnapshot(observe(params: params)) case "click": return try actionResult(params: params) { try click(params: $0) } case "performSecondaryAction": return try actionResult(params: params) { try performSecondaryAction(params: $0) } case "setValue": return try actionResult(params: params) { try setValue(params: $0) } case "typeText": return try actionResult(params: params) { try typeText(params: $0) } case "pressKey": return try actionResult(params: params) { try pressKey(params: $0) } case "hotkey": return try actionResult(params: params) { try hotkey(params: $0) } case "pasteText": return try actionResult(params: params) { try pasteText(params: $0) } case "scroll": return try actionResult(params: params) { try scroll(params: $0) } case "drag": return try actionResult(params: params) { try drag(params: $0) } default: throw ProviderError.coded("invalid_argument", "unknown method '\(method)'") } } private func actionResult(params requestParams: [String: JSONValue], action runAction: ([String: JSONValue]) throws -> [String: Any]) throws -> [String: Any] { let app = try resolveApp(try requiredString(requestParams, "app")) // Why: the main run loop keeps NSRunningApplication current while requests run on connection threads. let frontBefore = ForegroundDelivery.Observation( frontmost: app.app.isActive, focusedWindowID: app.app.isActive ? focusedWindowId(of: app) : nil ) // Why: raise once, before the action's reads, so the pre and post observations never raise again. let params = isForegroundRequest(requestParams) ? try recoverRequestedWindow(params: requestParams) : requestParams var action = try runAction(params) // Why: the action's currentSnapshot just cached this target's tree from before the action; // a subtree read can't be compared with the whole-window read that follows. let before = action["path"] as? String == "accessibility" && action["verification"] == nil ? (try? cachedSnapshot(params: params)).flatMap { $0.scope == nil ? $0 : nil } : nil let snapshot: Snapshot do { let after = try observe(params: params) if let before, after.showsSameState(as: before) { if let changed = try observeLateChange(before: before, params: params) { snapshot = changed } else { // Why: an AX call can succeed and do nothing, as AXPress does on many web controls (#31758). action["verification"] = unverifiedAction(reason: "suspected_noop") snapshot = after } } else { snapshot = after } } catch let error as ProviderError where (error.code == "window_not_found" || error.code == "window_stale") && hasRequestedWindowSelector(params) { var fallbackParams = params fallbackParams.removeValue(forKey: "windowId") fallbackParams.removeValue(forKey: "windowIndex") if action["verification"] == nil { action["verification"] = ["state": "unverified", "reason": "window_changed"] } snapshot = try observe(params: fallbackParams) } // Why: measured after the settle reads, since apps that activate themselves on AXPress (Electron, sheet openers) do it a turn later. let frontAfter = frontmostObservation(of: app) let tookForeground = ForegroundDelivery.tookForeground( before: frontBefore, after: frontAfter, targetWindowID: snapshot.windowId, restoreRequested: isForegroundRequest(params) ) action["delivery"] = tookForeground ? "foreground" : "background" return renderActionResult(action: action, snapshot: snapshot) } /// Re-reads the target for about a second, since apps often apply an action on a later run-loop turn. /// Returns the observation to report once something changed, or nil if nothing did. private func observeLateChange(before: Snapshot, params: [String: JSONValue]) throws -> Snapshot? { let quietParams = params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement } let deadline = Date().addingTimeInterval(1) while Date() < deadline { Thread.sleep(forTimeInterval: 0.2) let polled = try observe(params: quietParams) if !polled.showsSameState(as: before) { if params["noScreenshot"]?.bool == true { return polled } return try observe(params: params) } } return nil } /// Brings the requested window forward for --restore-window, once per request, and returns the params pinned /// to that window: raising reorders the window list, so a --window-index could name another window afterwards. private func recoverRequestedWindow(params: [String: JSONValue]) throws -> [String: JSONValue] { let app = try resolveApp(try requiredString(params, "app")) // Why: the window-index cache keys are not per app, so another app's snapshot can sit under the same index. let cached = try cachedSnapshot(params: params).flatMap { $0.app.pid == app.pid ? $0 : nil } // Why: a stale index fails after the raise, which would take over the user's screen for nothing. try requireCachedElements(cached: cached, requestedIndexes: try requestedElementIndexes(params)) let windows = windowInventory(app: app, appElement: AXUIElementCreateApplication(app.pid)).windows let byIndex = try requestedWindowIndex(params).flatMap { index in windows.filter { $0.candidate.layer == 0 }.dropFirst(index).first?.candidate.windowId } // Why: the cached snapshot is the window the agent's element indexes were read from, even if the index now names another. let windowId = try requestedWindowId(params) ?? cached?.windowId ?? byIndex let target = windowId.flatMap { id in windows.first { $0.candidate.windowId == id } } if let requested = try requestedWindowId(params), target == nil { throw ProviderError.coded("window_not_found", "window \(Int(requested)) of \(app.name) is closed or cannot be read, so nothing was raised; run list-windows and pick a listed window") } recoverWindow(app, windowId: windowId, windowBounds: target?.candidate.bounds ?? cached?.windowBounds, fallback: target?.element) guard let windowId else { return params } // Why: the raise reorders windows and focus, so later reads must not pick the target again; the id wins over // the index when both are set, and the index still keys the snapshot cache. var pinned = params pinned["windowId"] = .number(Double(windowId)) return pinned } /// On-screen windows, plus minimized, hidden and other-Space windows that accessibility can still read. private func windowInventory(app: AppDescriptor, appElement: AXUIElement) -> WindowInventory { let all = WindowCapture.candidates(pid: app.pid, includeOffScreen: true) let offScreen = all.filter { !$0.isOnScreen && $0.layer == 0 } var elements: [CGWindowID: AXUIElement] = [:] var unreadable = Set() if !offScreen.isEmpty { var listed: CFTypeRef? let status = AXUIElementCopyAttributeValue(appElement, kAXWindowsAttribute as CFString, &listed) for window in (listed as? [AXUIElement]) ?? [] { if let id = windowNumber(window) { elements[id] = window } } let unlisted = offScreen.map(\.windowId).filter { elements[$0] == nil } // Why: closed windows the app keeps belong to no Space; without SkyLight, try them all. let inSpaces = WindowSpaces.windowsInAnySpace(unlisted) ?? Set(unlisted) // Why: kAXWindowsAttribute leaves out windows on other Spaces; skip the lookup when the app stopped answering. if status != .cannotComplete { let live = Set(all.map(\.windowId)) elements.merge(otherSpaceWindows(pid: app.pid, windowIds: inSpaces, live: live)) { listed, _ in listed } } unreadable = inSpaces.subtracting(elements.keys) } var appHidden: Bool? let windows = all.compactMap { candidate -> ObservableWindow? in if candidate.isOnScreen { return ObservableWindow(candidate: candidate, element: nil, visibility: .onScreen) } guard candidate.layer == 0, let element = elements[candidate.windowId] else { return nil } let visibility: WindowVisibility if boolAttribute(element, kAXMinimizedAttribute as String) == true { visibility = .minimized } else { if appHidden == nil { appHidden = boolAttribute(appElement, kAXHiddenAttribute as String) == true } visibility = appHidden == true ? .appHidden : .otherSpace } return ObservableWindow(candidate: candidate, element: element, visibility: visibility) } return WindowInventory(windows: windows, unreadableOtherSpace: unreadable, windowIds: Set(all.map(\.windowId))) } private func otherSpaceWindows(pid: pid_t, windowIds: Set, live: Set) -> [CGWindowID: AXUIElement] { var lookups = otherSpaceLookups[pid] ?? OtherSpaceLookups() lookups.elements = lookups.elements.filter { live.contains($0.key) && windowNumber($0.value) == $0.key } lookups.misses = lookups.misses.filter { live.contains($0.key) } let wanted = windowIds.filter { id in lookups.elements[id] == nil && lookups.misses[id].map { Date().timeIntervalSince($0) >= 10 } ?? true } if !wanted.isEmpty { let found = RemoteAccessibilityWindows.windows(pid: pid, wanted: wanted) for id in wanted { if let element = found[id] { lookups.elements[id] = element } else { lookups.misses[id] = Date() } } } otherSpaceLookups = otherSpaceLookups.filter { pidIsLive($0.key) } otherSpaceLookups[pid] = lookups return lookups.elements.filter { windowIds.contains($0.key) } } private func observe(params: [String: JSONValue]) throws -> Snapshot { let query = try requiredString(params, "app") let windowId = try requestedWindowId(params) let windowIndex = try requestedWindowIndex(params) let app = try resolveApp(query) let namespace = snapshotNamespace(params) pruneSnapshotCache() let subtree = try requestedSubtree(params: params, app: app) let snapshot = try buildSnapshot( app: app, includeScreenshot: params["noScreenshot"]?.bool != true, windowId: subtree.map(\.base.windowId) ?? windowId, windowIndex: subtree == nil ? windowIndex : nil, subtree: subtree, previousSnapshot: { [snapshots] resolvedWindowId in // Why: indexes carry over from this caller's own last view of exactly this window. snapshots[namespacedSnapshotKey(namespace, snapshotCanonicalWindowIdKey(resolvedWindowId))] .flatMap { $0.app.pid == app.pid ? $0 : nil } } ) // Why: cached snapshots only validate element identity for follow-up // actions; retaining MB-scale screenshot base64 in the long-lived agent grows memory. rememberSnapshot( query: query, app: app, snapshot: snapshot.withoutScreenshotPayload(), params: params, windowIndex: windowIndex ) return snapshot } /// Resolves `rootElementIndex` against this caller's last snapshot of the window, whose indexes it extends. private func requestedSubtree(params: [String: JSONValue], app: AppDescriptor) throws -> (root: ElementRecord, base: Snapshot)? { guard let rootIndex = try optionalInteger(params, "rootElementIndex") else { return nil } guard let base = try cachedSnapshot(params: params), base.app.pid == app.pid else { throw ProviderError.coded("element_not_found", "--element-index needs a get-app-state snapshot of this app/window first") } return (try element(base, rootIndex), base) } private func rememberSnapshot( query: String, app: AppDescriptor, snapshot cachedSnapshot: Snapshot, params: [String: JSONValue], windowIndex: Int? ) { let keys = [query, app.name, app.bundleId ?? "", "pid:\(app.pid)"] .filter { !$0.isEmpty } .map { $0.lowercased() } let namespace = snapshotNamespace(params) var storedKeys: [String] = [] let canonicalWindowKey = snapshotCanonicalWindowIdKey(cachedSnapshot.windowId) if !isExplicitSnapshotNamespace(namespace) { snapshots[canonicalWindowKey.lowercased()] = cachedSnapshot storedKeys.append(canonicalWindowKey.lowercased()) } snapshots[namespacedSnapshotKey(namespace, canonicalWindowKey)] = cachedSnapshot storedKeys.append(namespacedSnapshotKey(namespace, canonicalWindowKey)) if let windowIndex { let canonicalWindowIndexKey = snapshotCanonicalWindowIndexKey(windowIndex) if !isExplicitSnapshotNamespace(namespace) { snapshots[canonicalWindowIndexKey.lowercased()] = cachedSnapshot storedKeys.append(canonicalWindowIndexKey.lowercased()) } snapshots[namespacedSnapshotKey(namespace, canonicalWindowIndexKey)] = cachedSnapshot storedKeys.append(namespacedSnapshotKey(namespace, canonicalWindowIndexKey)) } for key in keys { if !isExplicitSnapshotNamespace(namespace) { snapshots[key] = cachedSnapshot storedKeys.append(key) snapshots[snapshotWindowKey(key, cachedSnapshot.windowId)] = cachedSnapshot storedKeys.append(snapshotWindowKey(key, cachedSnapshot.windowId)) if let windowIndex { snapshots[snapshotWindowIndexKey(key, windowIndex)] = cachedSnapshot storedKeys.append(snapshotWindowIndexKey(key, windowIndex)) } } snapshots[namespacedSnapshotKey(namespace, key)] = cachedSnapshot storedKeys.append(namespacedSnapshotKey(namespace, key)) let namespacedWindowKey = namespacedSnapshotKey( namespace, snapshotWindowKey(key, cachedSnapshot.windowId) ) snapshots[namespacedWindowKey] = cachedSnapshot storedKeys.append(namespacedWindowKey) if let windowIndex { let namespacedWindowIndexKey = namespacedSnapshotKey( namespace, snapshotWindowIndexKey(key, windowIndex) ) snapshots[namespacedWindowIndexKey] = cachedSnapshot storedKeys.append(namespacedWindowIndexKey) } } snapshotEntries.append( CachedSnapshotEntry(snapshotId: cachedSnapshot.id, keys: storedKeys, createdAt: Date()) ) pruneSnapshotCache() } private func pruneSnapshotCache() { let now = Date() while let oldest = snapshotEntries.first, ComputerSnapshotCachePolicy.shouldPrune( entryCount: snapshotEntries.count, createdAt: oldest.createdAt, now: now ) { let expired = snapshotEntries.removeFirst() for key in expired.keys where snapshots[key]?.id == expired.snapshotId { snapshots.removeValue(forKey: key) } } } private func currentSnapshot(params: [String: JSONValue], needsFocus: Bool = false) throws -> Snapshot { pruneSnapshotCache() let cached = try cachedSnapshot(params: params) let requestedIndexes = try requestedElementIndexes(params) let observeParams = params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement } // Why: elements read with --element-index, focused ones included, may lie past the whole-window node limit, // so only their subtree can re-find them. if let scope = cached?.scope, requestedIndexes.isEmpty ? needsFocus : requestedIndexes.allSatisfy(scope.walkedIndexes.contains), let scoped = try? observe(params: observeParams.merging(["rootElementIndex": .number(Double(scope.rootIndex))]) { _, replacement in replacement }), !requestedIndexes.isEmpty || scoped.focusedElementId != nil { try validateRequestedElements(cached: cached, current: scoped, requestedIndexes: requestedIndexes) return scoped } // Why: cached AX frames can be stale after a window move or resize, and // stale geometry can turn an intended action into a misclick. let snapshot = try observe(params: observeParams) try validateRequestedElements(cached: cached, current: snapshot, requestedIndexes: requestedIndexes) return snapshot } private func currentKeyboardSnapshot(params: [String: JSONValue]) throws -> Snapshot { // Why: keyboard actions need fresh focused-element state, never pixels. try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement }, needsFocus: true) } private func cachedSnapshot(params: [String: JSONValue]) throws -> Snapshot? { guard let query = params["app"]?.string, !query.isEmpty else { return nil } let namespace = snapshotNamespace(params) if let targetWindowId = try requestedWindowId(params) { let canonicalKey = snapshotCanonicalWindowIdKey(targetWindowId) if let cached = snapshots[namespacedSnapshotKey(namespace, canonicalKey)] { return cached } if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[canonicalKey.lowercased()] { return cached } let windowKey = snapshotWindowKey(query.lowercased(), targetWindowId) if let cached = snapshots[namespacedSnapshotKey(namespace, windowKey)] { return cached } if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[windowKey] { return cached } return nil } if let targetWindowIndex = try requestedWindowIndex(params) { let canonicalKey = snapshotCanonicalWindowIndexKey(targetWindowIndex) if let cached = snapshots[namespacedSnapshotKey(namespace, canonicalKey)] { return cached } if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[canonicalKey.lowercased()] { return cached } let windowKey = snapshotWindowIndexKey(query.lowercased(), targetWindowIndex) if let cached = snapshots[namespacedSnapshotKey(namespace, windowKey)] { return cached } if !isExplicitSnapshotNamespace(namespace), let cached = snapshots[windowKey] { return cached } return nil } let key = query.lowercased() return snapshots[namespacedSnapshotKey(namespace, key)] ?? (isExplicitSnapshotNamespace(namespace) ? nil : snapshots[key]) } private func requestedElementIndexes(_ params: [String: JSONValue]) throws -> [Int] { try ["elementIndex", "fromElementIndex", "toElementIndex"].compactMap { key -> Int? in guard params[key]?.number != nil else { return nil } return try optionalInteger(params, key) } } private func requireCachedElements(cached: Snapshot?, requestedIndexes: [Int]) throws { guard !requestedIndexes.isEmpty else { return } guard let cached else { throw ProviderError.coded("element_not_found", "element indexes require a fresh get-app-state snapshot for this app/window") } for index in requestedIndexes where cached.elements[index] == nil { throw ProviderError.coded("element_not_found", "element \(index) is stale; run get-app-state again and use a fresh element index") } } private func validateRequestedElements(cached: Snapshot?, current: Snapshot, requestedIndexes: [Int]) throws { try requireCachedElements(cached: cached, requestedIndexes: requestedIndexes) guard let cached else { return } for index in requestedIndexes { // Why: a subtree read also carries the window's earlier records, which prove nothing about the element now. guard let expected = cached.elements[index], let actual = current.elements[index], current.scope?.walkedIndexes.contains(index) ?? true else { throw ProviderError.coded("element_not_found", "element \(index) is stale; run get-app-state again and use a fresh element index") } guard expected.signature == actual.signature else { throw ProviderError.coded("element_not_found", "element \(index) changed since the last snapshot; run get-app-state again and use a fresh element index") } } } private func ensureWindowStillAvailable(_ snapshot: Snapshot) throws { guard WindowCapture.candidates(pid: snapshot.app.pid).contains(where: { $0.windowId == snapshot.windowId }) else { throw ProviderError.coded("window_stale", "window \(Int(snapshot.windowId)) is no longer available; run get-app-state again to refresh the target window") } } private func listApps() -> [AppDescriptor] { var seen = Set() return NSWorkspace.shared.runningApplications .filter { !$0.isTerminated && $0.activationPolicy == .regular } .compactMap { app in guard let name = app.localizedName, !name.isEmpty else { return nil } let pid = app.processIdentifier guard pid > 0, pidIsLive(pid) else { return nil } let key = (app.bundleIdentifier ?? "pid:\(pid)").lowercased() guard seen.insert(key).inserted else { return nil } return AppDescriptor(name: name, bundleId: app.bundleIdentifier, pid: pid, app: app) } .sorted { lhs, rhs in if lhs.app.isActive != rhs.app.isActive { return lhs.app.isActive && !rhs.app.isActive } return lhs.name.localizedCaseInsensitiveCompare(rhs.name) == .orderedAscending } } private func renderListedApp(_ app: AppDescriptor) -> [String: Any] { [ "name": app.name, "bundleId": jsonNullable(app.bundleId), "pid": Int(app.pid), "isRunning": true, "lastUsedAt": NSNull(), "useCount": NSNull(), ] } private func providerHandshake() -> [String: Any] { [ "platform": "darwin", "provider": providerName, "providerVersion": providerVersion, "protocolVersion": providerProtocolVersion, "supports": [ "apps": [ "list": true, "bundleIds": true, "pids": true, "launch": true, ], "windows": [ "list": true, "targetById": true, "targetByIndex": true, "focus": false, "moveResize": false, ], "observation": [ "screenshot": true, "annotatedScreenshot": false, "elementFrames": true, "ocr": false, "disabledState": true, ], "actions": [ "click": true, "typeText": true, "typeTextElement": true, "pressKey": true, "hotkey": true, "pasteText": true, "scroll": true, "drag": true, "setValue": true, "performAction": true, ], "surfaces": [ "menus": false, "dialogs": false, "dock": false, "menubar": false, ], "input": [ "background": BackgroundInputTransport.isAvailable, ], ], ] } private func resolveApp(_ query: String) throws -> AppDescriptor { let trimmed = query.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { throw ProviderError.coded("invalid_argument", "app query must not be empty") } if let pid = parsePid(trimmed) { if let app = appByPid(pid) { try rejectBlockedApp(app) return app } throw ProviderError.coded("app_not_found", "app '\(trimmed)' not found") } if blockedBundleIds.contains(trimmed) { throw ProviderError.coded("app_blocked", "app '\(trimmed)' is blocked for safety") } if let app = listApps().first(where: { matches($0, query: trimmed) }) { try rejectBlockedApp(app) return app } throw ProviderError.coded("app_not_found", "app '\(trimmed)' not found") } private func rejectBlockedApp(_ app: AppDescriptor) throws { if let bundle = app.bundleId, blockedBundleIds.contains(bundle) { throw ProviderError.coded("app_blocked", "app '\(bundle)' is blocked for safety") } } private func launchApp(params: [String: JSONValue]) throws -> [String: Any] { let query = try requiredString(params, "app").trimmingCharacters(in: .whitespacesAndNewlines) let activate = params["activate"]?.bool == true // Why: Launch Services sends a running app a reopen event, and many apps answer it by opening a window. if let running = listApps().first(where: { matches($0, query: query) }) { try rejectBlockedApp(running) return renderLaunch(running, launched: false) } guard let url = installedApplicationURL(query) else { throw ProviderError.coded("app_not_found", "app '\(query)' is not running and no installed app has that name, bundle ID or path") } if let bundleId = Bundle(url: url)?.bundleIdentifier { if blockedBundleIds.contains(bundleId) { throw ProviderError.coded("app_blocked", "app '\(bundleId)' is blocked for safety") } // Why: a path or display-name query misses the running check above, and menu bar apps aren't listed. if let running = NSRunningApplication.runningApplications(withBundleIdentifier: bundleId) .first(where: { !$0.isTerminated }), let app = appByPid(running.processIdentifier) { return renderLaunch(app, launched: false) } } let pid: pid_t do { pid = try BlockingAsync.run(timeout: 30, timeoutMessage: "launching '\(query)' timed out") { let configuration = NSWorkspace.OpenConfiguration() configuration.activates = activate return try await NSWorkspace.shared.openApplication(at: url, configuration: configuration).processIdentifier } } catch let error as ProviderError { throw error } catch { throw ProviderError.coded("app_launch_failed", "could not launch '\(query)': \(error.localizedDescription)") } guard let app = appByPid(pid) else { throw ProviderError.coded("app_launch_failed", "'\(query)' exited right after it launched") } // Why: Launch Services reports the app once it checks in; activation, asked for or not, lands after it finishes launching. let deadline = Date().addingTimeInterval(5) while !app.app.isFinishedLaunching, !app.app.isTerminated, Date() < deadline { Thread.sleep(forTimeInterval: 0.05) } if app.app.isTerminated { throw ProviderError.coded("app_launch_failed", "'\(query)' quit while it was starting") } return renderLaunch(app, launched: true) } private func renderLaunch(_ app: AppDescriptor, launched: Bool) -> [String: Any] { // Why: measured, not taken from the request; an app may activate itself however it was started. let frontmost = NSWorkspace.shared.frontmostApplication?.processIdentifier == app.pid return [ "app": ["name": app.name, "bundleId": jsonNullable(app.bundleId), "pid": Int(app.pid)], "launched": launched, "delivery": frontmost ? "foreground" : "background", ] } private func listWindows(params: [String: JSONValue]) throws -> [String: Any] { let app = try resolveApp(try requiredString(params, "app")) let windows = windowInventory(app: app, appElement: AXUIElementCreateApplication(app.pid)).windows .filter { $0.candidate.layer == 0 } .enumerated() .map { index, window -> [String: Any] in let candidate = window.candidate return [ "index": index, "app": [ "name": app.name, "bundleId": jsonNullable(app.bundleId), "pid": Int(app.pid), ], "id": Int(candidate.windowId), "title": candidate.title ?? "", "x": Int(candidate.bounds.origin.x.rounded()), "y": Int(candidate.bounds.origin.y.rounded()), "width": Int(candidate.bounds.width.rounded()), "height": Int(candidate.bounds.height.rounded()), "isMinimized": window.visibility == .minimized, "isOffscreen": window.visibility != .onScreen, "screenIndex": jsonNullable(screenIndex(for: candidate.bounds)), "isMain": NSNull(), "platform": [ "layer": candidate.layer, "alpha": candidate.alpha, ], ] } return [ "app": renderListedApp(app), "windows": windows, ] } private func appByPid(_ pid: pid_t) -> AppDescriptor? { guard let app = NSRunningApplication(processIdentifier: pid), !app.isTerminated, let name = app.localizedName else { return nil } return AppDescriptor(name: name, bundleId: app.bundleIdentifier, pid: pid, app: app) } private func buildSnapshot( app: AppDescriptor, includeScreenshot: Bool, windowId: CGWindowID?, windowIndex: Int?, subtree: (root: ElementRecord, base: Snapshot)?, previousSnapshot: (CGWindowID) -> Snapshot? ) throws -> Snapshot { guard accessibilityTrustedSettled() else { // Why: agents retry failed observations. Only the explicit setup flow // should open macOS privacy prompts/settings; runtime calls stay quiet. throw ProviderError.coded( "permission_denied", "Accessibility permission is required for Orca Computer Use. Run `orca computer permissions` or open Settings > Computer Use, grant Accessibility to Orca Computer Use, then retry." ) } let appElement = AXUIElementCreateApplication(app.pid) var role: CFTypeRef? let probeStart = CFAbsoluteTimeGetCurrent() // Why: a hung app costs one timeout per AX read; walking it anyway held the request lock for every agent until the 60 s RPC timeout. if AccessibilityMessaging.timedOut(AXUIElementCopyAttributeValue(appElement, kAXRoleAttribute as CFString, &role), since: probeStart) { throw AccessibilityMessaging.appNotResponding(app.name) } enableManualAccessibilityIfNeeded(appElement, app: app) let inventory = windowInventory(app: app, appElement: appElement) let observable = inventory.windows let focused = try focusedWindow(appElement: appElement, app: app, inventory: inventory) let focusedTitle = stringAttribute(focused, kAXTitleAttribute as String) ?? app.name let canCaptureScreenshot = includeScreenshot && screenCaptureTrustedSettled() guard let capture = WindowCapture.resolve( candidates: observable.map(\.candidate), titleHint: focusedTitle, windowId: windowId, windowIndex: windowIndex, captureImage: canCaptureScreenshot ) else { if let windowId, inventory.unreadableOtherSpace.contains(windowId) { throw unreadableOtherSpaceError(app) } let reason = windowId != nil || windowIndex != nil ? "no window matching the requested selector; run list-windows and pick a listed one" : "no window to observe" throw ProviderError.coded("window_not_found", "app '\(app.name)' has \(reason)") } let target = observable.first { $0.candidate.windowId == capture.windowId } let visibility = target?.visibility ?? .onScreen guard let window = target?.element ?? matchingWindow(appElement: appElement, capture: capture, focused: focused, explicitTarget: windowId != nil || windowIndex != nil) else { throw ProviderError.coded("window_not_found", "could not match accessibility window to requested window; run get-app-state again or retry without a window selector") } // Why: a window first read after a large tree walk gets a remote-token id past the lookup's range, so keep the element // for when the window moves to another Space. otherSpaceLookups = otherSpaceLookups.filter { pidIsLive($0.key) } var lookups = otherSpaceLookups[app.pid] ?? OtherSpaceLookups() lookups.elements = lookups.elements.filter { inventory.windowIds.contains($0.key) } lookups.elements[capture.windowId] = window otherSpaceLookups[app.pid] = lookups let title = stringAttribute(window, kAXTitleAttribute as String) ?? capture.title ?? app.name let renderer = TreeRenderer( windowBounds: capture.bounds, focused: focusedElement(appElement: appElement), compactBrowserTabs: app.isKnownBrowser ) if let subtree { guard let ancestors = ancestorChain(of: subtree.root.element, in: window) else { throw ProviderError.coded("element_not_found", "element \(subtree.root.index) is no longer in this window; run get-app-state again and use a fresh element index") } renderer.render(subtree.root.element, ancestors: ancestors) } else { renderer.render(window) } if renderer.appStoppedResponding { throw AccessibilityMessaging.appNotResponding(app.name) } let previous = subtree?.base ?? previousSnapshot(capture.windowId) let indexes = renderer.carryOverIndexes(from: previous) var elements = renderer.records var elementOrder = indexes.order if let subtree { // Why: elements outside the subtree keep resolving, so the agent can still act on what the whole-window read showed. // Re-reading the same subtree replaces its last walk, so rows scrolled away don't pile up. let replaced = subtree.base.scope.flatMap { $0.rootIndex == subtree.root.index ? $0.walkedIndexes : nil } ?? [] let kept = subtree.base.elements.filter { !replaced.contains($0.key) && renderer.records[$0.key] == nil } elements = kept.merging(renderer.records) { _, walked in walked } elementOrder = subtree.base.elementOrder.filter { kept[$0] != nil } + indexes.order } let screenshot = includeScreenshot ? capture.screenshotPayload() : nil let screenshotStatus: ScreenshotStatus = if screenshot != nil { .captured } else if includeScreenshot && !canCaptureScreenshot { .failed("Screen Recording permission is required for Orca Computer Use; grant permission or pass --no-screenshot to inspect accessibility state only.") } else if includeScreenshot && visibility != .onScreen { .failed("the window \(visibility.reason), so it has no pixels to capture; the accessibility tree is still current. Pass --no-screenshot to skip the capture.") } else if includeScreenshot { .failed("window screenshot capture returned no image; retry with --no-screenshot if accessibility state is sufficient.") } else { .skipped } return Snapshot( id: UUID().uuidString, app: app, windowTitle: title, windowBounds: capture.bounds, windowId: capture.windowId, windowLayer: capture.layer, windowVisibility: visibility, treeText: renderTreeText(app: app, title: title, rootIndex: subtree?.root.index, lines: renderer.lines, focused: renderer.focusedSummary), focusedElementId: renderer.focusedElementId, screenshot: screenshot, screenshotStatus: screenshotStatus, screenshotScale: screenshotScale(screenshot: screenshot, bounds: capture.bounds), screenshotEngine: capture.image?.engine, elements: elements, elementOrder: elementOrder, nextFreshElementIndex: indexes.nextFreshIndex, truncated: renderer.truncated, maxDepthReached: renderer.maxDepthReached, scope: subtree.map { SnapshotScope(rootIndex: $0.root.index, walkedIndexes: Set(indexes.order)) }, appWindowIds: observable.filter { $0.visibility == .onScreen }.map(\.candidate.windowId).sorted() ) } private func renderSnapshot(_ snapshot: Snapshot) -> [String: Any] { var screenshot: Any = NSNull() if let payload = snapshot.screenshot { screenshot = [ "data": payload.data, "format": "png", "width": payload.width, "height": payload.height, "scale": payload.scale, ] } var snapshotPayload: [String: Any] = [ "id": snapshot.id, "app": [ "name": snapshot.app.name, "bundleId": jsonNullable(snapshot.app.bundleId), "pid": Int(snapshot.app.pid), ], "window": [ "id": Int(snapshot.windowId), "title": snapshot.windowTitle, "x": Int(snapshot.windowBounds.origin.x.rounded()), "y": Int(snapshot.windowBounds.origin.y.rounded()), "width": Int(snapshot.windowBounds.width.rounded()), "height": Int(snapshot.windowBounds.height.rounded()), "isMinimized": snapshot.windowVisibility == .minimized, "isOffscreen": snapshot.windowVisibility != .onScreen, "screenIndex": jsonNullable(screenIndex(for: snapshot.windowBounds)), "platform": [ "layer": snapshot.windowLayer, ], ], "coordinateSpace": "window", "treeText": snapshot.treeText, "elementCount": snapshot.scope?.walkedIndexes.count ?? snapshot.elements.count, "focusedElementId": snapshot.focusedElementId as Any, "truncation": [ "truncated": snapshot.truncated, "maxNodes": TreeRenderer.maxNodes, "maxDepth": TreeRenderer.maxDepth, "maxDepthReached": snapshot.maxDepthReached, ], ] if let scope = snapshot.scope { snapshotPayload["rootElementIndex"] = scope.rootIndex } return [ "snapshot": snapshotPayload, "screenshot": screenshot, "screenshotStatus": renderScreenshotStatus(snapshot.screenshotStatus, snapshot: snapshot), ] } private func renderActionResult(action: [String: Any], snapshot: Snapshot) -> [String: Any] { var result = renderSnapshot(snapshot) var metadata = action metadata["targetWindowId"] = Int(snapshot.windowId) result["action"] = metadata return result } private func click(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentSnapshot(params: params) let button = try mouseButton(params["mouseButton"]?.string) let count = try positiveInteger(params["clickCount"]?.number, defaultValue: 1, name: "clickCount") guard count <= SyntheticMouseClickDelivery.maxClickCount else { throw ProviderError.coded( "invalid_argument", "clickCount must be at most \(SyntheticMouseClickDelivery.maxClickCount)" ) } let modifiers = try KeyMap.parseModifiers(params["modifiers"]?.string) let foreground = isForegroundRequest(params) if let elementIndex = try optionalInteger(params, "elementIndex") { let record = try element(snapshot, elementIndex) try requireEnabled(record, snapshot: snapshot) if !foreground { try requireNoForegroundEffect(clickEffect(button: button, modifiers: modifiers, on: record.element)) } let rowSelection = AccessibilityRowSelection() let unselectedRow = modifiers.isEmpty && count <= 1 && button == .left && RowSelectionClick.isUnselectedRow(record.element, ax: rowSelection) if modifiers.isEmpty, count <= 1, button.hasAccessibilityAction, let actionName = try performClickAction(record: record, mouseButton: button) { guard unselectedRow else { return actionMetadata(path: "accessibility", actionName: actionName) } return rowClickMetadata( try RowSelectionClick.confirm(record.element, ax: rowSelection), path: "accessibility", actionName: actionName ) } if !foreground { try requireBackgroundInputReachable(snapshot, input: "pointer click") } let point = try pointerTarget(record, snapshot: snapshot) try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground, element: record.element) guard unselectedRow else { return actionMetadata( path: "synthetic", fallbackReason: "actionUnsupported", verification: unverifiedAction(reason: "synthetic_input") ) } return rowClickMetadata( try RowSelectionClick.confirm(record.element, ax: rowSelection), path: "synthetic", actionName: nil ) } if !foreground { try requireBackgroundInputReachable(snapshot, input: "pointer click") } let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot) try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground) return actionMetadata( path: "synthetic", verification: unverifiedAction(reason: "synthetic_input") ) } /// The selection reading back on verifies a row click; a row it never selected stays unverified. private func rowClickMetadata(_ outcome: RowSelectionClick.Outcome, path: String, actionName: String?) -> [String: Any] { let fallbackReason = path == "synthetic" ? "actionUnsupported" : nil switch outcome { case .selectedByClick: return actionMetadata(path: path, actionName: actionName, fallbackReason: fallbackReason, verification: verifiedAction(property: "selection")) case let .selectedByWrite(attribute): return actionMetadata(path: "accessibility", actionName: attribute, fallbackReason: fallbackReason, verification: verifiedAction(property: "selection")) case .unselected: // Why: an accessibility click leaves verification to the tree diff, which reports suspected_noop if nothing changed. return path == "synthetic" ? actionMetadata(path: path, fallbackReason: fallbackReason, verification: unverifiedAction(reason: "synthetic_input")) : actionMetadata(path: path, actionName: actionName) } } private func requireEnabled(_ record: ElementRecord, snapshot: Snapshot) throws { let ax = AccessibilityActionability(pid: snapshot.app.pid) let enabled = ElementActionability.waitUntilEnabled(record.element, ax: ax) if ax.unanswered { throw AccessibilityMessaging.appNotResponding(snapshot.app.name) } guard enabled else { throw notEnabledError(record) } } /// The screen point a pointer click on the element should use, once it is enabled, in view and on top there. private func pointerTarget(_ record: ElementRecord, snapshot: Snapshot) throws -> CGPoint { // Why: its own wait budget, so time spent enabling or trying AXPress cannot cut short the settle after a scroll. let ax = AccessibilityActionability(pid: snapshot.app.pid) let target = try ElementActionability.pointerTarget(record.element, windowFrame: snapshot.windowBounds, ax: ax) if ax.unanswered { throw AccessibilityMessaging.appNotResponding(snapshot.app.name) } switch target { case let .ready(point): return point case .notEnabled: throw notEnabledError(record) case .noFrame: throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame; no click was sent") case let .outOfView(scrolled): let reason = scrolled ? "is still out of view after scrolling it into view" : "is out of view and cannot be scrolled into view" throw ProviderError.coded("element_not_clickable", "element \(record.index) \(reason), so no click was sent; scroll its container, run get-app-state again and retry") case let .covered(by: cover): throw ProviderError.coded("element_not_clickable", "element \(record.index) is covered by \(describeElement(cover)), so no click was sent; close or finish what covers it, run get-app-state again and retry") } } private func notEnabledError(_ record: ElementRecord) -> ProviderError { .coded("element_not_enabled", "element \(record.index) is disabled, so no input was sent; do what enables it first, then run get-app-state and retry once its line no longer lists disabled") } private func performClickAction(record: ElementRecord, mouseButton: MouseButtonSelection) throws -> String? { if mouseButton == .right { return try performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil } for action in ["AXPress", "AXConfirm", "AXOpen"] { if try performAction(record.element, action) { return action } } return nil } private func performSecondaryAction(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentSnapshot(params: params) let record = try element(snapshot, try requiredInteger(params, "elementIndex")) let requested = try requiredString(params, "action") let action = record.actions.first { SnapshotRenderHeuristics.prettyAction($0).caseInsensitiveCompare(requested) == .orderedSame || $0.caseInsensitiveCompare(requested) == .orderedSame } guard let action else { throw ProviderError.coded("action_not_supported", "'\(requested)' is not a valid secondary action for element \(record.index)") } // Why: scrolling a control into view does not operate it. if action != "AXScrollToVisible" { try requireEnabled(record, snapshot: snapshot) } if !isForegroundRequest(params) { try requireNoForegroundEffect(ForegroundEffect.ofAction(action, target: BackgroundInputTransport.pressTarget(record.element))) } guard try performAction(record.element, action) else { throw ProviderError.coded("accessibility_error", "AXUIElementPerformAction(\(action)) failed") } return actionMetadata(path: "accessibility", actionName: action) } private func setValue(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentSnapshot(params: params) let record = try element(snapshot, try requiredInteger(params, "elementIndex")) let expected = try requiredStringAllowingEmpty(params, "value") try requireEnabled(record, snapshot: snapshot) try requireNotFileNameField(record) guard isSettable(record.element, kAXValueAttribute as String) else { throw ProviderError.coded("value_not_settable", "element \(record.index) is not settable") } let current = rawAttributeValue(record.element, kAXValueAttribute as String) let coercion = AttributeValueCoercion(existingValue: current, requested: expected) let result: AXError let writeStart = CFAbsoluteTimeGetCurrent() switch coercion.writeValue { case .string: result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, expected as CFString) case let .integer(value): result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, NSNumber(value: value)) case let .double(value): result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, NSNumber(value: value)) case let .boolean(value): result = AXUIElementSetAttributeValue(record.element, kAXValueAttribute as CFString, value ? kCFBooleanTrue : kCFBooleanFalse) } if AccessibilityMessaging.timedOut(result, since: writeStart) { throw AccessibilityMessaging.unanswered("the value write") } guard result == .success else { throw ProviderError.coded("accessibility_error", "AXUIElementSetAttributeValue failed with \(result.rawValue)") } let verification: [String: Any] let readback = settledValueReadback(record.element) { readback in if case .mismatch = coercion.compare(readback: readback) { return false } return true } switch coercion.compare(readback: readback) { case .match: verification = verifiedAction(property: "value") case let .mismatch(actualPreview): // Why: a secure field can read back masked, so its value neither disproves the write nor gets shown. verification = AXSnapshotReader().isSecureTextElement(record.element) ? unverifiedAction(reason: "readback_unsupported") : unverifiedAction(reason: "value_mismatch", actualPreview: actualPreview) case .unsupported: verification = unverifiedAction(reason: "readback_unsupported") } return actionMetadata(path: "accessibility", actionName: "AXSetValue", verification: verification) } private func typeText(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) let text = try requiredString(params, "text") if let elementIndex = try optionalInteger(params, "elementIndex") { return try typeTextIntoElement(try element(snapshot, elementIndex), text: text) } let focused = focusedRecord(snapshot)?.element if let focused, let verification = try TextInput.replaceSelection(focused, with: text) { return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification) } let insertion = TextInput.Insertion(into: focused, text: text) try deliverKeys(try withTransportErrors { try BackgroundInputTransport.textEvents(text) }, snapshot: snapshot, params: params) return actionMetadata( path: "synthetic", actionName: "typeText", verification: insertion?.verification() ?? unverifiedAction(reason: "synthetic_input") ) } private func typeTextIntoElement(_ record: ElementRecord, text: String) throws -> [String: Any] { try requireNotFileNameField(record) // Why: typing at focus instead would put the text in whatever field has focus, not the requested one. if BackgroundInputTransport.isChromiumNode(record.element) { throw ProviderError.coded("value_not_settable", "element \(record.index) is web content, which ignores accessibility text insertion; click it, then run type-text without --element-index") } guard isSettable(record.element, kAXValueAttribute as String), rawStringAttribute(record.element, kAXValueAttribute as String) != nil else { throw ProviderError.coded("value_not_settable", "element \(record.index) text is not settable; click it, then run type-text without --element-index") } guard let verification = try TextInput.replaceSelection(record.element, with: text, property: "elementText") else { throw ProviderError.coded("accessibility_error", "text insertion into element \(record.index) did not confirm; run get-app-state to check the field before retrying") } // Why: a secure field can read back masked, so its value neither disproves the write nor gets shown. if verification["state"] as? String != "verified", AXSnapshotReader().isSecureTextElement(record.element) { return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: unverifiedAction(reason: "readback_unsupported")) } return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification) } private func pressKey(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) try deliverKeys(try Input.keyEvents(try requiredString(params, "key")), snapshot: snapshot, params: params) return actionMetadata( path: "synthetic", actionName: "pressKey", verification: unverifiedAction(reason: "synthetic_input") ) } private func hotkey(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) let key = try requiredString(params, "key") if isSelectAllHotkey(key), let focused = focusedRecord(snapshot), TextInput.selectAll(focused.element) { return actionMetadata( path: "accessibility", actionName: "AXSelectAll", verification: TextInput.selectionVerification(focused.element) ) } try deliverKeys(try Input.keyEvents(key), snapshot: snapshot, params: params) return actionMetadata( path: "synthetic", actionName: "hotkey", verification: unverifiedAction(reason: "synthetic_input") ) } private func pasteText(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentKeyboardSnapshot(params: params) let text = try requiredString(params, "text") let focused = focusedRecord(snapshot)?.element if let focused, let verification = try TextInput.replaceSelection(focused, with: text) { return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification) } let pasteKeys = try Input.keyEvents("cmd+v") if !isForegroundRequest(params) { // Why: refuse before the clipboard is replaced, so clipboard history never records text that was not sent. try requireBackgroundInputReachable(snapshot, input: "paste") } let insertion = TextInput.Insertion(into: focused, text: text) try Input.withPasteboardText(text) { try deliverKeys(pasteKeys, snapshot: snapshot, params: params) } return actionMetadata( path: "clipboard", actionName: "paste", verification: insertion?.verification() ?? unverifiedAction(reason: "clipboard_paste") ) } private func scroll(params: [String: JSONValue]) throws -> [String: Any] { let snapshot = try currentSnapshot(params: params) let direction = try scrollDirection(try requiredString(params, "direction")) let pages = try positiveNumber(params["pages"]?.number, defaultValue: 1, name: "pages") if let elementIndex = try optionalInteger(params, "elementIndex") { let record = try element(snapshot, elementIndex) let action = "AXScroll\(direction.capitalized)ByPage" if pages.rounded() == pages, let pageCount = boundedInteger(pages, as: Int.self), record.actions.contains(action) { for _ in 0.. [String: Any] { let snapshot = try currentSnapshot(params: params) let start: CGPoint let end: CGPoint var fromRecord: ElementRecord? if let fromIndex = try optionalInteger(params, "fromElementIndex"), let toIndex = try optionalInteger(params, "toElementIndex") { let from = try element(snapshot, fromIndex) fromRecord = from let to = try element(snapshot, toIndex) guard let fromPoint = center(from.localFrame, in: snapshot.windowBounds), let toPoint = center(to.localFrame, in: snapshot.windowBounds) else { throw ProviderError.coded("element_not_found", "drag element has no frame") } start = fromPoint end = toPoint } else { start = try coordinatePoint(params: params, xKey: "fromX", yKey: "fromY", snapshot: snapshot) end = try coordinatePoint(params: params, xKey: "toX", yKey: "toY", snapshot: snapshot) } if isForegroundRequest(params) { try requireTargetWindowFocused(snapshot, restoreWindowRequested: true) try Input.drag(from: start, to: end, targetWindow: snapshot) } else { try requireBackgroundInputReachable(snapshot, input: "drag") // Why: the mouse-down lands on whatever is under the start point, which can open a menu over the user. guard let startElement = elementAtPoint(start, pid: snapshot.app.pid) ?? fromRecord?.element else { throw unidentifiedBackgroundTarget } try requireNoForegroundEffect(ForegroundEffect.ofPress(BackgroundInputTransport.pressTarget(startElement))) try withTransportErrors { try BackgroundInputTransport.drag(backgroundTarget(snapshot), from: start, to: end) } } return actionMetadata(path: "synthetic") } private func element(_ snapshot: Snapshot, _ index: Int) throws -> ElementRecord { guard let record = snapshot.elements[index] else { throw ProviderError.coded("element_not_found", "element \(index) is not in the current cached snapshot for \(snapshot.app.name); run get-app-state again and use a fresh element index") } return record } private func focusedRecord(_ snapshot: Snapshot) -> ElementRecord? { guard let focusedElementId = snapshot.focusedElementId else { return nil } return snapshot.elements[focusedElementId] } } private let blockedBundleIds: Set = [ "com.1password.1password", "com.1password.safari", "com.bitwarden.desktop", "com.dashlane.dashlanephonefinal", "com.lastpass.LastPass", "com.nordsec.nordpass", "me.proton.pass.electron", "me.proton.pass.catalyst", ] private func requiredString(_ params: [String: JSONValue], _ key: String) throws -> String { guard let value = params[key]?.string, !value.isEmpty else { throw ProviderError.coded("invalid_argument", "missing \(key)") } return value } private func requiredStringAllowingEmpty(_ params: [String: JSONValue], _ key: String) throws -> String { guard let value = params[key]?.string else { throw ProviderError.coded("invalid_argument", "missing \(key)") } return value } private func requiredNumber(_ params: [String: JSONValue], _ key: String) throws -> Double { guard let value = params[key]?.number, value.isFinite else { throw ProviderError.coded("invalid_argument", "missing \(key)") } return value } private func requiredInteger(_ params: [String: JSONValue], _ key: String) throws -> Int { guard let value = boundedInteger(try requiredNumber(params, key), as: Int.self) else { throw ProviderError.coded("invalid_argument", "\(key) is out of range") } return value } private func optionalInteger(_ params: [String: JSONValue], _ key: String) throws -> Int? { guard let raw = params[key]?.number else { return nil } guard let value = boundedInteger(raw, as: Int.self) else { throw ProviderError.coded("invalid_argument", "\(key) is out of range") } return value } private func positiveInteger(_ value: Double?, defaultValue: Int, name: String) throws -> Int { switch ActionArgumentValidation.positiveInteger(value, defaultValue: defaultValue, name: name) { case let .success(value): return value case let .failure(error): throw ProviderError.coded("invalid_argument", error.message) } } private func positiveNumber(_ value: Double?, defaultValue: Double, name: String) throws -> Double { switch ActionArgumentValidation.positiveNumber(value, defaultValue: defaultValue, name: name) { case let .success(value): return value case let .failure(error): throw ProviderError.coded("invalid_argument", error.message) } } private func scrollDirection(_ value: String) throws -> String { switch ActionArgumentValidation.scrollDirection(value) { case let .success(value): return value case let .failure(error): throw ProviderError.coded("invalid_argument", error.message) } } private func parsePid(_ query: String) -> pid_t? { guard query.hasPrefix("pid:") else { return nil } guard let pid = Int32(query.dropFirst(4)), pid > 0 else { return nil } return pid } /// An app that isn't running, by absolute path, bundle ID or bundle name in the usual install folders. private func installedApplicationURL(_ query: String) -> URL? { let path = (query as NSString).expandingTildeInPath if path.hasPrefix("/") { return FileManager.default.fileExists(atPath: path) ? URL(fileURLWithPath: path) : nil } if let url = NSWorkspace.shared.urlForApplication(withBundleIdentifier: query) { return url } let bundleName = query.lowercased().hasSuffix(".app") ? query : "\(query).app" let folders = [ "/Applications", "/Applications/Utilities", "/System/Applications", "/System/Applications/Utilities", NSHomeDirectory() + "/Applications", ] return folders .map { URL(fileURLWithPath: $0).appendingPathComponent(bundleName) } .first { FileManager.default.fileExists(atPath: $0.path) } } private func matches(_ app: AppDescriptor, query: String) -> Bool { app.name.caseInsensitiveCompare(query) == .orderedSame || app.bundleId?.caseInsensitiveCompare(query) == .orderedSame } private func pidIsLive(_ pid: pid_t) -> Bool { kill(pid, 0) == 0 } private func accessibilityTrusted() -> Bool { AXIsProcessTrusted() } private func accessibilityTrustedSettled() -> Bool { // Fresh helper processes can receive transient TCC preflight denials before the real grant settles. PermissionTrustSettling.settle(probe: accessibilityTrusted).settled } private func screenCaptureTrusted() -> Bool { CGPreflightScreenCaptureAccess() } private func screenCaptureTrustedSettled() -> Bool { PermissionTrustSettling.settle(timeoutMs: 2_000, probe: screenCaptureTrusted).settled } private func permissionStatusSnapshotSettled() -> PermissionStatusSnapshot { PermissionStatusSnapshotProbe.capture( accessibilityProbe: accessibilityTrustedSettled, screenshotsProbe: screenCaptureTrustedSettled ) } private func requestScreenCaptureAccess() -> Bool { CGRequestScreenCaptureAccess() } private func openAccessibilitySettings() { openSystemSettings("x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility") } private func openScreenRecordingSettings() { openSystemSettings("x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture") } private func openSystemSettings(_ value: String) { guard let url = URL(string: value) else { return } NSWorkspace.shared.open(url) } private func enableManualAccessibilityIfNeeded(_ appElement: AXUIElement, app: AppDescriptor) { guard app.needsManualAccessibilityMode else { return } if isSettable(appElement, "AXManualAccessibility") { _ = AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue) return } // Why: the older, broader switch, so only for apps that lack AXManualAccessibility and only when it is off. if boolAttribute(appElement, "AXEnhancedUserInterface") != true { _ = AXUIElementSetAttributeValue(appElement, "AXEnhancedUserInterface" as CFString, kCFBooleanTrue) } } private func focusedWindow(appElement: AXUIElement, app: AppDescriptor, inventory: WindowInventory) throws -> AXUIElement { let observable = inventory.windows let systemWide = AXUIElementCreateSystemWide() if let window = lookupUsableWindow(systemWide: systemWide, appElement: appElement, app: app) { return window } if observable.contains(where: { $0.visibility == .onScreen }) { var settledWindow: AXUIElement? let outcome = PermissionTrustSettling.settle { settledWindow = lookupUsableWindow( systemWide: systemWide, appElement: appElement, app: app ) return settledWindow != nil } if let window = settledWindow, outcome.settled { return window } var windows: CFTypeRef? if AXUIElementCopyAttributeValue(appElement, kAXWindowsAttribute as CFString, &windows) == .apiDisabled { throw ProviderError.coded("permission_denied", "app '\(app.name)' has visible windows but macOS refuses accessibility reads (AX reads stayed blocked for \(outcome.waitedMs)ms after retries). macOS Accessibility may need Orca Computer Use toggled off and on again in System Settings.") } } let offScreen = observable.filter { $0.element != nil } if let window = (offScreen.first { $0.visibility != .minimized } ?? offScreen.first)?.element { return window } if observable.contains(where: { $0.visibility == .onScreen }) { throw ProviderError.coded("window_not_found", "app '\(app.name)' has visible windows but exposes no accessibility window; it may still be starting up. Run get-app-state again shortly.") } if !inventory.unreadableOtherSpace.isEmpty { throw unreadableOtherSpaceError(app) } throw ProviderError.coded("window_not_found", "app '\(app.name)' has no open window, counting minimized, hidden and other-Space windows; open one in the app first.") } private func unreadableOtherSpaceError(_ app: AppDescriptor) -> ProviderError { .coded("window_not_found", "app '\(app.name)' has a window on another Space that accessibility cannot read yet: macOS leaves other-Space windows (full-screen ones included) out of accessibility until something has read them once. Ask the user to show that window (switch to its Space) and run get-app-state while it is showing, or ask the user before retrying with --restore-window.") } private func lookupUsableWindow( systemWide: AXUIElement, appElement: AXUIElement, app: AppDescriptor ) -> AXUIElement? { if let window = focusedSystemWindow(systemWide: systemWide, app: app) { return window } if let window = copyElement(appElement, kAXFocusedWindowAttribute as String), usableWindow(window) { return window } if let windows = copyArray(appElement, kAXWindowsAttribute as String) { return windows.first(where: usableWindow) } return nil } private func focusedSystemWindow(systemWide: AXUIElement, app: AppDescriptor) -> AXUIElement? { guard let focusedApp = copyElement(systemWide, kAXFocusedApplicationAttribute as String), pidAttribute(focusedApp) == app.pid else { return nil } if let window = copyElement(systemWide, kAXFocusedWindowAttribute as String), usableWindow(window) { return window } if let window = copyElement(focusedApp, kAXFocusedWindowAttribute as String), usableWindow(window) { return window } if let windows = copyArray(focusedApp, kAXWindowsAttribute as String) { return windows.first(where: usableWindow) } return nil } private func isTargetWindowFocused(_ snapshot: Snapshot) -> Bool { focusedSyntheticClickRecipient( targetPID: snapshot.app.pid, targetCandidates: { WindowCapture.candidates(pid: snapshot.app.pid) } ) == .focused(syntheticClickRecipient(pid: snapshot.app.pid, windowId: snapshot.windowId)) } private enum AXElementProbe { case value(AXUIElement) case absent case unavailable } private func copyElementProbe(_ element: AXUIElement, _ attribute: String) -> AXElementProbe { var value: CFTypeRef? switch AXUIElementCopyAttributeValue(element, attribute as CFString, &value) { case .success: guard let value else { return .unavailable } return .value(value as! AXUIElement) case .noValue: return .absent default: return .unavailable } } private func currentSyntheticClickRecipient( snapshot: Snapshot, point: CGPoint ) -> SyntheticMouseClickDelivery.RecipientObservation { let target = syntheticClickRecipient(pid: snapshot.app.pid, windowId: snapshot.windowId) var cachedTargetCandidates: [WindowCandidate]? func targetCandidates() -> [WindowCandidate] { if let cachedTargetCandidates { return cachedTargetCandidates } let candidates = WindowCapture.candidates(pid: snapshot.app.pid) cachedTargetCandidates = candidates return candidates } switch focusedSyntheticClickRecipient( targetPID: snapshot.app.pid, targetCandidates: targetCandidates ) { case let .focused(focused): guard focused == target else { return .focused(focused) } switch hitTestSyntheticClickRecipient( at: point, targetPID: snapshot.app.pid, targetCandidates: targetCandidates ) { case let .focused(recipient): return .focused(recipient) case .dismissed, .unavailable: return .unavailable } case .dismissed: return .dismissed case .unavailable: return .unavailable } } private func focusedSyntheticClickRecipient( targetPID: pid_t, targetCandidates: () -> [WindowCandidate] ) -> SyntheticMouseClickDelivery.RecipientObservation { let systemWide = AXUIElementCreateSystemWide() let focusedApp: AXUIElement switch copyElementProbe(systemWide, kAXFocusedApplicationAttribute as String) { case let .value(value): focusedApp = value case .absent: return .dismissed case .unavailable: return .unavailable } guard let ownerPID = pidAttribute(focusedApp) else { return .unavailable } let focusedWindow: AXUIElement switch copyElementProbe(focusedApp, kAXFocusedWindowAttribute as String) { case let .value(value): focusedWindow = value case .absent: guard ownerPID == targetPID else { return .unavailable } return .dismissed case .unavailable: return .unavailable } guard pidAttribute(focusedWindow) == ownerPID else { return .unavailable } return pointerRecipient(of: focusedWindow, targetPID: targetPID, targetCandidates: targetCandidates) } private func hitTestSyntheticClickRecipient( at point: CGPoint, targetPID: pid_t, targetCandidates: () -> [WindowCandidate] ) -> SyntheticMouseClickDelivery.RecipientObservation { guard Float(point.x).isFinite, Float(point.y).isFinite else { return .unavailable } let systemWide = AXUIElementCreateSystemWide() var hitElement: AXUIElement? switch AXUIElementCopyElementAtPosition( systemWide, Float(point.x), Float(point.y), &hitElement ) { case .success: break case .noValue: return .dismissed default: return .unavailable } guard let hitElement else { return .unavailable } return pointerRecipient(of: hitElement, targetPID: targetPID, targetCandidates: targetCandidates) } private func pointerRecipient( of element: AXUIElement, targetPID: pid_t, targetCandidates: () -> [WindowCandidate] ) -> SyntheticMouseClickDelivery.RecipientObservation { guard let ownerPID = pidAttribute(element), let window = containingWindow(element) else { return .unavailable } do { if let windowID = try AccessibilityWindowIdentity.windowID(of: window) { return .focused(syntheticClickRecipient(pid: ownerPID, windowId: windowID)) } } catch { return .unavailable } guard ownerPID == targetPID, let frame = absoluteFrame(window), let windows = copyArray(AXUIElementCreateApplication(ownerPID), kAXWindowsAttribute as String) else { return .unavailable } var matches: [AXUIElement] = [] for candidate in windows { guard let candidateFrame = absoluteFrame(candidate) else { return .unavailable } if windowFramesMatch(candidateFrame, frame) { matches.append(candidate) } } guard matches.count == 1, CFEqual(matches[0], window), let candidate = SyntheticMouseClickDelivery.uniqueWindowCandidate( from: targetCandidates(), matching: { windowFramesMatch($0.bounds, frame) } ) else { return .unavailable } return .focused(syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId)) } /// The element's parents from `window` down, or nil when it is no longer inside `window`. private func ancestorChain(of element: AXUIElement, in window: AXUIElement) -> [AXUIElement]? { var chain: [AXUIElement] = [] var current = element while !CFEqual(current, window) { guard chain.count < 256, let parent = copyElement(current, kAXParentAttribute as String) else { return nil } chain.append(parent) current = parent } return chain.reversed() } private func containingWindow(_ element: AXUIElement) -> AXUIElement? { var current = element for _ in 0..<64 { if stringAttribute(current, kAXRoleAttribute as String) == kAXWindowRole as String { return current } if let window = copyElement(current, kAXWindowAttribute as String) { return window } guard let parent = copyElement(current, kAXParentAttribute as String) else { return nil } current = parent } return nil } private func syntheticClickRecipient( pid: pid_t, windowId: CGWindowID ) -> SyntheticMouseClickDelivery.Recipient { SyntheticMouseClickDelivery.Recipient(ownerPID: pid, windowID: windowId) } private func requireTargetWindowFocused(_ snapshot: Snapshot, restoreWindowRequested: Bool) throws { guard let failure = KeyboardInputSafety.syntheticInputFocusFailure( targetWindowFocused: isTargetWindowFocused(snapshot), restoreWindowRequested: restoreWindowRequested ) else { return } switch failure { case .targetNotFocused: throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; nothing was sent. Use set-value for editable elements, or ask the user before retrying with --restore-window.") case .targetNotFocusedAfterRestore: throw ProviderError.coded("window_not_focused", "keyboard input requires the target \(snapshot.app.name) window to be focused; --restore-window was requested but the target is still not focused, so nothing was sent; ask the user to bring it forward or check Accessibility permissions") } } private func isForegroundRequest(_ params: [String: JSONValue]) -> Bool { params["restoreWindow"]?.bool == true } /// Refuses, before anything is sent, background pointer or key input the window cannot receive while off screen. private func requireBackgroundInputReachable(_ snapshot: Snapshot, input: String) throws { guard snapshot.windowVisibility != .onScreen else { return } throw ProviderError.coded("background_unavailable", "window \(Int(snapshot.windowId)) of \(snapshot.app.name) \(snapshot.windowVisibility.reason), so background \(input) cannot reach it without bringing it forward; nothing was sent. Use its elements instead (click --element-index, set-value, type-text --element-index). Ask the user before retrying with --restore-window.") } private func frontmostObservation(of app: AppDescriptor) -> ForegroundDelivery.Observation { let frontmost = boolAttribute(AXUIElementCreateApplication(app.pid), kAXFrontmostAttribute as String) == true return ForegroundDelivery.Observation(frontmost: frontmost, focusedWindowID: frontmost ? focusedWindowId(of: app) : nil) } private func focusedWindowId(of app: AppDescriptor) -> CGWindowID? { copyElement(AXUIElementCreateApplication(app.pid), kAXFocusedWindowAttribute as String).flatMap(windowNumber) } private func backgroundTarget(_ snapshot: Snapshot) -> BackgroundInputTransport.Target { BackgroundInputTransport.Target(pid: snapshot.app.pid, windowID: snapshot.windowId, windowFrame: snapshot.windowBounds) } private func withTransportErrors(_ body: () throws -> T) throws -> T { do { return try body() } catch let failure as BackgroundInputTransport.Failure { switch failure { case let .unavailable(missingSymbols): throw ProviderError.coded("background_unavailable", "background input is unavailable on this macOS version (missing \(missingSymbols.joined(separator: ", "))), so nothing was sent; foreground input would bring the app forward and take over the user's pointer and keyboard. Ask the user before retrying with --restore-window.") case .processNotFound: throw ProviderError.coded("app_not_found", "the target app quit before input was delivered") case .eventCreationFailed: throw ProviderError.coded("accessibility_error", "failed to create input event") case .targetWindowNotKey: throw ProviderError.coded("background_unavailable", "this app is in front and the target window is not its key window, so input to it would move the user's typing there; nothing was sent; use an element action (click --element-index, set-value) or wait until the user leaves the app. Ask the user before retrying with --restore-window.") case let .focusRecordFailed(status): throw ProviderError.coded("window_not_focused", "background key-window focus record failed (status \(status)); input may already have been delivered, so inspect current state before retrying") } } } private func clickPointer( at point: CGPoint, button: MouseButtonSelection, count: Int, modifiers: [KeyModifier], snapshot: Snapshot, foreground: Bool, element: AXUIElement? = nil ) throws { if foreground { try Input.click(at: point, button: button, count: count, modifiers: modifiers, targetWindow: snapshot) return } guard let hitElement = element ?? elementAtPoint(point, pid: snapshot.app.pid) else { throw unidentifiedBackgroundTarget } let hit = try backgroundClickHit(hitElement, snapshot: snapshot) // Why: an element click already checked its target before trying AXPress. if element == nil { try requireNoForegroundEffect(clickEffect(button: button, modifiers: modifiers, on: hitElement)) } try withTransportErrors { try BackgroundInputTransport.click( backgroundTarget(snapshot), at: point, button: button, clickCount: count, flags: eventFlags(modifiers), hit: hit ) } } private func elementAtPoint(_ point: CGPoint, pid: pid_t) -> AXUIElement? { var found: AXUIElement? guard Float(point.x).isFinite, Float(point.y).isFinite, AXUIElementCopyElementAtPosition(AXUIElementCreateApplication(pid), Float(point.x), Float(point.y), &found) == .success else { return nil } return found } private let unidentifiedBackgroundTarget = ProviderError.coded("background_unavailable", "cannot identify the background click target, so nothing was sent; run get-app-state again. If it keeps failing, a foreground click is the only route. Ask the user before retrying with --restore-window.") private func backgroundClickHit(_ element: AXUIElement, snapshot: Snapshot) throws -> BackgroundInputTransport.PointerHit { let recipient = pointerRecipient(of: element, targetPID: snapshot.app.pid) { WindowCapture.candidates(pid: snapshot.app.pid) } guard recipient == .focused(syntheticClickRecipient(pid: snapshot.app.pid, windowId: snapshot.windowId)) else { throw ProviderError.coded("background_unavailable", "the background click target does not identify the requested window, so nothing was sent; run get-app-state again. If it keeps failing, a foreground click is the only route. Ask the user before retrying with --restore-window.") } return BackgroundInputTransport.classifyPointerHit(element) } private func clickEffect(button: MouseButtonSelection, modifiers: [KeyModifier], on element: AXUIElement) -> ForegroundEffect? { ForegroundEffect.ofClick( button: button, control: modifiers.contains { $0.flag == .maskControl }, target: BackgroundInputTransport.pressTarget(element) ) } /// Refuses, before anything is sent, an action that would put a menu or dialog in front of the user. private func requireNoForegroundEffect(_ effect: ForegroundEffect?) throws { switch effect { case nil: return case .showsMenu: throw ProviderError.coded("background_unavailable", "this would open a menu over the user's screen and take their keyboard, so nothing was sent; for a popup menu, native combo box or web