From 594791256927aa0ad5f9ff50cdcb98f5219b1177 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Fri, 7 Aug 2026 05:41:07 -0400 Subject: [PATCH] fix(computer-use): settle macOS permission checks --- .../Sources/OrcaComputerUseMacOS/main.swift | 121 +++++++++++++----- .../PermissionTrustSettling.swift | 73 +++++++++++ .../ScreenCaptureProbeWindowSelection.swift | 22 ++++ .../PermissionTrustSettlingTests.swift | 59 +++++++++ ...ScreenCapturePermissionSettlingTests.swift | 79 ++++++++++++ ...reenCaptureProbeWindowSelectionTests.swift | 44 +++++++ 6 files changed, 366 insertions(+), 32 deletions(-) create mode 100644 native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/PermissionTrustSettling.swift create mode 100644 native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ScreenCaptureProbeWindowSelection.swift create mode 100644 native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/PermissionTrustSettlingTests.swift create mode 100644 native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ScreenCapturePermissionSettlingTests.swift create mode 100644 native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ScreenCaptureProbeWindowSelectionTests.swift diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index 7ecbdbf004a..34e9cc46f72 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -601,7 +601,7 @@ final class Provider { windowIndex: Int?, restoreWindow: Bool ) throws -> Snapshot { - guard accessibilityTrusted() else { + guard accessibilityTrustedSettled() else { // Why: agents retry failed observations. Only the explicit setup flow // should open macOS privacy prompts/settings; runtime calls stay quiet. throw ProviderError.coded( @@ -619,7 +619,7 @@ final class Provider { allowRecovery: restoreWindow ) let focusedTitle = stringAttribute(focused, kAXTitleAttribute as String) ?? app.name - let canCaptureScreenshot = includeScreenshot && screenCaptureTrusted() + let canCaptureScreenshot = includeScreenshot && screenCaptureTrustedSettled() guard let capture = WindowCapture.resolve( candidates: windowCandidates, titleHint: focusedTitle, @@ -1034,10 +1034,57 @@ private func accessibilityTrusted() -> Bool { AXIsProcessTrusted() } +private func accessibilityTrustedSettled() -> Bool { + // Fresh helper processes can receive transient TCC preflight denials before the real grant settles. + PermissionTrustSettling.settle(probe: accessibilityTrusted).settled +} + private func screenCaptureTrusted() -> Bool { CGPreflightScreenCaptureAccess() } +private func screenCaptureTrustedSettled() -> Bool { + PermissionTrustSettling.settleWithFallback( + finalTimeoutMs: 500, + probe: screenCaptureTrusted, + fallbackProbe: screenCaptureTrustedByCaptureProbe + ) +} + +private func screenCaptureTrustedByCaptureProbe() -> Bool { + guard let infos = CGWindowListCopyWindowInfo( + [.optionOnScreenOnly], + kCGNullWindowID + ) as? [[String: Any]] else { + return false + } + let windows = infos.compactMap { info -> ScreenCaptureProbeWindow? in + guard let layer = info[kCGWindowLayer as String] as? Int, + let ownerPid = info[kCGWindowOwnerPID as String] as? NSNumber, + let number = info[kCGWindowNumber as String] as? NSNumber + else { + return nil + } + return ScreenCaptureProbeWindow( + layer: layer, + ownerPid: ownerPid.int32Value, + windowId: number.uint32Value + ) + } + guard let windowId = ScreenCaptureProbeWindowSelection.firstCrossProcessNormalWindow( + ownPid: ProcessInfo.processInfo.processIdentifier, + windows: windows + ) else { + return false + } + return CGWindowListCreateImage( + .null, + [.optionIncludingWindow], + CGWindowID(windowId), + [.boundsIgnoreFraming] + ) != nil +} + private func requestScreenCaptureAccess() -> Bool { CGRequestScreenCaptureAccess() } @@ -1065,6 +1112,38 @@ private func enableManualAccessibilityIfNeeded(_ appElement: AXUIElement, app: A private func focusedWindow(appElement: AXUIElement, app: AppDescriptor, visibleWindowCount: Int, allowRecovery: Bool) throws -> AXUIElement { let systemWide = AXUIElementCreateSystemWide() + if let window = lookupUsableWindow(systemWide: systemWide, appElement: appElement, app: app) { + return window + } + if allowRecovery { + recoverWindow(app) + if let window = lookupUsableWindow(systemWide: systemWide, appElement: appElement, app: app) { + return window + } + } + if visibleWindowCount > 0 { + var settledWindow: AXUIElement? + let outcome = PermissionTrustSettling.settle { + settledWindow = lookupUsableWindow( + systemWide: systemWide, + appElement: appElement, + app: app + ) + return settledWindow != nil + } + if let window = settledWindow, outcome.settled { + return window + } + throw ProviderError.coded("permission_denied", "app '\(app.name)' has visible windows but no accessibility window (AX reads stayed blocked for \(outcome.waitedMs)ms after retries). macOS Accessibility may need Orca Computer Use toggled off and on again in System Settings.") + } + throw ProviderError.coded("window_not_found", "app '\(app.name)' has no accessibility window; make sure the app has a visible window, then retry with --restore-window.") +} + +private func lookupUsableWindow( + systemWide: AXUIElement, + appElement: AXUIElement, + app: AppDescriptor +) -> AXUIElement? { if let window = focusedSystemWindow(systemWide: systemWide, app: app) { return window } @@ -1072,31 +1151,9 @@ private func focusedWindow(appElement: AXUIElement, app: AppDescriptor, visibleW return window } if let windows = copyArray(appElement, kAXWindowsAttribute as String) { - if let window = windows.first(where: usableWindow) { - return window - } + return windows.first(where: usableWindow) } - if allowRecovery { - recoverWindow(app) - if let window = focusedSystemWindow(systemWide: systemWide, app: app) { - return window - } - if let window = copyElement(appElement, kAXFocusedWindowAttribute as String), usableWindow(window) { - return window - } - if let windows = copyArray(appElement, kAXWindowsAttribute as String) { - if let window = windows.first(where: usableWindow) { - return window - } - } - } - let permissionHint = visibleWindowCount > 0 - ? " The app has visible windows, so macOS Accessibility may need Orca Computer Use toggled off and on again in System Settings." - : "" - if visibleWindowCount > 0 { - throw ProviderError.coded("permission_denied", "app '\(app.name)' has visible windows but no accessibility window.\(permissionHint)") - } - throw ProviderError.coded("window_not_found", "app '\(app.name)' has no accessibility window; make sure the app has a visible window, then retry with --restore-window.") + return nil } private func focusedSystemWindow(systemWide: AXUIElement, app: AppDescriptor) -> AXUIElement? { @@ -3048,9 +3105,9 @@ private enum PermissionKind: CaseIterable { var isGranted: Bool { switch self { case .accessibility: - accessibilityTrusted() + accessibilityTrustedSettled() case .screenshots: - screenCaptureTrusted() + screenCaptureTrustedSettled() } } @@ -4039,14 +4096,14 @@ private func runPermissionCheck(initialPermission: PermissionKind? = nil) { } private func printPermissionStatus() { - let accessibility = accessibilityTrusted() ? "granted" : "not-granted" - let screenshots = screenCaptureTrusted() ? "granted" : "not-granted" + let accessibility = accessibilityTrustedSettled() ? "granted" : "not-granted" + let screenshots = screenCaptureTrustedSettled() ? "granted" : "not-granted" print(#"{"accessibility":"\#(accessibility)","screenshots":"\#(screenshots)"}"#) } private func writePermissionStatus(to path: String) { - let accessibility = accessibilityTrusted() ? "granted" : "not-granted" - let screenshots = screenCaptureTrusted() ? "granted" : "not-granted" + let accessibility = accessibilityTrustedSettled() ? "granted" : "not-granted" + let screenshots = screenCaptureTrustedSettled() ? "granted" : "not-granted" let text = #"{"accessibility":"\#(accessibility)","screenshots":"\#(screenshots)"}"# do { try text.write(toFile: path, atomically: true, encoding: .utf8) diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/PermissionTrustSettling.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/PermissionTrustSettling.swift new file mode 100644 index 00000000000..80612664454 --- /dev/null +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/PermissionTrustSettling.swift @@ -0,0 +1,73 @@ +import Foundation + +public enum PermissionTrustSettling { + public struct Outcome: Equatable { + public let settled: Bool + public let attempts: Int + public let waitedMs: Int + + public init(settled: Bool, attempts: Int, waitedMs: Int) { + self.settled = settled + self.attempts = attempts + self.waitedMs = waitedMs + } + } + + public static let defaultTimeoutMs = 1_500 + public static let defaultIntervalMs = 100 + + public static func settle( + timeoutMs: Int = defaultTimeoutMs, + intervalMs: Int = defaultIntervalMs, + sleepMs: (Int) -> Void = { interval in + Thread.sleep(forTimeInterval: TimeInterval(interval) / 1_000) + }, + probe: () -> Bool + ) -> Outcome { + precondition(timeoutMs >= 0, "timeoutMs must not be negative") + precondition(intervalMs > 0, "intervalMs must be positive") + var attempts = 0 + var waitedMs = 0 + while true { + attempts += 1 + if probe() { + return Outcome(settled: true, attempts: attempts, waitedMs: waitedMs) + } + if waitedMs >= timeoutMs { + return Outcome(settled: false, attempts: attempts, waitedMs: waitedMs) + } + let interval = min(intervalMs, timeoutMs - waitedMs) + sleepMs(interval) + waitedMs += interval + } + } + + public static func settleWithFallback( + initialTimeoutMs: Int = defaultTimeoutMs, + finalTimeoutMs: Int, + intervalMs: Int = defaultIntervalMs, + sleepMs: (Int) -> Void = { interval in + Thread.sleep(forTimeInterval: TimeInterval(interval) / 1_000) + }, + probe: () -> Bool, + fallbackProbe: () -> Bool + ) -> Bool { + if settle( + timeoutMs: initialTimeoutMs, + intervalMs: intervalMs, + sleepMs: sleepMs, + probe: probe + ).settled { + return true + } + if fallbackProbe() { + return true + } + return settle( + timeoutMs: finalTimeoutMs, + intervalMs: intervalMs, + sleepMs: sleepMs, + probe: probe + ).settled + } +} diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ScreenCaptureProbeWindowSelection.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ScreenCaptureProbeWindowSelection.swift new file mode 100644 index 00000000000..36b4346a8bc --- /dev/null +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ScreenCaptureProbeWindowSelection.swift @@ -0,0 +1,22 @@ +public struct ScreenCaptureProbeWindow: Equatable { + public let layer: Int + public let ownerPid: Int32 + public let windowId: UInt32 + + public init(layer: Int, ownerPid: Int32, windowId: UInt32) { + self.layer = layer + self.ownerPid = ownerPid + self.windowId = windowId + } +} + +public enum ScreenCaptureProbeWindowSelection { + public static func firstCrossProcessNormalWindow( + ownPid: Int32, + windows: [ScreenCaptureProbeWindow] + ) -> UInt32? { + windows.first { window in + window.layer == 0 && window.ownerPid != ownPid + }?.windowId + } +} diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/PermissionTrustSettlingTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/PermissionTrustSettlingTests.swift new file mode 100644 index 00000000000..c3f3956acd6 --- /dev/null +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/PermissionTrustSettlingTests.swift @@ -0,0 +1,59 @@ +import Testing +@testable import OrcaComputerUseMacOSCore + +@Suite("PermissionTrustSettling") +struct PermissionTrustSettlingTests { + @Test("already-trusted probe returns immediately") + func immediateSuccess() { + var sleeps: [Int] = [] + let outcome = PermissionTrustSettling.settle( + sleepMs: { sleeps.append($0) }, + probe: { true } + ) + + #expect(outcome == .init(settled: true, attempts: 1, waitedMs: 0)) + #expect(sleeps.isEmpty) + } + + @Test("transient denial settles after retries") + func lateGrant() { + var calls = 0 + var sleeps: [Int] = [] + let outcome = PermissionTrustSettling.settle( + sleepMs: { sleeps.append($0) }, + probe: { + calls += 1 + return calls >= 4 + } + ) + + #expect(outcome == .init(settled: true, attempts: 4, waitedMs: 300)) + #expect(sleeps == [100, 100, 100]) + } + + @Test("persistent denial stops at the timeout") + func neverGranted() { + let outcome = PermissionTrustSettling.settle( + timeoutMs: 1_000, + intervalMs: 300, + sleepMs: { _ in }, + probe: { false } + ) + + #expect(outcome == .init(settled: false, attempts: 5, waitedMs: 1_000)) + } + + @Test("final interval is clamped to the timeout") + func clampsFinalInterval() { + var sleeps: [Int] = [] + let outcome = PermissionTrustSettling.settle( + timeoutMs: 250, + intervalMs: 100, + sleepMs: { sleeps.append($0) }, + probe: { false } + ) + + #expect(sleeps == [100, 100, 50]) + #expect(outcome == .init(settled: false, attempts: 4, waitedMs: 250)) + } +} diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ScreenCapturePermissionSettlingTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ScreenCapturePermissionSettlingTests.swift new file mode 100644 index 00000000000..d53991d36d0 --- /dev/null +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ScreenCapturePermissionSettlingTests.swift @@ -0,0 +1,79 @@ +import Testing +@testable import OrcaComputerUseMacOSCore + +@Suite("Screen Capture permission settling") +struct ScreenCapturePermissionSettlingTests { + @Test("initial preflight success skips fallback") + func initialSuccess() { + var fallbackCalls = 0 + let trusted = PermissionTrustSettling.settleWithFallback( + finalTimeoutMs: 500, + sleepMs: { _ in }, + probe: { true }, + fallbackProbe: { + fallbackCalls += 1 + return true + } + ) + + #expect(trusted) + #expect(fallbackCalls == 0) + } + + @Test("capture fallback can establish trust") + func fallbackSuccess() { + var fallbackCalls = 0 + let trusted = PermissionTrustSettling.settleWithFallback( + initialTimeoutMs: 200, + finalTimeoutMs: 200, + intervalMs: 100, + sleepMs: { _ in }, + probe: { false }, + fallbackProbe: { + fallbackCalls += 1 + return true + } + ) + + #expect(trusted) + #expect(fallbackCalls == 1) + } + + @Test("preflight is retried after fallback failure") + func finalPreflightSuccess() { + var probeCalls = 0 + let trusted = PermissionTrustSettling.settleWithFallback( + initialTimeoutMs: 100, + finalTimeoutMs: 200, + intervalMs: 100, + sleepMs: { _ in }, + probe: { + probeCalls += 1 + return probeCalls == 4 + }, + fallbackProbe: { false } + ) + + #expect(trusted) + #expect(probeCalls == 4) + } + + @Test("persistent denial remains denied") + func persistentDenial() { + var fallbackCalls = 0 + let trusted = PermissionTrustSettling.settleWithFallback( + initialTimeoutMs: 100, + finalTimeoutMs: 100, + intervalMs: 100, + sleepMs: { _ in }, + probe: { false }, + fallbackProbe: { + fallbackCalls += 1 + return false + } + ) + + #expect(!trusted) + #expect(fallbackCalls == 1) + } +} diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ScreenCaptureProbeWindowSelectionTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ScreenCaptureProbeWindowSelectionTests.swift new file mode 100644 index 00000000000..96afd6fbccb --- /dev/null +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ScreenCaptureProbeWindowSelectionTests.swift @@ -0,0 +1,44 @@ +import Testing +@testable import OrcaComputerUseMacOSCore + +@Suite("Screen capture probe window selection") +struct ScreenCaptureProbeWindowSelectionTests { + @Test("own process windows cannot prove cross-app capture permission") + func excludesOwnWindows() { + let selected = ScreenCaptureProbeWindowSelection.firstCrossProcessNormalWindow( + ownPid: 42, + windows: [ + .init(layer: 0, ownerPid: 42, windowId: 1), + .init(layer: 0, ownerPid: 84, windowId: 2) + ] + ) + + #expect(selected == 2) + } + + @Test("non-normal windows are skipped") + func excludesNonNormalWindows() { + let selected = ScreenCaptureProbeWindowSelection.firstCrossProcessNormalWindow( + ownPid: 42, + windows: [ + .init(layer: 1, ownerPid: 84, windowId: 1), + .init(layer: 0, ownerPid: 84, windowId: 2) + ] + ) + + #expect(selected == 2) + } + + @Test("no cross-process normal window returns nil") + func noCandidate() { + let selected = ScreenCaptureProbeWindowSelection.firstCrossProcessNormalWindow( + ownPid: 42, + windows: [ + .init(layer: 0, ownerPid: 42, windowId: 1), + .init(layer: 1, ownerPid: 84, windowId: 2) + ] + ) + + #expect(selected == nil) + } +}