fix(computer): fence macOS HID coordinate clicks

2026-10-21 16:03:26 +00:00 · 2026-08-06 23:38:26 -04:00
parent 1aaa5cfbd9
commit efc160a6bb
10 changed files with 674 additions and 105 deletions
@@ -729,10 +729,16 @@ final class Provider {
        let snapshot = try currentSnapshot(params: params)
        let button = params["mouseButton"]?.string ?? "left"
        let count = try positiveInteger(params["clickCount"]?.number, defaultValue: 1, name: "clickCount")
+        guard count <= SyntheticMouseClickDelivery.maxClickCount else {
+            throw ProviderError.coded(
+                "invalid_argument",
+                "clickCount must be at most \(SyntheticMouseClickDelivery.maxClickCount)"
+            )
+        }
        let modifiers = try KeyMap.parseModifiers(params["modifiers"]?.string)
        // Why: agents expect a click into a target app to make the next
        // keyboard action safe, even when the click uses an AX action path.
-        recoverWindow(snapshot.app)
+        recoverWindow(snapshot.app, windowId: snapshot.windowId, windowBounds: snapshot.windowBounds)
        if let elementIndex = try optionalInteger(params, "elementIndex") {
            let record = try element(snapshot, elementIndex)
            if modifiers.isEmpty, count <= 1, let actionName = try performClickAction(record: record, mouseButton: button) {
@@ -743,7 +749,8 @@ final class Provider {
                    at: point,
                    button: mouseButton(button),
                    count: count,
-                    modifiers: modifiers
+                    modifiers: modifiers,
+                    targetWindow: snapshot
                )
                return actionMetadata(
                    path: "synthetic",
@@ -758,7 +765,8 @@ final class Provider {
            at: point,
            button: mouseButton(button),
            count: count,
-            modifiers: modifiers
+            modifiers: modifiers,
+            targetWindow: snapshot
        )
        return actionMetadata(
            path: "synthetic",
@@ -1123,6 +1131,124 @@ private func isTargetWindowFocused(_ snapshot: Snapshot) -> Bool {
    return !intersection.isNull && intersection.area >= min(frame.area, snapshot.windowBounds.area) * 0.75
 }

+private func currentSyntheticClickRecipient(
+    snapshot: Snapshot,
+    point: CGPoint
+) -> SyntheticMouseClickDelivery.Recipient? {
+    let target = syntheticClickRecipient(pid: snapshot.app.pid, windowId: snapshot.windowId)
+    var cachedTargetCandidates: [WindowCandidate]?
+    func targetCandidates() -> [WindowCandidate] {
+        if let cachedTargetCandidates { return cachedTargetCandidates }
+        let candidates = WindowCapture.candidates(pid: snapshot.app.pid)
+        cachedTargetCandidates = candidates
+        return candidates
+    }
+    if let focused = focusedSyntheticClickRecipient(
+        targetPID: snapshot.app.pid,
+        targetCandidates: targetCandidates
+    ) {
+        guard focused == target else { return focused }
+    } else {
+        guard NSWorkspace.shared.frontmostApplication?.processIdentifier == snapshot.app.pid else {
+            return nil
+        }
+    }
+    return hitTestSyntheticClickRecipient(
+        at: point,
+        targetPID: snapshot.app.pid,
+        targetCandidates: targetCandidates
+    )
+}
+
+private func focusedSyntheticClickRecipient(
+    targetPID: pid_t,
+    targetCandidates: () -> [WindowCandidate]
+) -> SyntheticMouseClickDelivery.Recipient? {
+    let systemWide = AXUIElementCreateSystemWide()
+    guard let focusedApp = copyElement(systemWide, kAXFocusedApplicationAttribute as String),
+          let ownerPID = pidAttribute(focusedApp),
+          let focusedWindow = copyElement(systemWide, kAXFocusedWindowAttribute as String) ??
+            copyElement(focusedApp, kAXFocusedWindowAttribute as String)
+    else {
+        return nil
+    }
+    if let windowId = windowNumber(focusedWindow) {
+        return syntheticClickRecipient(pid: ownerPID, windowId: windowId)
+    }
+    guard ownerPID == targetPID else { return nil }
+    guard let frame = absoluteFrame(focusedWindow),
+          let candidate = SyntheticMouseClickDelivery.uniqueWindowCandidate(
+            from: targetCandidates(),
+            matching: {
+              windowFramesMatch($0.bounds, frame)
+            }
+          )
+    else {
+        return nil
+    }
+    return syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId)
+}
+
+private func hitTestSyntheticClickRecipient(
+    at point: CGPoint,
+    targetPID: pid_t,
+    targetCandidates: () -> [WindowCandidate]
+) -> SyntheticMouseClickDelivery.Recipient? {
+    let systemWide = AXUIElementCreateSystemWide()
+    var hitElement: AXUIElement?
+    guard AXUIElementCopyElementAtPosition(
+        systemWide,
+        Float(point.x),
+        Float(point.y),
+        &hitElement
+    ) == .success,
+          let hitElement,
+          let ownerPID = pidAttribute(hitElement),
+          let window = containingWindow(hitElement)
+    else {
+        return nil
+    }
+    if let windowId = windowNumber(window) {
+        return syntheticClickRecipient(pid: ownerPID, windowId: windowId)
+    }
+    guard ownerPID == targetPID else { return nil }
+    guard let frame = absoluteFrame(window),
+          let candidate = SyntheticMouseClickDelivery.uniqueWindowCandidate(
+            from: targetCandidates(),
+            matching: {
+              windowFramesMatch($0.bounds, frame)
+            }
+          )
+    else {
+        return nil
+    }
+    return syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId)
+}
+
+private func containingWindow(_ element: AXUIElement) -> AXUIElement? {
+    var current = element
+    for _ in 0..<64 {
+        if stringAttribute(current, kAXRoleAttribute as String) == kAXWindowRole as String {
+            return current
+        }
+        if let window = copyElement(current, kAXWindowAttribute as String) {
+            return window
+        }
+        guard let parent = copyElement(current, kAXParentAttribute as String) else {
+            return nil
+        }
+        current = parent
+    }
+    return nil
+}
+
+private func syntheticClickRecipient(
+    pid: pid_t,
+    windowId: CGWindowID
+) -> SyntheticMouseClickDelivery.Recipient {
+    SyntheticMouseClickDelivery.Recipient(ownerPID: pid, windowID: windowId)
+}
+
 private func requireTargetWindowFocused(_ snapshot: Snapshot, restoreWindowRequested: Bool) throws {
    guard let failure = KeyboardInputSafety.syntheticInputFocusFailure(
        targetWindowFocused: isTargetWindowFocused(snapshot),
@@ -1163,14 +1289,44 @@ private func matchingWindow(appElement: AXUIElement, capture: WindowCapture, foc
    } ?? focused
 }

-private func recoverWindow(_ app: AppDescriptor) {
+private func recoverWindow(
+    _ app: AppDescriptor,
+    windowId: CGWindowID? = nil,
+    windowBounds: CGRect? = nil
+) {
    _ = app.app.unhide()
    _ = app.app.activate(options: [.activateAllWindows])
    if let bundleId = app.bundleId {
        openBundle(bundleId)
    }
    let appElement = AXUIElementCreateApplication(app.pid)
-    if let window = copyElement(appElement, kAXFocusedWindowAttribute as String) ?? copyArray(appElement, kAXWindowsAttribute as String)?.first {
+    let focusedWindow = copyElement(appElement, kAXFocusedWindowAttribute as String)
+    var cachedWindows: [AXUIElement]?
+    func windows() -> [AXUIElement] {
+        if let cachedWindows { return cachedWindows }
+        let value = copyArray(appElement, kAXWindowsAttribute as String) ?? []
+        cachedWindows = value
+        return value
+    }
+    let targetWindow: AXUIElement?
+    if let focusedWindow,
+       (windowId == nil && windowBounds == nil || windowMatchesCapture(
+           focusedWindow,
+           windowId: windowId,
+           windowBounds: windowBounds
+       )) {
+        targetWindow = focusedWindow
+    } else {
+        let exactWindow = windowId.flatMap { targetId in
+            windows().first { windowNumber($0) == targetId }
+        }
+        targetWindow = exactWindow ?? windowBounds.flatMap { targetBounds in
+            windows().first { window in
+                absoluteFrame(window).map { windowFramesMatch($0, targetBounds) } == true
+            }
+        }
+    }
+    if let window = targetWindow ?? focusedWindow ?? windows().first {
        _ = AXUIElementSetAttributeValue(window, kAXMinimizedAttribute as CFString, kCFBooleanFalse)
        _ = AXUIElementPerformAction(window, kAXRaiseAction as CFString)
        _ = AXUIElementSetAttributeValue(window, kAXMainAttribute as CFString, kCFBooleanTrue)
@@ -1179,6 +1335,24 @@ private func recoverWindow(_ app: AppDescriptor) {
    Thread.sleep(forTimeInterval: 0.4)
 }

+private func windowMatchesCapture(
+    _ window: AXUIElement,
+    windowId: CGWindowID?,
+    windowBounds: CGRect?
+) -> Bool {
+    if let windowId, windowNumber(window) == windowId { return true }
+    guard let windowBounds, let frame = absoluteFrame(window) else { return false }
+    return windowFramesMatch(frame, windowBounds)
+}
+
+private func windowFramesMatch(_ lhs: CGRect, _ rhs: CGRect) -> Bool {
+    let tolerance: CGFloat = 2
+    return abs(lhs.minX - rhs.minX) <= tolerance &&
+        abs(lhs.minY - rhs.minY) <= tolerance &&
+        abs(lhs.width - rhs.width) <= tolerance &&
+        abs(lhs.height - rhs.height) <= tolerance
+}
+
 private func openBundle(_ bundleId: String) {
    let process = Process()
    process.executableURL = URL(fileURLWithPath: "/usr/bin/open")
@@ -2294,7 +2468,8 @@ private enum Input {
        at point: CGPoint,
        button: MouseButton,
        count: Int,
-        modifiers: [KeyModifier]
+        modifiers: [KeyModifier],
+        targetWindow: Snapshot
    ) throws {
        guard let source = CGEventSource(stateID: .combinedSessionState) else {
            throw ProviderError.coded("accessibility_error", "failed to create event source")
@@ -2302,32 +2477,56 @@ private enum Input {
        let flags = modifiers.reduce(into: CGEventFlags()) { result, modifier in
            result.insert(modifier.flag)
        }
-        // Why HID tap + pacing: see SyntheticMouseClickDelivery (STA-3433).
-        for step in SyntheticMouseClickDelivery.steps(clickCount: count) {
-            let type: CGEventType
-            switch step {
-            case .move:
-                type = .mouseMoved
-            case .buttonDown:
-                type = button.downEvent
-            case .buttonUp:
-                type = button.upEvent
+        let target = syntheticClickRecipient(pid: targetWindow.app.pid, windowId: targetWindow.windowId)
+        do {
+            try SyntheticMouseClickDelivery.deliver(
+                clickCount: count,
+                target: target,
+                currentRecipient: {
+                    currentSyntheticClickRecipient(snapshot: targetWindow, point: point)
+                },
+                makeEvent: { step in
+                    let type: CGEventType
+                    switch step {
+                    case .move:
+                        type = .mouseMoved
+                    case .buttonDown:
+                        type = button.downEvent
+                    case .buttonUp:
+                        type = button.upEvent
+                    }
+                    guard let event = CGEvent(
+                        mouseEventSource: source,
+                        mouseType: type,
+                        mouseCursorPosition: point,
+                        mouseButton: button.cgButton
+                    ) else {
+                        throw ProviderError.coded("accessibility_error", "failed to create mouse event")
+                    }
+                    event.flags = flags
+                    let clickState = SyntheticMouseClickDelivery.clickState(for: step)
+                    if clickState > 0 {
+                        event.setIntegerValueField(.mouseEventClickState, value: clickState)
+                    }
+                    return event
+                },
+                post: { $0.post(tap: .cghidEventTap) },
+                pause: { _ = usleep($0) }
+            )
+        } catch let failure as SyntheticMouseClickDelivery.FenceFailure {
+            switch failure {
+            case let .recipientChanged(expected, actual, deliveredPresses):
+                let actualDescription = actual.map {
+                    "pid \($0.ownerPID) window \($0.windowID)"
+                } ?? "no focused window"
+                let recovery = deliveredPresses == 0
+                    ? "bring the target window forward, run get-app-state again, and retry"
+                    : "\(deliveredPresses) press(es) may already have been delivered; run get-app-state and verify state before retrying"
+                throw ProviderError.coded(
+                    "window_not_focused",
+                    "coordinate click aborted because target pid \(expected.ownerPID) window \(expected.windowID) is no longer the focused topmost recipient (current: \(actualDescription)); \(recovery)"
+                )
            }
-            guard let event = CGEvent(
-                mouseEventSource: source,
-                mouseType: type,
-                mouseCursorPosition: point,
-                mouseButton: button.cgButton
-            ) else {
-                throw ProviderError.coded("accessibility_error", "failed to create mouse event")
-            }
-            event.flags = flags
-            let clickState = SyntheticMouseClickDelivery.clickState(for: step)
-            if clickState > 0 {
-                event.setIntegerValueField(.mouseEventClickState, value: clickState)
-            }
-            event.post(tap: .cghidEventTap)
-            usleep(SyntheticMouseClickDelivery.interEventPauseMicroseconds)
        }
    }