Files
orca/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SyntheticMouseClickDelivery.swift
T
Kelvin Amoaba c31fdaa6a5 feat(computer): drive macOS apps in the background by default (#36005)
* feat(computer): drive macOS apps in the background by default

Clicks no longer activate and raise the target first. Pointer and key
input go to the target window's pid through a new BackgroundInputTransport:
mouse and scroll events carry the window id in field 51 and a window-local
location, keys run inside a focus-without-raise key-window scope, and
chords carry no Unicode string so menu key equivalents fire. Scroll and
drag now land (#27306); typed text no longer doubles in Electron (#17942).

The HID tap and focus fence stay for --restore-window only. Results
report action.delivery, the handshake reports supports.input.background,
and a missing private symbol or a popup click fails with
background_unavailable instead of falling back to the foreground.

* test(computer): read typed key strings back instead of diffing live-state events

* fix(computer): keep Unicode characters in one key event

* fix(computer): preserve active macOS key windows

Refuse key-window changes in the active app and surface failed
focus records, including cleanup after partially delivered input.

* fix(computer): guard macOS click routes before delivery

* fix(computer): let paste events read the clipboard

* fix(computer): classify clicks in the requested macOS window

* fix(computer): fence macOS scroll and drag recipients
2026-10-29 22:13:19 +00:00

155 lines
5.3 KiB
Swift

/// Foreground (--restore-window) clicks: HID taps behind a focused-recipient fence; background clicks use BackgroundInputTransport.
/// Pace down/up events or WindowServer can drop mouseUp.
public enum SyntheticMouseClickDelivery {
public static let maxClickCount = 3
public struct Recipient: Equatable, Sendable {
public let ownerPID: Int32
public let windowID: UInt32
public init(ownerPID: Int32, windowID: UInt32) {
self.ownerPID = ownerPID
self.windowID = windowID
}
}
public enum RecipientObservation: Equatable, Sendable {
case focused(Recipient)
case dismissed
case unavailable
var recipient: Recipient? {
guard case let .focused(recipient) = self else { return nil }
return recipient
}
}
public enum FenceFailure: Error, Equatable {
case recipientChanged(expected: Recipient, actual: Recipient?, deliveredPresses: Int)
}
public enum Step: Equatable {
case move
case buttonDown(pressIndex: Int)
case buttonUp(pressIndex: Int)
}
/// Pace events so WindowServer can route mouseUp.
public static let interEventPauseMicroseconds: UInt32 = 50_000
/// Paired presses use pressIndex as click state to register double/triple clicks.
public static func steps(clickCount: Int) -> [Step] {
var steps: [Step] = [.move]
for press in 1...min(max(clickCount, 1), maxClickCount) {
steps.append(.buttonDown(pressIndex: press))
steps.append(.buttonUp(pressIndex: press))
}
return steps
}
/// Click state field value for a step; 0 leaves the field unset.
public static func clickState(for step: Step) -> Int64 {
switch step {
case .move:
return 0
case let .buttonDown(pressIndex), let .buttonUp(pressIndex):
return Int64(pressIndex)
}
}
public static func uniqueWindowCandidate<Candidate>(
from candidates: [Candidate],
matching predicate: (Candidate) -> Bool
) -> Candidate? {
var match: Candidate?
for candidate in candidates where predicate(candidate) {
guard match == nil else { return nil }
match = candidate
}
return match
}
public static func deliverScroll<Event>(
move: Event,
wheel: Event,
target: Recipient,
currentObservation: () -> RecipientObservation,
post: (Event) -> Void,
pause: (UInt32) -> Void
) throws {
try requireRecipient(target, currentObservation: currentObservation)
post(move)
pause(interEventPauseMicroseconds)
try requireRecipient(target, currentObservation: currentObservation)
post(wheel)
}
public static func deliverDrag<Event>(
move: Event,
down: Event,
dragged: [Event],
up: Event,
target: Recipient,
currentObservation: () -> RecipientObservation,
post: (Event) -> Void,
pause: (UInt32) -> Void
) throws {
try requireRecipient(target, currentObservation: currentObservation)
post(move)
try requireRecipient(target, currentObservation: currentObservation)
post(down)
pause(interEventPauseMicroseconds)
for event in dragged {
post(event)
pause(10_000)
}
post(up)
}
private static func requireRecipient(_ target: Recipient, currentObservation: () -> RecipientObservation) throws {
let observation = currentObservation()
guard observation == .focused(target) else {
throw FenceFailure.recipientChanged(expected: target, actual: observation.recipient, deliveredPresses: 0)
}
}
public static func deliver<Event>(
clickCount: Int,
target: Recipient,
currentObservation: () -> RecipientObservation,
makeEvent: (Step) throws -> Event,
post: (Event) -> Void,
pause: (UInt32) -> Void
) throws {
post(try makeEvent(.move))
pause(interEventPauseMicroseconds)
let pressCount = min(max(clickCount, 1), maxClickCount)
for pressIndex in 1...pressCount {
let beforeDown = currentObservation()
guard beforeDown == .focused(target) else {
throw FenceFailure.recipientChanged(
expected: target,
actual: beforeDown.recipient,
deliveredPresses: pressIndex - 1
)
}
let down = try makeEvent(.buttonDown(pressIndex: pressIndex))
let up = try makeEvent(.buttonUp(pressIndex: pressIndex))
post(down)
pause(interEventPauseMicroseconds)
post(up)
let afterUp = currentObservation()
// A final mouse-up may dismiss the target, but an unavailable probe is unsafe.
let finalDismissal = pressIndex == pressCount && afterUp == .dismissed
guard afterUp == .focused(target) || finalDismissal else {
throw FenceFailure.recipientChanged(
expected: target,
actual: afterUp.recipient,
deliveredPresses: pressIndex
)
}
pause(interEventPauseMicroseconds)
}
}
}