mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
* feat(computer): drive macOS apps in the background by default Clicks no longer activate and raise the target first. Pointer and key input go to the target window's pid through a new BackgroundInputTransport: mouse and scroll events carry the window id in field 51 and a window-local location, keys run inside a focus-without-raise key-window scope, and chords carry no Unicode string so menu key equivalents fire. Scroll and drag now land (#27306); typed text no longer doubles in Electron (#17942). The HID tap and focus fence stay for --restore-window only. Results report action.delivery, the handshake reports supports.input.background, and a missing private symbol or a popup click fails with background_unavailable instead of falling back to the foreground. * test(computer): read typed key strings back instead of diffing live-state events * fix(computer): keep Unicode characters in one key event * fix(computer): preserve active macOS key windows Refuse key-window changes in the active app and surface failed focus records, including cleanup after partially delivered input. * fix(computer): guard macOS click routes before delivery * fix(computer): let paste events read the clipboard * fix(computer): classify clicks in the requested macOS window * fix(computer): fence macOS scroll and drag recipients
155 lines
5.3 KiB
Swift
155 lines
5.3 KiB
Swift
/// Foreground (--restore-window) clicks: HID taps behind a focused-recipient fence; background clicks use BackgroundInputTransport.
|
|
/// Pace down/up events or WindowServer can drop mouseUp.
|
|
public enum SyntheticMouseClickDelivery {
|
|
public static let maxClickCount = 3
|
|
|
|
public struct Recipient: Equatable, Sendable {
|
|
public let ownerPID: Int32
|
|
public let windowID: UInt32
|
|
|
|
public init(ownerPID: Int32, windowID: UInt32) {
|
|
self.ownerPID = ownerPID
|
|
self.windowID = windowID
|
|
}
|
|
}
|
|
|
|
public enum RecipientObservation: Equatable, Sendable {
|
|
case focused(Recipient)
|
|
case dismissed
|
|
case unavailable
|
|
|
|
var recipient: Recipient? {
|
|
guard case let .focused(recipient) = self else { return nil }
|
|
return recipient
|
|
}
|
|
}
|
|
|
|
public enum FenceFailure: Error, Equatable {
|
|
case recipientChanged(expected: Recipient, actual: Recipient?, deliveredPresses: Int)
|
|
}
|
|
|
|
public enum Step: Equatable {
|
|
case move
|
|
case buttonDown(pressIndex: Int)
|
|
case buttonUp(pressIndex: Int)
|
|
}
|
|
|
|
/// Pace events so WindowServer can route mouseUp.
|
|
public static let interEventPauseMicroseconds: UInt32 = 50_000
|
|
|
|
/// Paired presses use pressIndex as click state to register double/triple clicks.
|
|
public static func steps(clickCount: Int) -> [Step] {
|
|
var steps: [Step] = [.move]
|
|
for press in 1...min(max(clickCount, 1), maxClickCount) {
|
|
steps.append(.buttonDown(pressIndex: press))
|
|
steps.append(.buttonUp(pressIndex: press))
|
|
}
|
|
return steps
|
|
}
|
|
|
|
/// Click state field value for a step; 0 leaves the field unset.
|
|
public static func clickState(for step: Step) -> Int64 {
|
|
switch step {
|
|
case .move:
|
|
return 0
|
|
case let .buttonDown(pressIndex), let .buttonUp(pressIndex):
|
|
return Int64(pressIndex)
|
|
}
|
|
}
|
|
|
|
public static func uniqueWindowCandidate<Candidate>(
|
|
from candidates: [Candidate],
|
|
matching predicate: (Candidate) -> Bool
|
|
) -> Candidate? {
|
|
var match: Candidate?
|
|
for candidate in candidates where predicate(candidate) {
|
|
guard match == nil else { return nil }
|
|
match = candidate
|
|
}
|
|
return match
|
|
}
|
|
|
|
public static func deliverScroll<Event>(
|
|
move: Event,
|
|
wheel: Event,
|
|
target: Recipient,
|
|
currentObservation: () -> RecipientObservation,
|
|
post: (Event) -> Void,
|
|
pause: (UInt32) -> Void
|
|
) throws {
|
|
try requireRecipient(target, currentObservation: currentObservation)
|
|
post(move)
|
|
pause(interEventPauseMicroseconds)
|
|
try requireRecipient(target, currentObservation: currentObservation)
|
|
post(wheel)
|
|
}
|
|
|
|
public static func deliverDrag<Event>(
|
|
move: Event,
|
|
down: Event,
|
|
dragged: [Event],
|
|
up: Event,
|
|
target: Recipient,
|
|
currentObservation: () -> RecipientObservation,
|
|
post: (Event) -> Void,
|
|
pause: (UInt32) -> Void
|
|
) throws {
|
|
try requireRecipient(target, currentObservation: currentObservation)
|
|
post(move)
|
|
try requireRecipient(target, currentObservation: currentObservation)
|
|
post(down)
|
|
pause(interEventPauseMicroseconds)
|
|
for event in dragged {
|
|
post(event)
|
|
pause(10_000)
|
|
}
|
|
post(up)
|
|
}
|
|
|
|
private static func requireRecipient(_ target: Recipient, currentObservation: () -> RecipientObservation) throws {
|
|
let observation = currentObservation()
|
|
guard observation == .focused(target) else {
|
|
throw FenceFailure.recipientChanged(expected: target, actual: observation.recipient, deliveredPresses: 0)
|
|
}
|
|
}
|
|
|
|
public static func deliver<Event>(
|
|
clickCount: Int,
|
|
target: Recipient,
|
|
currentObservation: () -> RecipientObservation,
|
|
makeEvent: (Step) throws -> Event,
|
|
post: (Event) -> Void,
|
|
pause: (UInt32) -> Void
|
|
) throws {
|
|
post(try makeEvent(.move))
|
|
pause(interEventPauseMicroseconds)
|
|
let pressCount = min(max(clickCount, 1), maxClickCount)
|
|
for pressIndex in 1...pressCount {
|
|
let beforeDown = currentObservation()
|
|
guard beforeDown == .focused(target) else {
|
|
throw FenceFailure.recipientChanged(
|
|
expected: target,
|
|
actual: beforeDown.recipient,
|
|
deliveredPresses: pressIndex - 1
|
|
)
|
|
}
|
|
let down = try makeEvent(.buttonDown(pressIndex: pressIndex))
|
|
let up = try makeEvent(.buttonUp(pressIndex: pressIndex))
|
|
post(down)
|
|
pause(interEventPauseMicroseconds)
|
|
post(up)
|
|
let afterUp = currentObservation()
|
|
// A final mouse-up may dismiss the target, but an unavailable probe is unsafe.
|
|
let finalDismissal = pressIndex == pressCount && afterUp == .dismissed
|
|
guard afterUp == .focused(target) || finalDismissal else {
|
|
throw FenceFailure.recipientChanged(
|
|
expected: target,
|
|
actual: afterUp.recipient,
|
|
deliveredPresses: pressIndex
|
|
)
|
|
}
|
|
pause(interEventPauseMicroseconds)
|
|
}
|
|
}
|
|
}
|