mirror of
https://github.com/stablyai/orca.git
synced 2026-10-30 16:47:34 +00:00
feat(computer): drive macOS apps in the background by default (#36005)
* feat(computer): drive macOS apps in the background by default Clicks no longer activate and raise the target first. Pointer and key input go to the target window's pid through a new BackgroundInputTransport: mouse and scroll events carry the window id in field 51 and a window-local location, keys run inside a focus-without-raise key-window scope, and chords carry no Unicode string so menu key equivalents fire. Scroll and drag now land (#27306); typed text no longer doubles in Electron (#17942). The HID tap and focus fence stay for --restore-window only. Results report action.delivery, the handshake reports supports.input.background, and a missing private symbol or a popup click fails with background_unavailable instead of falling back to the foreground. * test(computer): read typed key strings back instead of diffing live-state events * fix(computer): keep Unicode characters in one key event * fix(computer): preserve active macOS key windows Refuse key-window changes in the active app and surface failed focus records, including cleanup after partially delivered input. * fix(computer): guard macOS click routes before delivery * fix(computer): let paste events read the clipboard * fix(computer): classify clicks in the requested macOS window * fix(computer): fence macOS scroll and drag recipients
This commit is contained in:
Kelvin Amoaba
@@ -344,11 +344,6 @@ function verifyNativeArgumentGuardrails() {
|
||||
'macOS typeText must try verified focused AX text replacement before synthetic keyboard fallback'
|
||||
)
|
||||
}
|
||||
if (!macos.includes('even when the click uses an AX action path')) {
|
||||
failures.push(
|
||||
'macOS clicks must activate the target window before accessibility or pointer input'
|
||||
)
|
||||
}
|
||||
if (
|
||||
!macos.includes('if visibleWindowCount > 0') ||
|
||||
!macos.includes('ProviderError.coded("permission_denied", "app') ||
|
||||
|
||||
@@ -73,6 +73,8 @@ orca computer perform-secondary-action --app <app> --element-index <i> --action
|
||||
|
||||
Prefer semantic actions (`click`, `set-value`, `perform-secondary-action`) over raw `type-text` or `press-key` — they target accessibility elements directly and survive focus changes that keyboard input doesn't.
|
||||
|
||||
On macOS, actions run in the background: the target app isn't brought forward, your frontmost app and key window stay put, and the pointer doesn't move, so you can keep working while an agent drives another window. Each action result reports `action.delivery` (`background` or `foreground`). Pass `--restore-window` only when an app ignores background input, such as drags in Chromium or Electron apps or games; it brings the app forward and uses the real pointer and keyboard. Popup menus and web `<select>` controls are set with `set-value` instead of a click. On Linux and Windows, clicks, scrolling, drags and keyboard input still bring the window forward.
|
||||
|
||||
When accessibility targeting fails, fall back carefully to coordinates:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -244,6 +244,8 @@ final class Provider {
|
||||
|
||||
private func actionResult(params: [String: JSONValue], action runAction: () throws -> [String: Any]) throws -> [String: Any] {
|
||||
var action = try runAction()
|
||||
// Why: only --restore-window activates the app or posts to the HID tap; every other route leaves the user's desktop alone.
|
||||
action["delivery"] = isForegroundRequest(params) ? "foreground" : "background"
|
||||
do {
|
||||
return try renderActionResult(action: action, snapshot: observe(params: params))
|
||||
} catch let error as ProviderError where (error.code == "window_not_found" || error.code == "window_stale") && hasRequestedWindowSelector(params) {
|
||||
@@ -372,8 +374,7 @@ final class Provider {
|
||||
}
|
||||
|
||||
private func currentKeyboardSnapshot(params: [String: JSONValue]) throws -> Snapshot {
|
||||
// Why: AX text replacement/select-all do not post global input, so only
|
||||
// synthetic fallback paths require the target window to be focused.
|
||||
// Why: keyboard actions need fresh focused-element state, never pixels.
|
||||
try currentSnapshot(params: params.merging(["noScreenshot": .bool(true)]) { _, replacement in replacement })
|
||||
}
|
||||
|
||||
@@ -517,6 +518,9 @@ final class Provider {
|
||||
"dock": false,
|
||||
"menubar": false,
|
||||
],
|
||||
"input": [
|
||||
"background": BackgroundInputTransport.isAvailable,
|
||||
],
|
||||
],
|
||||
]
|
||||
}
|
||||
@@ -736,11 +740,15 @@ final class Provider {
|
||||
)
|
||||
}
|
||||
let modifiers = try KeyMap.parseModifiers(params["modifiers"]?.string)
|
||||
// Why: agents expect a click into a target app to make the next
|
||||
// keyboard action safe, even when the click uses an AX action path.
|
||||
recoverWindow(snapshot.app, windowId: snapshot.windowId, windowBounds: snapshot.windowBounds)
|
||||
let foreground = isForegroundRequest(params)
|
||||
if foreground {
|
||||
recoverWindow(snapshot.app, windowId: snapshot.windowId, windowBounds: snapshot.windowBounds)
|
||||
}
|
||||
if let elementIndex = try optionalInteger(params, "elementIndex") {
|
||||
let record = try element(snapshot, elementIndex)
|
||||
if !foreground {
|
||||
try requireBackgroundClickAllowed(backgroundClickHit(record.element, snapshot: snapshot))
|
||||
}
|
||||
if modifiers.isEmpty,
|
||||
count <= 1,
|
||||
button.hasAccessibilityAction,
|
||||
@@ -748,13 +756,7 @@ final class Provider {
|
||||
return actionMetadata(path: "accessibility", actionName: actionName)
|
||||
}
|
||||
if let point = center(record.localFrame, in: snapshot.windowBounds) {
|
||||
try Input.click(
|
||||
at: point,
|
||||
button: button,
|
||||
count: count,
|
||||
modifiers: modifiers,
|
||||
targetWindow: snapshot
|
||||
)
|
||||
try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground, element: record.element)
|
||||
return actionMetadata(
|
||||
path: "synthetic",
|
||||
fallbackReason: "actionUnsupported",
|
||||
@@ -764,13 +766,7 @@ final class Provider {
|
||||
throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame")
|
||||
}
|
||||
let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot)
|
||||
try Input.click(
|
||||
at: point,
|
||||
button: button,
|
||||
count: count,
|
||||
modifiers: modifiers,
|
||||
targetWindow: snapshot
|
||||
)
|
||||
try clickPointer(at: point, button: button, count: count, modifiers: modifiers, snapshot: snapshot, foreground: foreground)
|
||||
return actionMetadata(
|
||||
path: "synthetic",
|
||||
verification: unverifiedAction(reason: "synthetic_input")
|
||||
@@ -844,8 +840,7 @@ final class Provider {
|
||||
if let focused = focusedRecord(snapshot), let verification = TextInput.replaceSelection(focused.element, with: text) {
|
||||
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
|
||||
}
|
||||
try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true)
|
||||
try Input.typeText(text, pid: snapshot.app.pid)
|
||||
try deliverKeys(try withTransportErrors { try BackgroundInputTransport.textEvents(text) }, snapshot: snapshot, params: params)
|
||||
return actionMetadata(
|
||||
path: "synthetic",
|
||||
actionName: "typeText",
|
||||
@@ -855,8 +850,7 @@ final class Provider {
|
||||
|
||||
private func pressKey(params: [String: JSONValue]) throws -> [String: Any] {
|
||||
let snapshot = try currentKeyboardSnapshot(params: params)
|
||||
try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true)
|
||||
try Input.pressKey(try requiredString(params, "key"), pid: snapshot.app.pid)
|
||||
try deliverKeys(try Input.keyEvents(try requiredString(params, "key")), snapshot: snapshot, params: params)
|
||||
return actionMetadata(
|
||||
path: "synthetic",
|
||||
actionName: "pressKey",
|
||||
@@ -874,8 +868,7 @@ final class Provider {
|
||||
verification: TextInput.selectionVerification(focused.element)
|
||||
)
|
||||
}
|
||||
try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true)
|
||||
try Input.pressKey(key, pid: snapshot.app.pid)
|
||||
try deliverKeys(try Input.keyEvents(key), snapshot: snapshot, params: params)
|
||||
return actionMetadata(
|
||||
path: "synthetic",
|
||||
actionName: "hotkey",
|
||||
@@ -889,8 +882,10 @@ final class Provider {
|
||||
if let focused = focusedRecord(snapshot), let verification = TextInput.replaceSelection(focused.element, with: text) {
|
||||
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
|
||||
}
|
||||
try requireTargetWindowFocused(snapshot, restoreWindowRequested: params["restoreWindow"]?.bool == true)
|
||||
try Input.pasteText(text, pid: snapshot.app.pid)
|
||||
let pasteKeys = try Input.keyEvents("cmd+v")
|
||||
try Input.withPasteboardText(text) {
|
||||
try deliverKeys(pasteKeys, snapshot: snapshot, params: params)
|
||||
}
|
||||
return actionMetadata(
|
||||
path: "clipboard",
|
||||
actionName: "paste",
|
||||
@@ -915,11 +910,11 @@ final class Provider {
|
||||
guard let point = center(record.localFrame, in: snapshot.windowBounds) else {
|
||||
throw ProviderError.coded("element_not_found", "element \(record.index) has no scrollable frame")
|
||||
}
|
||||
try Input.scroll(pid: snapshot.app.pid, at: point, direction: direction, pages: pages)
|
||||
try scrollPointer(at: point, direction: direction, pages: pages, snapshot: snapshot, params: params)
|
||||
return actionMetadata(path: "synthetic", fallbackReason: "actionUnsupported")
|
||||
}
|
||||
let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot)
|
||||
try Input.scroll(pid: snapshot.app.pid, at: point, direction: direction, pages: pages)
|
||||
try scrollPointer(at: point, direction: direction, pages: pages, snapshot: snapshot, params: params)
|
||||
return actionMetadata(path: "synthetic")
|
||||
}
|
||||
|
||||
@@ -942,7 +937,14 @@ final class Provider {
|
||||
start = try coordinatePoint(params: params, xKey: "fromX", yKey: "fromY", snapshot: snapshot)
|
||||
end = try coordinatePoint(params: params, xKey: "toX", yKey: "toY", snapshot: snapshot)
|
||||
}
|
||||
try Input.drag(pid: snapshot.app.pid, from: start, to: end)
|
||||
if isForegroundRequest(params) {
|
||||
try requireTargetWindowFocused(snapshot, restoreWindowRequested: true)
|
||||
try Input.drag(from: start, to: end, targetWindow: snapshot)
|
||||
} else {
|
||||
try withTransportErrors {
|
||||
try BackgroundInputTransport.drag(backgroundTarget(snapshot), from: start, to: end)
|
||||
}
|
||||
}
|
||||
return actionMetadata(path: "synthetic")
|
||||
}
|
||||
|
||||
@@ -1278,6 +1280,7 @@ private func hitTestSyntheticClickRecipient(
|
||||
targetPID: pid_t,
|
||||
targetCandidates: () -> [WindowCandidate]
|
||||
) -> SyntheticMouseClickDelivery.RecipientObservation {
|
||||
guard Float(point.x).isFinite, Float(point.y).isFinite else { return .unavailable }
|
||||
let systemWide = AXUIElementCreateSystemWide()
|
||||
var hitElement: AXUIElement?
|
||||
switch AXUIElementCopyElementAtPosition(
|
||||
@@ -1293,20 +1296,37 @@ private func hitTestSyntheticClickRecipient(
|
||||
default:
|
||||
return .unavailable
|
||||
}
|
||||
guard let hitElement,
|
||||
let ownerPID = pidAttribute(hitElement),
|
||||
let window = containingWindow(hitElement)
|
||||
else { return .unavailable }
|
||||
if let windowId = windowNumber(window) {
|
||||
return .focused(syntheticClickRecipient(pid: ownerPID, windowId: windowId))
|
||||
guard let hitElement else { return .unavailable }
|
||||
return pointerRecipient(of: hitElement, targetPID: targetPID, targetCandidates: targetCandidates)
|
||||
}
|
||||
|
||||
private func pointerRecipient(
|
||||
of element: AXUIElement,
|
||||
targetPID: pid_t,
|
||||
targetCandidates: () -> [WindowCandidate]
|
||||
) -> SyntheticMouseClickDelivery.RecipientObservation {
|
||||
guard let ownerPID = pidAttribute(element), let window = containingWindow(element) else {
|
||||
return .unavailable
|
||||
}
|
||||
guard ownerPID == targetPID else { return .unavailable }
|
||||
guard let frame = absoluteFrame(window),
|
||||
var numberValue: CFTypeRef?
|
||||
let numberStatus = AXUIElementCopyAttributeValue(window, "AXWindowNumber" as CFString, &numberValue)
|
||||
if numberStatus == .success {
|
||||
guard let number = numberValue as? NSNumber else { return .unavailable }
|
||||
return .focused(syntheticClickRecipient(pid: ownerPID, windowId: CGWindowID(number.uint32Value)))
|
||||
}
|
||||
guard numberStatus == .attributeUnsupported || numberStatus == .noValue else { return .unavailable }
|
||||
guard ownerPID == targetPID, let frame = absoluteFrame(window),
|
||||
let windows = copyArray(AXUIElementCreateApplication(ownerPID), kAXWindowsAttribute as String)
|
||||
else { return .unavailable }
|
||||
var matches: [AXUIElement] = []
|
||||
for candidate in windows {
|
||||
guard let candidateFrame = absoluteFrame(candidate) else { return .unavailable }
|
||||
if windowFramesMatch(candidateFrame, frame) { matches.append(candidate) }
|
||||
}
|
||||
guard matches.count == 1, CFEqual(matches[0], window),
|
||||
let candidate = SyntheticMouseClickDelivery.uniqueWindowCandidate(
|
||||
from: targetCandidates(),
|
||||
matching: {
|
||||
windowFramesMatch($0.bounds, frame)
|
||||
}
|
||||
matching: { windowFramesMatch($0.bounds, frame) }
|
||||
)
|
||||
else { return .unavailable }
|
||||
return .focused(syntheticClickRecipient(pid: ownerPID, windowId: candidate.windowId))
|
||||
@@ -1351,6 +1371,124 @@ private func requireTargetWindowFocused(_ snapshot: Snapshot, restoreWindowReque
|
||||
}
|
||||
}
|
||||
|
||||
private func isForegroundRequest(_ params: [String: JSONValue]) -> Bool {
|
||||
params["restoreWindow"]?.bool == true
|
||||
}
|
||||
|
||||
private func backgroundTarget(_ snapshot: Snapshot) -> BackgroundInputTransport.Target {
|
||||
BackgroundInputTransport.Target(pid: snapshot.app.pid, windowID: snapshot.windowId, windowFrame: snapshot.windowBounds)
|
||||
}
|
||||
|
||||
private func withTransportErrors<T>(_ body: () throws -> T) throws -> T {
|
||||
do {
|
||||
return try body()
|
||||
} catch let failure as BackgroundInputTransport.Failure {
|
||||
switch failure {
|
||||
case let .unavailable(missingSymbols):
|
||||
throw ProviderError.coded("background_unavailable", "background input is unavailable on this macOS version (missing \(missingSymbols.joined(separator: ", "))); retry with --restore-window to bring the app forward and use foreground input")
|
||||
case .processNotFound:
|
||||
throw ProviderError.coded("app_not_found", "the target app quit before input was delivered")
|
||||
case .eventCreationFailed:
|
||||
throw ProviderError.coded("accessibility_error", "failed to create input event")
|
||||
case .targetWindowNotKey:
|
||||
throw ProviderError.coded("background_unavailable", "background input cannot change the key window of the active app; retry with --restore-window to target this window explicitly")
|
||||
case let .focusRecordFailed(status):
|
||||
throw ProviderError.coded("window_not_focused", "background key-window focus record failed (status \(status)); input may already have been delivered, so inspect current state before retrying")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func clickPointer(
|
||||
at point: CGPoint,
|
||||
button: MouseButtonSelection,
|
||||
count: Int,
|
||||
modifiers: [KeyModifier],
|
||||
snapshot: Snapshot,
|
||||
foreground: Bool,
|
||||
element: AXUIElement? = nil
|
||||
) throws {
|
||||
if foreground {
|
||||
try Input.click(at: point, button: button, count: count, modifiers: modifiers, targetWindow: snapshot)
|
||||
return
|
||||
}
|
||||
let hitElement: AXUIElement
|
||||
if let element {
|
||||
hitElement = element
|
||||
} else {
|
||||
var found: AXUIElement?
|
||||
guard Float(point.x).isFinite, Float(point.y).isFinite,
|
||||
AXUIElementCopyElementAtPosition(AXUIElementCreateApplication(snapshot.app.pid), Float(point.x), Float(point.y), &found) == .success,
|
||||
let found
|
||||
else {
|
||||
throw ProviderError.coded("background_unavailable", "cannot identify the background click target; run get-app-state again or retry with --restore-window")
|
||||
}
|
||||
hitElement = found
|
||||
}
|
||||
let hit = try backgroundClickHit(hitElement, snapshot: snapshot)
|
||||
try requireBackgroundClickAllowed(hit)
|
||||
try withTransportErrors {
|
||||
try BackgroundInputTransport.click(
|
||||
backgroundTarget(snapshot),
|
||||
at: point,
|
||||
button: button,
|
||||
clickCount: count,
|
||||
flags: eventFlags(modifiers),
|
||||
hit: hit
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private func backgroundClickHit(_ element: AXUIElement, snapshot: Snapshot) throws -> BackgroundInputTransport.PointerHit {
|
||||
let recipient = pointerRecipient(of: element, targetPID: snapshot.app.pid) {
|
||||
WindowCapture.candidates(pid: snapshot.app.pid)
|
||||
}
|
||||
guard recipient == .focused(syntheticClickRecipient(pid: snapshot.app.pid, windowId: snapshot.windowId)) else {
|
||||
throw ProviderError.coded("background_unavailable", "the background click target does not identify the requested window; run get-app-state again or retry with --restore-window")
|
||||
}
|
||||
return BackgroundInputTransport.classifyPointerHit(element)
|
||||
}
|
||||
|
||||
private func requireBackgroundClickAllowed(_ hit: BackgroundInputTransport.PointerHit) throws {
|
||||
guard hit != .popupMenu else {
|
||||
throw ProviderError.coded("background_unavailable", "a background click on a popup menu or web <select> opens the menu off screen or wedges the app's input; set the option with set-value on the popup element, or retry with --restore-window for a foreground click")
|
||||
}
|
||||
}
|
||||
|
||||
private func scrollPointer(at point: CGPoint, direction: String, pages: Double, snapshot: Snapshot, params: [String: JSONValue]) throws {
|
||||
guard let delta = boundedInteger(max(1, (12 * pages).rounded()), as: Int32.self) else {
|
||||
throw ProviderError.coded("invalid_argument", "pages is out of range")
|
||||
}
|
||||
let wheel1: Int32 = direction == "up" ? delta : direction == "down" ? -delta : 0
|
||||
let wheel2: Int32 = direction == "left" ? delta : direction == "right" ? -delta : 0
|
||||
if isForegroundRequest(params) {
|
||||
try requireTargetWindowFocused(snapshot, restoreWindowRequested: true)
|
||||
try Input.scroll(at: point, wheel1: wheel1, wheel2: wheel2, targetWindow: snapshot)
|
||||
return
|
||||
}
|
||||
try withTransportErrors {
|
||||
try BackgroundInputTransport.scroll(backgroundTarget(snapshot), at: point, wheel1: wheel1, wheel2: wheel2)
|
||||
}
|
||||
}
|
||||
|
||||
private func deliverKeys(_ events: [CGEvent], snapshot: Snapshot, params: [String: JSONValue]) throws {
|
||||
if isForegroundRequest(params) {
|
||||
try requireTargetWindowFocused(snapshot, restoreWindowRequested: true)
|
||||
for event in events {
|
||||
event.post(tap: .cghidEventTap)
|
||||
}
|
||||
return
|
||||
}
|
||||
try withTransportErrors {
|
||||
try BackgroundInputTransport.sendKeys(events, to: backgroundTarget(snapshot))
|
||||
}
|
||||
}
|
||||
|
||||
private func eventFlags(_ modifiers: [KeyModifier]) -> CGEventFlags {
|
||||
modifiers.reduce(into: CGEventFlags()) { result, modifier in
|
||||
result.insert(modifier.flag)
|
||||
}
|
||||
}
|
||||
|
||||
private func matchingWindow(appElement: AXUIElement, capture: WindowCapture, focused: AXUIElement, explicitTarget: Bool) -> AXUIElement? {
|
||||
guard let windows = copyArray(appElement, kAXWindowsAttribute as String) else {
|
||||
return nil
|
||||
@@ -1700,43 +1838,6 @@ private func screenshotScale(screenshot: ScreenshotPayload?, bounds: CGRect) ->
|
||||
)
|
||||
}
|
||||
|
||||
extension MouseButtonSelection {
|
||||
// Why: macOS has no dedicated middle-button event family; it rides `otherMouse*`
|
||||
// with the button number carried by `mouseButton:` on the event constructor.
|
||||
var cgButton: CGMouseButton {
|
||||
switch self {
|
||||
case .left:
|
||||
return .left
|
||||
case .right:
|
||||
return .right
|
||||
case .middle:
|
||||
return .center
|
||||
}
|
||||
}
|
||||
|
||||
var downEvent: CGEventType {
|
||||
switch self {
|
||||
case .left:
|
||||
return .leftMouseDown
|
||||
case .right:
|
||||
return .rightMouseDown
|
||||
case .middle:
|
||||
return .otherMouseDown
|
||||
}
|
||||
}
|
||||
|
||||
var upEvent: CGEventType {
|
||||
switch self {
|
||||
case .left:
|
||||
return .leftMouseUp
|
||||
case .right:
|
||||
return .rightMouseUp
|
||||
case .middle:
|
||||
return .otherMouseUp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func mouseButton(_ raw: String?) throws -> MouseButtonSelection {
|
||||
switch ActionArgumentValidation.mouseButton(raw) {
|
||||
case let .success(button):
|
||||
@@ -2567,11 +2668,9 @@ private enum Input {
|
||||
guard let source = CGEventSource(stateID: .combinedSessionState) else {
|
||||
throw ProviderError.coded("accessibility_error", "failed to create event source")
|
||||
}
|
||||
let flags = modifiers.reduce(into: CGEventFlags()) { result, modifier in
|
||||
result.insert(modifier.flag)
|
||||
}
|
||||
let flags = eventFlags(modifiers)
|
||||
let target = syntheticClickRecipient(pid: targetWindow.app.pid, windowId: targetWindow.windowId)
|
||||
do {
|
||||
try withRecipientErrors {
|
||||
try SyntheticMouseClickDelivery.deliver(
|
||||
clickCount: count,
|
||||
target: target,
|
||||
@@ -2606,6 +2705,12 @@ private enum Input {
|
||||
post: { $0.post(tap: .cghidEventTap) },
|
||||
pause: { _ = usleep($0) }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private static func withRecipientErrors(_ body: () throws -> Void) throws {
|
||||
do {
|
||||
try body()
|
||||
} catch let failure as SyntheticMouseClickDelivery.FenceFailure {
|
||||
switch failure {
|
||||
case let .recipientChanged(expected, actual, deliveredPresses):
|
||||
@@ -2617,79 +2722,59 @@ private enum Input {
|
||||
: "\(deliveredPresses) press(es) may already have been delivered; run get-app-state and verify state before retrying"
|
||||
throw ProviderError.coded(
|
||||
"window_not_focused",
|
||||
"coordinate click aborted because target pid \(expected.ownerPID) window \(expected.windowID) is no longer the focused topmost recipient (current: \(actualDescription)); \(recovery)"
|
||||
"pointer input aborted because target pid \(expected.ownerPID) window \(expected.windowID) is no longer the focused topmost recipient (current: \(actualDescription)); \(recovery)"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static func scroll(pid: pid_t, at point: CGPoint, direction: String, pages: Double) throws {
|
||||
guard let delta = boundedInteger(max(1, (12 * pages).rounded()), as: Int32.self) else {
|
||||
throw ProviderError.coded("invalid_argument", "pages is out of range")
|
||||
}
|
||||
let wheel1: Int32 = direction == "up" ? delta : direction == "down" ? -delta : 0
|
||||
let wheel2: Int32 = direction == "left" ? delta : direction == "right" ? -delta : 0
|
||||
guard let event = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 2, wheel1: wheel1, wheel2: wheel2, wheel3: 0) else {
|
||||
static func scroll(at point: CGPoint, wheel1: Int32, wheel2: Int32, targetWindow: Snapshot) throws {
|
||||
let move = try mouseEvent(.mouseMoved, at: point)
|
||||
guard let wheel = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 2, wheel1: wheel1, wheel2: wheel2, wheel3: 0) else {
|
||||
throw ProviderError.coded("accessibility_error", "failed to create scroll event")
|
||||
}
|
||||
event.location = point
|
||||
event.postToPid(pid)
|
||||
}
|
||||
|
||||
static func drag(pid: pid_t, from start: CGPoint, to end: CGPoint) throws {
|
||||
guard let source = CGEventSource(stateID: .combinedSessionState) else {
|
||||
throw ProviderError.coded("accessibility_error", "failed to create event source")
|
||||
}
|
||||
try mouse(.mouseMoved, source: source, point: start, button: .left, pid: pid)
|
||||
try mouse(.leftMouseDown, source: source, point: start, button: .left, pid: pid)
|
||||
for step in 1...10 {
|
||||
let progress = CGFloat(step) / 10
|
||||
try mouse(
|
||||
.leftMouseDragged,
|
||||
source: source,
|
||||
point: CGPoint(x: start.x + (end.x - start.x) * progress, y: start.y + (end.y - start.y) * progress),
|
||||
button: .left,
|
||||
pid: pid
|
||||
wheel.location = point
|
||||
try withRecipientErrors {
|
||||
try SyntheticMouseClickDelivery.deliverScroll(
|
||||
move: move, wheel: wheel,
|
||||
target: syntheticClickRecipient(pid: targetWindow.app.pid, windowId: targetWindow.windowId),
|
||||
currentObservation: { currentSyntheticClickRecipient(snapshot: targetWindow, point: point) },
|
||||
post: { $0.post(tap: .cghidEventTap) },
|
||||
pause: { _ = usleep($0) }
|
||||
)
|
||||
}
|
||||
try mouse(.leftMouseUp, source: source, point: end, button: .left, pid: pid)
|
||||
}
|
||||
|
||||
static func typeText(_ text: String, pid: pid_t) throws {
|
||||
for unit in text.utf16 {
|
||||
var char = unit
|
||||
guard let down = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: true),
|
||||
let up = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false)
|
||||
else {
|
||||
throw ProviderError.coded("accessibility_error", "failed to create keyboard event")
|
||||
}
|
||||
down.keyboardSetUnicodeString(stringLength: 1, unicodeString: &char)
|
||||
up.keyboardSetUnicodeString(stringLength: 1, unicodeString: &char)
|
||||
down.post(tap: .cghidEventTap)
|
||||
up.post(tap: .cghidEventTap)
|
||||
static func drag(from start: CGPoint, to end: CGPoint, targetWindow: Snapshot) throws {
|
||||
let move = try mouseEvent(.mouseMoved, at: start)
|
||||
let down = try mouseEvent(.leftMouseDown, at: start)
|
||||
let up = try mouseEvent(.leftMouseUp, at: end)
|
||||
let dragged = try (1...10).map { step in
|
||||
let progress = CGFloat(step) / 10
|
||||
return try mouseEvent(.leftMouseDragged, at: CGPoint(x: start.x + (end.x - start.x) * progress, y: start.y + (end.y - start.y) * progress))
|
||||
}
|
||||
try withRecipientErrors {
|
||||
try SyntheticMouseClickDelivery.deliverDrag(
|
||||
move: move, down: down, dragged: dragged, up: up,
|
||||
target: syntheticClickRecipient(pid: targetWindow.app.pid, windowId: targetWindow.windowId),
|
||||
currentObservation: { currentSyntheticClickRecipient(snapshot: targetWindow, point: start) },
|
||||
post: { $0.post(tap: .cghidEventTap) },
|
||||
pause: { _ = usleep($0) }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
static func pressKey(_ key: String, pid: pid_t) throws {
|
||||
static func keyEvents(_ key: String) throws -> [CGEvent] {
|
||||
let parsed = try KeyMap.parse(key)
|
||||
var flags = CGEventFlags()
|
||||
var pressedModifiers: [KeyModifier] = []
|
||||
defer {
|
||||
for modifier in pressedModifiers.reversed() {
|
||||
flags.remove(modifier.flag)
|
||||
try? keyEvent(modifier.keyCode, down: false, flags: flags, pid: pid)
|
||||
}
|
||||
return try withTransportErrors {
|
||||
try BackgroundInputTransport.chordEvents(
|
||||
keyCode: parsed.keyCode,
|
||||
modifiers: parsed.modifiers.map { (keyCode: $0.keyCode, flag: $0.flag) }
|
||||
)
|
||||
}
|
||||
for modifier in parsed.modifiers {
|
||||
flags.insert(modifier.flag)
|
||||
try keyEvent(modifier.keyCode, down: true, flags: flags, pid: pid)
|
||||
pressedModifiers.append(modifier)
|
||||
}
|
||||
try keyEvent(parsed.keyCode, down: true, flags: flags, pid: pid)
|
||||
try keyEvent(parsed.keyCode, down: false, flags: flags, pid: pid)
|
||||
}
|
||||
|
||||
static func pasteText(_ text: String, pid: pid_t) throws {
|
||||
static func withPasteboardText(_ text: String, _ body: () throws -> Void) throws {
|
||||
let pasteboard = NSPasteboard.general
|
||||
let previousItems: [NSPasteboardItem] = pasteboard.pasteboardItems?.map { item in
|
||||
let copy = NSPasteboardItem()
|
||||
@@ -2702,36 +2787,28 @@ private enum Input {
|
||||
} ?? []
|
||||
pasteboard.clearContents()
|
||||
pasteboard.setString(text, forType: .string)
|
||||
let changeCount = pasteboard.changeCount
|
||||
defer {
|
||||
pasteboard.clearContents()
|
||||
if !previousItems.isEmpty {
|
||||
pasteboard.writeObjects(previousItems)
|
||||
// Why: posting Cmd+V returns before the receiver reads the pasteboard.
|
||||
usleep(200_000)
|
||||
if pasteboard.changeCount == changeCount {
|
||||
pasteboard.clearContents()
|
||||
if !previousItems.isEmpty {
|
||||
pasteboard.writeObjects(previousItems)
|
||||
}
|
||||
}
|
||||
}
|
||||
try pressKey("cmd+v", pid: pid)
|
||||
try body()
|
||||
}
|
||||
|
||||
private static func mouse(
|
||||
_ type: CGEventType,
|
||||
source: CGEventSource,
|
||||
point: CGPoint,
|
||||
button: CGMouseButton,
|
||||
flags: CGEventFlags = [],
|
||||
pid: pid_t
|
||||
) throws {
|
||||
guard let event = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: point, mouseButton: button) else {
|
||||
private static func mouseEvent(_ type: CGEventType, at point: CGPoint) throws -> CGEvent {
|
||||
guard point.x.isFinite, point.y.isFinite else {
|
||||
throw ProviderError.coded("invalid_argument", "pointer coordinates are out of range")
|
||||
}
|
||||
guard let event = CGEvent(mouseEventSource: nil, mouseType: type, mouseCursorPosition: point, mouseButton: .left) else {
|
||||
throw ProviderError.coded("accessibility_error", "failed to create mouse event")
|
||||
}
|
||||
event.flags = flags
|
||||
event.postToPid(pid)
|
||||
}
|
||||
|
||||
private static func keyEvent(_ keyCode: CGKeyCode, down: Bool, flags: CGEventFlags, pid: pid_t) throws {
|
||||
guard let event = CGEvent(keyboardEventSource: nil, virtualKey: keyCode, keyDown: down) else {
|
||||
throw ProviderError.coded("accessibility_error", "failed to create key event")
|
||||
}
|
||||
event.flags = flags
|
||||
event.post(tap: .cghidEventTap)
|
||||
return event
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+421
@@ -0,0 +1,421 @@
|
||||
import AppKit
|
||||
import ApplicationServices
|
||||
import CoreGraphics
|
||||
import Darwin
|
||||
|
||||
/// Delivers pointer and key input to one window of an app without activating it, raising it,
|
||||
/// changing the user's key window or moving the real pointer. It owns every private symbol it needs.
|
||||
public enum BackgroundInputTransport {
|
||||
public struct Target: Equatable, Sendable {
|
||||
public let pid: pid_t
|
||||
public let windowID: CGWindowID
|
||||
/// Window frame in global top-left screen points, as CGWindowList reports it.
|
||||
public let windowFrame: CGRect
|
||||
|
||||
public init(pid: pid_t, windowID: CGWindowID, windowFrame: CGRect) {
|
||||
self.pid = pid
|
||||
self.windowID = windowID
|
||||
self.windowFrame = windowFrame
|
||||
}
|
||||
}
|
||||
|
||||
public enum Failure: Error, Equatable {
|
||||
case unavailable(missingSymbols: [String])
|
||||
case processNotFound
|
||||
case eventCreationFailed
|
||||
case targetWindowNotKey
|
||||
case focusRecordFailed(status: Int32)
|
||||
}
|
||||
|
||||
/// What the app's accessibility tree has under a point, which decides how a pointer event may be sent.
|
||||
public enum PointerHit: Equatable, Sendable {
|
||||
/// WebKit content (Safari, WKWebView): drops clicks into a window that is not key.
|
||||
case webKitContent
|
||||
/// A popup menu or web `<select>`: a background click opens menu tracking off screen or wedges Chromium input.
|
||||
case popupMenu
|
||||
case other
|
||||
}
|
||||
|
||||
/// Down/up pacing; WindowServer can drop a mouseUp that follows its mouseDown too closely.
|
||||
static let pressPauseMicroseconds: UInt32 = 28_000
|
||||
static let windowIDField = CGEventField(rawValue: 51)!
|
||||
|
||||
public static var missingSymbols: [String] {
|
||||
PrivateSymbols.shared.missing
|
||||
}
|
||||
|
||||
public static var isAvailable: Bool {
|
||||
missingSymbols.isEmpty
|
||||
}
|
||||
|
||||
// MARK: Pointer
|
||||
|
||||
public static func click(
|
||||
_ target: Target,
|
||||
at point: CGPoint,
|
||||
button: MouseButtonSelection,
|
||||
clickCount: Int,
|
||||
flags: CGEventFlags,
|
||||
hit: PointerHit
|
||||
) throws {
|
||||
let symbols = try PrivateSymbols.shared.require()
|
||||
let clicks = {
|
||||
for clickState in 1...max(clickCount, 1) {
|
||||
try press(target, at: point, button: button, clickState: Int64(clickState), flags: flags, symbols: symbols)
|
||||
}
|
||||
}
|
||||
// Why: WebKit ignores the first click into a window that is not key.
|
||||
if hit == .webKitContent {
|
||||
try withKeyWindow(target, clicks)
|
||||
} else {
|
||||
try clicks()
|
||||
}
|
||||
}
|
||||
|
||||
/// One down/up pair whose click state is its position in a double or triple click.
|
||||
public static func press(
|
||||
_ target: Target,
|
||||
at point: CGPoint,
|
||||
button: MouseButtonSelection,
|
||||
clickState: Int64,
|
||||
flags: CGEventFlags
|
||||
) throws {
|
||||
try press(target, at: point, button: button, clickState: clickState, flags: flags, symbols: PrivateSymbols.shared.require())
|
||||
}
|
||||
|
||||
public static func scroll(_ target: Target, at point: CGPoint, wheel1: Int32, wheel2: Int32, units: CGScrollEventUnit = .line) throws {
|
||||
let symbols = try PrivateSymbols.shared.require()
|
||||
guard let event = CGEvent(
|
||||
scrollWheelEvent2Source: nil,
|
||||
units: units,
|
||||
wheelCount: 2,
|
||||
wheel1: wheel1,
|
||||
wheel2: wheel2,
|
||||
wheel3: 0
|
||||
) else {
|
||||
throw Failure.eventCreationFailed
|
||||
}
|
||||
stamp(event, target: target, at: point, symbols: symbols)
|
||||
event.postToPid(target.pid)
|
||||
}
|
||||
|
||||
/// Runs inside the key-window scope: NSTextView only tracks a drag selection in a key window.
|
||||
/// Chromium still ignores it, because it reads the held button from the hardware state.
|
||||
public static func drag(_ target: Target, from start: CGPoint, to end: CGPoint) throws {
|
||||
let symbols = try PrivateSymbols.shared.require()
|
||||
try withKeyWindow(target) {
|
||||
try mouseEvent(.leftMouseDown, target: target, at: start, button: .left, clickState: 1, flags: [], symbols: symbols).postToPid(target.pid)
|
||||
usleep(pressPauseMicroseconds)
|
||||
for step in 1...10 {
|
||||
let progress = CGFloat(step) / 10
|
||||
let point = CGPoint(x: start.x + (end.x - start.x) * progress, y: start.y + (end.y - start.y) * progress)
|
||||
try mouseEvent(.leftMouseDragged, target: target, at: point, button: .left, clickState: 1, flags: [], symbols: symbols).postToPid(target.pid)
|
||||
usleep(10_000)
|
||||
}
|
||||
try mouseEvent(.leftMouseUp, target: target, at: end, button: .left, clickState: 1, flags: [], symbols: symbols).postToPid(target.pid)
|
||||
}
|
||||
}
|
||||
|
||||
/// Mouse event routed to `target`'s window: AppKit drops pid-posted mouse events unless field 51
|
||||
/// carries the window number and the window location is window-local (fields 91/92 do nothing).
|
||||
static func mouseEvent(
|
||||
_ type: CGEventType,
|
||||
target: Target,
|
||||
at point: CGPoint,
|
||||
button: MouseButtonSelection,
|
||||
clickState: Int64,
|
||||
flags: CGEventFlags
|
||||
) throws -> CGEvent {
|
||||
try mouseEvent(type, target: target, at: point, button: button.cgButton, clickState: clickState, flags: flags, symbols: PrivateSymbols.shared.require())
|
||||
}
|
||||
|
||||
public static func classifyPointerHit(_ element: AXUIElement) -> PointerHit {
|
||||
var current = element
|
||||
for depth in 0..<48 {
|
||||
let role = stringAttribute(current, kAXRoleAttribute as String)
|
||||
// Why: the hit lands on a popup's label or the popup itself, never deeper than its parent.
|
||||
if depth < 2, role == kAXPopUpButtonRole as String || role == kAXMenuButtonRole as String {
|
||||
return .popupMenu
|
||||
}
|
||||
if role == "AXWebArea" {
|
||||
// Why: Chromium accepts window-stamped clicks as is; only WebKit needs the key-window scope,
|
||||
// which fires the app's own did-become-active code (Slack, for one, marks messages read).
|
||||
return isChromiumNode(current) ? .other : .webKitContent
|
||||
}
|
||||
if role == kAXWindowRole as String { return .other }
|
||||
var parent: CFTypeRef?
|
||||
guard AXUIElementCopyAttributeValue(current, kAXParentAttribute as CFString, &parent) == .success,
|
||||
let parent,
|
||||
CFGetTypeID(parent) == AXUIElementGetTypeID()
|
||||
else {
|
||||
return .other
|
||||
}
|
||||
// SAFETY: the CFTypeID check above proves this is an AXUIElement.
|
||||
current = parent as! AXUIElement
|
||||
}
|
||||
return .other
|
||||
}
|
||||
|
||||
// MARK: Keys
|
||||
|
||||
/// Keys of a chord without a Unicode string: AppKit only matches menu key equivalents (⌘A/C/V/Z)
|
||||
/// when the key events carry none.
|
||||
public static func chordEvents(keyCode: CGKeyCode, modifiers: [(keyCode: CGKeyCode, flag: CGEventFlags)]) throws -> [CGEvent] {
|
||||
var events: [CGEvent] = []
|
||||
var flags = CGEventFlags()
|
||||
for modifier in modifiers {
|
||||
flags.insert(modifier.flag)
|
||||
events.append(try keyEvent(modifier.keyCode, down: true, flags: flags))
|
||||
}
|
||||
events.append(try keyEvent(keyCode, down: true, flags: flags))
|
||||
events.append(try keyEvent(keyCode, down: false, flags: flags))
|
||||
for modifier in modifiers.reversed() {
|
||||
flags.remove(modifier.flag)
|
||||
events.append(try keyEvent(modifier.keyCode, down: false, flags: flags))
|
||||
}
|
||||
return events
|
||||
}
|
||||
|
||||
/// Unicode text as key events; the string rides the key-down only, or Electron inserts it twice (#17942).
|
||||
public static func textEvents(_ text: String) throws -> [CGEvent] {
|
||||
var events: [CGEvent] = []
|
||||
for character in text {
|
||||
var units = Array(String(character).utf16)
|
||||
let down = try keyEvent(0, down: true, flags: [])
|
||||
down.keyboardSetUnicodeString(stringLength: units.count, unicodeString: &units)
|
||||
events.append(down)
|
||||
events.append(try keyEvent(0, down: false, flags: []))
|
||||
}
|
||||
return events
|
||||
}
|
||||
|
||||
/// Posts the keys to the pid while `target` is the app's key window.
|
||||
public static func sendKeys(_ events: [CGEvent], to target: Target) throws {
|
||||
try withKeyWindow(target) {
|
||||
for event in events {
|
||||
event.postToPid(target.pid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Refuses to change an active app's key window; inactive apps receive a temporary key-window scope.
|
||||
static func withKeyWindow(_ target: Target, _ body: () throws -> Void) throws {
|
||||
let symbols = try PrivateSymbols.shared.require()
|
||||
let wasActive = NSRunningApplication(processIdentifier: target.pid)?.isActive == true
|
||||
if wasActive {
|
||||
let appElement = AXUIElementCreateApplication(target.pid)
|
||||
var focused: CFTypeRef?
|
||||
guard AXUIElementCopyAttributeValue(appElement, kAXFocusedWindowAttribute as CFString, &focused) == .success,
|
||||
let focused, CFGetTypeID(focused) == AXUIElementGetTypeID()
|
||||
else { throw Failure.targetWindowNotKey }
|
||||
// SAFETY: the CFTypeID check above proves focused is an AXUIElement.
|
||||
let window = focused as! AXUIElement
|
||||
var number: CFTypeRef?
|
||||
guard AXUIElementCopyAttributeValue(window, "AXWindowNumber" as CFString, &number) == .success,
|
||||
let number = number as? NSNumber, number.uint32Value == target.windowID
|
||||
else { throw Failure.targetWindowNotKey }
|
||||
try body()
|
||||
return
|
||||
}
|
||||
var psn = ProcessSerialNumber()
|
||||
guard symbols.getProcessForPID(target.pid, &psn) == noErr else {
|
||||
throw Failure.processNotFound
|
||||
}
|
||||
func defocus() throws {
|
||||
// Why: focus records and input events travel different paths, so let the input arrive first.
|
||||
usleep(50_000)
|
||||
try postRecord(keyFocusRecord(windowID: target.windowID, focus: false), psn: psn, symbols: symbols)
|
||||
}
|
||||
do {
|
||||
try postRecord(keyFocusRecord(windowID: target.windowID, focus: true), psn: psn, symbols: symbols)
|
||||
usleep(20_000)
|
||||
for kind: UInt8 in [0x01, 0x02] {
|
||||
try postRecord(makeKeyRecord(windowID: target.windowID, kind: kind), psn: psn, symbols: symbols)
|
||||
}
|
||||
usleep(50_000)
|
||||
try body()
|
||||
} catch {
|
||||
try defocus()
|
||||
throw error
|
||||
}
|
||||
try defocus()
|
||||
}
|
||||
|
||||
/// yabai/cua focus-without-raise record: 0x0d at 0x08, window id at 0x3c, 1 = focus / 2 = defocus at 0x8a.
|
||||
static func keyFocusRecord(windowID: CGWindowID, focus: Bool) -> [UInt8] {
|
||||
var record = [UInt8](repeating: 0, count: 0xF8)
|
||||
record[0x04] = 0xF8
|
||||
record[0x08] = 0x0D
|
||||
writeWindowID(windowID, into: &record)
|
||||
record[0x8A] = focus ? 0x01 : 0x02
|
||||
return record
|
||||
}
|
||||
|
||||
/// yabai/cua make-key record; posted as the 0x01/0x02 pair after the focus record.
|
||||
static func makeKeyRecord(windowID: CGWindowID, kind: UInt8) -> [UInt8] {
|
||||
var record = [UInt8](repeating: 0, count: 0xF8)
|
||||
record[0x04] = 0xF8
|
||||
record[0x08] = kind
|
||||
record[0x3A] = 0x10
|
||||
writeWindowID(windowID, into: &record)
|
||||
for index in 0x20..<0x30 {
|
||||
record[index] = 0xFF
|
||||
}
|
||||
return record
|
||||
}
|
||||
|
||||
// MARK: Internals
|
||||
|
||||
private static func press(
|
||||
_ target: Target,
|
||||
at point: CGPoint,
|
||||
button: MouseButtonSelection,
|
||||
clickState: Int64,
|
||||
flags: CGEventFlags,
|
||||
symbols: PrivateSymbols.Loaded
|
||||
) throws {
|
||||
try mouseEvent(button.downEvent, target: target, at: point, button: button.cgButton, clickState: clickState, flags: flags, symbols: symbols).postToPid(target.pid)
|
||||
usleep(pressPauseMicroseconds)
|
||||
try mouseEvent(button.upEvent, target: target, at: point, button: button.cgButton, clickState: clickState, flags: flags, symbols: symbols).postToPid(target.pid)
|
||||
usleep(pressPauseMicroseconds)
|
||||
}
|
||||
|
||||
private static func mouseEvent(
|
||||
_ type: CGEventType,
|
||||
target: Target,
|
||||
at point: CGPoint,
|
||||
button: CGMouseButton,
|
||||
clickState: Int64,
|
||||
flags: CGEventFlags,
|
||||
symbols: PrivateSymbols.Loaded
|
||||
) throws -> CGEvent {
|
||||
guard let event = CGEvent(mouseEventSource: nil, mouseType: type, mouseCursorPosition: point, mouseButton: button) else {
|
||||
throw Failure.eventCreationFailed
|
||||
}
|
||||
event.flags = flags
|
||||
event.setIntegerValueField(.mouseEventClickState, value: clickState)
|
||||
stamp(event, target: target, at: point, symbols: symbols)
|
||||
return event
|
||||
}
|
||||
|
||||
private static func stamp(_ event: CGEvent, target: Target, at point: CGPoint, symbols: PrivateSymbols.Loaded) {
|
||||
event.location = point
|
||||
event.setIntegerValueField(windowIDField, value: Int64(target.windowID))
|
||||
symbols.setWindowLocation(event, CGPoint(x: point.x - target.windowFrame.minX, y: point.y - target.windowFrame.minY))
|
||||
}
|
||||
|
||||
private static func postRecord(_ record: [UInt8], psn: ProcessSerialNumber, symbols: PrivateSymbols.Loaded) throws {
|
||||
var record = record
|
||||
var recordPSN = psn
|
||||
let status = symbols.postEventRecordTo(&recordPSN, &record)
|
||||
guard status == noErr else { throw Failure.focusRecordFailed(status: status) }
|
||||
}
|
||||
|
||||
private static func keyEvent(_ keyCode: CGKeyCode, down: Bool, flags: CGEventFlags) throws -> CGEvent {
|
||||
guard let event = CGEvent(keyboardEventSource: nil, virtualKey: keyCode, keyDown: down) else {
|
||||
throw Failure.eventCreationFailed
|
||||
}
|
||||
event.flags = flags
|
||||
return event
|
||||
}
|
||||
|
||||
private static func writeWindowID(_ windowID: CGWindowID, into record: inout [UInt8]) {
|
||||
withUnsafeBytes(of: windowID.littleEndian) { bytes in
|
||||
for (offset, byte) in bytes.enumerated() {
|
||||
record[0x3C + offset] = byte
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static func isChromiumNode(_ element: AXUIElement) -> Bool {
|
||||
var names: CFArray?
|
||||
guard AXUIElementCopyAttributeNames(element, &names) == .success, let names = names as? [String] else {
|
||||
return false
|
||||
}
|
||||
return names.contains("ChromeAXNodeId")
|
||||
}
|
||||
|
||||
private static func stringAttribute(_ element: AXUIElement, _ attribute: String) -> String? {
|
||||
var value: CFTypeRef?
|
||||
guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else {
|
||||
return nil
|
||||
}
|
||||
return value as? String
|
||||
}
|
||||
}
|
||||
|
||||
extension MouseButtonSelection {
|
||||
// Why: macOS has no dedicated middle-button event family; it rides `otherMouse*`
|
||||
// with the button number carried by `mouseButton:` on the event constructor.
|
||||
public var cgButton: CGMouseButton {
|
||||
switch self {
|
||||
case .left: return .left
|
||||
case .right: return .right
|
||||
case .middle: return .center
|
||||
}
|
||||
}
|
||||
|
||||
public var downEvent: CGEventType {
|
||||
switch self {
|
||||
case .left: return .leftMouseDown
|
||||
case .right: return .rightMouseDown
|
||||
case .middle: return .otherMouseDown
|
||||
}
|
||||
}
|
||||
|
||||
public var upEvent: CGEventType {
|
||||
switch self {
|
||||
case .left: return .leftMouseUp
|
||||
case .right: return .rightMouseUp
|
||||
case .middle: return .otherMouseUp
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Private CoreGraphics/SkyLight/HIServices symbols, resolved once with dlsym (SIP stays on).
|
||||
private final class PrivateSymbols: Sendable {
|
||||
typealias SetWindowLocation = @convention(c) (CGEvent, CGPoint) -> Void
|
||||
typealias PostEventRecordTo = @convention(c) (UnsafeMutablePointer<ProcessSerialNumber>, UnsafeMutablePointer<UInt8>) -> Int32
|
||||
typealias GetProcessForPID = @convention(c) (pid_t, UnsafeMutablePointer<ProcessSerialNumber>) -> OSStatus
|
||||
|
||||
struct Loaded: Sendable {
|
||||
let setWindowLocation: SetWindowLocation
|
||||
let postEventRecordTo: PostEventRecordTo
|
||||
let getProcessForPID: GetProcessForPID
|
||||
}
|
||||
|
||||
static let shared = PrivateSymbols()
|
||||
|
||||
let loaded: Loaded?
|
||||
let missing: [String]
|
||||
|
||||
private init() {
|
||||
let skyLight = dlopen("/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight", RTLD_LAZY)
|
||||
let applicationServices = dlopen("/System/Library/Frameworks/ApplicationServices.framework/ApplicationServices", RTLD_LAZY)
|
||||
var missing: [String] = []
|
||||
func resolve<T>(_ handle: UnsafeMutableRawPointer?, _ name: String, as type: T.Type) -> T? {
|
||||
guard let symbol = dlsym(handle, name) ?? dlsym(UnsafeMutableRawPointer(bitPattern: -2), name) else {
|
||||
missing.append(name)
|
||||
return nil
|
||||
}
|
||||
// SAFETY: each name is a C function whose ABI matches `type` (checked against the SDK and cua/yabai).
|
||||
return unsafeBitCast(symbol, to: type)
|
||||
}
|
||||
let setWindowLocation = resolve(skyLight, "CGEventSetWindowLocation", as: SetWindowLocation.self)
|
||||
let postEventRecordTo = resolve(skyLight, "SLPSPostEventRecordTo", as: PostEventRecordTo.self)
|
||||
let getProcessForPID = resolve(applicationServices, "GetProcessForPID", as: GetProcessForPID.self)
|
||||
if let setWindowLocation, let postEventRecordTo, let getProcessForPID {
|
||||
loaded = Loaded(setWindowLocation: setWindowLocation, postEventRecordTo: postEventRecordTo, getProcessForPID: getProcessForPID)
|
||||
} else {
|
||||
loaded = nil
|
||||
}
|
||||
self.missing = missing
|
||||
}
|
||||
|
||||
func require() throws -> Loaded {
|
||||
guard let loaded else {
|
||||
throw BackgroundInputTransport.Failure.unavailable(missingSymbols: missing)
|
||||
}
|
||||
return loaded
|
||||
}
|
||||
}
|
||||
+45
-1
@@ -1,4 +1,4 @@
|
||||
/// STA-3433: use HID taps; CGEventPostToPid lacks window association and AppKit drops presses.
|
||||
/// Foreground (--restore-window) clicks: HID taps behind a focused-recipient fence; background clicks use BackgroundInputTransport.
|
||||
/// Pace down/up events or WindowServer can drop mouseUp.
|
||||
public enum SyntheticMouseClickDelivery {
|
||||
public static let maxClickCount = 3
|
||||
@@ -69,6 +69,50 @@ public enum SyntheticMouseClickDelivery {
|
||||
return match
|
||||
}
|
||||
|
||||
public static func deliverScroll<Event>(
|
||||
move: Event,
|
||||
wheel: Event,
|
||||
target: Recipient,
|
||||
currentObservation: () -> RecipientObservation,
|
||||
post: (Event) -> Void,
|
||||
pause: (UInt32) -> Void
|
||||
) throws {
|
||||
try requireRecipient(target, currentObservation: currentObservation)
|
||||
post(move)
|
||||
pause(interEventPauseMicroseconds)
|
||||
try requireRecipient(target, currentObservation: currentObservation)
|
||||
post(wheel)
|
||||
}
|
||||
|
||||
public static func deliverDrag<Event>(
|
||||
move: Event,
|
||||
down: Event,
|
||||
dragged: [Event],
|
||||
up: Event,
|
||||
target: Recipient,
|
||||
currentObservation: () -> RecipientObservation,
|
||||
post: (Event) -> Void,
|
||||
pause: (UInt32) -> Void
|
||||
) throws {
|
||||
try requireRecipient(target, currentObservation: currentObservation)
|
||||
post(move)
|
||||
try requireRecipient(target, currentObservation: currentObservation)
|
||||
post(down)
|
||||
pause(interEventPauseMicroseconds)
|
||||
for event in dragged {
|
||||
post(event)
|
||||
pause(10_000)
|
||||
}
|
||||
post(up)
|
||||
}
|
||||
|
||||
private static func requireRecipient(_ target: Recipient, currentObservation: () -> RecipientObservation) throws {
|
||||
let observation = currentObservation()
|
||||
guard observation == .focused(target) else {
|
||||
throw FenceFailure.recipientChanged(expected: target, actual: observation.recipient, deliveredPresses: 0)
|
||||
}
|
||||
}
|
||||
|
||||
public static func deliver<Event>(
|
||||
clickCount: Int,
|
||||
target: Recipient,
|
||||
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
import CoreGraphics
|
||||
import XCTest
|
||||
@testable import OrcaComputerUseMacOSCore
|
||||
|
||||
final class BackgroundInputTransportTests: XCTestCase {
|
||||
private let target = BackgroundInputTransport.Target(
|
||||
pid: 4242,
|
||||
windowID: 9001,
|
||||
windowFrame: CGRect(x: 300, y: 200, width: 520, height: 560)
|
||||
)
|
||||
|
||||
func testMouseEventsNameTheWindowAndCarryAWindowLocalLocation() throws {
|
||||
try XCTSkipUnless(BackgroundInputTransport.isAvailable, "private symbols missing: \(BackgroundInputTransport.missingSymbols)")
|
||||
let point = CGPoint(x: 340, y: 275)
|
||||
for type in [CGEventType.leftMouseDown, .rightMouseUp, .otherMouseDown, .leftMouseDragged] {
|
||||
let event = try BackgroundInputTransport.mouseEvent(type, target: target, at: point, button: .left, clickState: 2, flags: .maskCommand)
|
||||
XCTAssertEqual(event.getIntegerValueField(CGEventField(rawValue: 51)!), 9001, "\(type)")
|
||||
XCTAssertEqual(try windowLocation(of: event), CGPoint(x: 40, y: 75), "\(type)")
|
||||
XCTAssertEqual(event.location, point)
|
||||
XCTAssertEqual(event.getIntegerValueField(.mouseEventClickState), 2)
|
||||
XCTAssertTrue(event.flags.contains(.maskCommand))
|
||||
}
|
||||
}
|
||||
|
||||
func testChordKeysCarryOnlyKeyCodesAndFlags() throws {
|
||||
let events = try BackgroundInputTransport.chordEvents(keyCode: 0, modifiers: [(keyCode: 55, flag: .maskCommand)])
|
||||
XCTAssertEqual(events.map { $0.getIntegerValueField(.keyboardEventKeycode) }, [55, 0, 0, 55])
|
||||
XCTAssertEqual(events.map { $0.flags.contains(.maskCommand) }, [true, true, true, false])
|
||||
}
|
||||
|
||||
func testTypedTextRidesTheKeyDownOnly() throws {
|
||||
let bareKeyUp = try XCTUnwrap(CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false))
|
||||
for (text, characters) in [("hé", ["h", "é"]), ("🚀e\u{301}👩💻", ["🚀", "e\u{301}", "👩💻"])] {
|
||||
let events = try BackgroundInputTransport.textEvents(text)
|
||||
XCTAssertEqual(events.map { $0.type }, characters.flatMap { _ in [CGEventType.keyDown, .keyUp] })
|
||||
XCTAssertEqual(events.filter { $0.type == .keyDown }.map(unicodeString), characters)
|
||||
// Why: an unset payload reads back as the keyboard layout's translation of key code 0.
|
||||
for event in events where event.type == .keyUp {
|
||||
XCTAssertEqual(unicodeString(event), unicodeString(bareKeyUp))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func windowLocation(of event: CGEvent) throws -> CGPoint {
|
||||
let skyLight = dlopen("/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight", RTLD_LAZY)
|
||||
let symbol = try XCTUnwrap(dlsym(skyLight, "CGEventGetWindowLocation"))
|
||||
typealias GetWindowLocation = @convention(c) (CGEvent) -> CGPoint
|
||||
// SAFETY: CGEventGetWindowLocation takes a CGEventRef and returns a CGPoint.
|
||||
return unsafeBitCast(symbol, to: GetWindowLocation.self)(event)
|
||||
}
|
||||
|
||||
private func unicodeString(_ event: CGEvent) -> String {
|
||||
var length = 0
|
||||
var buffer = [UniChar](repeating: 0, count: 8)
|
||||
event.keyboardGetUnicodeString(maxStringLength: buffer.count, actualStringLength: &length, unicodeString: &buffer)
|
||||
return String(utf16CodeUnits: buffer, count: length)
|
||||
}
|
||||
}
|
||||
+56
@@ -2,6 +2,62 @@ import XCTest
|
||||
@testable import OrcaComputerUseMacOSCore
|
||||
|
||||
final class SyntheticMouseClickDeliveryTests: XCTestCase {
|
||||
func testScrollAndDragFenceTheOriginBeforeMoveAndBeforeMutation() {
|
||||
let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101)
|
||||
let sibling = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 202)
|
||||
for drag in [false, true] {
|
||||
for loss in [SyntheticMouseClickDelivery.RecipientObservation.focused(sibling), .unavailable, .dismissed] {
|
||||
for afterMove in [false, true] {
|
||||
var observations: [SyntheticMouseClickDelivery.RecipientObservation] = afterMove ? [.focused(target), loss] : [loss]
|
||||
var posted: [String] = []
|
||||
let observe = { observations.isEmpty ? .unavailable : observations.removeFirst() }
|
||||
XCTAssertThrowsError(try {
|
||||
if drag {
|
||||
try SyntheticMouseClickDelivery.deliverDrag(
|
||||
move: "move", down: "down", dragged: ["drag"], up: "up",
|
||||
target: target, currentObservation: observe,
|
||||
post: { posted.append($0) }, pause: { _ in }
|
||||
)
|
||||
} else {
|
||||
try SyntheticMouseClickDelivery.deliverScroll(
|
||||
move: "move", wheel: "wheel",
|
||||
target: target, currentObservation: observe,
|
||||
post: { posted.append($0) }, pause: { _ in }
|
||||
)
|
||||
}
|
||||
}())
|
||||
XCTAssertEqual(posted, afterMove ? ["move"] : [])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testScrollWaitsBeforeRecheckAndDragDoesNotRecheckAfterDown() throws {
|
||||
let target = SyntheticMouseClickDelivery.Recipient(ownerPID: 41, windowID: 101)
|
||||
for drag in [false, true] {
|
||||
var trace: [String] = []
|
||||
let observe = {
|
||||
trace.append("recipient")
|
||||
return SyntheticMouseClickDelivery.RecipientObservation.focused(target)
|
||||
}
|
||||
let post = { (event: String) in trace.append(event) }
|
||||
let pause = { (_: UInt32) in trace.append("pause") }
|
||||
if drag {
|
||||
try SyntheticMouseClickDelivery.deliverDrag(
|
||||
move: "move", down: "down", dragged: ["drag"], up: "up",
|
||||
target: target, currentObservation: observe, post: post, pause: pause
|
||||
)
|
||||
XCTAssertEqual(trace, ["recipient", "move", "recipient", "down", "pause", "drag", "pause", "up"])
|
||||
} else {
|
||||
try SyntheticMouseClickDelivery.deliverScroll(
|
||||
move: "move", wheel: "wheel", target: target,
|
||||
currentObservation: observe, post: post, pause: pause
|
||||
)
|
||||
XCTAssertEqual(trace, ["recipient", "move", "pause", "recipient", "wheel"])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testSingleClickPlanPairsDownAndUpAfterMove() {
|
||||
XCTAssertEqual(
|
||||
SyntheticMouseClickDelivery.steps(clickCount: 1),
|
||||
|
||||
@@ -95,9 +95,25 @@ printf '%s' "$TEXT" | ORCA computer set-value --app <app> --element-index <index
|
||||
- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; synthetic keyboard delivery is reported as unverified, so inspect the returned state before assuming text landed.
|
||||
- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.
|
||||
- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.
|
||||
- Some actions work in background apps, but this is app-dependent. If success does not change the UI, refresh state and choose a more semantic action or restore/focus the window.
|
||||
- Coordinates are window-local; use coordinates from the latest screenshot/state for the same target window.
|
||||
|
||||
## Background and Foreground Input
|
||||
|
||||
On macOS, every action runs in the background by default: the target app is not brought forward, the user's frontmost app and key window stay put, and the real pointer does not move. The user can keep working while you act. `capabilities` reports this as `supports.input.background`, and each action result reports `action.delivery` as `background` or `foreground`.
|
||||
|
||||
On Linux and Windows, keyboard and pointer input still need the target window in front; a `window_not_focused` refusal there means retry with `--restore-window`.
|
||||
|
||||
Do not pass `--restore-window` by default. Use it only when foreground input is needed, and tell the user first because it takes over their screen, keyboard focus and pointer:
|
||||
|
||||
- An action fails with `background_unavailable` that does not name a popup.
|
||||
- A background action reports success but the UI does not change, and a semantic action (`set-value`, `perform-secondary-action`) cannot do the job.
|
||||
- Drags in Chromium or Electron apps, and apps that read the hardware pointer (games, OpenGL/Metal views, Java/Tk), which ignore background pointer events.
|
||||
- The window is minimized or hidden. Accessibility actions still work there; pointer and keyboard events do not.
|
||||
|
||||
Popup menus and web `<select>` controls: set the option with `set-value` on the popup element. Background clicks on them are refused because they open the menu off screen or wedge the app's input.
|
||||
|
||||
On Linux and Windows, clicks, scrolling, drags and keyboard input still bring the target window forward.
|
||||
|
||||
## Screenshots
|
||||
|
||||
`get-app-state` and actions request screenshots by default unless `--no-screenshot` is
|
||||
@@ -120,12 +136,12 @@ On Linux and Windows, screenshots may come from the visible desktop region for t
|
||||
|
||||
## App Notes
|
||||
|
||||
Browsers: for Edge, Chrome, Safari, and similar browser windows, set the address/search field directly, then press Return. Do not assume raw typing went to the address bar. Use `--restore-window` when the browser is not already frontmost. Large tab strips may show only the active tab plus an "inactive browser tabs omitted" marker; treat that as intentional noise reduction and operate on the current page/address bar unless the user asked to manage tabs.
|
||||
Browsers: for Edge, Chrome, Safari, and similar browser windows, set the address/search field directly, then press Return. Do not assume raw typing went to the address bar. On Linux and Windows, use `--restore-window` when the browser is not already frontmost; on macOS this works in the background. Large tab strips may show only the active tab plus an "inactive browser tabs omitted" marker; treat that as intentional noise reduction and operate on the current page/address bar unless the user asked to manage tabs.
|
||||
|
||||
For browser-hosted forms such as Gmail compose, verify the focused UI element after each field action. Page text fields can expose accessibility actions without moving DOM focus; if a click or `set-value` does not change the focused receiver, use `Tab` / `Shift+Tab` from a known focused field or window-local coordinates from a fresh screenshot. Prefer `paste-text` into the verified focused field for draft bodies, then inspect the returned state before continuing.
|
||||
|
||||
```text
|
||||
ORCA computer get-app-state --app com.microsoft.edgemac --restore-window --json
|
||||
ORCA computer get-app-state --app com.microsoft.edgemac --json
|
||||
ORCA computer set-value --app com.microsoft.edgemac --element-index <addressBarIndex> --value "test123" --json
|
||||
ORCA computer press-key --app com.microsoft.edgemac --key Return --json
|
||||
```
|
||||
@@ -139,6 +155,7 @@ Slack: the accessibility tree may be shallow while the screenshot contains usefu
|
||||
- `app_not_found`: run `list-apps` and retry with the bundle ID. If the target is a web app such as Gmail, choose the desktop browser app/window that contains it; do not retry `ORCA computer ... --app Gmail` unchanged because `orca computer` app selectors refer to desktop apps, not website names.
|
||||
- `app_blocked`: stop; the target is intentionally blocked from computer-use.
|
||||
- `window_not_found` / `window_stale`: run `list-windows`, choose a current selector, then rerun `get-app-state`.
|
||||
- `background_unavailable`: for a popup or `<select>`, use `set-value` on it; otherwise retry once with `--restore-window` after telling the user it will take over their screen.
|
||||
- `window_not_focused`: retry once with `--restore-window`; if the message says restore was already requested, stop retrying restore and bring the app forward manually or check permissions. For editable fields prefer `set-value`, then inspect before assuming keyboard input worked.
|
||||
- `element_not_found`: index is stale; run `get-app-state` again.
|
||||
- `unsupported_capability`: the provider or desktop environment cannot do that action; use a semantic alternative or install the missing dependency if the message names one.
|
||||
|
||||
@@ -112,9 +112,25 @@ const COMPUTER_USE_MARKDOWN = [
|
||||
"- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; synthetic keyboard delivery is reported as unverified, so inspect the returned state before assuming text landed.",
|
||||
"- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.",
|
||||
"- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.",
|
||||
"- Some actions work in background apps, but this is app-dependent. If success does not change the UI, refresh state and choose a more semantic action or restore/focus the window.",
|
||||
"- Coordinates are window-local; use coordinates from the latest screenshot/state for the same target window.",
|
||||
"",
|
||||
"## Background and Foreground Input",
|
||||
"",
|
||||
"On macOS, every action runs in the background by default: the target app is not brought forward, the user's frontmost app and key window stay put, and the real pointer does not move. The user can keep working while you act. `capabilities` reports this as `supports.input.background`, and each action result reports `action.delivery` as `background` or `foreground`.",
|
||||
"",
|
||||
"On Linux and Windows, keyboard and pointer input still need the target window in front; a `window_not_focused` refusal there means retry with `--restore-window`.",
|
||||
"",
|
||||
"Do not pass `--restore-window` by default. Use it only when foreground input is needed, and tell the user first because it takes over their screen, keyboard focus and pointer:",
|
||||
"",
|
||||
"- An action fails with `background_unavailable` that does not name a popup.",
|
||||
"- A background action reports success but the UI does not change, and a semantic action (`set-value`, `perform-secondary-action`) cannot do the job.",
|
||||
"- Drags in Chromium or Electron apps, and apps that read the hardware pointer (games, OpenGL/Metal views, Java/Tk), which ignore background pointer events.",
|
||||
"- The window is minimized or hidden. Accessibility actions still work there; pointer and keyboard events do not.",
|
||||
"",
|
||||
"Popup menus and web `<select>` controls: set the option with `set-value` on the popup element. Background clicks on them are refused because they open the menu off screen or wedge the app's input.",
|
||||
"",
|
||||
"On Linux and Windows, clicks, scrolling, drags and keyboard input still bring the target window forward.",
|
||||
"",
|
||||
"## Screenshots",
|
||||
"",
|
||||
"`get-app-state` and actions request screenshots by default unless `--no-screenshot` is",
|
||||
@@ -137,12 +153,12 @@ const COMPUTER_USE_MARKDOWN = [
|
||||
"",
|
||||
"## App Notes",
|
||||
"",
|
||||
"Browsers: for Edge, Chrome, Safari, and similar browser windows, set the address/search field directly, then press Return. Do not assume raw typing went to the address bar. Use `--restore-window` when the browser is not already frontmost. Large tab strips may show only the active tab plus an \"inactive browser tabs omitted\" marker; treat that as intentional noise reduction and operate on the current page/address bar unless the user asked to manage tabs.",
|
||||
"Browsers: for Edge, Chrome, Safari, and similar browser windows, set the address/search field directly, then press Return. Do not assume raw typing went to the address bar. On Linux and Windows, use `--restore-window` when the browser is not already frontmost; on macOS this works in the background. Large tab strips may show only the active tab plus an \"inactive browser tabs omitted\" marker; treat that as intentional noise reduction and operate on the current page/address bar unless the user asked to manage tabs.",
|
||||
"",
|
||||
"For browser-hosted forms such as Gmail compose, verify the focused UI element after each field action. Page text fields can expose accessibility actions without moving DOM focus; if a click or `set-value` does not change the focused receiver, use `Tab` / `Shift+Tab` from a known focused field or window-local coordinates from a fresh screenshot. Prefer `paste-text` into the verified focused field for draft bodies, then inspect the returned state before continuing.",
|
||||
"",
|
||||
"```text",
|
||||
"ORCA computer get-app-state --app com.microsoft.edgemac --restore-window --json",
|
||||
"ORCA computer get-app-state --app com.microsoft.edgemac --json",
|
||||
"ORCA computer set-value --app com.microsoft.edgemac --element-index <addressBarIndex> --value \"test123\" --json",
|
||||
"ORCA computer press-key --app com.microsoft.edgemac --key Return --json",
|
||||
"```",
|
||||
@@ -156,6 +172,7 @@ const COMPUTER_USE_MARKDOWN = [
|
||||
"- `app_not_found`: run `list-apps` and retry with the bundle ID. If the target is a web app such as Gmail, choose the desktop browser app/window that contains it; do not retry `ORCA computer ... --app Gmail` unchanged because `orca computer` app selectors refer to desktop apps, not website names.",
|
||||
"- `app_blocked`: stop; the target is intentionally blocked from computer-use.",
|
||||
"- `window_not_found` / `window_stale`: run `list-windows`, choose a current selector, then rerun `get-app-state`.",
|
||||
"- `background_unavailable`: for a popup or `<select>`, use `set-value` on it; otherwise retry once with `--restore-window` after telling the user it will take over their screen.",
|
||||
"- `window_not_focused`: retry once with `--restore-window`; if the message says restore was already requested, stop retrying restore and bring the app forward manually or check permissions. For editable fields prefer `set-value`, then inspect before assuming keyboard input worked.",
|
||||
"- `element_not_found`: index is stale; run `get-app-state` again.",
|
||||
"- `unsupported_capability`: the provider or desktop environment cannot do that action; use a semantic alternative or install the missing dependency if the message names one.",
|
||||
|
||||
@@ -236,6 +236,7 @@ function formatComputerCapabilities(value: ComputerProviderCapabilities): string
|
||||
` Actions: ${Object.entries(value.supports.actions)
|
||||
.filter(([, enabled]) => enabled)
|
||||
.map(([name]) => name)
|
||||
.join(', ')}`
|
||||
.join(', ')}`,
|
||||
...(value.supports.input ? [` Input: background=${value.supports.input.background}`] : [])
|
||||
].join('\n')
|
||||
}
|
||||
|
||||
@@ -35,6 +35,11 @@ export function computerUseErrorRecoveryData(
|
||||
'Retry once with `--restore-window`.',
|
||||
'If `--restore-window` was already used, stop retrying restore; bring the app forward manually, check permissions, or prefer `set-value` for editable fields.'
|
||||
)
|
||||
case 'background_unavailable':
|
||||
return recoverWith(
|
||||
'If the message names a popup menu or `<select>`, set its option with `orca computer set-value` on that element instead of clicking.',
|
||||
'Otherwise retry once with `--restore-window`; that brings the app forward and uses foreground input, so tell the user it will take over their screen.'
|
||||
)
|
||||
case 'window_stale':
|
||||
return recoverWith(
|
||||
'Run `orca computer list-windows --app <app> --json` and choose a current window selector.',
|
||||
|
||||
@@ -3,6 +3,7 @@ export const COMPUTER_ERROR_CODES = {
|
||||
app_blocked: 'app_blocked',
|
||||
window_not_found: 'window_not_found',
|
||||
window_not_focused: 'window_not_focused',
|
||||
background_unavailable: 'background_unavailable',
|
||||
window_stale: 'window_stale',
|
||||
provider_incompatible: 'provider_incompatible',
|
||||
unsupported_capability: 'unsupported_capability',
|
||||
@@ -89,6 +90,8 @@ export type ComputerActionMetadata = {
|
||||
fallbackReason?: string | null
|
||||
targetWindowId?: number | null
|
||||
targetWindowIndex?: number | null
|
||||
// Absent from providers that predate background input (Linux, Windows, older macOS helpers).
|
||||
delivery?: 'background' | 'foreground'
|
||||
verification?: ComputerActionVerification
|
||||
}
|
||||
|
||||
@@ -164,6 +167,10 @@ export type ComputerProviderCapabilities = {
|
||||
dock: boolean
|
||||
menubar: boolean
|
||||
}
|
||||
// Absent from providers that predate background input.
|
||||
input?: {
|
||||
background: boolean
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user