fix(computer): deliver macOS coordinate clicks via the HID event tap (STA-3433) (#12839)

Mouse events posted with CGEventPostToPid reach the target app with no window association, so AppKit never routes the press to a view: hover states fire but the control is never activated, and the mouseUp is dropped outright when posted back-to-back. Post click events to the HID event tap instead (as keyboard synthesis already does), pace them, and stamp mouseEventClickState so multi-clicks register. Synthetic clicks now also report verification unverified/synthetic_input from the helper itself, matching the other synthetic actions.
2026-09-21 16:02:20 +00:00 · 2026-08-05 22:14:18 -07:00
parent 89f00658d2
commit c2da0e47f9
5 changed files with 200 additions and 10 deletions
@@ -28,7 +28,10 @@ describe('computer-use modifier safety', () => {
    )

    expect(mouseInput).toContain('event.flags = flags')
-    expect(clickInput.match(/flags: flags/gu)).toHaveLength(3)
+    // Every click event flows through the shared delivery plan and carries
+    // the modifier flags on the mouse event itself.
+    expect(clickInput).toContain('SyntheticMouseClickDelivery.steps(clickCount: count)')
+    expect(clickInput).toContain('event.flags = flags')
    expect(clickInput).not.toContain('down: true')
  })

@@ -740,25 +740,30 @@ final class Provider {
            }
            if let point = center(record.localFrame, in: snapshot.windowBounds) {
                try Input.click(
-                    pid: snapshot.app.pid,
                    at: point,
                    button: mouseButton(button),
                    count: count,
                    modifiers: modifiers
                )
-                return actionMetadata(path: "synthetic", fallbackReason: "actionUnsupported")
+                return actionMetadata(
+                    path: "synthetic",
+                    fallbackReason: "actionUnsupported",
+                    verification: unverifiedAction(reason: "synthetic_input")
+                )
            }
            throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame")
        }
        let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot)
        try Input.click(
-            pid: snapshot.app.pid,
            at: point,
            button: mouseButton(button),
            count: count,
            modifiers: modifiers
        )
-        return actionMetadata(path: "synthetic")
+        return actionMetadata(
+            path: "synthetic",
+            verification: unverifiedAction(reason: "synthetic_input")
+        )
    }

    private func performClickAction(record: ElementRecord, mouseButton: String) throws -> String? {
@@ -2286,7 +2291,6 @@ private func resizePng(_ image: CGImage, scale: CGFloat) -> BoundedPNG? {

 private enum Input {
    static func click(
-        pid: pid_t,
        at point: CGPoint,
        button: MouseButton,
        count: Int,
@@ -2298,10 +2302,32 @@ private enum Input {
        let flags = modifiers.reduce(into: CGEventFlags()) { result, modifier in
            result.insert(modifier.flag)
        }
-        for _ in 0..<max(count, 1) {
-            try mouse(.mouseMoved, source: source, point: point, button: button.cgButton, flags: flags, pid: pid)
-            try mouse(button.downEvent, source: source, point: point, button: button.cgButton, flags: flags, pid: pid)
-            try mouse(button.upEvent, source: source, point: point, button: button.cgButton, flags: flags, pid: pid)
+        // Why HID tap + pacing: see SyntheticMouseClickDelivery (STA-3433).
+        for step in SyntheticMouseClickDelivery.steps(clickCount: count) {
+            let type: CGEventType
+            switch step {
+            case .move:
+                type = .mouseMoved
+            case .buttonDown:
+                type = button.downEvent
+            case .buttonUp:
+                type = button.upEvent
+            }
+            guard let event = CGEvent(
+                mouseEventSource: source,
+                mouseType: type,
+                mouseCursorPosition: point,
+                mouseButton: button.cgButton
+            ) else {
+                throw ProviderError.coded("accessibility_error", "failed to create mouse event")
+            }
+            event.flags = flags
+            let clickState = SyntheticMouseClickDelivery.clickState(for: step)
+            if clickState > 0 {
+                event.setIntegerValueField(.mouseEventClickState, value: clickState)
+            }
+            event.post(tap: .cghidEventTap)
+            usleep(SyntheticMouseClickDelivery.interEventPauseMicroseconds)
        }
    }

@@ -0,0 +1,40 @@
+/// Event plan for synthetic mouse clicks (STA-3433).
+///
+/// Clicks must be posted to the HID event tap, not `CGEventPostToPid`:
+/// pid-targeted mouse events reach the app with no window association, so
+/// AppKit never routes the press to a view (hover fires, activation never
+/// happens). The window server also drops a mouseUp posted back-to-back
+/// with its mouseDown, so consecutive events need a pause between them.
+public enum SyntheticMouseClickDelivery {
+    public enum Step: Equatable {
+        case move
+        case buttonDown(pressIndex: Int)
+        case buttonUp(pressIndex: Int)
+    }
+
+    /// Pause after posting each event; unpaced posts race the window
+    /// server's routing and the mouseUp is silently dropped.
+    public static let interEventPauseMicroseconds: UInt32 = 50_000
+
+    /// One move, then a paired down/up per press. `pressIndex` becomes the
+    /// event's click state so repeated presses register as double/triple
+    /// clicks instead of independent single clicks.
+    public static func steps(clickCount: Int) -> [Step] {
+        var steps: [Step] = [.move]
+        for press in 1...max(clickCount, 1) {
+            steps.append(.buttonDown(pressIndex: press))
+            steps.append(.buttonUp(pressIndex: press))
+        }
+        return steps
+    }
+
+    /// Click state field value for a step; 0 leaves the field unset.
+    public static func clickState(for step: Step) -> Int64 {
+        switch step {
+        case .move:
+            return 0
+        case let .buttonDown(pressIndex), let .buttonUp(pressIndex):
+            return Int64(pressIndex)
+        }
+    }
+}
@@ -0,0 +1,43 @@
+import XCTest
+@testable import OrcaComputerUseMacOSCore
+
+final class SyntheticMouseClickDeliveryTests: XCTestCase {
+    func testSingleClickPlanPairsDownAndUpAfterMove() {
+        XCTAssertEqual(
+            SyntheticMouseClickDelivery.steps(clickCount: 1),
+            [.move, .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1)]
+        )
+    }
+
+    func testMultiClickPlanNumbersEachPressForClickState() {
+        XCTAssertEqual(
+            SyntheticMouseClickDelivery.steps(clickCount: 2),
+            [
+                .move,
+                .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1),
+                .buttonDown(pressIndex: 2), .buttonUp(pressIndex: 2),
+            ]
+        )
+    }
+
+    func testNonPositiveClickCountStillDeliversOnePress() {
+        for count in [0, -3] {
+            XCTAssertEqual(
+                SyntheticMouseClickDelivery.steps(clickCount: count),
+                [.move, .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1)]
+            )
+        }
+    }
+
+    func testClickStateMatchesPressIndexAndSkipsMove() {
+        XCTAssertEqual(SyntheticMouseClickDelivery.clickState(for: .move), 0)
+        XCTAssertEqual(SyntheticMouseClickDelivery.clickState(for: .buttonDown(pressIndex: 1)), 1)
+        XCTAssertEqual(SyntheticMouseClickDelivery.clickState(for: .buttonUp(pressIndex: 2)), 2)
+    }
+
+    func testInterEventPauseIsNonZero() {
+        // Unpaced posts race the window server and the mouseUp is dropped,
+        // turning the click into a hover-only no-op (STA-3433).
+        XCTAssertGreaterThan(SyntheticMouseClickDelivery.interEventPauseMicroseconds, 0)
+    }
+}
@@ -71,6 +71,84 @@ describe.skipIf(!isMac || !e2eOptIn)('computer-use macOS e2e (TextEdit)', () =>
    expect(after.result.snapshot.treeText).toContain(marker)
  })

+  test('coordinate double-click activates a control, not just hover (STA-3433)', async () => {
+    // Ten identical short lines: any first-lines click hits a word, and the
+    // whole document stays inside the treeText value preview after editing.
+    const filler = Array(10).fill('wordword').join('\n')
+    await runOrcaCli([
+      'computer',
+      'hotkey',
+      '--app',
+      'TextEdit',
+      '--key',
+      'CmdOrCtrl+A',
+      '--no-screenshot'
+    ])
+    await runOrcaCli([
+      'computer',
+      'paste-text',
+      '--app',
+      'TextEdit',
+      '--text',
+      filler,
+      '--no-screenshot'
+    ])
+
+    // Word-select via coordinates: 40px in, 70px down lands inside a
+    // "wordword" on one of the first lines whether or not the ruler is shown.
+    const clicked = parseJsonOutput<{ result: ComputerActionResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'click',
+          '--app',
+          'TextEdit',
+          '--x',
+          '40',
+          '--y',
+          '70',
+          '--click-count',
+          '2',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(clicked.result.action?.path).toBe('synthetic')
+    expect(clicked.result.action?.verification).toMatchObject({
+      state: 'unverified',
+      reason: 'synthetic_input'
+    })
+
+    const marker = `zz${Date.now()}zz`
+    await runOrcaCli([
+      'computer',
+      'type-text',
+      '--app',
+      'TextEdit',
+      '--text',
+      marker,
+      '--no-screenshot'
+    ])
+
+    const after = parseJsonOutput<{ result: ComputerSnapshotResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'get-app-state',
+          '--app',
+          'TextEdit',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    // The double-click selected a mid-document word, so the typed marker must
+    // be followed by more filler. A dropped press leaves the caret at the end
+    // of the document and the marker only ever appends (the STA-3433 no-op).
+    expect(after.result.snapshot.treeText).toMatch(new RegExp(`${marker}\\s+wordword`))
+  })
+
  test('paste-text and hotkey verify TextEdit text replacement', async () => {
    const first = parseJsonOutput<{ result: ComputerActionResult }>(
      (