diff --git a/config/scripts/computer-use-modifier-safety.test.mjs b/config/scripts/computer-use-modifier-safety.test.mjs index f39015796a9..32fdd94e938 100644 --- a/config/scripts/computer-use-modifier-safety.test.mjs +++ b/config/scripts/computer-use-modifier-safety.test.mjs @@ -28,7 +28,10 @@ describe('computer-use modifier safety', () => { ) expect(mouseInput).toContain('event.flags = flags') - expect(clickInput.match(/flags: flags/gu)).toHaveLength(3) + // Every click event flows through the shared delivery plan and carries + // the modifier flags on the mouse event itself. + expect(clickInput).toContain('SyntheticMouseClickDelivery.steps(clickCount: count)') + expect(clickInput).toContain('event.flags = flags') expect(clickInput).not.toContain('down: true') }) diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift index 50ecae7d024..7236f7bab2c 100644 --- a/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift @@ -740,25 +740,30 @@ final class Provider { } if let point = center(record.localFrame, in: snapshot.windowBounds) { try Input.click( - pid: snapshot.app.pid, at: point, button: mouseButton(button), count: count, modifiers: modifiers ) - return actionMetadata(path: "synthetic", fallbackReason: "actionUnsupported") + return actionMetadata( + path: "synthetic", + fallbackReason: "actionUnsupported", + verification: unverifiedAction(reason: "synthetic_input") + ) } throw ProviderError.coded("element_not_clickable", "element \(record.index) has no clickable frame") } let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot) try Input.click( - pid: snapshot.app.pid, at: point, button: mouseButton(button), count: count, modifiers: modifiers ) - return actionMetadata(path: "synthetic") + return actionMetadata( + path: "synthetic", + verification: unverifiedAction(reason: "synthetic_input") + ) } private func performClickAction(record: ElementRecord, mouseButton: String) throws -> String? { @@ -2286,7 +2291,6 @@ private func resizePng(_ image: CGImage, scale: CGFloat) -> BoundedPNG? { private enum Input { static func click( - pid: pid_t, at point: CGPoint, button: MouseButton, count: Int, @@ -2298,10 +2302,32 @@ private enum Input { let flags = modifiers.reduce(into: CGEventFlags()) { result, modifier in result.insert(modifier.flag) } - for _ in 0.. 0 { + event.setIntegerValueField(.mouseEventClickState, value: clickState) + } + event.post(tap: .cghidEventTap) + usleep(SyntheticMouseClickDelivery.interEventPauseMicroseconds) } } diff --git a/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SyntheticMouseClickDelivery.swift b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SyntheticMouseClickDelivery.swift new file mode 100644 index 00000000000..d61e56af35c --- /dev/null +++ b/native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/SyntheticMouseClickDelivery.swift @@ -0,0 +1,40 @@ +/// Event plan for synthetic mouse clicks (STA-3433). +/// +/// Clicks must be posted to the HID event tap, not `CGEventPostToPid`: +/// pid-targeted mouse events reach the app with no window association, so +/// AppKit never routes the press to a view (hover fires, activation never +/// happens). The window server also drops a mouseUp posted back-to-back +/// with its mouseDown, so consecutive events need a pause between them. +public enum SyntheticMouseClickDelivery { + public enum Step: Equatable { + case move + case buttonDown(pressIndex: Int) + case buttonUp(pressIndex: Int) + } + + /// Pause after posting each event; unpaced posts race the window + /// server's routing and the mouseUp is silently dropped. + public static let interEventPauseMicroseconds: UInt32 = 50_000 + + /// One move, then a paired down/up per press. `pressIndex` becomes the + /// event's click state so repeated presses register as double/triple + /// clicks instead of independent single clicks. + public static func steps(clickCount: Int) -> [Step] { + var steps: [Step] = [.move] + for press in 1...max(clickCount, 1) { + steps.append(.buttonDown(pressIndex: press)) + steps.append(.buttonUp(pressIndex: press)) + } + return steps + } + + /// Click state field value for a step; 0 leaves the field unset. + public static func clickState(for step: Step) -> Int64 { + switch step { + case .move: + return 0 + case let .buttonDown(pressIndex), let .buttonUp(pressIndex): + return Int64(pressIndex) + } + } +} diff --git a/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/SyntheticMouseClickDeliveryTests.swift b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/SyntheticMouseClickDeliveryTests.swift new file mode 100644 index 00000000000..d80a5e9e8ee --- /dev/null +++ b/native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/SyntheticMouseClickDeliveryTests.swift @@ -0,0 +1,43 @@ +import XCTest +@testable import OrcaComputerUseMacOSCore + +final class SyntheticMouseClickDeliveryTests: XCTestCase { + func testSingleClickPlanPairsDownAndUpAfterMove() { + XCTAssertEqual( + SyntheticMouseClickDelivery.steps(clickCount: 1), + [.move, .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1)] + ) + } + + func testMultiClickPlanNumbersEachPressForClickState() { + XCTAssertEqual( + SyntheticMouseClickDelivery.steps(clickCount: 2), + [ + .move, + .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1), + .buttonDown(pressIndex: 2), .buttonUp(pressIndex: 2), + ] + ) + } + + func testNonPositiveClickCountStillDeliversOnePress() { + for count in [0, -3] { + XCTAssertEqual( + SyntheticMouseClickDelivery.steps(clickCount: count), + [.move, .buttonDown(pressIndex: 1), .buttonUp(pressIndex: 1)] + ) + } + } + + func testClickStateMatchesPressIndexAndSkipsMove() { + XCTAssertEqual(SyntheticMouseClickDelivery.clickState(for: .move), 0) + XCTAssertEqual(SyntheticMouseClickDelivery.clickState(for: .buttonDown(pressIndex: 1)), 1) + XCTAssertEqual(SyntheticMouseClickDelivery.clickState(for: .buttonUp(pressIndex: 2)), 2) + } + + func testInterEventPauseIsNonZero() { + // Unpaced posts race the window server and the mouseUp is dropped, + // turning the click into a hover-only no-op (STA-3433). + XCTAssertGreaterThan(SyntheticMouseClickDelivery.interEventPauseMicroseconds, 0) + } +} diff --git a/tests/e2e/computer-mac.e2e.ts b/tests/e2e/computer-mac.e2e.ts index 6e6c0a60873..6aa162cee40 100644 --- a/tests/e2e/computer-mac.e2e.ts +++ b/tests/e2e/computer-mac.e2e.ts @@ -71,6 +71,84 @@ describe.skipIf(!isMac || !e2eOptIn)('computer-use macOS e2e (TextEdit)', () => expect(after.result.snapshot.treeText).toContain(marker) }) + test('coordinate double-click activates a control, not just hover (STA-3433)', async () => { + // Ten identical short lines: any first-lines click hits a word, and the + // whole document stays inside the treeText value preview after editing. + const filler = Array(10).fill('wordword').join('\n') + await runOrcaCli([ + 'computer', + 'hotkey', + '--app', + 'TextEdit', + '--key', + 'CmdOrCtrl+A', + '--no-screenshot' + ]) + await runOrcaCli([ + 'computer', + 'paste-text', + '--app', + 'TextEdit', + '--text', + filler, + '--no-screenshot' + ]) + + // Word-select via coordinates: 40px in, 70px down lands inside a + // "wordword" on one of the first lines whether or not the ruler is shown. + const clicked = parseJsonOutput<{ result: ComputerActionResult }>( + ( + await runOrcaCli([ + 'computer', + 'click', + '--app', + 'TextEdit', + '--x', + '40', + '--y', + '70', + '--click-count', + '2', + '--no-screenshot', + '--json' + ]) + ).stdout + ) + expect(clicked.result.action?.path).toBe('synthetic') + expect(clicked.result.action?.verification).toMatchObject({ + state: 'unverified', + reason: 'synthetic_input' + }) + + const marker = `zz${Date.now()}zz` + await runOrcaCli([ + 'computer', + 'type-text', + '--app', + 'TextEdit', + '--text', + marker, + '--no-screenshot' + ]) + + const after = parseJsonOutput<{ result: ComputerSnapshotResult }>( + ( + await runOrcaCli([ + 'computer', + 'get-app-state', + '--app', + 'TextEdit', + '--no-screenshot', + '--json' + ]) + ).stdout + ) + // The double-click selected a mid-document word, so the typed marker must + // be followed by more filler. A dropped press leaves the caret at the end + // of the document and the marker only ever appends (the STA-3433 no-op). + expect(after.result.snapshot.treeText).toMatch(new RegExp(`${marker}\\s+wordword`)) + }) + test('paste-text and hotkey verify TextEdit text replacement', async () => { const first = parseJsonOutput<{ result: ComputerActionResult }>( (