feat(computer-use): support macOS middle click and stop the silent left-click fallback (#14721)

* feat(computer-use): support macOS middle click and gate the AX click path

`--mouse-button middle` already validated end-to-end through the CLI, the
zod schema, and the provider validator, and both the Windows and Linux
providers honored it. Only the macOS provider rejected it outright with
"middle-click is not yet supported", so the flag was a dead end on the one
platform that has no fallback.

Two changes:

- Add `.middle` to the macOS button mapping. macOS has no dedicated middle
  event family, so it rides `otherMouseDown`/`otherMouseUp` with the button
  number carried by `mouseButton: .center`; that constructor argument is
  honored for exactly the `otherMouse*` types, so no extra field write is
  needed.
- Validate the requested button before the accessibility fast path, and skip
  that path for buttons it cannot express. Previously the raw string was read
  unvalidated, and `performClickAction` only special-cased `right`, so
  `click --mouse-button middle --element-index N` (no modifiers, count 1) fell
  through to `AXPress` — a left click — and reported success with
  `path: "accessibility"`. Any unrecognized button string did the same. This
  matches guards the Windows and Linux providers already had.

The button enum moves into `OrcaComputerUseMacOSCore` so it is unit-testable;
`main.swift` keeps only the CoreGraphics mapping.

Also documents `--mouse-button` in the computer-use skill guide, which never
mentioned the flag, so agents on Windows and Linux had no way to discover it.

* test(computer-use): cover macOS middle click in the real-desktop e2e suite

* test(computer-use): prove macOS middle-click delivery

config/scripts/computer-use-mouse-button-routing.test.mjs

@@ -0,0 +1,65 @@
+import { readFileSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+import { describe, expect, it } from 'vitest'
+
+const projectDir = resolve(import.meta.dirname, '../..')
+
+function source(path) {
+  return readFileSync(join(projectDir, path), 'utf8')
+}
+
+function sourceBetween(contents, startMarker, endMarker) {
+  const start = contents.indexOf(startMarker)
+  const end = contents.indexOf(endMarker, start + startMarker.length)
+  if (start === -1 || end === -1) {
+    throw new Error(`Missing source boundary: ${startMarker} → ${endMarker}`)
+  }
+  return contents.slice(start, end)
+}
+
+describe('computer-use mouse button routing', () => {
+  it('maps the macOS middle button onto the otherMouse event family', () => {
+    const macOS = source('native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift')
+    const mapping = sourceBetween(
+      macOS,
+      'extension MouseButtonSelection {',
+      'private func mouseButton('
+    )
+
+    expect(mapping).toContain('return .center')
+    expect(mapping).toContain('return .otherMouseDown')
+    expect(mapping).toContain('return .otherMouseUp')
+    // A middle press posted as a left event type would silently left-click.
+    expect(mapping).not.toContain('case .middle:\n            return .leftMouseDown')
+  })
+
+  it('validates the macOS mouse button before any accessibility shortcut runs', () => {
+    const macOS = source('native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift')
+    const click = sourceBetween(
+      macOS,
+      'private func click(params:',
+      'private func performClickAction('
+    )
+
+    expect(click).toContain('let button = try mouseButton(params["mouseButton"]?.string)')
+    expect(click).toContain('button.hasAccessibilityAction')
+    // An unvalidated raw string reaches AXPress and reports a left click as success.
+    expect(click).not.toContain('params["mouseButton"]?.string ?? "left"')
+  })
+
+  it('keeps every platform from resolving a middle click through its accessibility path', () => {
+    const windows = source('native/computer-use-windows/runtime.ps1')
+    const windowsClick = sourceBetween(
+      windows,
+      '$handledByPattern = $false',
+      'if (-not $handledByPattern)'
+    )
+
+    expect(windowsClick).toContain('$Operation.mouse_button -ne "middle"')
+
+    const linux = source('native/computer-use-linux/runtime.py')
+    const linuxClick = sourceBetween(linux, 'has_modifiers = bool(', 'if not handled:')
+
+    expect(linuxClick).toContain('operation.get("mouse_button", "left") == "left"')
+  })
+})

native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift

@@ -727,7 +727,7 @@ final class Provider {
 
     private func click(params: [String: JSONValue]) throws -> [String: Any] {
         let snapshot = try currentSnapshot(params: params)
-        let button = params["mouseButton"]?.string ?? "left"
+        let button = try mouseButton(params["mouseButton"]?.string)
         let count = try positiveInteger(params["clickCount"]?.number, defaultValue: 1, name: "clickCount")
         guard count <= SyntheticMouseClickDelivery.maxClickCount else {
             throw ProviderError.coded(
@@ -741,13 +741,16 @@ final class Provider {
         recoverWindow(snapshot.app, windowId: snapshot.windowId, windowBounds: snapshot.windowBounds)
         if let elementIndex = try optionalInteger(params, "elementIndex") {
             let record = try element(snapshot, elementIndex)
-            if modifiers.isEmpty, count <= 1, let actionName = try performClickAction(record: record, mouseButton: button) {
+            if modifiers.isEmpty,
+               count <= 1,
+               button.hasAccessibilityAction,
+               let actionName = try performClickAction(record: record, mouseButton: button) {
                 return actionMetadata(path: "accessibility", actionName: actionName)
             }
             if let point = center(record.localFrame, in: snapshot.windowBounds) {
                 try Input.click(
                     at: point,
-                    button: mouseButton(button),
+                    button: button,
                     count: count,
                     modifiers: modifiers,
                     targetWindow: snapshot
@@ -763,7 +766,7 @@ final class Provider {
         let point = try coordinatePoint(params: params, xKey: "x", yKey: "y", snapshot: snapshot)
         try Input.click(
             at: point,
-            button: mouseButton(button),
+            button: button,
             count: count,
             modifiers: modifiers,
             targetWindow: snapshot
@@ -774,8 +777,8 @@ final class Provider {
         )
     }
 
-    private func performClickAction(record: ElementRecord, mouseButton: String) throws -> String? {
-        if mouseButton == "right" {
+    private func performClickAction(record: ElementRecord, mouseButton: MouseButtonSelection) throws -> String? {
+        if mouseButton == .right {
             return performAction(record.element, "AXShowMenu") ? "AXShowMenu" : nil
         }
         for action in ["AXPress", "AXConfirm", "AXOpen"] {
@@ -1634,16 +1637,17 @@ private func screenshotScale(screenshot: ScreenshotPayload?, bounds: CGRect) ->
     )
 }
 
-private enum MouseButton {
-    case left
-    case right
-
+extension MouseButtonSelection {
+    // Why: macOS has no dedicated middle-button event family; it rides `otherMouse*`
+    // with the button number carried by `mouseButton:` on the event constructor.
     var cgButton: CGMouseButton {
         switch self {
         case .left:
             return .left
         case .right:
             return .right
+        case .middle:
+            return .center
         }
     }
 
@@ -1653,6 +1657,8 @@ private enum MouseButton {
             return .leftMouseDown
         case .right:
             return .rightMouseDown
+        case .middle:
+            return .otherMouseDown
         }
     }
 
@@ -1662,20 +1668,18 @@ private enum MouseButton {
             return .leftMouseUp
         case .right:
             return .rightMouseUp
+        case .middle:
+            return .otherMouseUp
         }
     }
 }
 
-private func mouseButton(_ raw: String?) throws -> MouseButton {
-    switch raw ?? "left" {
-    case "left":
-        return .left
-    case "right":
-        return .right
-    case "middle":
-        throw ProviderError.coded("invalid_argument", "middle-click is not yet supported")
-    case let value:
-        throw ProviderError.coded("invalid_argument", "unsupported mouse button '\(value)'")
+private func mouseButton(_ raw: String?) throws -> MouseButtonSelection {
+    switch ActionArgumentValidation.mouseButton(raw) {
+    case let .success(button):
+        return button
+    case let .failure(error):
+        throw ProviderError.coded("invalid_argument", error.message)
     }
 }
 
@@ -2492,7 +2496,7 @@ private func resizePng(_ image: CGImage, scale: CGFloat) -> BoundedPNG? {
 private enum Input {
     static func click(
         at point: CGPoint,
-        button: MouseButton,
+        button: MouseButtonSelection,
         count: Int,
         modifiers: [KeyModifier],
         targetWindow: Snapshot

native/computer-use-macos/Sources/OrcaComputerUseMacOSCore/ActionArgumentValidation.swift

@@ -6,6 +6,18 @@ public struct ActionArgumentValidationError: Error, Equatable {
     }
 }
 
+public enum MouseButtonSelection: String, Equatable, Sendable, CaseIterable {
+    case left
+    case right
+    case middle
+
+    /// Why: AXPress/AXShowMenu only model primary and secondary intent, so a middle
+    /// click has no accessibility equivalent and must reach the app as real events.
+    public var hasAccessibilityAction: Bool {
+        self != .middle
+    }
+}
+
 public enum ActionArgumentValidation {
     public static func positiveInteger(
         _ value: Double?,
@@ -31,6 +43,16 @@ public enum ActionArgumentValidation {
         return .success(value)
     }
 
+    public static func mouseButton(
+        _ value: String?
+    ) -> Result<MouseButtonSelection, ActionArgumentValidationError> {
+        guard let value else { return .success(.left) }
+        guard let button = MouseButtonSelection(rawValue: value) else {
+            return .failure(ActionArgumentValidationError("unsupported mouse button '\(value)'"))
+        }
+        return .success(button)
+    }
+
     public static func scrollDirection(_ value: String) -> Result<String, ActionArgumentValidationError> {
         switch value {
         case "up", "down", "left", "right":

native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/ActionArgumentValidationTests.swift

@@ -54,6 +54,30 @@ final class ActionArgumentValidationTests: XCTestCase {
         )
     }
 
+    func testMouseButtonDefaultsToLeftAndAcceptsEveryButton() {
+        XCTAssertEqual(try ActionArgumentValidation.mouseButton(nil).get(), .left)
+        XCTAssertEqual(try ActionArgumentValidation.mouseButton("left").get(), .left)
+        XCTAssertEqual(try ActionArgumentValidation.mouseButton("right").get(), .right)
+        XCTAssertEqual(try ActionArgumentValidation.mouseButton("middle").get(), .middle)
+    }
+
+    func testMouseButtonRejectsUnknownButtons() {
+        XCTAssertEqual(
+            failureMessage(ActionArgumentValidation.mouseButton("primary")),
+            "unsupported mouse button 'primary'"
+        )
+        XCTAssertEqual(
+            failureMessage(ActionArgumentValidation.mouseButton("")),
+            "unsupported mouse button ''"
+        )
+    }
+
+    func testOnlyMiddleButtonLacksAnAccessibilityAction() {
+        XCTAssertTrue(MouseButtonSelection.left.hasAccessibilityAction)
+        XCTAssertTrue(MouseButtonSelection.right.hasAccessibilityAction)
+        XCTAssertFalse(MouseButtonSelection.middle.hasAccessibilityAction)
+    }
+
     func testScrollDirectionRejectsUnknownDirections() {
         XCTAssertEqual(try ActionArgumentValidation.scrollDirection("down").get(), "down")
         XCTAssertEqual(

skill-guides/computer-use.md

@@ -70,6 +70,8 @@ ORCA computer get-app-state --app <app> --restore-window --json
 ORCA computer click --app <app> --element-index <index> --json
 ORCA computer click --app <app> --x 100 --y 100 --json
 ORCA computer click --app <app> --x 100 --y 100 --modifiers CmdOrCtrl+Shift --json
+ORCA computer click --app <app> --element-index <index> --mouse-button right --json
+ORCA computer click --app <app> --element-index <index> --mouse-button middle --json
 ORCA computer perform-secondary-action --app <app> --element-index <index> --action <name> --json
 ORCA computer set-value --app <app> --element-index <index> --value "text" --json
 ORCA computer type-text --app <app> --text "text" --json

src/cli/handlers/computer-action-routing.test.ts

@@ -80,6 +80,32 @@ describe('orca computer action CLI routing', () => {
     })
   })
 
+  it('forwards a middle click to the runtime instead of silently downgrading it', async () => {
+    queueFixtures(callMock, okFixture('req_click', sampleSnapshot()))
+
+    await main(
+      [
+        'computer',
+        'click',
+        '--session',
+        'manual',
+        '--app',
+        'Finder',
+        '--element-index',
+        '3',
+        '--mouse-button',
+        'middle',
+        '--json'
+      ],
+      '/tmp/repo/src'
+    )
+
+    expect(callMock).toHaveBeenCalledWith(
+      'computer.click',
+      expect.objectContaining({ mouseButton: 'middle' })
+    )
+  })
+
   it('prints session and window context in action follow-up commands', async () => {
     queueFixtures(callMock, okFixture('req_click', sampleSnapshot()))
 

src/main/computer/desktop-script-provider-action-errors.test.ts

@@ -161,6 +161,12 @@ describe('DesktopScriptProviderClient action errors', () => {
       code: 'invalid_argument',
       message: expect.stringContaining('Unsupported direction')
     })
+    await expect(
+      client.action('click', { app: 'Text Editor', elementIndex: 0, mouseButton: 'wheel' })
+    ).rejects.toMatchObject({
+      code: 'invalid_argument',
+      message: expect.stringContaining('Unsupported mouseButton')
+    })
     await expect(
       client.action('drag', { app: 'Text Editor', fromX: 1, fromY: 2 })
     ).rejects.toMatchObject({

tests/e2e/computer-mac-safari.e2e.ts

@@ -168,6 +168,54 @@ describe.skipIf(!isMac || !e2eOptIn)('computer-use macOS e2e (Safari web app)',
     expect(saved.result.action?.actionName).toBe('AXPress')
     expect(saved.result.snapshot.treeText).toContain(`Draft ready: ${recipient} / ${body}`)
   })
+
+  test('delivers an element-index middle click to the browser receiver', async () => {
+    const targetArgs = await safariFixtureWindowTargetArgs(fixture.title)
+    const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'get-app-state',
+          '--app',
+          'com.apple.Safari',
+          ...targetArgs,
+          '--restore-window',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+    expect(before.result.snapshot.treeText).toContain('Middle click waiting')
+
+    const receiverIndex = findRoleIndex(
+      before.result.snapshot.treeText,
+      'button Middle click receiver'
+    )
+    expect(receiverIndex).toBeGreaterThanOrEqual(0)
+
+    const middle = parseJsonOutput<{ result: ComputerActionResult }>(
+      (
+        await runOrcaCli([
+          'computer',
+          'click',
+          '--app',
+          'com.apple.Safari',
+          ...targetArgs,
+          '--element-index',
+          String(receiverIndex),
+          '--mouse-button',
+          'middle',
+          '--restore-window',
+          '--no-screenshot',
+          '--json'
+        ])
+      ).stdout
+    )
+
+    expect(middle.result.action?.path).toBe('synthetic')
+    expect(middle.result.action?.fallbackReason).toBe('actionUnsupported')
+    expect(middle.result.snapshot.treeText).toContain('Middle click received')
+  })
 })
 
 async function safariFixtureWindowTargetArgs(title: string): Promise<string[]> {

tests/e2e/helpers/computer-driver.ts

@@ -283,6 +283,8 @@ function safariDraftFixtureHtml(title: string): string {
     '<label>Body <textarea id="body" aria-label="Body"></textarea></label>',
     "<button id=\"save\" onclick=\"document.getElementById('status').textContent = 'Draft ready: ' + document.getElementById('recipient').value + ' / ' + document.getElementById('body').value\">Save draft</button>",
     '<p id="status" role="status">Draft empty</p>',
+    '<button id="middle-click-target" onauxclick="if (event.button === 1) document.getElementById(\'middle-click-status\').textContent = \'Middle click received\'">Middle click receiver</button>',
+    '<p id="middle-click-status" role="status">Middle click waiting</p>',
     '</main>',
     '</body>',
     '</html>'