mirror of
https://github.com/stablyai/orca.git
synced 2026-09-21 16:02:20 +00:00
Brennan Benson
66dfdc456f
* feat(computer-use): support macOS middle click and gate the AX click path `--mouse-button middle` already validated end-to-end through the CLI, the zod schema, and the provider validator, and both the Windows and Linux providers honored it. Only the macOS provider rejected it outright with "middle-click is not yet supported", so the flag was a dead end on the one platform that has no fallback. Two changes: - Add `.middle` to the macOS button mapping. macOS has no dedicated middle event family, so it rides `otherMouseDown`/`otherMouseUp` with the button number carried by `mouseButton: .center`; that constructor argument is honored for exactly the `otherMouse*` types, so no extra field write is needed. - Validate the requested button before the accessibility fast path, and skip that path for buttons it cannot express. Previously the raw string was read unvalidated, and `performClickAction` only special-cased `right`, so `click --mouse-button middle --element-index N` (no modifiers, count 1) fell through to `AXPress` — a left click — and reported success with `path: "accessibility"`. Any unrecognized button string did the same. This matches guards the Windows and Linux providers already had. The button enum moves into `OrcaComputerUseMacOSCore` so it is unit-testable; `main.swift` keeps only the CoreGraphics mapping. Also documents `--mouse-button` in the computer-use skill guide, which never mentioned the flag, so agents on Windows and Linux had no way to discover it. * test(computer-use): cover macOS middle click in the real-desktop e2e suite * test(computer-use): prove macOS middle-click delivery
66 lines
2.5 KiB
JavaScript
66 lines
2.5 KiB
JavaScript
import { readFileSync } from 'node:fs'
|
|
import { join, resolve } from 'node:path'
|
|
import { describe, expect, it } from 'vitest'
|
|
|
|
const projectDir = resolve(import.meta.dirname, '../..')
|
|
|
|
function source(path) {
|
|
return readFileSync(join(projectDir, path), 'utf8')
|
|
}
|
|
|
|
function sourceBetween(contents, startMarker, endMarker) {
|
|
const start = contents.indexOf(startMarker)
|
|
const end = contents.indexOf(endMarker, start + startMarker.length)
|
|
if (start === -1 || end === -1) {
|
|
throw new Error(`Missing source boundary: ${startMarker} → ${endMarker}`)
|
|
}
|
|
return contents.slice(start, end)
|
|
}
|
|
|
|
describe('computer-use mouse button routing', () => {
|
|
it('maps the macOS middle button onto the otherMouse event family', () => {
|
|
const macOS = source('native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift')
|
|
const mapping = sourceBetween(
|
|
macOS,
|
|
'extension MouseButtonSelection {',
|
|
'private func mouseButton('
|
|
)
|
|
|
|
expect(mapping).toContain('return .center')
|
|
expect(mapping).toContain('return .otherMouseDown')
|
|
expect(mapping).toContain('return .otherMouseUp')
|
|
// A middle press posted as a left event type would silently left-click.
|
|
expect(mapping).not.toContain('case .middle:\n return .leftMouseDown')
|
|
})
|
|
|
|
it('validates the macOS mouse button before any accessibility shortcut runs', () => {
|
|
const macOS = source('native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift')
|
|
const click = sourceBetween(
|
|
macOS,
|
|
'private func click(params:',
|
|
'private func performClickAction('
|
|
)
|
|
|
|
expect(click).toContain('let button = try mouseButton(params["mouseButton"]?.string)')
|
|
expect(click).toContain('button.hasAccessibilityAction')
|
|
// An unvalidated raw string reaches AXPress and reports a left click as success.
|
|
expect(click).not.toContain('params["mouseButton"]?.string ?? "left"')
|
|
})
|
|
|
|
it('keeps every platform from resolving a middle click through its accessibility path', () => {
|
|
const windows = source('native/computer-use-windows/runtime.ps1')
|
|
const windowsClick = sourceBetween(
|
|
windows,
|
|
'$handledByPattern = $false',
|
|
'if (-not $handledByPattern)'
|
|
)
|
|
|
|
expect(windowsClick).toContain('$Operation.mouse_button -ne "middle"')
|
|
|
|
const linux = source('native/computer-use-linux/runtime.py')
|
|
const linuxClick = sourceBetween(linux, 'has_modifiers = bool(', 'if not handled:')
|
|
|
|
expect(linuxClick).toContain('operation.get("mouse_button", "left") == "left"')
|
|
})
|
|
})
|