mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 08:02:28 +00:00
* feat(computer-use): support macOS middle click and gate the AX click path `--mouse-button middle` already validated end-to-end through the CLI, the zod schema, and the provider validator, and both the Windows and Linux providers honored it. Only the macOS provider rejected it outright with "middle-click is not yet supported", so the flag was a dead end on the one platform that has no fallback. Two changes: - Add `.middle` to the macOS button mapping. macOS has no dedicated middle event family, so it rides `otherMouseDown`/`otherMouseUp` with the button number carried by `mouseButton: .center`; that constructor argument is honored for exactly the `otherMouse*` types, so no extra field write is needed. - Validate the requested button before the accessibility fast path, and skip that path for buttons it cannot express. Previously the raw string was read unvalidated, and `performClickAction` only special-cased `right`, so `click --mouse-button middle --element-index N` (no modifiers, count 1) fell through to `AXPress` — a left click — and reported success with `path: "accessibility"`. Any unrecognized button string did the same. This matches guards the Windows and Linux providers already had. The button enum moves into `OrcaComputerUseMacOSCore` so it is unit-testable; `main.swift` keeps only the CoreGraphics mapping. Also documents `--mouse-button` in the computer-use skill guide, which never mentioned the flag, so agents on Windows and Linux had no way to discover it. * test(computer-use): cover macOS middle click in the real-desktop e2e suite * test(computer-use): prove macOS middle-click delivery
291 lines
9.8 KiB
TypeScript
291 lines
9.8 KiB
TypeScript
import { afterEach, describe, expect, it } from 'vitest'
|
|
import {
|
|
CLIPBOARD_TEXT_WRITE_MAX_BYTES,
|
|
CLIPBOARD_TEXT_WRITE_TOO_LARGE_ERROR
|
|
} from '../../shared/clipboard-text'
|
|
import {
|
|
createDesktopScriptProviderClient,
|
|
expectDesktopProviderSubprocessStartCount,
|
|
mockBridgeResponse,
|
|
resetDesktopScriptProviderTestHarness,
|
|
sampleBridgeSnapshot,
|
|
sampleCapabilities
|
|
} from './desktop-script-provider-test-harness'
|
|
|
|
describe('DesktopScriptProviderClient action errors', () => {
|
|
afterEach(resetDesktopScriptProviderTestHarness)
|
|
|
|
it('maps action-specific bridge errors to actionable codes', async () => {
|
|
mockBridgeResponse({ ok: false, error: 'element value is not settable' })
|
|
mockBridgeResponse({ ok: false, error: 'Raise is not a valid secondary action' })
|
|
|
|
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
|
|
|
|
await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'value_not_settable'
|
|
})
|
|
await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'action_not_supported'
|
|
})
|
|
})
|
|
|
|
it('maps native validation bridge errors to invalid arguments', async () => {
|
|
mockBridgeResponse({ ok: false, error: 'unsupported scroll direction: diagonal' })
|
|
mockBridgeResponse({ ok: false, error: 'x is required' })
|
|
mockBridgeResponse({ ok: false, error: 'text is required' })
|
|
mockBridgeResponse({ ok: false, error: 'click_count must be a positive integer' })
|
|
mockBridgeResponse({ ok: false, error: 'Unsupported key: Nope' })
|
|
|
|
const client = await createDesktopScriptProviderClient('windows', '/tmp/runtime.ps1')
|
|
|
|
await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'invalid_argument'
|
|
})
|
|
await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'invalid_argument'
|
|
})
|
|
await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'invalid_argument'
|
|
})
|
|
await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'invalid_argument'
|
|
})
|
|
await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'invalid_argument'
|
|
})
|
|
})
|
|
|
|
it('does not confuse permission setup errors with invalid arguments', async () => {
|
|
mockBridgeResponse({ ok: false, error: 'Accessibility permission is required' })
|
|
|
|
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
|
|
|
|
await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'permission_denied'
|
|
})
|
|
})
|
|
|
|
it('does not confuse accessibility permission failures with missing windows', async () => {
|
|
mockBridgeResponse({
|
|
ok: false,
|
|
error:
|
|
"app 'Text Editor' has visible windows but no accessibility window. Accessibility permission may need to be toggled."
|
|
})
|
|
|
|
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
|
|
|
|
await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'permission_denied'
|
|
})
|
|
})
|
|
|
|
it('maps keyboard focus bridge errors to window_not_focused', async () => {
|
|
mockBridgeResponse({
|
|
ok: false,
|
|
error:
|
|
'window_not_focused: keyboard input requires the target window to be focused; retry with --restore-window'
|
|
})
|
|
|
|
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
|
|
|
|
await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'window_not_focused'
|
|
})
|
|
})
|
|
|
|
it('maps bridge screenshot capture errors to screenshot_failed', async () => {
|
|
mockBridgeResponse({
|
|
ok: false,
|
|
error: 'screenshot_failed: Screen Recording permission is required'
|
|
})
|
|
|
|
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
|
|
|
|
await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'screenshot_failed'
|
|
})
|
|
})
|
|
|
|
it('maps missing Linux keyboard dependencies to unsupported capability', async () => {
|
|
mockBridgeResponse({
|
|
ok: false,
|
|
error: 'GDK is required for non-character key synthesis'
|
|
})
|
|
|
|
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
|
|
|
|
await expect(client.snapshot({ app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'unsupported_capability'
|
|
})
|
|
})
|
|
|
|
it('rejects actions that the provider does not advertise', async () => {
|
|
mockBridgeResponse({
|
|
ok: true,
|
|
capabilities: sampleCapabilities({
|
|
hotkey: false,
|
|
pasteText: false
|
|
})
|
|
})
|
|
|
|
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
|
|
|
|
await expect(
|
|
client.action('pasteText', { app: 'Text Editor', text: 'hello' })
|
|
).rejects.toMatchObject({
|
|
code: 'unsupported_capability',
|
|
message: expect.stringContaining('actions.pasteText')
|
|
})
|
|
})
|
|
|
|
it('rejects malformed action payloads before launching the native bridge', async () => {
|
|
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
|
|
|
|
await expect(client.action('click', { elementIndex: 0 })).rejects.toMatchObject({
|
|
code: 'invalid_argument',
|
|
message: expect.stringContaining('Missing app')
|
|
})
|
|
await expect(client.action('click', { app: 'Text Editor' })).rejects.toMatchObject({
|
|
code: 'invalid_argument',
|
|
message: expect.stringContaining('Click requires')
|
|
})
|
|
await expect(
|
|
client.action('click', { app: 'Text Editor', windowId: 1, windowIndex: 0, x: 1, y: 2 })
|
|
).rejects.toMatchObject({
|
|
code: 'invalid_argument',
|
|
message: expect.stringContaining('either windowId or windowIndex')
|
|
})
|
|
await expect(
|
|
client.action('scroll', { app: 'Text Editor', elementIndex: 0, direction: 'diagonal' })
|
|
).rejects.toMatchObject({
|
|
code: 'invalid_argument',
|
|
message: expect.stringContaining('Unsupported direction')
|
|
})
|
|
await expect(
|
|
client.action('click', { app: 'Text Editor', elementIndex: 0, mouseButton: 'wheel' })
|
|
).rejects.toMatchObject({
|
|
code: 'invalid_argument',
|
|
message: expect.stringContaining('Unsupported mouseButton')
|
|
})
|
|
await expect(
|
|
client.action('drag', { app: 'Text Editor', fromX: 1, fromY: 2 })
|
|
).rejects.toMatchObject({
|
|
code: 'invalid_argument',
|
|
message: expect.stringContaining('Drag coordinates')
|
|
})
|
|
await expect(
|
|
client.action('performSecondaryAction', { app: 'Text Editor', elementIndex: 0 })
|
|
).rejects.toMatchObject({
|
|
code: 'invalid_argument',
|
|
message: expect.stringContaining('Missing action')
|
|
})
|
|
await expect(client.action('typeText', { app: 'Text Editor', text: '' })).rejects.toMatchObject(
|
|
{
|
|
code: 'invalid_argument',
|
|
message: expect.stringContaining('Missing text')
|
|
}
|
|
)
|
|
await expect(
|
|
client.action('pasteText', {
|
|
app: 'Text Editor',
|
|
text: ['native-provider-secret', 'x'.repeat(CLIPBOARD_TEXT_WRITE_MAX_BYTES + 1)].join('')
|
|
})
|
|
).rejects.toMatchObject({
|
|
code: 'invalid_argument',
|
|
message: CLIPBOARD_TEXT_WRITE_TOO_LARGE_ERROR
|
|
})
|
|
await expect(
|
|
client.action('pressKey', { app: 'Text Editor', key: 'CmdOrCtrl+V' })
|
|
).rejects.toMatchObject({
|
|
code: 'invalid_argument',
|
|
message: expect.stringContaining('Press-key accepts one key only')
|
|
})
|
|
await expect(client.action('hotkey', { app: 'Text Editor', key: 'A' })).rejects.toMatchObject({
|
|
code: 'invalid_argument',
|
|
message: expect.stringContaining('Hotkey requires')
|
|
})
|
|
await expect(
|
|
client.action('setValue', { app: 'Text Editor', elementIndex: 0 })
|
|
).rejects.toMatchObject({
|
|
code: 'invalid_argument',
|
|
message: expect.stringContaining('Missing value')
|
|
})
|
|
expectDesktopProviderSubprocessStartCount(0)
|
|
})
|
|
|
|
it('rejects non-keyboard actions that the provider does not advertise', async () => {
|
|
mockBridgeResponse({
|
|
ok: true,
|
|
snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
|
|
})
|
|
mockBridgeResponse({
|
|
ok: true,
|
|
capabilities: sampleCapabilities({
|
|
setValue: false
|
|
})
|
|
})
|
|
|
|
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
|
|
|
|
await client.snapshot({ app: 'Text Editor' })
|
|
await expect(
|
|
client.action('setValue', {
|
|
app: 'Text Editor',
|
|
elementIndex: 0,
|
|
value: 'changed',
|
|
noScreenshot: true
|
|
})
|
|
).rejects.toMatchObject({
|
|
code: 'unsupported_capability',
|
|
message: expect.stringContaining('actions.setValue')
|
|
})
|
|
})
|
|
|
|
it('uses provider action metadata when bridge reports the actual path', async () => {
|
|
mockBridgeResponse({
|
|
ok: true,
|
|
snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
|
|
})
|
|
mockBridgeResponse({
|
|
ok: true,
|
|
capabilities: sampleCapabilities()
|
|
})
|
|
mockBridgeResponse({
|
|
ok: true,
|
|
action: {
|
|
path: 'accessibility',
|
|
actionName: 'Press',
|
|
fallbackReason: null
|
|
},
|
|
snapshot: sampleBridgeSnapshot('Text Editor', 'changed')
|
|
})
|
|
|
|
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
|
|
|
|
await client.snapshot({ app: 'Text Editor' })
|
|
await expect(
|
|
client.action('click', {
|
|
app: 'Text Editor',
|
|
elementIndex: 0,
|
|
noScreenshot: true
|
|
})
|
|
).resolves.toMatchObject({
|
|
action: {
|
|
path: 'accessibility',
|
|
actionName: 'Press'
|
|
}
|
|
})
|
|
})
|
|
|
|
it('explains that missing element indexes require a fresh snapshot', async () => {
|
|
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
|
|
|
|
await expect(
|
|
client.action('click', { app: 'Text Editor', elementIndex: 4 })
|
|
).rejects.toMatchObject({
|
|
code: 'element_not_found',
|
|
message: expect.stringContaining('run get-app-state again')
|
|
})
|
|
})
|
|
})
|