mirror of
https://github.com/stablyai/orca.git
synced 2026-10-21 08:03:33 +00:00
Brennan Benson
c2da0e47f9
Mouse events posted with CGEventPostToPid reach the target app with no window association, so AppKit never routes the press to a view: hover states fire but the control is never activated, and the mouseUp is dropped outright when posted back-to-back. Post click events to the HID event tap instead (as keyboard synthesis already does), pace them, and stamp mouseEventClickState so multi-clicks register. Synthetic clicks now also report verification unverified/synthetic_input from the helper itself, matching the other synthetic actions.
76 lines
3.1 KiB
JavaScript
76 lines
3.1 KiB
JavaScript
import { readFileSync } from 'node:fs'
|
|
import { join, resolve } from 'node:path'
|
|
import { describe, expect, it } from 'vitest'
|
|
|
|
const projectDir = resolve(import.meta.dirname, '../..')
|
|
|
|
function source(path) {
|
|
return readFileSync(join(projectDir, path), 'utf8')
|
|
}
|
|
|
|
function sourceBetween(contents, startMarker, endMarker) {
|
|
const start = contents.indexOf(startMarker)
|
|
const end = contents.indexOf(endMarker, start + startMarker.length)
|
|
if (start < 0 || end < 0) {
|
|
throw new Error(`Missing source boundary: ${startMarker} → ${endMarker}`)
|
|
}
|
|
return contents.slice(start, end)
|
|
}
|
|
|
|
describe('computer-use modifier safety', () => {
|
|
it('uses mouse-event flags instead of held modifier keys on macOS', () => {
|
|
const macOS = source('native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift')
|
|
const clickInput = sourceBetween(macOS, 'static func click(', 'static func scroll(')
|
|
const mouseInput = sourceBetween(
|
|
macOS,
|
|
'private static func mouse(',
|
|
'private static func keyEvent('
|
|
)
|
|
|
|
expect(mouseInput).toContain('event.flags = flags')
|
|
// Every click event flows through the shared delivery plan and carries
|
|
// the modifier flags on the mouse event itself.
|
|
expect(clickInput).toContain('SyntheticMouseClickDelivery.steps(clickCount: count)')
|
|
expect(clickInput).toContain('event.flags = flags')
|
|
expect(clickInput).not.toContain('down: true')
|
|
})
|
|
|
|
it('submits each modified Windows click in a closed, timed SendInput batch', () => {
|
|
const windows = source('native/computer-use-windows/runtime.ps1')
|
|
const modifiedClick = sourceBetween(
|
|
windows,
|
|
'public static void SendModifiedClick',
|
|
'private static INPUT KeyboardInput'
|
|
)
|
|
const mouseClick = sourceBetween(
|
|
windows,
|
|
'function Send-OrcaMouseClick',
|
|
'function Send-OrcaDrag'
|
|
)
|
|
|
|
expect(modifiedClick).toContain('SendInput((uint)values.Length, values')
|
|
expect(modifiedClick).toContain('SendInput((uint)releaseValues.Length, releaseValues')
|
|
expect(modifiedClick).toContain('if (sent != (uint)values.Length)')
|
|
expect(modifiedClick).toContain('releases.Add(MouseInput(mouseInput, mouseUp))')
|
|
expect(modifiedClick).not.toContain('int count')
|
|
expect(mouseClick).toMatch(
|
|
/for \(\$i = 0; \$i -lt \$clickCount; \$i\+\+\) \{\s+\[OrcaDesktopWin32\]::SendModifiedClick\(/
|
|
)
|
|
expect(mouseClick).toContain('if ($i + 1 -lt $clickCount) { Start-Sleep -Milliseconds 35 }')
|
|
expect(windows).not.toContain('keybd_event')
|
|
})
|
|
|
|
it('keeps Linux modifier release in the xdotool sequence and a fallback', () => {
|
|
const linux = source('native/computer-use-linux/runtime.py')
|
|
const modifiedClick = sourceBetween(linux, 'def modified_click_at(', 'def scroll_at(')
|
|
|
|
expect(modifiedClick).toContain('command.extend(["keyup", modifier])')
|
|
expect(modifiedClick).toContain('is_wayland')
|
|
expect(modifiedClick).toContain('modified clicks require xdotool on an X11 session')
|
|
expect(modifiedClick).toContain('finally:')
|
|
expect(modifiedClick).toContain('check=False')
|
|
expect(modifiedClick).toContain('timeout=5')
|
|
expect(modifiedClick).toContain('timeout=2')
|
|
})
|
|
})
|