starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-09-21 16:02:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(computer-use): make modifier clicks interruption-safe (#11451)

Browse Source 
* fix(computer-use): make modifier clicks interruption-safe

* fix(computer-use): pace modified Windows multiclicks

* fix(computer-use): address modifier safety review
This commit is contained in:
Neil
2026-07-29 18:29:10 -07:00
committed by GitHub
parent 5517bfcbd2
commit d0f341ad69
26 changed files with 527 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/computer-e2e.yml
+2
View File
@@ -9,6 +9,7 @@ on:
- 'config/scripts/build-windows-cli-launcher.mjs'
- 'config/scripts/build-windows-cli-launcher.test.mjs'
- 'config/scripts/computer-e2e-workflow.test.mjs'
- 'config/scripts/computer-use-modifier-safety.test.mjs'
- 'config/scripts/computer-use-skill-guidance.test.mjs'
- 'config/scripts/computer-use-smoke.mjs'
- 'config/scripts/computer-use-smoke.test.mjs'
@@ -84,6 +85,7 @@ jobs:
config/scripts/build-windows-cli-launcher.test.mjs
src/main/ssh/ssh-remote-cli-launcher.test.ts
config/scripts/computer-e2e-workflow.test.mjs
config/scripts/computer-use-modifier-safety.test.mjs
config/scripts/computer-use-skill-guidance.test.mjs
config/scripts/computer-use-smoke.test.mjs
src/main/computer/computer-provider-lifecycle.test.ts
config/scripts/computer-e2e-workflow.test.mjs
+2
View File
@@ -47,6 +47,7 @@ describe('computer-use e2e workflow', () => {
expect(triggerPaths).toEqual(
expect.arrayContaining([
'config/scripts/computer-e2e-workflow.test.mjs',
'config/scripts/computer-use-modifier-safety.test.mjs',
'config/scripts/computer-use-skill-guidance.test.mjs',
'config/scripts/computer-use-smoke.mjs',
'config/scripts/computer-use-smoke.test.mjs',
@@ -72,6 +73,7 @@ describe('computer-use e2e workflow', () => {
const regressionRun = nativeSmokeRuns.find((run) => run.includes('pnpm vitest run'))
const expectedRegressionFiles = [
'config/scripts/computer-e2e-workflow.test.mjs',
'config/scripts/computer-use-modifier-safety.test.mjs',
'config/scripts/computer-use-skill-guidance.test.mjs',
'config/scripts/computer-use-smoke.test.mjs',
'src/main/computer/computer-provider-lifecycle.test.ts',
config/scripts/computer-use-modifier-safety.test.mjs
+72
View File
@@ -0,0 +1,72 @@
import { readFileSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { describe, expect, it } from 'vitest'
const projectDir = resolve(import.meta.dirname, '../..')
function source(path) {
return readFileSync(join(projectDir, path), 'utf8')
}
function sourceBetween(contents, startMarker, endMarker) {
const start = contents.indexOf(startMarker)
const end = contents.indexOf(endMarker, start + startMarker.length)
if (start < 0 || end < 0) {
throw new Error(`Missing source boundary: ${startMarker} → ${endMarker}`)
}
return contents.slice(start, end)
}
describe('computer-use modifier safety', () => {
it('uses mouse-event flags instead of held modifier keys on macOS', () => {
const macOS = source('native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift')
const clickInput = sourceBetween(macOS, 'static func click(', 'static func scroll(')
const mouseInput = sourceBetween(
macOS,
'private static func mouse(',
'private static func keyEvent('
)
expect(mouseInput).toContain('event.flags = flags')
expect(clickInput.match(/flags: flags/gu)).toHaveLength(3)
expect(clickInput).not.toContain('down: true')
})
it('submits each modified Windows click in a closed, timed SendInput batch', () => {
const windows = source('native/computer-use-windows/runtime.ps1')
const modifiedClick = sourceBetween(
windows,
'public static void SendModifiedClick',
'private static INPUT KeyboardInput'
)
const mouseClick = sourceBetween(
windows,
'function Send-OrcaMouseClick',
'function Send-OrcaDrag'
)
expect(modifiedClick).toContain('SendInput((uint)values.Length, values')
expect(modifiedClick).toContain('SendInput((uint)releaseValues.Length, releaseValues')
expect(modifiedClick).toContain('if (sent != (uint)values.Length)')
expect(modifiedClick).toContain('releases.Add(MouseInput(mouseInput, mouseUp))')
expect(modifiedClick).not.toContain('int count')
expect(mouseClick).toMatch(
/for \(\$i = 0; \$i -lt \$clickCount; \$i\+\+\) \{\s+\[OrcaDesktopWin32\]::SendModifiedClick\(/
)
expect(mouseClick).toContain('if ($i + 1 -lt $clickCount) { Start-Sleep -Milliseconds 35 }')
expect(windows).not.toContain('keybd_event')
})
it('keeps Linux modifier release in the xdotool sequence and a fallback', () => {
const linux = source('native/computer-use-linux/runtime.py')
const modifiedClick = sourceBetween(linux, 'def modified_click_at(', 'def scroll_at(')
expect(modifiedClick).toContain('command.extend(["keyup", modifier])')
expect(modifiedClick).toContain('is_wayland')
expect(modifiedClick).toContain('modified clicks require xdotool on an X11 session')
expect(modifiedClick).toContain('finally:')
expect(modifiedClick).toContain('check=False')
expect(modifiedClick).toContain('timeout=5')
expect(modifiedClick).toContain('timeout=2')
})
})
config/scripts/computer-use-skill-guidance.test.mjs
+11
View File
@@ -1,6 +1,7 @@
import { readFileSync } from 'node:fs'
import { join, resolve } from 'node:path'
import { describe, expect, it } from 'vitest'
import { BUNDLED_SKILL_GUIDES } from '../../src/cli/bundled-skill-guides'
const projectDir = resolve(import.meta.dirname, '../..')
// Why: computer-use now ships a hybrid discovery stub, so its version-sensitive command
@@ -8,6 +9,7 @@ const projectDir = resolve(import.meta.dirname, '../..')
// installable stub projection is checked separately below.
const guidePath = join(projectDir, 'skill-guides', 'computer-use.md')
const stubPath = join(projectDir, 'skills', 'computer-use', 'SKILL.md')
const bundledGuide = BUNDLED_SKILL_GUIDES.find((guide) => guide.name === 'computer-use')?.markdown
describe('computer-use skill guidance', () => {
it('keeps web-app targeting on the computer-use surface', () => {
@@ -44,6 +46,15 @@ describe('computer-use skill guidance', () => {
expect(skill).toContain('`result.snapshot.treeText`')
expect(skill).not.toContain('`result.elements`')
})
it('requires atomic modifier-click actions in the source and bundled guide', () => {
expect(bundledGuide).toBeDefined()
for (const skill of [readFileSync(guidePath, 'utf8'), bundledGuide]) {
expect(skill).toContain('click --modifiers <chord>')
expect(skill).toContain('Never synthesize separate modifier-down and modifier-up commands')
}
})
})
describe('computer-use install stub', () => {
native/computer-use-linux/runtime.py
+46 -3
View File
@@ -822,19 +822,60 @@ def require_non_empty_string(value, name):
return str(value)
def click_at(x, y, button, count):
def click_at(x, y, button, count, modifiers=None):
button = (button or "left").lower()
buttons = {"left": ("b1p", "b1r"), "right": ("b3p", "b3r"), "middle": ("b2p", "b2r")}
if button not in buttons:
raise RuntimeError(f"unsupported mouse button: {button}")
parsed_count = require_positive_integer(1 if count is None else count, "click_count")
modifier_keys = click_modifier_keys(modifiers)
if modifier_keys:
modified_click_at(x, y, button, parsed_count, modifier_keys)
return
down, up = buttons[button]
for _ in range(require_positive_integer(1 if count is None else count, "click_count")):
for _ in range(parsed_count):
Atspi.generate_mouse_event(round(x), round(y), "abs")
Atspi.generate_mouse_event(round(x), round(y), down)
time.sleep(0.03)
Atspi.generate_mouse_event(round(x), round(y), up)
def click_modifier_keys(raw):
if raw is None:
return []
aliases = {
"ctrl": "ctrl", "control": "ctrl", "cmdorctrl": "ctrl", "commandorcontrol": "ctrl",
"shift": "shift", "alt": "alt", "option": "alt",
"meta": "super", "super": "super", "win": "super", "cmd": "super", "command": "super",
}
parts = [part.strip().lower() for part in str(raw).split("+")]
if not parts or any(not part or part not in aliases for part in parts):
raise RuntimeError("click modifiers require modifier keys only")
return list(dict.fromkeys(aliases[part] for part in parts))
def modified_click_at(x, y, button, count, modifier_keys):
xdotool = shutil.which("xdotool")
is_wayland = os.environ.get("XDG_SESSION_TYPE", "").lower() == "wayland"
if not xdotool or is_wayland:
raise RuntimeError("modified clicks require xdotool on an X11 session")
button_number = {"left": "1", "middle": "2", "right": "3"}[button]
command = [xdotool, "mousemove", "--sync", str(round(x)), str(round(y))]
for modifier in modifier_keys:
command.extend(["keydown", modifier])
command.extend(["click", "--repeat", str(count), "--delay", "35", button_number])
for modifier in reversed(modifier_keys):
command.extend(["keyup", modifier])
try:
subprocess.run(command, check=True, timeout=5)
finally:
subprocess.run(
[xdotool, *[item for modifier in reversed(modifier_keys) for item in ("keyup", modifier)]],
check=False,
timeout=2,
)
def scroll_at(x, y, direction, pages):
if direction is None or str(direction).strip() == "":
raise RuntimeError("direction is required")
@@ -1024,12 +1065,14 @@ def run_operation(operation):
if operation.get("click_count") is not None
else 1
)
handled = operation.get("mouse_button", "left") == "left" and click_count <= 1 and perform_action(node, preferred)
has_modifiers = bool(str(operation.get("modifiers") or "").strip())
handled = not has_modifiers and operation.get("mouse_button", "left") == "left" and click_count <= 1 and perform_action(node, preferred)
if not handled:
click_at(
*screen_point(bounds, saved, operation.get("x"), operation.get("y"), node),
operation.get("mouse_button", "left"),
click_count,
operation.get("modifiers"),
)
action = {"path": "synthetic", "actionName": None, "fallbackReason": "actionUnsupported"}
else:
native/computer-use-macos/Sources/OrcaComputerUseMacOS/main.swift
+71 -22
View File
@@ -729,12 +729,13 @@ final class Provider {
let snapshot = try currentSnapshot(params: params)
let button = params["mouseButton"]?.string ?? "left"
let count = try positiveInteger(params["clickCount"]?.number, defaultValue: 1, name: "clickCount")
let modifiers = try KeyMap.parseModifiers(params["modifiers"]?.string)
// Why: agents expect a click into a target app to make the next
// keyboard action safe, even when the click uses an AX action path.
recoverWindow(snapshot.app)
if let elementIndex = try optionalInteger(params, "elementIndex") {
let record = try element(snapshot, elementIndex)
if count <= 1, let actionName = try performClickAction(record: record, mouseButton: button) {
if modifiers.isEmpty, count <= 1, let actionName = try performClickAction(record: record, mouseButton: button) {
return actionMetadata(path: "accessibility", actionName: actionName)
}
if let point = center(record.localFrame, in: snapshot.windowBounds) {
@@ -742,7 +743,8 @@ final class Provider {
pid: snapshot.app.pid,
at: point,
button: mouseButton(button),
count: count
count: count,
modifiers: modifiers
)
return actionMetadata(path: "synthetic", fallbackReason: "actionUnsupported")
}
@@ -753,7 +755,8 @@ final class Provider {
pid: snapshot.app.pid,
at: point,
button: mouseButton(button),
count: count
count: count,
modifiers: modifiers
)
return actionMetadata(path: "synthetic")
}
@@ -2282,14 +2285,23 @@ private func resizePng(_ image: CGImage, scale: CGFloat) -> BoundedPNG? {
}
private enum Input {
static func click(pid: pid_t, at point: CGPoint, button: MouseButton, count: Int) throws {
static func click(
pid: pid_t,
at point: CGPoint,
button: MouseButton,
count: Int,
modifiers: [KeyModifier]
) throws {
guard let source = CGEventSource(stateID: .combinedSessionState) else {
throw ProviderError.coded("accessibility_error", "failed to create event source")
}
let flags = modifiers.reduce(into: CGEventFlags()) { result, modifier in
result.insert(modifier.flag)
}
for _ in 0..<max(count, 1) {
try mouse(.mouseMoved, source: source, point: point, button: button.cgButton, pid: pid)
try mouse(button.downEvent, source: source, point: point, button: button.cgButton, pid: pid)
try mouse(button.upEvent, source: source, point: point, button: button.cgButton, pid: pid)
try mouse(.mouseMoved, source: source, point: point, button: button.cgButton, flags: flags, pid: pid)
try mouse(button.downEvent, source: source, point: point, button: button.cgButton, flags: flags, pid: pid)
try mouse(button.upEvent, source: source, point: point, button: button.cgButton, flags: flags, pid: pid)
}
}
@@ -2343,16 +2355,20 @@ private enum Input {
static func pressKey(_ key: String, pid: pid_t) throws {
let parsed = try KeyMap.parse(key)
var flags = CGEventFlags()
var pressedModifiers: [KeyModifier] = []
defer {
for modifier in pressedModifiers.reversed() {
flags.remove(modifier.flag)
try? keyEvent(modifier.keyCode, down: false, flags: flags, pid: pid)
}
}
for modifier in parsed.modifiers {
flags.insert(modifier.flag)
try keyEvent(modifier.keyCode, down: true, flags: flags, pid: pid)
pressedModifiers.append(modifier)
}
try keyEvent(parsed.keyCode, down: true, flags: flags, pid: pid)
try keyEvent(parsed.keyCode, down: false, flags: flags, pid: pid)
for modifier in parsed.modifiers.reversed() {
try keyEvent(modifier.keyCode, down: false, flags: flags, pid: pid)
flags.remove(modifier.flag)
}
}
static func pasteText(_ text: String, pid: pid_t) throws {
@@ -2377,10 +2393,18 @@ private enum Input {
try pressKey("cmd+v", pid: pid)
}
private static func mouse(_ type: CGEventType, source: CGEventSource, point: CGPoint, button: CGMouseButton, pid: pid_t) throws {
private static func mouse(
_ type: CGEventType,
source: CGEventSource,
point: CGPoint,
button: CGMouseButton,
flags: CGEventFlags = [],
pid: pid_t
) throws {
guard let event = CGEvent(mouseEventSource: source, mouseType: type, mouseCursorPosition: point, mouseButton: button) else {
throw ProviderError.coded("accessibility_error", "failed to create mouse event")
}
event.flags = flags
event.postToPid(pid)
}
@@ -2497,16 +2521,9 @@ private enum KeyMap {
var modifiers: [KeyModifier] = []
var keyName: String?
for part in parts {
switch part {
case "cmd", "command", "meta", "super", "cmdorctrl", "commandorcontrol":
modifiers.append(KeyModifier(keyCode: 55, flag: .maskCommand))
case "ctrl", "control":
modifiers.append(KeyModifier(keyCode: 59, flag: .maskControl))
case "alt", "option":
modifiers.append(KeyModifier(keyCode: 58, flag: .maskAlternate))
case "shift":
modifiers.append(KeyModifier(keyCode: 56, flag: .maskShift))
default:
if let modifier = modifier(part) {
modifiers.append(modifier)
} else {
keyName = part
}
}
@@ -2516,6 +2533,38 @@ private enum KeyMap {
return ParsedKey(keyCode: keyCode, modifiers: modifiers)
}
static func parseModifiers(_ spec: String?) throws -> [KeyModifier] {
guard let spec else {
return []
}
let parts = spec.split(separator: "+", omittingEmptySubsequences: false)
.map { String($0).trimmingCharacters(in: .whitespacesAndNewlines).lowercased() }
guard !parts.isEmpty, !parts.contains(where: \.isEmpty) else {
throw ProviderError.coded("invalid_argument", "click modifiers require modifier keys only")
}
return try parts.map { part in
guard let modifier = modifier(part) else {
throw ProviderError.coded("invalid_argument", "unsupported click modifier '\(part)'")
}
return modifier
}
}
private static func modifier(_ part: String) -> KeyModifier? {
switch part {
case "cmd", "command", "meta", "super", "win", "cmdorctrl", "commandorcontrol":
return KeyModifier(keyCode: 55, flag: .maskCommand)
case "ctrl", "control":
return KeyModifier(keyCode: 59, flag: .maskControl)
case "alt", "option":
return KeyModifier(keyCode: 58, flag: .maskAlternate)
case "shift":
return KeyModifier(keyCode: 56, flag: .maskShift)
default:
return nil
}
}
private static let codes: [String: CGKeyCode] = [
"a": 0, "s": 1, "d": 2, "f": 3, "h": 4, "g": 5, "z": 6, "x": 7, "c": 8, "v": 9,
"b": 11, "q": 12, "w": 13, "e": 14, "r": 15, "y": 16, "t": 17, "1": 18, "2": 19,
native/computer-use-macos/Tests/OrcaComputerUseMacOSTests/AgentEntrypointSourceSafetyTests.swift
+28
View File
@@ -18,4 +18,32 @@ final class AgentEntrypointSourceSafetyTests: XCTestCase {
XCTAssertFalse(source.contains("unlink(tokenPath)"))
XCTAssertFalse(source.contains("unlink(socketPath)"))
}
func testSyntheticModifiersHaveGuaranteedReleaseAndModifiedClicksUseFlags() throws {
let source = try agentEntrypointSource()
XCTAssertTrue(source.contains("var pressedModifiers: [KeyModifier] = []"))
XCTAssertTrue(source.contains(
"""
defer {
for modifier in pressedModifiers.reversed() {
flags.remove(modifier.flag)
try? keyEvent(modifier.keyCode, down: false, flags: flags, pid: pid)
"""
))
XCTAssertTrue(source.contains("event.flags = flags\n event.postToPid(pid)"))
}
private func agentEntrypointSource() throws -> String {
let testFile = URL(fileURLWithPath: #filePath)
let packageRoot = testFile
.deletingLastPathComponent()
.deletingLastPathComponent()
.deletingLastPathComponent()
let mainPath = packageRoot
.appendingPathComponent("Sources")
.appendingPathComponent("OrcaComputerUseMacOS")
.appendingPathComponent("main.swift")
return try String(contentsOf: mainPath, encoding: .utf8)
}
}
native/computer-use-windows/runtime.ps1
+113 -7
View File
@@ -16,6 +16,7 @@ Add-Type -AssemblyName System.Windows.Forms
Add-Type -TypeDefinition @"
using System;
using System.Collections.Generic;
using System.Runtime.InteropServices;
public static class OrcaDesktopWin32 {
@@ -33,6 +34,39 @@ public static class OrcaDesktopWin32 {
public int Y;
}
[StructLayout(LayoutKind.Sequential)]
public struct INPUT {
public uint type;
public INPUTUNION data;
}
[StructLayout(LayoutKind.Explicit)]
public struct INPUTUNION {
[FieldOffset(0)]
public MOUSEINPUT mouse;
[FieldOffset(0)]
public KEYBDINPUT keyboard;
}
[StructLayout(LayoutKind.Sequential)]
public struct MOUSEINPUT {
public int dx;
public int dy;
public uint mouseData;
public uint flags;
public uint time;
public UIntPtr extraInfo;
}
[StructLayout(LayoutKind.Sequential)]
public struct KEYBDINPUT {
public ushort virtualKey;
public ushort scanCode;
public uint flags;
public uint time;
public UIntPtr extraInfo;
}
[DllImport("user32.dll")]
public static extern bool GetWindowRect(IntPtr hwnd, out RECT rect);
@@ -56,6 +90,54 @@ public static class OrcaDesktopWin32 {
[DllImport("user32.dll")]
public static extern void mouse_event(uint dwFlags, uint dx, uint dy, int dwData, UIntPtr dwExtraInfo);
[DllImport("user32.dll")]
public static extern uint SendInput(uint count, INPUT[] inputs, int size);
public static void SendModifiedClick(byte[] modifiers, uint mouseDown, uint mouseUp) {
const uint keyboardInput = 1;
const uint mouseInput = 0;
const uint keyUp = 0x0002;
var inputs = new List<INPUT>();
foreach (var modifier in modifiers) {
inputs.Add(KeyboardInput(keyboardInput, modifier, 0));
}
inputs.Add(MouseInput(mouseInput, mouseDown));
inputs.Add(MouseInput(mouseInput, mouseUp));
for (var index = modifiers.Length - 1; index >= 0; index--) {
inputs.Add(KeyboardInput(keyboardInput, modifiers[index], keyUp));
}
var values = inputs.ToArray();
var sent = SendInput((uint)values.Length, values, Marshal.SizeOf(typeof(INPUT)));
if (sent != (uint)values.Length) {
var releases = new List<INPUT>();
releases.Add(MouseInput(mouseInput, mouseUp));
for (var index = modifiers.Length - 1; index >= 0; index--) {
releases.Add(KeyboardInput(keyboardInput, modifiers[index], keyUp));
}
var releaseValues = releases.ToArray();
SendInput((uint)releaseValues.Length, releaseValues, Marshal.SizeOf(typeof(INPUT)));
throw new InvalidOperationException("SendInput did not complete the modified click");
}
}
private static INPUT KeyboardInput(uint type, byte virtualKey, uint flags) {
return new INPUT {
type = type,
data = new INPUTUNION {
keyboard = new KEYBDINPUT { virtualKey = virtualKey, flags = flags }
}
};
}
private static INPUT MouseInput(uint type, uint flags) {
return new INPUT {
type = type,
data = new INPUTUNION {
mouse = new MOUSEINPUT { flags = flags }
}
};
}
}
"@
@@ -912,7 +994,7 @@ function Get-OrcaElementScreenPoint($Element) {
$null
}
function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY, [string]$Button, [int]$Count) {
function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY, [string]$Button, [int]$Count, [string]$Modifiers) {
[void][OrcaDesktopWin32]::SetForegroundWindow($WindowHandle)
[void][OrcaDesktopWin32]::SetCursorPos($ScreenX, $ScreenY)
$buttonName = if ([string]::IsNullOrWhiteSpace($Button)) { "left" } else { $Button.ToLowerInvariant() }
@@ -923,10 +1005,23 @@ function Send-OrcaMouseClick([IntPtr]$WindowHandle, [int]$ScreenX, [int]$ScreenY
default { throw "unsupported mouse button: $Button" }
}
for ($i = 0; $i -lt (Get-OrcaPositiveInteger $Count "click_count"); $i++) {
[OrcaDesktopWin32]::mouse_event($down, 0, 0, 0, [UIntPtr]::Zero)
Start-Sleep -Milliseconds 35
[OrcaDesktopWin32]::mouse_event($up, 0, 0, 0, [UIntPtr]::Zero)
$modifierKeys = @(Get-OrcaClickModifierVirtualKeys $Modifiers)
$clickCount = Get-OrcaPositiveInteger $Count "click_count"
if ($modifierKeys.Count -eq 0) {
for ($i = 0; $i -lt $clickCount; $i++) {
[OrcaDesktopWin32]::mouse_event($down, 0, 0, 0, [UIntPtr]::Zero)
Start-Sleep -Milliseconds 35
[OrcaDesktopWin32]::mouse_event($up, 0, 0, 0, [UIntPtr]::Zero)
}
return
}
for ($i = 0; $i -lt $clickCount; $i++) {
[OrcaDesktopWin32]::SendModifiedClick(
[byte[]]$modifierKeys,
[uint32]$down,
[uint32]$up
)
if ($i + 1 -lt $clickCount) { Start-Sleep -Milliseconds 35 }
}
}
@@ -990,6 +1085,16 @@ function Get-OrcaModifierVirtualKey([string]$Modifier) {
}
}
function Get-OrcaClickModifierVirtualKeys([string]$Modifiers) {
if ([string]::IsNullOrWhiteSpace($Modifiers)) { return @() }
$parts = @($Modifiers.Split("+") | ForEach-Object { $_.Trim() })
$emptyParts = @($parts | Where-Object { [string]::IsNullOrWhiteSpace($_) })
if ($parts.Count -eq 0 -or $emptyParts.Count -gt 0) {
throw "Click modifiers require modifier keys only"
}
@($parts | ForEach-Object { Get-OrcaModifierVirtualKey $_ })
}
function Send-OrcaHotkey([IntPtr]$WindowHandle, [string]$KeySpec) {
$parts = @($KeySpec.Split("+") | ForEach-Object { $_.Trim() } | Where-Object { -not [string]::IsNullOrWhiteSpace($_) })
if ($parts.Count -eq 0) { throw "Unsupported key: $KeySpec" }
@@ -1106,13 +1211,14 @@ function Invoke-OrcaOperation($Operation) {
Restore-OrcaWindow $process
$handledByPattern = $false
$clickCount = Get-OrcaPositiveInteger $Operation.click_count "click_count"
if ($null -ne $element -and $Operation.mouse_button -ne "right" -and $Operation.mouse_button -ne "middle" -and $clickCount -le 1) {
$hasModifiers = -not [string]::IsNullOrWhiteSpace([string]$Operation.modifiers)
if (-not $hasModifiers -and $null -ne $element -and $Operation.mouse_button -ne "right" -and $Operation.mouse_button -ne "middle" -and $clickCount -le 1) {
$handledByPattern = Invoke-OrcaPrimaryAction $element
}
if (-not $handledByPattern) {
$point = Get-OrcaElementScreenPoint $element
if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }
Send-OrcaMouseClick $handle $point.x $point.y $Operation.mouse_button $clickCount
Send-OrcaMouseClick $handle $point.x $point.y $Operation.mouse_button $clickCount $Operation.modifiers
$action = [pscustomobject]@{ path = "synthetic"; actionName = $null; fallbackReason = "actionUnsupported" }
} else {
$action = [pscustomobject]@{ path = "accessibility"; actionName = "primaryAction"; fallbackReason = $null }
skill-guides/computer-use.md
+2
View File
@@ -69,6 +69,7 @@ ORCA computer get-app-state --app <app> --json
ORCA computer get-app-state --app <app> --restore-window --json
ORCA computer click --app <app> --element-index <index> --json
ORCA computer click --app <app> --x 100 --y 100 --json
ORCA computer click --app <app> --x 100 --y 100 --modifiers CmdOrCtrl+Shift --json
ORCA computer perform-secondary-action --app <app> --element-index <index> --action <name> --json
ORCA computer set-value --app <app> --element-index <index> --value "text" --json
ORCA computer type-text --app <app> --text "text" --json
@@ -95,6 +96,7 @@ printf '%s' "$TEXT" | ORCA computer set-value --app <app> --element-index <index
- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.
- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; synthetic keyboard delivery is reported as unverified, so inspect the returned state before assuming text landed.
- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.
- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.
- Some actions work in background apps, but this is app-dependent. If success does not change the UI, refresh state and choose a more semantic action or restore/focus the window.
- Prefer `set-value` for text fields that expose values; it can report verified value writes when the provider can read the refreshed value.
- Coordinates are window-local; use coordinates from the latest screenshot/state for the same target window.
src/cli/bundled-skill-guides.ts
+1 -1
View File
File diff suppressed because one or more lines are too long
src/cli/handlers/computer-action-flags.ts
+12 -1
View File
@@ -1,4 +1,5 @@
import {
computerUseClickModifiersValidationMessage,
computerUseHotkeyValidationMessage,
computerUsePressKeyValidationMessage
} from '../../shared/computer-use-key-spec'
@@ -60,16 +61,26 @@ export function getComputerClickActionFlags(flags: Map<string, string | boolean>
y?: number
clickCount?: number
mouseButton?: string
modifiers?: string
} {
const rawModifiers = flags.get('modifiers')
const modifiers = typeof rawModifiers === 'string' ? rawModifiers : undefined
const result = {
elementIndex: getOptionalNonNegativeIntegerFlag(flags, 'element-index'),
x: getOptionalNumberFlag(flags, 'x'),
y: getOptionalNumberFlag(flags, 'y'),
clickCount: getOptionalPositiveIntegerFlag(flags, 'click-count'),
mouseButton: getOptionalStringFlag(flags, 'mouse-button')
mouseButton: getOptionalStringFlag(flags, 'mouse-button'),
modifiers
}
validateElementOrCoordinates('Click', result.elementIndex, result.x, result.y)
validateMouseButton(result.mouseButton)
if (modifiers !== undefined) {
const message = computerUseClickModifiersValidationMessage(modifiers)
if (message) {
throw new RuntimeClientError('invalid_argument', message)
}
}
return result
}
src/cli/handlers/computer-action-routing.test.ts
+3
View File
@@ -59,6 +59,8 @@ describe('orca computer action CLI routing', () => {
'Finder',
'--element-index',
'3',
'--modifiers',
'CmdOrCtrl+Shift',
'--json'
],
'/tmp/repo/src'
@@ -73,6 +75,7 @@ describe('orca computer action CLI routing', () => {
y: undefined,
clickCount: undefined,
mouseButton: undefined,
modifiers: 'CmdOrCtrl+Shift',
noScreenshot: undefined
})
})
src/cli/handlers/computer-action-validation.test.ts
+37
View File
@@ -289,6 +289,43 @@ describe('orca computer action CLI validation', () => {
vi.mocked(console.error).mockClear()
process.exitCode = undefined
await main(
[
'computer',
'click',
'--app',
'Finder',
'--element-index',
'1',
'--modifiers',
'CmdOrCtrl+A'
],
'/tmp/repo/src'
)
expect(callMock).not.toHaveBeenCalled()
expect(vi.mocked(console.error).mock.calls[0][0]).toContain(
'Click modifiers accept modifier keys only'
)
expect(process.exitCode).toBe(1)
vi.mocked(console.error).mockClear()
process.exitCode = undefined
await main(
['computer', 'click', '--app', 'Finder', '--element-index', '1', '--modifiers', ''],
'/tmp/repo/src'
)
expect(callMock).not.toHaveBeenCalled()
expect(vi.mocked(console.error).mock.calls[0][0]).toContain(
'Click modifiers accept modifier keys only'
)
expect(process.exitCode).toBe(1)
vi.mocked(console.error).mockClear()
process.exitCode = undefined
await main(
['computer', 'scroll', '--app', 'Finder', '--element-index', '1', '--direction', 'diagonal'],
'/tmp/repo/src'
src/cli/handlers/computer.test.ts
+8
View File
@@ -72,6 +72,14 @@ describe('orca computer observation CLI handlers', () => {
expect(pressKeyOutput).toContain('Single key, e.g. Return, Escape, Tab, Left, or PageUp')
})
it('documents atomic modified clicks', async () => {
await main(['computer', 'click', '--help'], '/tmp/repo')
const output = vi.mocked(console.log).mock.calls[0][0]
expect(output).toContain('--modifiers <chord>')
expect(output).toContain('Modifier keys held only for this click')
})
it('passes list-apps through without resolving a worktree', async () => {
queueFixtures(callMock, okFixture('req_apps', { apps: [] }))
src/cli/help.ts
+1
View File
@@ -523,6 +523,7 @@ export function formatFlagHelp(flag: string): string {
limit: '--limit <n> Maximum number of rows to return',
mode: '--mode <mode> Mode such as edit, diff, or both',
'mouse-button': '--mouse-button <btn> Mouse button: left, right, or middle',
modifiers: '--modifiers <chord> Modifier keys held only for this click',
name: '--name <name> Name for the new worktree or automation',
'no-parent': '--no-parent Force no parent lineage for unrelated work',
'no-screenshot': '--no-screenshot Skip screenshot capture after the operation',
src/cli/specs/computer.ts
+4 -3
View File
@@ -44,16 +44,17 @@ export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [
},
{
path: ['computer', 'click'],
summary: 'Click an app element or window coordinate',
summary: 'Click an app element or window coordinate, optionally with modifiers',
usage:
'orca computer click --app <app> (--element-index <n> | --x <x> --y <y>) [--window-id <id> | --window-index <n>] [--click-count <n>] [--mouse-button <left|right|middle>] [--restore-window] [--no-screenshot] [--json]',
'orca computer click --app <app> (--element-index <n> | --x <x> --y <y>) [--window-id <id> | --window-index <n>] [--click-count <n>] [--mouse-button <left|right|middle>] [--modifiers <modifier-chord>] [--restore-window] [--no-screenshot] [--json]',
allowedFlags: [
...COMPUTER_ACTION_FLAGS,
'element-index',
'x',
'y',
'click-count',
'mouse-button'
'mouse-button',
'modifiers'
]
},
{
src/main/computer/computer-provider-action-validation.ts
+13
View File
@@ -1,4 +1,5 @@
import {
computerUseClickModifiersValidationMessage,
computerUseHotkeyValidationMessage,
computerUsePressKeyValidationMessage
} from '../../shared/computer-use-key-spec'
@@ -27,6 +28,7 @@ export async function validateComputerProviderActionParams(
validateElementOrCoordinates('Click', params)
validatePositiveInteger(params, 'clickCount')
validateMouseButton(params)
validateClickModifiers(params)
return app
case 'performSecondaryAction':
requireNonNegativeInteger(params, 'elementIndex')
@@ -159,6 +161,17 @@ function validateMouseButton(params: Record<string, unknown>): void {
}
}
function validateClickModifiers(params: Record<string, unknown>): void {
if (params.modifiers === undefined) {
return
}
const modifiers = requireNonEmptyString(params, 'modifiers')
const message = computerUseClickModifiersValidationMessage(modifiers)
if (message) {
throw new RuntimeClientError('invalid_argument', message)
}
}
function validateScrollDirection(params: Record<string, unknown>): void {
const direction = requireNonEmptyString(params, 'direction')
if (direction !== 'up' && direction !== 'down' && direction !== 'left' && direction !== 'right') {
src/main/computer/desktop-script-provider-actions.test.ts
+13 -4
View File
@@ -186,10 +186,18 @@ describe('DesktopScriptProviderClient actions', () => {
ok: true,
capabilities: sampleCapabilities()
})
mockBridgeResponse({
ok: true,
snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
})
mockBridgeResponse(
{
ok: true,
snapshot: sampleBridgeSnapshot('Text Editor', 'initial')
},
(operation) => {
expect(operation).toMatchObject({
tool: 'click',
modifiers: 'CmdOrCtrl+Shift'
})
}
)
const client = await createDesktopScriptProviderClient('linux', '/tmp/runtime.py')
@@ -197,6 +205,7 @@ describe('DesktopScriptProviderClient actions', () => {
const result = await client.action('click', {
app: 'Text Editor',
elementIndex: 0,
modifiers: 'CmdOrCtrl+Shift',
noScreenshot: true
})
src/main/computer/desktop-script-provider-bridge.ts
+1 -1
View File
@@ -112,7 +112,7 @@ export function mapBridgeError(message: string): RuntimeClientError {
return new RuntimeClientError('app_blocked', text)
}
if (
/unsupported capability|hotkey.*require|paste_text requires|GDK is required for non-character key synthesis/i.test(
/unsupported capability|hotkey.*require|paste_text requires|modified clicks require xdotool|GDK is required for non-character key synthesis/i.test(
text
)
) {
src/main/computer/desktop-script-provider-client.ts
+1
View File
@@ -152,6 +152,7 @@ export class DesktopScriptProviderClient {
to_y: optionalNumberParam(params, 'toY'),
click_count: optionalNumberParam(params, 'clickCount'),
mouse_button: optionalStringParam(params, 'mouseButton'),
modifiers: optionalStringParam(params, 'modifiers'),
action: optionalStringParam(params, 'action'),
direction: optionalStringParam(params, 'direction'),
pages: optionalNumberParam(params, 'pages'),
src/main/computer/desktop-script-provider-types.ts
+1
View File
@@ -134,6 +134,7 @@ export type BridgeRequest = {
to_y?: number
click_count?: number
mouse_button?: string
modifiers?: string
action?: string
direction?: string
pages?: number
src/main/computer/macos-native-provider-client.test.ts
+10
View File
@@ -253,6 +253,16 @@ describe('MacOSNativeProviderClient', () => {
code: 'invalid_argument',
message: expect.stringContaining('Click requires')
})
await expect(
client.action('click', {
app: 'TextEdit',
elementIndex: 0,
modifiers: 'CmdOrCtrl+A'
})
).rejects.toMatchObject({
code: 'invalid_argument',
message: expect.stringContaining('Click modifiers accept modifier keys only')
})
await expect(client.action('typeText', { app: 'TextEdit', text: '' })).rejects.toMatchObject({
code: 'invalid_argument',
message: expect.stringContaining('Missing text')
src/main/runtime/rpc/methods/computer-actions.test.ts
+33
View File
@@ -101,6 +101,37 @@ describe('computer action RPC methods', () => {
).toThrow()
})
it('accepts modifier-only click chords and rejects embedded keys', () => {
expect(
findMethod('computer.click').params!.parse({
app: 'Finder',
elementIndex: 0,
modifiers: 'CmdOrCtrl+Shift'
})
).toMatchObject({ modifiers: 'CmdOrCtrl+Shift' })
expect(() =>
findMethod('computer.click').params!.parse({
app: 'Finder',
elementIndex: 0,
modifiers: 'CmdOrCtrl+A'
})
).toThrow(/Click modifiers accept modifier keys only/)
expect(() =>
findMethod('computer.click').params!.parse({
app: 'Finder',
elementIndex: 0,
modifiers: ''
})
).toThrow(/Click modifiers accept modifier keys only/)
expect(() =>
findMethod('computer.click').params!.parse({
app: 'Finder',
elementIndex: 0,
modifiers: true
})
).toThrow()
})
it('rejects modifier chords on press-key but allows literal plus', () => {
expect(() =>
findMethod('computer.pressKey').params!.parse({
@@ -126,6 +157,7 @@ describe('computer action RPC methods', () => {
elementIndex: 0,
clickCount: 2,
mouseButton: 'left',
modifiers: 'CmdOrCtrl+Shift',
noScreenshot: true
})
await call('computer.performSecondaryAction', {
@@ -147,6 +179,7 @@ describe('computer action RPC methods', () => {
elementIndex: 0,
clickCount: 2,
mouseButton: 'left',
modifiers: 'CmdOrCtrl+Shift',
noScreenshot: true
})
expect(computerMocks.callComputerSidecarAction).toHaveBeenNthCalledWith(
src/main/runtime/rpc/methods/computer-schemas.ts
+9 -1
View File
@@ -1,5 +1,6 @@
import { z } from 'zod'
import {
computerUseClickModifiersValidationMessage,
computerUseHotkeyValidationMessage,
computerUsePressKeyValidationMessage
} from '../../../../shared/computer-use-key-spec'
@@ -67,7 +68,8 @@ export const Click = ComputerObserveTargetBase.extend({
x: OptionalFiniteNumber,
y: OptionalFiniteNumber,
clickCount: OptionalPositiveInt,
mouseButton: z.enum(['left', 'right', 'middle']).optional()
mouseButton: z.enum(['left', 'right', 'middle']).optional(),
modifiers: z.string().optional()
}).superRefine((value, ctx) => {
validateComputerTarget(value, ctx)
const hasElement = value.elementIndex !== undefined
@@ -91,6 +93,12 @@ export const Click = ComputerObserveTargetBase.extend({
message: 'Click accepts either --element-index or coordinate flags, not both'
})
}
if (value.modifiers !== undefined) {
const message = computerUseClickModifiersValidationMessage(value.modifiers)
if (message) {
ctx.addIssue({ code: 'custom', message })
}
}
})
export const PerformSecondaryAction = ComputerObserveTargetBase.extend({
src/shared/computer-use-key-spec.test.ts
+19
View File
@@ -1,6 +1,7 @@
import { describe, expect, it } from 'vitest'
import {
computerUseClickModifiersValidationMessage,
computerUseHotkeyValidationMessage,
computerUsePressKeyValidationMessage
} from './computer-use-key-spec'
@@ -34,4 +35,22 @@ describe('computer-use key specs', () => {
expect(computerUsePressKeyValidationMessage('Ctrl+Shift+P')).toEqual(expected)
expect(computerUsePressKeyValidationMessage('')).toEqual(expected)
})
it('accepts modifier-only click chords', () => {
expect(computerUseClickModifiersValidationMessage('CmdOrCtrl')).toBeNull()
expect(computerUseClickModifiersValidationMessage('CmdOrCtrl+Shift')).toBeNull()
expect(computerUseClickModifiersValidationMessage('Control + Alt')).toBeNull()
})
it('rejects empty, key-bearing, and malformed click modifier chords', () => {
const expected = expect.stringContaining('Click modifiers accept modifier keys only')
expect(computerUseClickModifiersValidationMessage('')).toEqual(expected)
expect(computerUseClickModifiersValidationMessage('CmdOrCtrl+A')).toEqual(expected)
expect(computerUseClickModifiersValidationMessage('Cmd-Or-Ctrl')).toEqual(expected)
expect(computerUseClickModifiersValidationMessage('Ctrl++Shift')).toEqual(expected)
expect(computerUseClickModifiersValidationMessage('Ctrl+Shift+Alt+Meta+Super')).toEqual(
expected
)
})
})
src/shared/computer-use-key-spec.ts
+14
View File
@@ -17,6 +17,8 @@ const HOTKEY_HINT =
'Hotkey requires a modifier and one key, e.g. CmdOrCtrl+A. Use press-key for a single key.'
const PRESS_KEY_HINT =
'Press-key accepts one key only, e.g. Return, Escape, Tab, or +. Use hotkey for modifier combinations.'
const CLICK_MODIFIERS_HINT =
'Click modifiers accept modifier keys only, e.g. CmdOrCtrl or CmdOrCtrl+Shift.'
export function computerUseHotkeyValidationMessage(key: string): string | null {
const parts = key.split('+').map((part) => part.trim())
@@ -49,6 +51,18 @@ export function computerUsePressKeyValidationMessage(key: string): string | null
return null
}
export function computerUseClickModifiersValidationMessage(modifiers: string): string | null {
const parts = modifiers.split('+').map((part) => part.trim())
if (
parts.length === 0 ||
parts.length > 4 ||
parts.some((part) => part.length === 0 || !HOTKEY_MODIFIERS.has(part.toLowerCase()))
) {
return CLICK_MODIFIERS_HINT
}
return null
}
function normalizeHotkeyPart(part: string): string {
return part.toLowerCase().replace(/[\s_-]/g, '')
}