mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
feat(computer): type into a named text field without focusing it on Linux and macOS
This commit is contained in:
@@ -62,11 +62,11 @@ Tasks cover explicit text targeting with another field logically focused, Unicod
|
||||
entry plus submit, select-all replacement plus submit, and color choice plus one
|
||||
click. Interference probes remain separate from task latency comparisons.
|
||||
|
||||
Explicit-target typing is a native capability probe. Orca's public CLI/RPC currently
|
||||
accepts typing at the focused field and does not expose an element target for text.
|
||||
Failure of this probe identifies a capability to consider adding; it does not prove
|
||||
a violation of the current public typing contract. Click-then-type tasks exercise
|
||||
the existing sequence, although this benchmark still bypasses public transports.
|
||||
Explicit-target typing is a native capability probe. Orca's public CLI/RPC accepts
|
||||
an element target for text (`type-text --element-index`) on providers that report
|
||||
`actions.typeTextElement` (macOS and Linux); Windows types only at the focused
|
||||
field. Click-then-type tasks exercise the existing sequence, although this
|
||||
benchmark still bypasses public transports.
|
||||
|
||||
The Linux Electron job also records restoration stages in `focus-probe/results.json`.
|
||||
It records the AT-SPI focus acknowledgment, selected ACTIVE state, manual legacy PID
|
||||
|
||||
@@ -77,6 +77,7 @@ App names work when unambiguous (`--app Spotify`). Use `--app pid:<number>` only
|
||||
orca computer click --app <app> --element-index <i> --json
|
||||
orca computer set-value --app <app> --element-index <i> --value "text" --json
|
||||
orca computer type-text --app <app> --text "text" --json
|
||||
orca computer type-text --app <app> --element-index <i> --text "text" --json
|
||||
orca computer press-key --app <app> --key Return --json
|
||||
orca computer hotkey --app <app> --key CmdOrCtrl+A --json
|
||||
orca computer paste-text --app <app> --text "text" --json
|
||||
@@ -89,6 +90,8 @@ Prefer semantic actions (`click`, `set-value`, `perform-secondary-action`) over
|
||||
|
||||
`hotkey` and `press-key` refuse shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session, such as `Cmd+Q`, `Shift+Cmd+Q`, `Ctrl+Cmd+Q` and `Option+Cmd+Esc` on macOS, `Alt+F4` and `Win+L` on Windows, and `Ctrl+Q`, `Alt+F4`, `Super+L` and `Ctrl+Alt+Delete` on Linux. The error code is `system_shortcut_blocked`. Pass `--allow-system-shortcut` to send one anyway.
|
||||
|
||||
`type-text --element-index <i>` inserts the text at that field's caret without focusing it or bringing its window forward. It works on macOS and Linux; on Windows, click the field and run `type-text` without the index.
|
||||
|
||||
On macOS, actions run in the background: the target app isn't brought forward, your frontmost app and key window stay put, and the pointer doesn't move, so you can keep working while an agent drives another window. Each action result reports `action.delivery` (`background` or `foreground`). Pass `--restore-window` only when an app ignores background input, such as drags in Chromium or Electron apps or games; it brings the app forward and uses the real pointer and keyboard. Popup menus and web `<select>` controls are set with `set-value` instead of a click. On Linux, clicks, scrolling, drags and keyboard input still bring the window forward. On Windows, element clicks, `set-value` (including sliders), element scrolls and secondary actions (`Invoke`, `Toggle`, `Select`, `Expand`, `Collapse`, `ScrollToVisible`) go through UI Automation when the control supports them, without moving the pointer or sending keys. Some apps still bring their own window forward when driven this way (WPF apps do for most of these), so `action.delivery` reports `foreground` whenever the target window came forward. Anything else brings the window forward.
|
||||
|
||||
When accessibility targeting fails, fall back carefully to coordinates:
|
||||
|
||||
@@ -1292,6 +1292,7 @@ def handshake_response():
|
||||
"actions": {
|
||||
"click": True,
|
||||
"typeText": True,
|
||||
"typeTextElement": True,
|
||||
"pressKey": True,
|
||||
"hotkey": has_hotkey,
|
||||
"pasteText": has_clipboard,
|
||||
@@ -1752,8 +1753,12 @@ def text_selection(text):
|
||||
|
||||
|
||||
def insert_focused_text(window, value):
|
||||
# Why: EditableText needs no window focus and still works on Wayland, where key synthesis is a no-op.
|
||||
node = first_descendant(window, lambda candidate: has_state(candidate, Atspi.StateType.FOCUSED))
|
||||
return insert_text(node, value, "focusedText")
|
||||
|
||||
|
||||
def insert_text(node, value, verified_property):
|
||||
# Why: EditableText needs no window focus and still works on Wayland, where key synthesis is a no-op.
|
||||
if node is None or not has_state(node, Atspi.StateType.EDITABLE):
|
||||
return None
|
||||
editable = attempt(node.get_editable_text_iface)
|
||||
@@ -1787,7 +1792,18 @@ def insert_focused_text(window, value):
|
||||
raise RuntimeError("text_caret_unconfirmed: text was inserted but caret movement did not confirm success; inspect state before retrying")
|
||||
action = {"path": "accessibility", "actionName": "insertText", "fallbackReason": None}
|
||||
if attempt(lambda: Atspi.Text.get_text(text, start, caret)) == value:
|
||||
action["verification"] = {"state": "verified", "property": "focusedText"}
|
||||
action["verification"] = {"state": "verified", "property": verified_property}
|
||||
return action
|
||||
|
||||
|
||||
def insert_element_text(node, saved, value):
|
||||
index = saved.get("index")
|
||||
if node is None:
|
||||
raise RuntimeError(f"element {index} changed since the last snapshot; run get-app-state again and use a fresh element index")
|
||||
action = insert_text(node, value, "elementText")
|
||||
# Why: typing at focus instead would put the text in whatever field has focus, not the requested one.
|
||||
if action is None:
|
||||
raise RuntimeError(f"element {index} text is not settable; click it, then run type-text without --element-index")
|
||||
return action
|
||||
|
||||
|
||||
@@ -1950,7 +1966,10 @@ def run_operation(operation):
|
||||
action = {"path": "synthetic", "actionName": "drag", "fallbackReason": None}
|
||||
elif tool == "type_text":
|
||||
text = require_non_empty_string(operation.get("text"), "text")
|
||||
action = insert_focused_text(window, text)
|
||||
if saved:
|
||||
action = insert_element_text(node, saved, text)
|
||||
else:
|
||||
action = insert_focused_text(window, text)
|
||||
if action is None:
|
||||
require_keyboard_focus(window, operation)
|
||||
type_text(text, window)
|
||||
|
||||
@@ -596,6 +596,7 @@ final class Provider {
|
||||
"actions": [
|
||||
"click": true,
|
||||
"typeText": true,
|
||||
"typeTextElement": true,
|
||||
"pressKey": true,
|
||||
"hotkey": true,
|
||||
"pasteText": true,
|
||||
@@ -1047,6 +1048,9 @@ final class Provider {
|
||||
private func typeText(params: [String: JSONValue]) throws -> [String: Any] {
|
||||
let snapshot = try currentKeyboardSnapshot(params: params)
|
||||
let text = try requiredString(params, "text")
|
||||
if let elementIndex = try optionalInteger(params, "elementIndex") {
|
||||
return try typeTextIntoElement(try element(snapshot, elementIndex), text: text)
|
||||
}
|
||||
let focused = focusedRecord(snapshot)?.element
|
||||
if let focused, let verification = try TextInput.replaceSelection(focused, with: text) {
|
||||
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
|
||||
@@ -1060,6 +1064,22 @@ final class Provider {
|
||||
)
|
||||
}
|
||||
|
||||
private func typeTextIntoElement(_ record: ElementRecord, text: String) throws -> [String: Any] {
|
||||
// Why: typing at focus instead would put the text in whatever field has focus, not the requested one.
|
||||
if BackgroundInputTransport.isChromiumNode(record.element) {
|
||||
throw ProviderError.coded("value_not_settable", "element \(record.index) is web content, which ignores accessibility text insertion; click it, then run type-text without --element-index")
|
||||
}
|
||||
guard isSettable(record.element, kAXValueAttribute as String),
|
||||
rawStringAttribute(record.element, kAXValueAttribute as String) != nil
|
||||
else {
|
||||
throw ProviderError.coded("value_not_settable", "element \(record.index) text is not settable; click it, then run type-text without --element-index")
|
||||
}
|
||||
guard let verification = try TextInput.replaceSelection(record.element, with: text, property: "elementText") else {
|
||||
throw ProviderError.coded("accessibility_error", "text insertion into element \(record.index) did not confirm; run get-app-state to check the field before retrying")
|
||||
}
|
||||
return actionMetadata(path: "accessibility", actionName: "AXReplaceSelection", verification: verification)
|
||||
}
|
||||
|
||||
private func pressKey(params: [String: JSONValue]) throws -> [String: Any] {
|
||||
let snapshot = try currentKeyboardSnapshot(params: params)
|
||||
try deliverKeys(try Input.keyEvents(try requiredString(params, "key")), snapshot: snapshot, params: params)
|
||||
@@ -3263,7 +3283,7 @@ private enum Input {
|
||||
}
|
||||
|
||||
private enum TextInput {
|
||||
static func replaceSelection(_ element: AXUIElement, with text: String) throws -> [String: Any]? {
|
||||
static func replaceSelection(_ element: AXUIElement, with text: String, property: String = "focusedText") throws -> [String: Any]? {
|
||||
// Why: Chromium applies AX value writes without the input events pages listen to; a contenteditable gets none, so editors like Discord's never see the text.
|
||||
guard !BackgroundInputTransport.isChromiumNode(element),
|
||||
isSettable(element, kAXValueAttribute as String),
|
||||
@@ -3297,7 +3317,7 @@ private enum TextInput {
|
||||
guard readback == next else {
|
||||
return unverifiedAction(reason: "accessibility_action_unasserted")
|
||||
}
|
||||
return verifiedAction(property: "focusedText")
|
||||
return verifiedAction(property: property)
|
||||
}
|
||||
|
||||
/// Text sent as keys or a paste: what the focused field should read once it lands.
|
||||
|
||||
@@ -1171,6 +1171,7 @@ function Get-OrcaHandshake {
|
||||
actions = [pscustomobject]@{
|
||||
click = $true
|
||||
typeText = $true
|
||||
typeTextElement = $false
|
||||
pressKey = $true
|
||||
hotkey = $true
|
||||
pasteText = $true
|
||||
|
||||
@@ -98,6 +98,7 @@ ORCA computer click --app <app> --element-index <index> --mouse-button middle --
|
||||
ORCA computer perform-secondary-action --app <app> --element-index <index> --action <name> --json
|
||||
ORCA computer set-value --app <app> --element-index <index> --value "text" --json
|
||||
ORCA computer type-text --app <app> --text "text" --json
|
||||
ORCA computer type-text --app <app> --element-index <index> --text "text" --json
|
||||
ORCA computer press-key --app <app> --key Return --json
|
||||
ORCA computer hotkey --app <app> --key CmdOrCtrl+A --json
|
||||
ORCA computer paste-text --app <app> --text "text" --json
|
||||
@@ -128,6 +129,7 @@ printf '%s' "$TEXT" | ORCA computer set-value --app <app> --element-index <index
|
||||
- Prefer semantic actions: `set-value` for editable fields, `click` for controls, and `perform-secondary-action` only for listed action names.
|
||||
- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.
|
||||
- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; unless the result says `verified focusedText`, inspect the returned state before assuming text landed.
|
||||
- `type-text --element-index <index>` inserts at that text field's caret, replacing its selection, without focusing it or bringing its window forward, and reads the text back. It works on macOS and Linux; where `capabilities` lacks `actions.typeTextElement` it fails with `unsupported_capability`, so click the field and type without the index. A `value_not_settable` error means that element does not accept text insertion.
|
||||
- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.
|
||||
- Shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session (such as `Cmd+Q` on macOS, `Alt+F4` on Windows and Linux, `Ctrl+Q` on Linux) are refused with `system_shortcut_blocked`. The error says what the shortcut would do; reach that goal another way, such as the window's own close button or menu item. Add `--allow-system-shortcut` only when the user asked for that exact shortcut.
|
||||
- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.
|
||||
|
||||
@@ -143,6 +143,15 @@ export async function getComputerTextActionFlags(flags: Map<string, string | boo
|
||||
return { text: await getTextPayload(flags, 'text') }
|
||||
}
|
||||
|
||||
export async function getComputerTypeTextActionFlags(
|
||||
flags: Map<string, string | boolean>
|
||||
): Promise<{ elementIndex?: number; text: string }> {
|
||||
return {
|
||||
elementIndex: getOptionalNonNegativeIntegerFlag(flags, 'element-index'),
|
||||
...(await getComputerTextActionFlags(flags))
|
||||
}
|
||||
}
|
||||
|
||||
export function getComputerKeyActionFlags(flags: Map<string, string | boolean>): {
|
||||
key: string
|
||||
allowSystemShortcut?: boolean
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const callMock = vi.fn()
|
||||
|
||||
vi.mock('../runtime/client', () => {
|
||||
class RuntimeClient {
|
||||
call = callMock
|
||||
getCliStatus = vi.fn()
|
||||
openOrca = vi.fn()
|
||||
}
|
||||
return { RuntimeClient }
|
||||
})
|
||||
|
||||
import { main } from '../index'
|
||||
import { okFixture, queueFixtures } from '../runtime/fixtures/runtime-rpc-fixtures'
|
||||
|
||||
describe('orca computer type-text element target', () => {
|
||||
beforeEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
callMock.mockReset()
|
||||
vi.spyOn(console, 'log').mockImplementation(() => {})
|
||||
vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
process.exitCode = undefined
|
||||
})
|
||||
|
||||
it('refuses element-targeted type-text on a runtime that does not report it', async () => {
|
||||
queueFixtures(
|
||||
callMock,
|
||||
okFixture('req_capabilities', {
|
||||
platform: 'win32',
|
||||
provider: 'orca-computer-use-windows',
|
||||
providerVersion: '1.0.0',
|
||||
protocolVersion: 1,
|
||||
supports: { actions: { typeText: true, setValue: true } }
|
||||
})
|
||||
)
|
||||
|
||||
await main(
|
||||
[
|
||||
'computer',
|
||||
'type-text',
|
||||
'--session',
|
||||
'manual',
|
||||
'--app',
|
||||
'Notepad',
|
||||
'--element-index',
|
||||
'3',
|
||||
'--text',
|
||||
'hello',
|
||||
'--json'
|
||||
],
|
||||
'/tmp/repo/src'
|
||||
)
|
||||
|
||||
expect(callMock).toHaveBeenCalledTimes(1)
|
||||
expect(callMock).toHaveBeenCalledWith('computer.capabilities', {})
|
||||
const output = JSON.parse(vi.mocked(console.log).mock.calls[0][0])
|
||||
expect(output.error).toMatchObject({ code: 'unsupported_capability' })
|
||||
expect(process.exitCode).toBe(1)
|
||||
})
|
||||
})
|
||||
@@ -33,8 +33,10 @@ import {
|
||||
getComputerScrollActionFlags,
|
||||
getComputerSecondaryActionFlags,
|
||||
getComputerSetValueActionFlags,
|
||||
getComputerTextActionFlags
|
||||
getComputerTextActionFlags,
|
||||
getComputerTypeTextActionFlags
|
||||
} from './computer-action-flags'
|
||||
import { computerTypeTextTargetUnsupportedMessage } from '../../shared/computer-use-type-text-target'
|
||||
|
||||
export const COMPUTER_HANDLERS: Record<string, CommandHandler> = {
|
||||
'computer capabilities': async ({ client, json }) => {
|
||||
@@ -169,8 +171,19 @@ export const COMPUTER_HANDLERS: Record<string, CommandHandler> = {
|
||||
'computer type-text': async ({ flags, client, cwd, json }) => {
|
||||
assertComputerAppFlag(flags)
|
||||
const observeFlags = getComputerActionObserveFlags(flags)
|
||||
const actionParams = await getComputerTextActionFlags(flags)
|
||||
const actionParams = await getComputerTypeTextActionFlags(flags)
|
||||
const target = await getComputerCommandTarget(flags, cwd, client)
|
||||
if (actionParams.elementIndex !== undefined) {
|
||||
// Why: older runtimes strip elementIndex, so ask first rather than type into the focused field.
|
||||
const capabilities = await client.call<ComputerProviderCapabilities>(
|
||||
'computer.capabilities',
|
||||
{}
|
||||
)
|
||||
const unsupported = computerTypeTextTargetUnsupportedMessage(capabilities.result)
|
||||
if (unsupported) {
|
||||
throw new RuntimeClientError('unsupported_capability', unsupported)
|
||||
}
|
||||
}
|
||||
const result = await client.call<ComputerActionResult>('computer.typeText', {
|
||||
...target,
|
||||
...actionParams,
|
||||
|
||||
@@ -191,7 +191,7 @@ export const ROOT_HELP_TEXT_PRIMARY = [
|
||||
' computer perform-secondary-action Run an advertised accessibility action',
|
||||
' computer scroll Scroll an app element',
|
||||
' computer drag Drag between app elements or window coordinates',
|
||||
' computer type-text Type literal text at the current app focus',
|
||||
' computer type-text Type literal text at the current app focus or into an element',
|
||||
' computer press-key Press a single key such as Return or Escape',
|
||||
' computer hotkey Press a shortcut combination such as CmdOrCtrl+A',
|
||||
' computer paste-text Paste text through the native clipboard path',
|
||||
|
||||
@@ -1117,6 +1117,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
|
||||
"ORCA computer perform-secondary-action --app <app> --element-index <index> --action <name> --json",
|
||||
"ORCA computer set-value --app <app> --element-index <index> --value \"text\" --json",
|
||||
"ORCA computer type-text --app <app> --text \"text\" --json",
|
||||
"ORCA computer type-text --app <app> --element-index <index> --text \"text\" --json",
|
||||
"ORCA computer press-key --app <app> --key Return --json",
|
||||
"ORCA computer hotkey --app <app> --key CmdOrCtrl+A --json",
|
||||
"ORCA computer paste-text --app <app> --text \"text\" --json",
|
||||
@@ -1147,6 +1148,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
|
||||
"- Prefer semantic actions: `set-value` for editable fields, `click` for controls, and `perform-secondary-action` only for listed action names.",
|
||||
"- After any UI-changing action, use the returned state or rerun `get-app-state` before choosing the next element index.",
|
||||
"- Use `type-text` only after focusing a field and confirming the app has a focused text receiver; unless the result says `verified focusedText`, inspect the returned state before assuming text landed.",
|
||||
"- `type-text --element-index <index>` inserts at that text field's caret, replacing its selection, without focusing it or bringing its window forward, and reads the text back. It works on macOS and Linux; where `capabilities` lacks `actions.typeTextElement` it fails with `unsupported_capability`, so click the field and type without the index. A `value_not_settable` error means that element does not accept text insertion.",
|
||||
"- Use `press-key` for single/navigation keys such as Return, Escape, Tab, and arrows. Use `hotkey` only for one modifier chord plus one key, such as `CmdOrCtrl+A` or `CmdOrCtrl+Shift+P`; prefer `CmdOrCtrl+...` for cross-platform combos.",
|
||||
"- Shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session (such as `Cmd+Q` on macOS, `Alt+F4` on Windows and Linux, `Ctrl+Q` on Linux) are refused with `system_shortcut_blocked`. The error says what the shortcut would do; reach that goal another way, such as the window's own close button or menu item. Add `--allow-system-shortcut` only when the user asked for that exact shortcut.",
|
||||
"- Use `click --modifiers <chord>` for modifier-clicks. Never synthesize separate modifier-down and modifier-up commands around a click; interruption can leave a modifier logically held.",
|
||||
|
||||
@@ -112,10 +112,10 @@ export const COMPUTER_COMMAND_SPECS: CommandSpec[] = [
|
||||
},
|
||||
{
|
||||
path: ['computer', 'type-text'],
|
||||
summary: 'Type literal text at the current app focus',
|
||||
summary: 'Type literal text at the current app focus or into an element',
|
||||
usage:
|
||||
'orca computer type-text --app <app> (--text <text> | --text-stdin) [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
|
||||
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'text', 'text-stdin']
|
||||
'orca computer type-text --app <app> (--text <text> | --text-stdin) [--element-index <n>] [--window-id <id> | --window-index <n>] [--restore-window] [--no-screenshot] [--full-tree] [--json]',
|
||||
allowedFlags: [...COMPUTER_ACTION_FLAGS, 'element-index', 'text', 'text-stdin']
|
||||
},
|
||||
{
|
||||
path: ['computer', 'press-key'],
|
||||
|
||||
@@ -45,6 +45,9 @@ export async function validateComputerProviderActionParams(
|
||||
validateDragTarget(params)
|
||||
return app
|
||||
case 'typeText':
|
||||
if (params.elementIndex !== undefined) {
|
||||
requireNonNegativeInteger(params, 'elementIndex')
|
||||
}
|
||||
requireNonEmptyString(params, 'text')
|
||||
return app
|
||||
case 'pasteText':
|
||||
|
||||
@@ -3,6 +3,7 @@ import { currentComputerProvider, shutdownComputerProviders } from './computer-p
|
||||
import { RuntimeClientError } from './runtime-client-error'
|
||||
import { ComputerTreeBaselines } from './computer-snapshot-tree-diff'
|
||||
import type { ComputerProviderCapabilities } from '../../shared/computer-use-runtime-types'
|
||||
import { computerTypeTextTargetUnsupportedMessage } from '../../shared/computer-use-type-text-target'
|
||||
|
||||
type SidecarRequest = {
|
||||
id: number
|
||||
@@ -77,6 +78,12 @@ async function dispatch(method: string, params: Record<string, unknown>): Promis
|
||||
return treeBaselines.apply(params, await provider.action('drag', params))
|
||||
}
|
||||
case 'typeText': {
|
||||
if (params.elementIndex !== undefined) {
|
||||
const unsupported = computerTypeTextTargetUnsupportedMessage(await provider.capabilities())
|
||||
if (unsupported) {
|
||||
throw new RuntimeClientError('unsupported_capability', unsupported)
|
||||
}
|
||||
}
|
||||
return treeBaselines.apply(params, await provider.action('typeText', params))
|
||||
}
|
||||
case 'pressKey': {
|
||||
|
||||
@@ -110,7 +110,7 @@ export type ComputerActionMetadata = {
|
||||
export type ComputerActionVerification =
|
||||
| {
|
||||
state: 'verified'
|
||||
property: 'focusedText' | 'selection' | 'value'
|
||||
property: 'focusedText' | 'elementText' | 'selection' | 'value'
|
||||
}
|
||||
| {
|
||||
state: 'unverified'
|
||||
@@ -171,6 +171,8 @@ export type ComputerProviderCapabilities = {
|
||||
actions: {
|
||||
click: boolean
|
||||
typeText: boolean
|
||||
// Absent from providers that predate element-targeted typing; they type at focus and ignore the element.
|
||||
typeTextElement?: boolean
|
||||
pressKey: boolean
|
||||
hotkey: boolean
|
||||
pasteText: boolean
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
import type { ComputerProviderCapabilities } from './computer-use-runtime-types'
|
||||
|
||||
// Why: a runtime or provider without element typing drops the target and types into whatever field has focus.
|
||||
export function computerTypeTextTargetUnsupportedMessage(
|
||||
capabilities: ComputerProviderCapabilities
|
||||
): string | null {
|
||||
if (capabilities.supports.actions.typeTextElement === true) {
|
||||
return null
|
||||
}
|
||||
return `${capabilities.provider} cannot type into a specific element (actions.typeTextElement); click the field, then run type-text without --element-index`
|
||||
}
|
||||
@@ -190,6 +190,7 @@ export const Drag = ComputerObserveTargetBase.extend({
|
||||
})
|
||||
|
||||
export const TypeText = ComputerObserveTargetBase.extend({
|
||||
elementIndex: OptionalNonNegativeInt,
|
||||
text: requiredString('Missing text')
|
||||
}).superRefine(validateComputerTarget)
|
||||
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'vitest'
|
||||
import type { ComputerActionResult, ComputerSnapshotResult } from '../../src/shared/runtime-types'
|
||||
import type {
|
||||
ComputerActionResult,
|
||||
ComputerListWindowsResult,
|
||||
ComputerSnapshotResult
|
||||
} from '../../src/shared/runtime-types'
|
||||
import {
|
||||
ensureOrcaRuntimeLaunched,
|
||||
ensureGeditLaunched,
|
||||
findRoleIndex,
|
||||
killGedit,
|
||||
openGeditWindow,
|
||||
parseJsonOutput,
|
||||
runOrcaCli
|
||||
} from './helpers/computer-driver'
|
||||
@@ -12,24 +17,28 @@ import {
|
||||
const isLinux = process.platform === 'linux'
|
||||
const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'
|
||||
|
||||
async function focusGeditDocument(): Promise<void> {
|
||||
const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
|
||||
const GEDIT_TEXT_ROLE = /^\s*(\d+)\s+(text|text entry|entry|document|edit area)(?:\s|$)/im
|
||||
|
||||
async function geditState(windowArgs: string[], restore: boolean): Promise<ComputerSnapshotResult> {
|
||||
return parseJsonOutput<{ result: ComputerSnapshotResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'get-app-state',
|
||||
'--app',
|
||||
'gedit',
|
||||
'--restore-window',
|
||||
...windowArgs,
|
||||
...(restore ? ['--restore-window'] : []),
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
)
|
||||
const textIndex = findRoleIndex(
|
||||
before.result.snapshot.treeText,
|
||||
/^\s*(\d+)\s+(text|text entry|entry|document|edit area)(?:\s|$)/im
|
||||
)
|
||||
).result
|
||||
}
|
||||
|
||||
async function focusGeditDocument(windowArgs: string[] = []): Promise<void> {
|
||||
const before = await geditState(windowArgs, true)
|
||||
const textIndex = findRoleIndex(before.snapshot.treeText, GEDIT_TEXT_ROLE)
|
||||
expect(textIndex).toBeGreaterThanOrEqual(0)
|
||||
|
||||
await runOrcaCli([
|
||||
@@ -37,6 +46,7 @@ async function focusGeditDocument(): Promise<void> {
|
||||
'click',
|
||||
'--app',
|
||||
'gedit',
|
||||
...windowArgs,
|
||||
'--element-index',
|
||||
String(textIndex),
|
||||
'--restore-window',
|
||||
@@ -45,6 +55,15 @@ async function focusGeditDocument(): Promise<void> {
|
||||
])
|
||||
}
|
||||
|
||||
async function geditWindowArgs(title: string): Promise<string[]> {
|
||||
const listed = parseJsonOutput<{ result: ComputerListWindowsResult }>(
|
||||
(await runOrcaCli(['computer', 'list-windows', '--app', 'gedit', '--json'])).stdout
|
||||
)
|
||||
const window = listed.result.windows.find((candidate) => candidate.title.includes(title))
|
||||
expect(window).toBeDefined()
|
||||
return ['--window-index', String(window!.index)]
|
||||
}
|
||||
|
||||
describe.skipIf(!isLinux || !e2eOptIn)('computer-use Linux e2e (gedit)', () => {
|
||||
beforeAll(async () => {
|
||||
await ensureOrcaRuntimeLaunched()
|
||||
@@ -117,4 +136,41 @@ describe.skipIf(!isLinux || !e2eOptIn)('computer-use Linux e2e (gedit)', () => {
|
||||
)
|
||||
expect(after.result.snapshot.treeText).toContain(marker)
|
||||
})
|
||||
|
||||
test('type-text --element-index inserts into a background window field without focusing it', async () => {
|
||||
await openGeditWindow('gedit-front.txt')
|
||||
const targetWindow = await geditWindowArgs('gedit-target.txt')
|
||||
const frontWindow = await geditWindowArgs('gedit-front.txt')
|
||||
await focusGeditDocument(frontWindow)
|
||||
|
||||
const target = await geditState(targetWindow, false)
|
||||
const documentIndex = findRoleIndex(target.snapshot.treeText, GEDIT_TEXT_ROLE)
|
||||
expect(documentIndex).toBeGreaterThanOrEqual(0)
|
||||
|
||||
const marker = `orca-linux-element-${Date.now()}-élan-🚀`
|
||||
const typed = parseJsonOutput<{ result: ComputerActionResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'type-text',
|
||||
'--app',
|
||||
'gedit',
|
||||
...targetWindow,
|
||||
'--element-index',
|
||||
String(documentIndex),
|
||||
'--text',
|
||||
marker,
|
||||
'--no-screenshot',
|
||||
'--full-tree',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
)
|
||||
expect(typed.result.action).toMatchObject({
|
||||
path: 'accessibility',
|
||||
verification: { state: 'verified', property: 'elementText' }
|
||||
})
|
||||
expect(typed.result.snapshot.treeText).toContain(marker)
|
||||
expect((await geditState(frontWindow, false)).snapshot.treeText).not.toContain(marker)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -19,6 +19,7 @@ let safariDraftTitle: string | null = null
|
||||
let linuxTempDir: string | null = null
|
||||
let windowsTempDir: string | null = null
|
||||
let geditProcess: ChildProcess | null = null
|
||||
let extraGeditProcesses: ChildProcess[] = []
|
||||
let notepadProcess: ChildProcess | null = null
|
||||
let notepadAppSelector: string | null = null
|
||||
|
||||
@@ -161,15 +162,32 @@ export async function ensureGeditLaunched(): Promise<void> {
|
||||
await waitForComputerWindowTitle('gedit', fileName, 15000)
|
||||
}
|
||||
|
||||
/** Opens a second window in the test-owned gedit instance. */
|
||||
export async function openGeditWindow(fileName: string): Promise<void> {
|
||||
if (!linuxTempDir) {
|
||||
throw new Error('gedit has not been launched')
|
||||
}
|
||||
const filePath = join(linuxTempDir, fileName)
|
||||
await writeFile(filePath, 'seed', 'utf8')
|
||||
const launcher = spawn('gedit', ['--new-window', filePath], { detached: true, stdio: 'ignore' })
|
||||
launcher.unref()
|
||||
extraGeditProcesses.push(launcher)
|
||||
await waitForComputerWindowTitle('gedit', fileName, 15000)
|
||||
}
|
||||
|
||||
export async function killGedit(): Promise<void> {
|
||||
if (geditProcess?.pid) {
|
||||
for (const owned of [geditProcess, ...extraGeditProcesses]) {
|
||||
if (!owned?.pid) {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
process.kill(-geditProcess.pid, 'SIGTERM')
|
||||
process.kill(-owned.pid, 'SIGTERM')
|
||||
} catch {
|
||||
// The test-owned gedit process may already be closed.
|
||||
}
|
||||
geditProcess = null
|
||||
}
|
||||
geditProcess = null
|
||||
extraGeditProcesses = []
|
||||
if (linuxTempDir) {
|
||||
await rm(linuxTempDir, { force: true, recursive: true })
|
||||
linuxTempDir = null
|
||||
|
||||
Reference in New Issue
Block a user