Files
orca/tests/e2e/computer-windows-background.e2e.ts
Kelvin Amoaba a406364654 fix(computer): click, set and scroll Windows buttons, text boxes, combo boxes and lists without bringing the window forward (#42237)
* fix(computer): press, set and scroll Windows native controls without activating the window

Measured on Windows CI: UI Automation's Invoke, Toggle, ValuePattern and
ScrollPattern focus a background WinForms control first, which activates its
window. Native controls now go through their own messages instead, each
owner-checked: BM_CLICK for buttons, check boxes and radio buttons, select-all
plus EM_REPLACESEL for set-value on edits, CB_SETCURSEL with CBN_SELCHANGE for
combo boxes (exact-text match, read back after the app's handler runs) and
LB_SETTOPINDEX for list box pages.

* fix(computer): refuse read-only Windows documents before typing keys, document the caret fallback

* fix(computer): keep Windows native routes to measured controls and refuse partial writes

BM_CLICK only for WinForms buttons, multi-column lists stay on ScrollPattern, a
list already at its end reports success, edits refuse text past their limit,
the combo's owner is checked right before the selection, a cleared selection
reads as a mismatch and a disabled control is element_not_enabled.

* fix(computer): hold Windows edit limits without a readback, require a complete combo search

* fix(computer): read a long Windows edit's length from the field for its limit check

* test(computer): share the Windows fixture action helpers
2026-11-14 02:45:10 +00:00

312 lines
11 KiB
TypeScript

import { execFile } from 'node:child_process'
import { promisify } from 'node:util'
import { afterAll, beforeAll, describe, expect, test } from 'vitest'
import type { ComputerActionResult, ComputerLaunchAppResult } from '../../src/shared/runtime-types'
import {
ensureOrcaRuntimeLaunched,
parseJsonOutput,
runOrcaCli,
stopOrcaRuntime
} from './helpers/computer-driver'
import { elementIndex, runPatternAction } from './helpers/windows-fixture-actions'
import {
launchWindowsFormsFixture,
launchWindowsSemanticFixture,
type WindowsFormsFixture,
type WindowsFormsFixtureState,
type WindowsSemanticFixture
} from './helpers/windows-semantic-controls-fixture'
const isWindows = process.platform === 'win32'
const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'
let fixture: WindowsSemanticFixture | null = null
function requireFixture(): WindowsSemanticFixture {
if (!fixture) {
throw new Error('Windows semantic controls fixture has not been launched')
}
return fixture
}
// The fixture shows real windows and hands foreground back to its holder, so this runs only on CI.
describe.skipIf(!isWindows || !e2eOptIn || process.env.CI !== 'true')(
'computer-use Windows UI Automation actions',
() => {
beforeAll(async () => {
await ensureOrcaRuntimeLaunched()
fixture = await launchWindowsSemanticFixture()
})
afterAll(async () => {
await fixture?.close()
fixture = null
await stopOrcaRuntime()
})
test('scroll pages a list through ScrollPattern and reports whether the window came forward', async () => {
const { app } = requireFixture()
const item = await elementIndex(app, /^\s*(\d+)\s+list item Semantic item 0(?:,|$)/m)
await runPatternAction(
requireFixture(),
['scroll', '--app', app, '--element-index', String(item), '--direction', 'down'],
(state) => state.listOffset > 0,
'list scrolled down'
)
})
test('ScrollToVisible brings an off-screen list item into view in the background', async () => {
const { app } = requireFixture()
const item = await elementIndex(app, /^\s*(\d+)\s+list item Semantic item 150(?:,|$)/m)
const { activated } = await runPatternAction(
requireFixture(),
[
'perform-secondary-action',
'--app',
app,
'--element-index',
String(item),
'--action',
'ScrollToVisible'
],
(state) => state.listOffset <= 150 && 150 < state.listOffset + state.listViewport,
'item 150 in the list viewport'
)
expect(activated).toBe(false)
})
test('Expand and Collapse drive a tree item through ExpandCollapsePattern', async () => {
const { app } = requireFixture()
const collapsed = await elementIndex(
app,
/^\s*(\d+)\s+tree view item Semantic branch,.*Secondary Actions:.*\bExpand\b/m
)
await runPatternAction(
requireFixture(),
[
'perform-secondary-action',
'--app',
app,
'--element-index',
String(collapsed),
'--action',
'Expand'
],
(state) => state.branchExpanded,
'branch expanded'
)
const expanded = await elementIndex(
app,
/^\s*(\d+)\s+tree view item Semantic branch,.*Secondary Actions:.*\bCollapse\b/m
)
await runPatternAction(
requireFixture(),
[
'perform-secondary-action',
'--app',
app,
'--element-index',
String(expanded),
'--action',
'Collapse'
],
(state) => !state.branchExpanded,
'branch collapsed'
)
})
test('set-value moves a slider through RangeValuePattern', async () => {
const { app } = requireFixture()
const slider = await elementIndex(app, /^\s*(\d+)\s+slider Semantic level(?:,|$)/m)
const { result } = await runPatternAction(
requireFixture(),
['set-value', '--app', app, '--element-index', String(slider), '--value', '42'],
(state) => state.level === 42,
'slider at 42'
)
expect(result.action?.verification).toMatchObject({ state: 'verified', property: 'value' })
})
}
)
let formsFixture: WindowsFormsFixture | null = null
function requireFormsFixture(): WindowsFormsFixture {
if (!formsFixture) {
throw new Error('Windows Forms fixture has not been launched')
}
return formsFixture
}
// Runs an action on the background WinForms window and checks it never came forward.
async function runInBackground(
args: (app: string) => string[],
effect: (state: WindowsFormsFixtureState) => boolean,
label: string
): Promise<ComputerActionResult> {
const target = requireFormsFixture()
const { result, activated } = await runPatternAction(target, args(target.app), effect, label)
expect(activated, JSON.stringify(result.action)).toBe(false)
return result
}
async function formsIndex(line: RegExp): Promise<string> {
return String(await elementIndex(requireFormsFixture().app, line))
}
describe.skipIf(!isWindows || !e2eOptIn || process.env.CI !== 'true')(
'computer-use Windows Forms actions',
() => {
beforeAll(async () => {
await ensureOrcaRuntimeLaunched()
formsFixture = await launchWindowsFormsFixture()
})
afterAll(async () => {
await formsFixture?.close()
formsFixture = null
await stopOrcaRuntime()
})
test('type-text --element-index types into background text boxes at their caret', async () => {
for (const [seed, text, field, expected] of [
['seed-name', ' café Ω 漢字', 'name', 'seed-name café Ω 漢字'],
['seed-notes', '\nsecond line', 'notes', 'seed-notes\r\nsecond line']
] as const) {
const index = await formsIndex(
new RegExp(`^\\s*(\\d+)\\s+(?:edit|document)\\b.*${seed}`, 'm')
)
const result = await runInBackground(
(app) => ['type-text', '--app', app, '--element-index', index, '--text', text],
(state) => state[field] === expected,
`${seed} typed`
)
expect(result.action).toMatchObject({
actionName: 'EM_REPLACESEL',
verification: { state: 'verified', property: 'elementText' }
})
}
})
test('click presses a button and ticks a check box', async () => {
const button = await formsIndex(/^\s*(\d+)\s+button Semantic press/m)
await runInBackground(
(app) => ['click', '--app', app, '--element-index', button],
(state) => state.presses === 1,
'button pressed'
)
const check = await formsIndex(/^\s*(\d+)\s+check box Semantic check/m)
await runInBackground(
(app) => ['click', '--app', app, '--element-index', check],
(state) => state.checked,
'check box ticked'
)
})
test('set-value writes a text box and picks a combo box option', async () => {
const field = await formsIndex(/^\s*(\d+)\s+edit\b.*seed-name/m)
await runInBackground(
(app) => ['set-value', '--app', app, '--element-index', field, '--value', 'replaced'],
(state) => state.name === 'replaced',
'text box replaced'
)
const combo = await formsIndex(/^\s*(\d+)\s+combo box\b/m)
const result = await runInBackground(
(app) => ['set-value', '--app', app, '--element-index', combo, '--value', 'Gamma'],
(state) => state.combo === 'Gamma',
'combo box at Gamma'
)
expect(result.action?.verification).toMatchObject({ state: 'verified', property: 'value' })
})
test('scroll pages a list box by its own rows', async () => {
const row = await formsIndex(/^\s*(\d+)\s+list item Row 0\b/m)
await runInBackground(
(app) => ['scroll', '--app', app, '--element-index', row, '--direction', 'down'],
(state) => state.listTop > 0,
'list scrolled'
)
})
test('keys refuse in the background and send nothing', async () => {
const { app, readState, waitForState } = requireFormsFixture()
const before = await waitForState((state) => !state.active, 'fixture window inactive')
const failure = await runOrcaCli([
'computer',
'press-key',
'--app',
app,
'--key',
'Return',
'--no-screenshot',
'--json'
]).then(
() => 'press-key unexpectedly succeeded',
(error: unknown) => (error instanceof Error ? error.message : String(error))
)
expect(failure).toMatch(/"code":\s*"background_unavailable"/)
expect(failure).toContain('type-text --element-index')
const after = await readState()
expect(after.activations).toBe(before.activations)
expect(after.notes).toBe(before.notes)
})
}
)
async function foregroundWindowOwnerPid(): Promise<number> {
const { stdout } = await promisify(execFile)(
'powershell.exe',
[
'-NoProfile',
'-NonInteractive',
'-Command',
[
'Add-Type -Name Foreground -Namespace OrcaE2E -MemberDefinition \'[DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint pid);\'',
'$owner = [uint32]0',
'[void][OrcaE2E.Foreground]::GetWindowThreadProcessId([OrcaE2E.Foreground]::GetForegroundWindow(), [ref]$owner)',
'$owner'
].join('; ')
],
{ windowsHide: true }
)
return Number.parseInt(stdout.trim(), 10)
}
// Character Map is a classic single-process Win32 app on every Windows image, so its pid owns its window.
describe.skipIf(!isWindows || !e2eOptIn || process.env.CI !== 'true')(
'computer-use Windows launch-app',
() => {
let launchedPid: number | null = null
beforeAll(async () => {
await ensureOrcaRuntimeLaunched()
})
afterAll(async () => {
if (launchedPid !== null) {
await promisify(execFile)('taskkill.exe', ['/PID', String(launchedPid), '/F'], {
windowsHide: true
}).catch(() => {})
}
await stopOrcaRuntime()
})
test('launch-app starts an app and reports where it ended up', async () => {
const result = parseJsonOutput<{ result: ComputerLaunchAppResult }>(
(await runOrcaCli(['computer', 'launch-app', 'charmap', '--json'])).stdout
).result
launchedPid = result.app.pid
expect(result.launched).toBe(true)
// CI never holds a foreground lock (no user input), so any app may take the foreground here;
// what must hold is that delivery reports what happened.
const ownsForeground = (await foregroundWindowOwnerPid()) === result.app.pid
expect(result.delivery).toBe(ownsForeground ? 'foreground' : 'background')
})
}
)