import { afterAll, beforeAll, describe, expect, test } from 'vitest' import type { ComputerActionResult, ComputerSnapshotResult } from '../../src/shared/runtime-types' import { ensureOrcaRuntimeLaunched, findRoleIndex, parseJsonOutput, runOrcaCli, stopOrcaRuntime } from './helpers/computer-driver' import { launchWindowsSemanticFixture, type WindowsSemanticFixture, type WindowsSemanticFixtureState } from './helpers/windows-semantic-controls-fixture' const isWindows = process.platform === 'win32' const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1' let fixture: WindowsSemanticFixture | null = null function requireFixture(): WindowsSemanticFixture { if (!fixture) { throw new Error('Windows semantic controls fixture has not been launched') } return fixture } async function treeText(app: string): Promise { return parseJsonOutput<{ result: ComputerSnapshotResult }>( (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json'])) .stdout ).result.snapshot.treeText } async function elementIndex(app: string, line: RegExp): Promise { const text = await treeText(app) const index = findRoleIndex(text, line) expect(index, text).toBeGreaterThanOrEqual(0) return index } // Runs a UI Automation action, waits for its effect, and checks action.delivery against whether the window was activated. async function runPatternAction( args: string[], effect: (state: WindowsSemanticFixtureState) => boolean, label: string ): Promise<{ result: ComputerActionResult; activated: boolean }> { const { waitForState } = requireFixture() const before = await waitForState((state) => !state.active, 'fixture window inactive') const result = parseJsonOutput<{ result: ComputerActionResult }>( (await runOrcaCli(['computer', ...args, '--no-screenshot', '--json'])).stdout ).result expect(result.action?.path).toBe('accessibility') const after = await waitForState(effect, label) const activated = after.activations > before.activations expect(result.action?.delivery, after.events.join('\n')).toBe( activated ? 'foreground' : 'background' ) return { result, activated } } // The fixture shows real windows and hands foreground back to its holder, so this runs only on CI. describe.skipIf(!isWindows || !e2eOptIn || process.env.CI !== 'true')( 'computer-use Windows UI Automation actions', () => { beforeAll(async () => { await ensureOrcaRuntimeLaunched() fixture = await launchWindowsSemanticFixture() }) afterAll(async () => { await fixture?.close() fixture = null await stopOrcaRuntime() }) test('scroll pages a list through ScrollPattern and reports whether the window came forward', async () => { const { app } = requireFixture() const item = await elementIndex(app, /^\s*(\d+)\s+list item Semantic item 0(?:,|$)/m) await runPatternAction( ['scroll', '--app', app, '--element-index', String(item), '--direction', 'down'], (state) => state.listOffset > 0, 'list scrolled down' ) }) test('ScrollToVisible brings an off-screen list item into view in the background', async () => { const { app } = requireFixture() const item = await elementIndex(app, /^\s*(\d+)\s+list item Semantic item 150(?:,|$)/m) const { activated } = await runPatternAction( [ 'perform-secondary-action', '--app', app, '--element-index', String(item), '--action', 'ScrollToVisible' ], (state) => state.listOffset <= 150 && 150 < state.listOffset + state.listViewport, 'item 150 in the list viewport' ) expect(activated).toBe(false) }) test('Expand and Collapse drive a tree item through ExpandCollapsePattern', async () => { const { app } = requireFixture() const collapsed = await elementIndex( app, /^\s*(\d+)\s+tree view item Semantic branch,.*Secondary Actions:.*\bExpand\b/m ) await runPatternAction( [ 'perform-secondary-action', '--app', app, '--element-index', String(collapsed), '--action', 'Expand' ], (state) => state.branchExpanded, 'branch expanded' ) const expanded = await elementIndex( app, /^\s*(\d+)\s+tree view item Semantic branch,.*Secondary Actions:.*\bCollapse\b/m ) await runPatternAction( [ 'perform-secondary-action', '--app', app, '--element-index', String(expanded), '--action', 'Collapse' ], (state) => !state.branchExpanded, 'branch collapsed' ) }) test('set-value moves a slider through RangeValuePattern', async () => { const { app } = requireFixture() const slider = await elementIndex(app, /^\s*(\d+)\s+slider Semantic level(?:,|$)/m) const { result } = await runPatternAction( ['set-value', '--app', app, '--element-index', String(slider), '--value', '42'], (state) => state.level === 42, 'slider at 42' ) expect(result.action?.verification).toMatchObject({ state: 'verified', property: 'value' }) }) } )