orca/tests/e2e/computer-windows-background.e2e.ts

import { afterAll, beforeAll, describe, expect, test } from 'vitest'
import type { ComputerActionResult, ComputerSnapshotResult } from '../../src/shared/runtime-types'
import {
  ensureOrcaRuntimeLaunched,
  findRoleIndex,
  parseJsonOutput,
  runOrcaCli,
  stopOrcaRuntime
} from './helpers/computer-driver'
import {
  launchWindowsSemanticFixture,
  type WindowsSemanticFixture,
  type WindowsSemanticFixtureState
} from './helpers/windows-semantic-controls-fixture'

const isWindows = process.platform === 'win32'
const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'

let fixture: WindowsSemanticFixture | null = null

function requireFixture(): WindowsSemanticFixture {
  if (!fixture) {
    throw new Error('Windows semantic controls fixture has not been launched')
  }
  return fixture
}

async function treeText(app: string): Promise<string> {
  return parseJsonOutput<{ result: ComputerSnapshotResult }>(
    (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json']))
      .stdout
  ).result.snapshot.treeText
}

async function elementIndex(app: string, line: RegExp): Promise<number> {
  const text = await treeText(app)
  const index = findRoleIndex(text, line)
  expect(index, text).toBeGreaterThanOrEqual(0)
  return index
}

// Runs a UI Automation action, waits for its effect, and checks action.delivery against whether the window was activated.
async function runPatternAction(
  args: string[],
  effect: (state: WindowsSemanticFixtureState) => boolean,
  label: string
): Promise<{ result: ComputerActionResult; activated: boolean }> {
  const { waitForState } = requireFixture()
  const before = await waitForState((state) => !state.active, 'fixture window inactive')
  const result = parseJsonOutput<{ result: ComputerActionResult }>(
    (await runOrcaCli(['computer', ...args, '--no-screenshot', '--json'])).stdout
  ).result
  expect(result.action?.path).toBe('accessibility')
  const after = await waitForState(effect, label)
  const activated = after.activations > before.activations
  expect(result.action?.delivery, after.events.join('\n')).toBe(
    activated ? 'foreground' : 'background'
  )
  return { result, activated }
}

// The fixture shows real windows and hands foreground back to its holder, so this runs only on CI.
describe.skipIf(!isWindows || !e2eOptIn || process.env.CI !== 'true')(
  'computer-use Windows UI Automation actions',
  () => {
    beforeAll(async () => {
      await ensureOrcaRuntimeLaunched()
      fixture = await launchWindowsSemanticFixture()
    })

    afterAll(async () => {
      await fixture?.close()
      fixture = null
      await stopOrcaRuntime()
    })

    test('scroll pages a list through ScrollPattern and reports whether the window came forward', async () => {
      const { app } = requireFixture()
      const item = await elementIndex(app, /^\s*(\d+)\s+list item Semantic item 0(?:,|$)/m)

      await runPatternAction(
        ['scroll', '--app', app, '--element-index', String(item), '--direction', 'down'],
        (state) => state.listOffset > 0,
        'list scrolled down'
      )
    })

    test('ScrollToVisible brings an off-screen list item into view in the background', async () => {
      const { app } = requireFixture()
      const item = await elementIndex(app, /^\s*(\d+)\s+list item Semantic item 150(?:,|$)/m)

      const { activated } = await runPatternAction(
        [
          'perform-secondary-action',
          '--app',
          app,
          '--element-index',
          String(item),
          '--action',
          'ScrollToVisible'
        ],
        (state) => state.listOffset <= 150 && 150 < state.listOffset + state.listViewport,
        'item 150 in the list viewport'
      )
      expect(activated).toBe(false)
    })

    test('Expand and Collapse drive a tree item through ExpandCollapsePattern', async () => {
      const { app } = requireFixture()
      const collapsed = await elementIndex(
        app,
        /^\s*(\d+)\s+tree view item Semantic branch,.*Secondary Actions:.*\bExpand\b/m
      )
      await runPatternAction(
        [
          'perform-secondary-action',
          '--app',
          app,
          '--element-index',
          String(collapsed),
          '--action',
          'Expand'
        ],
        (state) => state.branchExpanded,
        'branch expanded'
      )

      const expanded = await elementIndex(
        app,
        /^\s*(\d+)\s+tree view item Semantic branch,.*Secondary Actions:.*\bCollapse\b/m
      )
      await runPatternAction(
        [
          'perform-secondary-action',
          '--app',
          app,
          '--element-index',
          String(expanded),
          '--action',
          'Collapse'
        ],
        (state) => !state.branchExpanded,
        'branch collapsed'
      )
    })

    test('set-value moves a slider through RangeValuePattern', async () => {
      const { app } = requireFixture()
      const slider = await elementIndex(app, /^\s*(\d+)\s+slider Semantic level(?:,|$)/m)

      const { result } = await runPatternAction(
        ['set-value', '--app', app, '--element-index', String(slider), '--value', '42'],
        (state) => state.level === 42,
        'slider at 42'
      )
      expect(result.action?.verification).toMatchObject({ state: 'verified', property: 'value' })
    })
  }
)