orca/tests/e2e/computer-windows.e2e.ts

import { afterAll, beforeAll, describe, expect, test } from 'vitest'
import type {
  ComputerActionResult,
  ComputerListAppsResult,
  ComputerListWindowsResult,
  ComputerSnapshotResult
} from '../../src/shared/runtime-types'
import {
  ensureOrcaRuntimeLaunched,
  ensureNotepadLaunched,
  findRoleIndex,
  getNotepadAppSelector,
  killNotepad,
  parseJsonOutput,
  runOrcaCli,
  stopOrcaRuntime
} from './helpers/computer-driver'
import {
  editableRolePattern,
  focusNotepadDocument,
  waitForNotepadText
} from './helpers/notepad-document'
import {
  closeWindowFixture,
  countMagentaPixelRatio,
  launchHungWindowFixture,
  launchWindowFixture
} from './helpers/windows-form-fixture'

const isWindows = process.platform === 'win32'
const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'
describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Notepad)', () => {
  beforeAll(async () => {
    await ensureOrcaRuntimeLaunched()
    await ensureNotepadLaunched()
  })

  afterAll(async () => {
    await killNotepad()
    await stopOrcaRuntime()
  })

  test('list-apps includes the test-owned Notepad process', async () => {
    const result = await runOrcaCli(['computer', 'list-apps', '--json'])
    const envelope = parseJsonOutput<{ result: ComputerListAppsResult }>(result.stdout)
    const pid = Number.parseInt(getNotepadAppSelector().slice(4), 10)
    const notepadApp = envelope.result.apps.find((app) => app.pid === pid)

    expect(notepadApp).toMatchObject({ pid, bundleId: 'notepad' })
    expect(notepadApp?.name.toLowerCase()).toBe('notepad')
  })

  test('list-windows returns a targetable Notepad window', async () => {
    const app = getNotepadAppSelector()
    const result = await runOrcaCli(['computer', 'list-windows', '--app', app, '--json'])
    const envelope = parseJsonOutput<{ result: ComputerListWindowsResult }>(result.stdout)

    expect(envelope.result.windows).toHaveLength(1)
    expect(envelope.result.windows[0]).toMatchObject({
      index: 0,
      app: expect.objectContaining({ pid: Number.parseInt(app.slice(4), 10) }),
      id: expect.any(Number),
      title: expect.stringContaining('Notepad'),
      width: expect.any(Number),
      height: expect.any(Number)
    })
  })

  test('Notepad exposes a basic accessibility tree', async () => {
    const result = await runOrcaCli([
      'computer',
      'get-app-state',
      '--app',
      getNotepadAppSelector(),
      '--json'
    ])
    const envelope = parseJsonOutput<{ result: ComputerSnapshotResult }>(result.stdout)

    expect(envelope.result.snapshot.app.name.toLowerCase()).toBe('notepad')
    expect(envelope.result.snapshot.window.title).toContain('Notepad')
    expect(envelope.result.snapshot.elementCount).toBeGreaterThan(0)
    expect(envelope.result.snapshot.coordinateSpace).toBe('window')
    expect(envelope.result.snapshot.truncation?.truncated).toBe(false)
    expect(envelope.result.screenshotStatus.state).toBe('captured')
    expect(envelope.result.screenshot?.format).toBe('png')
    expect(envelope.result.screenshot?.data).toBeUndefined()
    expect(envelope.result.screenshot?.dataOmitted).toBe(true)
    expect(envelope.result.screenshot?.path).toContain('orca-computer-use')
  })

  test('paste-text mutates the test-owned document', async () => {
    const app = getNotepadAppSelector()
    const marker = `orca-windows-paste-${Date.now()}`
    await focusNotepadDocument(app)
    const action = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'paste-text',
          '--app',
          app,
          '--text',
          marker,
          '--restore-window',
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(action.result.action?.path).toBe('clipboard')
    expect(action.result.action?.verification).toMatchObject({
      state: 'unverified',
      reason: 'clipboard_paste'
    })

    const after = await waitForNotepadText(app, marker)
    expect(after.snapshot.treeText).toContain(marker)
  })

  test('Unicode payloads survive paste-text', async () => {
    const app = getNotepadAppSelector()
    const unicode = `orca unicode café Ω 漢字 ${Date.now()}`
    await focusNotepadDocument(app)
    const pasted = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'paste-text',
          '--app',
          app,
          '--text',
          unicode,
          '--restore-window',
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(pasted.result.snapshot.treeText).toContain(unicode)
  })

  test('hotkey and paste-text can replace the document selection', async () => {
    const app = getNotepadAppSelector()
    const first = `orca-windows-first-${Date.now()}`
    await focusNotepadDocument(app)
    await runOrcaCli([
      'computer',
      'paste-text',
      '--app',
      app,
      '--text',
      first,
      '--restore-window',
      '--no-screenshot',
      '--json'
    ])

    const selectAll = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'hotkey',
          '--app',
          app,
          '--key',
          'CmdOrCtrl+A',
          '--restore-window',
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(selectAll.result.action?.actionName).toBe('hotkey')

    const marker = `orca-windows-replaced-${Date.now()}`
    const second = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'paste-text',
          '--app',
          app,
          '--text',
          marker,
          '--restore-window',
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(second.result.snapshot.treeText).toContain(marker)
    expect(second.result.snapshot.treeText).not.toContain(first)
  })

  test('click and type-text send synthetic input to the document', async () => {
    const app = getNotepadAppSelector()
    const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
      (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json']))
        .stdout
    )
    const documentIndex = findRoleIndex(before.result.snapshot.treeText, editableRolePattern)
    expect(documentIndex).toBeGreaterThanOrEqual(0)

    await runOrcaCli([
      'computer',
      'click',
      '--app',
      app,
      '--element-index',
      String(documentIndex),
      '--restore-window',
      '--no-screenshot',
      '--json'
    ])

    const marker = ` typed-${Date.now()}`
    const typed = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'type-text',
          '--app',
          app,
          '--text',
          marker,
          '--restore-window',
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(typed.result.action?.path).toBe('synthetic')
    expect(typed.result.snapshot.treeText).toContain(marker.trim())
  })

  test('a covered window screenshot shows the window, not what covers it', async () => {
    const app = getNotepadAppSelector()
    const notepadWindow = parseJsonOutput<{ result: ComputerListWindowsResult }>(
      (await runOrcaCli(['computer', 'list-windows', '--app', app, '--json'])).stdout
    ).result.windows[0]
    const bounds = {
      x: notepadWindow.x ?? 0,
      y: notepadWindow.y ?? 0,
      width: notepadWindow.width,
      height: notepadWindow.height
    }
    const cover = await launchWindowFixture(
      [
        "$form.Text = 'orca-e2e-cover'",
        "$form.FormBorderStyle = 'None'",
        "$form.StartPosition = 'Manual'",
        '$form.TopMost = $true',
        '$form.ShowInTaskbar = $false',
        '$form.BackColor = [System.Drawing.Color]::FromArgb(255, 0, 255)',
        `$form.Bounds = New-Object System.Drawing.Rectangle(${bounds.x}, ${bounds.y}, ${bounds.width}, ${bounds.height})`
      ].join('\n')
    )
    try {
      // Prove the cover really hides Notepad, so the screenshot check below can fail.
      expect(await countMagentaPixelRatio(bounds)).toBeGreaterThan(0.9)
      const state = parseJsonOutput<{ result: ComputerSnapshotResult }>(
        (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--json'])).stdout
      )
      const screenshotPath = state.result.screenshot?.path
      if (!screenshotPath) {
        throw new Error(`no screenshot: ${JSON.stringify(state.result.screenshotStatus)}`)
      }
      // A screen copy of the covered bounds would be almost entirely the magenta cover.
      expect(await countMagentaPixelRatio(screenshotPath)).toBeLessThan(0.01)
    } finally {
      await closeWindowFixture(cover)
    }
  })

  test('a hung window fails fast with a timeout error and does not stall later calls', async () => {
    const hung = await launchHungWindowFixture()
    try {
      const started = Date.now()
      const failure = await runOrcaCli([
        'computer',
        'get-app-state',
        '--app',
        `pid:${hung.pid}`,
        '--no-screenshot',
        '--json'
      ]).then(
        () => 'get-app-state unexpectedly succeeded',
        (error: unknown) => (error instanceof Error ? error.message : String(error))
      )
      expect(failure).toMatch(/window_not_responding/)
      expect(failure).toMatch(/"code":\s*"action_timeout"/)
      // The provider request timeout is 30 s; the hang probe answers within its 5 s bound.
      expect(Date.now() - started).toBeLessThan(15_000)

      const apps = parseJsonOutput<{ result: ComputerListAppsResult }>(
        (await runOrcaCli(['computer', 'list-apps', '--json'])).stdout
      )
      expect(apps.result.apps.some((entry) => entry.pid === hung.pid)).toBe(true)
    } finally {
      await closeWindowFixture(hung)
    }
  })
})