orca/tests/e2e/computer-windows.e2e.ts

import { readFile } from 'node:fs/promises'
import { afterAll, beforeAll, describe, expect, test } from 'vitest'
import type {
  ComputerActionResult,
  ComputerListAppsResult,
  ComputerListWindowsResult,
  ComputerSnapshotResult
} from '../../src/shared/runtime-types'
import {
  ensureOrcaRuntimeLaunched,
  ensureNotepadLaunched,
  findRoleIndex,
  getNotepadAppSelector,
  killNotepad,
  parseJsonOutput,
  runOrcaCli,
  stopOrcaRuntime
} from './helpers/computer-driver'
import { editableRolePattern, focusNotepadDocument } from './helpers/notepad-document'
import {
  closeWindowFixture,
  countMagentaPixelRatio,
  launchHungWindowFixture,
  launchWindowFixture
} from './helpers/windows-form-fixture'

const isWindows = process.platform === 'win32'
const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'
describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Notepad)', () => {
  let notepadPath = ''

  async function saveNotepad() {
    await runOrcaCli([
      'computer',
      'hotkey',
      '--app',
      getNotepadAppSelector(),
      '--key',
      'CmdOrCtrl+S',
      '--restore-window',
      '--no-screenshot',
      '--json'
    ])
  }

  beforeAll(async () => {
    await ensureOrcaRuntimeLaunched()
    notepadPath = await ensureNotepadLaunched()
  })

  afterAll(async () => {
    await killNotepad()
    await stopOrcaRuntime()
  })

  test('hotkey and Unicode paste-text replace the document selection', async () => {
    const app = getNotepadAppSelector()
    const first = `orca-windows-first-${Date.now()}`
    await focusNotepadDocument(app)
    await runOrcaCli([
      'computer',
      'paste-text',
      '--app',
      app,
      '--text',
      first,
      '--restore-window',
      '--no-screenshot',
      '--json'
    ])

    await runOrcaCli([
      'computer',
      'hotkey',
      '--app',
      app,
      '--key',
      'CmdOrCtrl+A',
      '--restore-window',
      '--no-screenshot',
      '--json'
    ])

    const marker = `orca-windows-replaced-café Ω 漢字 ${Date.now()}`
    await runOrcaCli([
      'computer',
      'paste-text',
      '--app',
      app,
      '--text',
      marker,
      '--restore-window',
      '--no-screenshot',
      '--json'
    ])
    await saveNotepad()
    await expect.poll(() => readFile(notepadPath, 'utf8')).toBe(marker)
  })

  test('click and type-text send synthetic input to the document', async () => {
    const app = getNotepadAppSelector()
    const before = parseJsonOutput<{ result: ComputerSnapshotResult }>(
      (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json']))
        .stdout
    )
    const documentIndex = findRoleIndex(before.result.snapshot.treeText, editableRolePattern)
    expect(documentIndex).toBeGreaterThanOrEqual(0)

    await runOrcaCli([
      'computer',
      'click',
      '--app',
      app,
      '--element-index',
      String(documentIndex),
      '--restore-window',
      '--no-screenshot',
      '--json'
    ])

    const marker = ` typed-${Date.now()}`
    const typed = parseJsonOutput<{ result: ComputerActionResult }>(
      (
        await runOrcaCli([
          'computer',
          'type-text',
          '--app',
          app,
          '--text',
          marker,
          '--restore-window',
          '--no-screenshot',
          '--json'
        ])
      ).stdout
    )
    expect(typed.result.action?.path).toBe('synthetic')
    await saveNotepad()
    await expect.poll(() => readFile(notepadPath, 'utf8')).toContain(marker.trim())
  })

  test('a covered window screenshot shows the window, not what covers it', async () => {
    const app = getNotepadAppSelector()
    const notepadWindow = parseJsonOutput<{ result: ComputerListWindowsResult }>(
      (await runOrcaCli(['computer', 'list-windows', '--app', app, '--json'])).stdout
    ).result.windows[0]
    const bounds = {
      x: notepadWindow.x ?? 0,
      y: notepadWindow.y ?? 0,
      width: notepadWindow.width,
      height: notepadWindow.height
    }
    const cover = await launchWindowFixture(
      [
        "$form.Text = 'orca-e2e-cover'",
        "$form.FormBorderStyle = 'None'",
        "$form.StartPosition = 'Manual'",
        '$form.TopMost = $true',
        '$form.ShowInTaskbar = $false',
        '$form.BackColor = [System.Drawing.Color]::FromArgb(255, 0, 255)',
        `$form.Bounds = New-Object System.Drawing.Rectangle(${bounds.x}, ${bounds.y}, ${bounds.width}, ${bounds.height})`
      ].join('\n')
    )
    try {
      // Prove the cover really hides Notepad, so the screenshot check below can fail.
      expect(await countMagentaPixelRatio(bounds)).toBeGreaterThan(0.9)
      const state = parseJsonOutput<{ result: ComputerSnapshotResult }>(
        (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--json'])).stdout
      )
      const screenshotPath = state.result.screenshot?.path
      if (!screenshotPath) {
        throw new Error(`no screenshot: ${JSON.stringify(state.result.screenshotStatus)}`)
      }
      // A screen copy of the covered bounds would be almost entirely the magenta cover.
      expect(await countMagentaPixelRatio(screenshotPath)).toBeLessThan(0.01)
    } finally {
      await closeWindowFixture(cover)
    }
  })

  test('a hung window reports a timeout error and later calls still succeed', async () => {
    const hung = await launchHungWindowFixture()
    try {
      const failure = await runOrcaCli([
        'computer',
        'get-app-state',
        '--app',
        `pid:${hung.pid}`,
        '--no-screenshot',
        '--json'
      ]).then(
        () => 'get-app-state unexpectedly succeeded',
        (error: unknown) => (error instanceof Error ? error.message : String(error))
      )
      expect(failure).toMatch(/window_not_responding/)
      expect(failure).toMatch(/"code":\s*"action_timeout"/)

      const apps = parseJsonOutput<{ result: ComputerListAppsResult }>(
        (await runOrcaCli(['computer', 'list-apps', '--json'])).stdout
      )
      expect(apps.result.apps.some((entry) => entry.pid === hung.pid)).toBe(true)
    } finally {
      await closeWindowFixture(hung)
    }
  })
})