From 98fadae4d804778cef5ee3fc0834d2f9190102cf Mon Sep 17 00:00:00 2001 From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:17:33 +0000 Subject: [PATCH] fix(computer): keep the agent's window id as the target window on Windows (#43162) The Windows helper checked an agent's --window-id against Process.MainWindowHandle, a z-order guess that moves to a tooltip or another window of the same app. The window id is now the target: a live, visible top-level window of the target process, carried through the tree read, frame, screenshot, raise, keyboard input and the observe after the action. Co-authored-by: toodeceptive <137078450+toodeceptive@users.noreply.github.com> --- native/computer-use-windows/runtime.ps1 | 200 ++++++++++++++++-------- tests/e2e/computer-windows.e2e.ts | 88 ++++++++++- 2 files changed, 221 insertions(+), 67 deletions(-) diff --git a/native/computer-use-windows/runtime.ps1 b/native/computer-use-windows/runtime.ps1 index 90504088a97..80f97aaa425 100644 --- a/native/computer-use-windows/runtime.ps1 +++ b/native/computer-use-windows/runtime.ps1 @@ -104,6 +104,67 @@ public static class OrcaDesktopWin32 { [DllImport("user32.dll")] public static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint processId); + [DllImport("user32.dll")] + private static extern bool IsWindowVisible(IntPtr hwnd); + + [DllImport("user32.dll")] + private static extern IntPtr GetAncestor(IntPtr hwnd, uint flags); + + [DllImport("user32.dll")] + private static extern IntPtr GetWindow(IntPtr hwnd, uint command); + + [DllImport("user32.dll", EntryPoint = "GetClassNameW", CharSet = CharSet.Unicode)] + private static extern int GetClassName(IntPtr hwnd, System.Text.StringBuilder className, int maxCount); + + [DllImport("user32.dll", EntryPoint = "GetWindowTextW", CharSet = CharSet.Unicode)] + private static extern int GetWindowText(IntPtr hwnd, System.Text.StringBuilder text, int maxCount); + + [DllImport("user32.dll", EntryPoint = "GetWindowTextLengthW")] + private static extern int GetWindowTextLength(IntPtr hwnd); + + private delegate bool EnumWindowsProc(IntPtr hwnd, IntPtr lParam); + + [DllImport("user32.dll")] + private static extern bool EnumWindows(EnumWindowsProc callback, IntPtr lParam); + + // A visible top-level window of the process; tooltips, menus and their shadows are transient popups, not targets. + public static bool IsTargetWindow(IntPtr hwnd, int processId) { + const uint rootAncestor = 2; + if (hwnd == IntPtr.Zero || !IsWindow(hwnd) || !IsWindowVisible(hwnd) || GetAncestor(hwnd, rootAncestor) != hwnd) { + return false; + } + uint ownerProcessId; + GetWindowThreadProcessId(hwnd, out ownerProcessId); + if (ownerProcessId != (uint)processId) { + return false; + } + var className = new System.Text.StringBuilder(256); + GetClassName(hwnd, className, className.Capacity); + var name = className.ToString(); + return name != "tooltips_class32" && name != "#32768" && name != "SysShadow"; + } + + // Process.MainWindowHandle's rule (first visible unowned window in z-order) without transient popups. + public static IntPtr FindMainTargetWindow(int processId) { + const uint ownerWindow = 4; + var found = IntPtr.Zero; + EnumWindows((hwnd, _) => { + if (GetWindow(hwnd, ownerWindow) != IntPtr.Zero || !IsTargetWindow(hwnd, processId)) { + return true; + } + found = hwnd; + return false; + }, IntPtr.Zero); + return found; + } + + // Reads the caption Windows stores, without sending the target a message, so a hung window cannot block it. + public static string GetWindowTitle(IntPtr hwnd) { + var title = new System.Text.StringBuilder(GetWindowTextLength(hwnd) + 1); + GetWindowText(hwnd, title, title.Capacity); + return title.ToString(); + } + [DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y); @@ -660,7 +721,8 @@ function Find-OrcaProcess([string]$Query) { $needle = "" if ($null -ne $Query) { $needle = $Query.Trim() } if ([string]::IsNullOrWhiteSpace($needle)) { throw 'appNotFound("")' } - if ($needle.StartsWith("pid:", [System.StringComparison]::OrdinalIgnoreCase)) { + $byPid = $needle.StartsWith("pid:", [System.StringComparison]::OrdinalIgnoreCase) + if ($byPid) { $needle = $needle.Substring(4) } @@ -673,6 +735,8 @@ function Find-OrcaProcess([string]$Query) { return $match } } + # Why: a pid: query names one process; matching its digits against window titles would pick another app. + if ($byPid) { throw "appNotFound(`"$Query`")" } $processNeedle = $needle if ($processNeedle.EndsWith(".exe", [System.StringComparison]::OrdinalIgnoreCase)) { @@ -693,8 +757,8 @@ function Find-OrcaProcess([string]$Query) { throw "appNotFound(`"$Query`")" } -function Assert-OrcaProcessAllowed($Process) { - $values = @($Process.ProcessName, $Process.MainWindowTitle) | ForEach-Object { ([string]$_).ToLowerInvariant() } +function Assert-OrcaProcessAllowed($Process, [string]$WindowTitle = "") { + $values = @($Process.ProcessName, $Process.MainWindowTitle, $WindowTitle) | ForEach-Object { ([string]$_).ToLowerInvariant() } foreach ($fragment in $BlockedAppFragments) { foreach ($value in $values) { if ($value.Contains($fragment)) { @@ -721,14 +785,31 @@ function Test-OrcaBrowserProcess($Process) { $browserProcesses -contains $name } -function Get-OrcaRootElement($Process) { - if ($Process.MainWindowHandle -eq 0) { - throw "No top-level UI Automation window is available for $($Process.ProcessName)." +# Why: MainWindowHandle is a z-order guess that moves to a tooltip or another window, so the id the agent holds is the window. +function Resolve-OrcaTargetWindow($Process, $WindowId, $WindowIndex) { + if ($null -ne $WindowIndex -and [int]$WindowIndex -ne 0) { + throw "windowNotFound(`"$WindowIndex`")" } - if ([OrcaDesktopWin32]::IsWindowHung([IntPtr]$Process.MainWindowHandle, $WindowResponseTimeoutMilliseconds)) { + if ($null -ne $WindowId) { + $handle = [IntPtr][int64]$WindowId + if (-not [OrcaDesktopWin32]::IsTargetWindow($handle, [int]$Process.Id)) { + throw "windowNotFound(`"$WindowId`")" + } + } else { + $handle = [OrcaDesktopWin32]::FindMainTargetWindow([int]$Process.Id) + if ($handle -eq [IntPtr]::Zero) { + throw "No top-level UI Automation window is available for $($Process.ProcessName)." + } + } + Assert-OrcaProcessAllowed $Process ([OrcaDesktopWin32]::GetWindowTitle($handle)) + $handle +} + +function Get-OrcaRootElement($Process, [IntPtr]$WindowHandle) { + if ([OrcaDesktopWin32]::IsWindowHung($WindowHandle, $WindowResponseTimeoutMilliseconds)) { throw "window_not_responding: $(Get-OrcaAppName $Process) is not responding; nothing was read from or sent to it" } - [Windows.Automation.AutomationElement]::FromHandle([IntPtr]$Process.MainWindowHandle) + [Windows.Automation.AutomationElement]::FromHandle($WindowHandle) } function Assert-OrcaInputAllowed($Process) { @@ -739,9 +820,9 @@ function Assert-OrcaInputAllowed($Process) { throw "target_elevated: $(Get-OrcaAppName $Process) runs as administrator, so Windows blocks input to it from Orca; no input was sent" } -function Get-OrcaWindowFrame($Process, $RootElement) { +function Get-OrcaWindowFrame([IntPtr]$WindowHandle, $RootElement) { $rect = New-Object OrcaDesktopWin32+RECT - if ([OrcaDesktopWin32]::GetWindowRect([IntPtr]$Process.MainWindowHandle, [ref]$rect)) { + if ([OrcaDesktopWin32]::GetWindowRect($WindowHandle, [ref]$rect)) { return New-OrcaFrame $rect.Left $rect.Top ($rect.Right - $rect.Left) ($rect.Bottom - $rect.Top) } @@ -754,10 +835,6 @@ function Get-OrcaWindowFrame($Process, $RootElement) { $null } -function Get-OrcaWindowId($Process) { - [int64]$Process.MainWindowHandle -} - function Get-OrcaAppName($Process) { if ($Process.ProcessName -eq "ApplicationFrameHost" -and -not [string]::IsNullOrWhiteSpace($Process.MainWindowTitle)) { return [string]$Process.MainWindowTitle @@ -774,15 +851,6 @@ function New-OrcaAppRecord($Process) { } } -function Assert-OrcaWindowTarget($Process, $WindowId, $WindowIndex) { - if ($null -ne $WindowIndex -and [int]$WindowIndex -ne 0) { - throw "windowNotFound(`"$WindowIndex`")" - } - if ($null -ne $WindowId -and [int64]$WindowId -ne (Get-OrcaWindowId $Process)) { - throw "windowNotFound(`"$WindowId`")" - } -} - function Invoke-OrcaWindowPatternActivation($RootElement) { # Why: reapplying the current visual state lets the window's provider activate it when Windows refuses our raise (agent-desktop #651). # Only for maximized windows: re-applying Normal restores a snapped window's pre-snap size. @@ -794,24 +862,22 @@ function Invoke-OrcaWindowPatternActivation($RootElement) { } # Brings only the target window to the front; returns $null once it is the foreground window, otherwise why not. -function Restore-OrcaWindow($Process, $RootElement) { - $handle = [IntPtr]$Process.MainWindowHandle - if ($handle -eq [IntPtr]::Zero) { return "it has no top-level window" } - if ([OrcaDesktopWin32]::IsWindowCloaked($handle)) { +function Restore-OrcaWindow([IntPtr]$Handle, $RootElement) { + if ([OrcaDesktopWin32]::IsWindowCloaked($Handle)) { return "it is on another virtual desktop, and raising it would switch the user's desktop" } # Why: SW_RESTORE also un-maximizes, which moves the frame the agent targeted. - if ([OrcaDesktopWin32]::IsIconic($handle)) { - [void][OrcaDesktopWin32]::ShowWindow($handle, 9) + if ([OrcaDesktopWin32]::IsIconic($Handle)) { + [void][OrcaDesktopWin32]::ShowWindow($Handle, 9) } - if (Test-OrcaWindowFocused $handle) { return $null } + if (Test-OrcaWindowFocused $Handle) { return $null } # Why: Windows often refuses a background process's raise; after one refusal try the UIA route, then one more raise. - [void][OrcaDesktopWin32]::SetForegroundWindow($handle) - if (Wait-OrcaWindowFocused $handle 250) { return $null } + [void][OrcaDesktopWin32]::SetForegroundWindow($Handle) + if (Wait-OrcaWindowFocused $Handle 250) { return $null } Invoke-OrcaWindowPatternActivation $RootElement - if (Wait-OrcaWindowFocused $handle 250) { return $null } - [void][OrcaDesktopWin32]::SetForegroundWindow($handle) - if (Wait-OrcaWindowFocused $handle 250) { return $null } + if (Wait-OrcaWindowFocused $Handle 250) { return $null } + [void][OrcaDesktopWin32]::SetForegroundWindow($Handle) + if (Wait-OrcaWindowFocused $Handle 250) { return $null } "Windows refused to bring it to the front" } @@ -852,12 +918,12 @@ function Assert-OrcaKeyboardFocus([IntPtr]$WindowHandle, $Operation, $RestoreFai } # Raises the target for real mouse input, then returns its frame read after the raise (restoring moves it). -function Restore-OrcaPointerWindow($Process, $RootElement) { - $failure = Restore-OrcaWindow $Process $RootElement +function Restore-OrcaPointerWindow($Process, [IntPtr]$WindowHandle, $RootElement) { + $failure = Restore-OrcaWindow $WindowHandle $RootElement if ($null -ne $failure) { throw "window_not_focused: restoreWindow was requested but $(Get-OrcaAppName $Process) did not come to the front ($failure); no pointer input was sent" } - Get-OrcaWindowFrame $Process $RootElement + Get-OrcaWindowFrame $WindowHandle $RootElement } function New-OrcaForegroundRefusal($Process, [string]$Need, [string]$Alternative = "", [string]$Takes = "moves the user's pointer") { @@ -1109,7 +1175,7 @@ function Test-OrcaTextPatternReadOnly($Element) { } } -function Invoke-OrcaElementTextInsertion($Element, $Record, $Process, $RootElement, $Operation, [string]$Text) { +function Invoke-OrcaElementTextInsertion($Element, $Record, $Process, [IntPtr]$WindowHandle, $RootElement, $Operation, [string]$Text) { if ($null -eq $Element) { throw "stale element: element $($Record.index) is gone; run get-app-state again and use a fresh element index" } @@ -1130,12 +1196,12 @@ function Invoke-OrcaElementTextInsertion($Element, $Record, $Process, $RootEleme if (-not [bool]$Operation.restoreWindow) { throw (New-OrcaForegroundRefusal $Process "element $index accepts text only as keys, which go only to the window in front" "" "takes the user's keyboard") } - $failure = Restore-OrcaWindow $Process $RootElement - Assert-OrcaKeyboardFocus ([IntPtr]$Process.MainWindowHandle) $Operation $failure $Process + $failure = Restore-OrcaWindow $WindowHandle $RootElement + Assert-OrcaKeyboardFocus $WindowHandle $Operation $failure $Process try { $Element.SetFocus() } catch { throw "window_not_focused: element $index did not take keyboard focus; no text was sent" } - Send-OrcaText ([IntPtr]$Process.MainWindowHandle) $Text + Send-OrcaText $WindowHandle $Text [pscustomobject]@{ path = "synthetic"; actionName = "typeText"; fallbackReason = "actionUnsupported"; verification = [pscustomobject]@{ state = "unverified"; reason = "synthetic_input" } } } @@ -1602,12 +1668,12 @@ function Get-OrcaScreenshot($ScreenshotRequest, $WindowFrame, [IntPtr]$WindowHan function New-OrcaSnapshot([string]$Query, $ScreenshotRequest, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false, $TreeRoot = $null) { $process = Find-OrcaProcess $Query - Assert-OrcaWindowTarget $process $WindowId $WindowIndex + $handle = Resolve-OrcaTargetWindow $process $WindowId $WindowIndex # Root first: its hang probe must run before ShowWindow, which blocks on a hung window. - $root = Get-OrcaRootElement $process + $root = Get-OrcaRootElement $process $handle # Why: observation works behind other windows, so a refused raise still returns the snapshot. - if ($RestoreWindow) { [void](Restore-OrcaWindow $process $root) } - $windowFrame = Get-OrcaWindowFrame $process $root + if ($RestoreWindow) { [void](Restore-OrcaWindow $handle $root) } + $windowFrame = Get-OrcaWindowFrame $handle $root $walkRoot = $root if ($null -ne $TreeRoot) { $walkRoot = Find-OrcaElement $root $TreeRoot @@ -1616,13 +1682,13 @@ function New-OrcaSnapshot([string]$Query, $ScreenshotRequest, $WindowId = $null, } } $tree = Render-OrcaTree $walkRoot $windowFrame (Test-OrcaBrowserProcess $process) - $screenshot = Get-OrcaScreenshot $ScreenshotRequest $windowFrame ([IntPtr]$process.MainWindowHandle) + $screenshot = Get-OrcaScreenshot $ScreenshotRequest $windowFrame $handle [pscustomobject]@{ snapshotId = [guid]::NewGuid().ToString() app = New-OrcaAppRecord $process - windowTitle = $process.MainWindowTitle - windowId = Get-OrcaWindowId $process + windowTitle = [OrcaDesktopWin32]::GetWindowTitle($handle) + windowId = [int64]$handle windowBounds = $windowFrame screenshotBase64 = if ($null -ne $screenshot) { $screenshot.base64 } else { $null } screenshotFormat = if ($null -ne $screenshot) { $screenshot.format } else { $null } @@ -1648,8 +1714,9 @@ function Get-OrcaAppList { function Get-OrcaWindowList([string]$Query) { $process = Find-OrcaProcess $Query - $root = Get-OrcaRootElement $process - $windowFrame = Get-OrcaWindowFrame $process $root + $handle = Resolve-OrcaTargetWindow $process $null $null + $root = Get-OrcaRootElement $process $handle + $windowFrame = Get-OrcaWindowFrame $handle $root $x = $null $y = $null $width = 0 @@ -1666,8 +1733,8 @@ function Get-OrcaWindowList([string]$Query) { windows = @([pscustomobject]@{ index = 0 app = $app - id = Get-OrcaWindowId $process - title = $process.MainWindowTitle + id = [int64]$handle + title = [OrcaDesktopWin32]::GetWindowTitle($handle) x = $x y = $y width = $width @@ -1675,7 +1742,7 @@ function Get-OrcaWindowList([string]$Query) { isMinimized = $false isOffscreen = $false screenIndex = $null - platform = [pscustomobject]@{ backend = "uia"; nativeWindowHandle = Get-OrcaWindowId $process } + platform = [pscustomobject]@{ backend = "uia"; nativeWindowHandle = [int64]$handle } }) } } @@ -2480,17 +2547,16 @@ function Invoke-OrcaOperation($Operation) { $process = Find-OrcaProcess $Operation.app Assert-OrcaInputAllowed $process - Assert-OrcaWindowTarget $process $Operation.windowId $Operation.windowIndex - $root = Get-OrcaRootElement $process + $handle = Resolve-OrcaTargetWindow $process $Operation.windowId $Operation.windowIndex + $root = Get-OrcaRootElement $process $handle $restoreWindow = [bool]$Operation.restoreWindow - $windowFrame = if ($null -ne $Operation.windowBounds) { $Operation.windowBounds } else { Get-OrcaWindowFrame $process $root } + $windowFrame = if ($null -ne $Operation.windowBounds) { $Operation.windowBounds } else { Get-OrcaWindowFrame $handle $root } $element = Find-OrcaElement $root $Operation.element $fromElement = Find-OrcaElement $root $Operation.fromElement $toElement = Find-OrcaElement $root $Operation.toElement - $handle = [IntPtr]$process.MainWindowHandle $typesIntoElement = $Operation.tool -eq "type_text" -and $null -ne $Operation.element if ($Operation.tool -in @("type_text", "press_key", "hotkey", "paste_text") -and -not $typesIntoElement) { - $restoreFailure = if ($restoreWindow) { Restore-OrcaWindow $process $root } else { $null } + $restoreFailure = if ($restoreWindow) { Restore-OrcaWindow $handle $root } else { $null } Assert-OrcaKeyboardFocus $handle $Operation $restoreFailure $process } $foregroundBefore = [OrcaDesktopWin32]::GetForegroundWindow() @@ -2527,7 +2593,7 @@ function Invoke-OrcaOperation($Operation) { if (-not $restoreWindow) { throw (New-OrcaForegroundRefusal $process $pointerNeed $alternative) } - $windowFrame = Restore-OrcaPointerWindow $process $root + $windowFrame = Restore-OrcaPointerWindow $process $handle $root $point = Get-OrcaPointerPoint $element $Operation.element $Operation.x $Operation.y "x" "y" $windowFrame $cursor = Get-OrcaCursorPosition try { @@ -2574,7 +2640,7 @@ function Invoke-OrcaOperation($Operation) { throw (New-OrcaForegroundRefusal $process "nothing under $where scrolls $($Operation.direction) through UI Automation, so this scroll needs the real mouse wheel") } # Why: the wheel scrolls whatever is under the pointer, so the window must be on top. - $windowFrame = Restore-OrcaPointerWindow $process $root + $windowFrame = Restore-OrcaPointerWindow $process $handle $root $point = Get-OrcaPointerPoint $element $Operation.element $Operation.x $Operation.y "x" "y" $windowFrame $cursor = Get-OrcaCursorPosition try { @@ -2596,7 +2662,7 @@ function Invoke-OrcaOperation($Operation) { if (-not $restoreWindow) { throw (New-OrcaForegroundRefusal $process "a drag needs the real mouse, because UI Automation has no drag") } - $windowFrame = Restore-OrcaPointerWindow $process $root + $windowFrame = Restore-OrcaPointerWindow $process $handle $root $from = Get-OrcaPointerPoint $fromElement $Operation.fromElement $Operation.from_x $Operation.from_y "from_x" "from_y" $windowFrame $to = Get-OrcaPointerPoint $toElement $Operation.toElement $Operation.to_x $Operation.to_y "to_x" "to_y" $windowFrame $cursor = Get-OrcaCursorPosition @@ -2610,7 +2676,7 @@ function Invoke-OrcaOperation($Operation) { "type_text" { $text = Get-OrcaRequiredString $Operation.text "text" if ($typesIntoElement) { - $action = Invoke-OrcaElementTextInsertion $element $Operation.element $process $root $Operation $text + $action = Invoke-OrcaElementTextInsertion $element $Operation.element $process $handle $root $Operation $text } else { Send-OrcaText $handle $text $action = [pscustomobject]@{ path = "synthetic"; actionName = "typeText"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "synthetic_input" } } @@ -2647,14 +2713,16 @@ function Invoke-OrcaOperation($Operation) { try { try { - $snapshot = New-OrcaSnapshot $Operation.app $screenshotRequest $Operation.windowId $Operation.windowIndex + # Why: the observe reads the process and window the input went to, even if the action retitled it. + $snapshot = New-OrcaSnapshot "pid:$($process.Id)" $screenshotRequest ([int64]$handle) $null } catch { if ($_.Exception.Message -like "window_not_responding:*") { throw } - if ($null -eq $Operation.windowId -and $null -eq $Operation.windowIndex) { throw } + # Why: only a target window the action closed may hand the observe to the app's main window. + if ([OrcaDesktopWin32]::IsTargetWindow($handle, [int]$process.Id)) { throw } if ($null -eq $action.verification) { $action | Add-Member -NotePropertyName verification -NotePropertyValue ([pscustomobject]@{ state = "unverified"; reason = "window_changed" }) } - $snapshot = New-OrcaSnapshot $Operation.app $screenshotRequest $null $null + $snapshot = New-OrcaSnapshot "pid:$($process.Id)" $screenshotRequest $null $null } } catch { # Why: the input was already sent, so the pre-action "nothing was sent" wording would invite a duplicate retry. diff --git a/tests/e2e/computer-windows.e2e.ts b/tests/e2e/computer-windows.e2e.ts index e083df394e7..3a5409326fc 100644 --- a/tests/e2e/computer-windows.e2e.ts +++ b/tests/e2e/computer-windows.e2e.ts @@ -1,5 +1,7 @@ +import { existsSync } from 'node:fs' import { readFile } from 'node:fs/promises' -import { afterAll, beforeAll, describe, expect, test } from 'vitest' +import { join } from 'node:path' +import { afterAll, beforeAll, describe, expect, test, vi } from 'vitest' import type { ComputerActionResult, ComputerListAppsResult, @@ -178,6 +180,90 @@ describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Notepad)', ( } }) + test('a window id keeps its window when another window of the app becomes the main one', async () => { + // A topmost second window comes first in z-order, so Process.MainWindowHandle moves to it. + const fixture = await launchWindowFixture( + [ + "$form.Text = 'orca-e2e-target'", + '$button = New-Object System.Windows.Forms.Button', + "$button.Text = 'Target press'", + "$button.Add_Click({ $form.Text = 'orca-e2e-pressed' })", + '$form.Controls.Add($button)', + "$targetPath = Join-Path $PSScriptRoot 'target-window'" + ].join('\n'), + [ + '$second = New-Object System.Windows.Forms.Form', + "$second.Text = 'orca-e2e-second'", + '$second.TopMost = $true', + '$second.Show()', + '[IO.File]::WriteAllText($targetPath, [string][int64]$form.Handle)' + ].join('; ') + ) + try { + const targetPath = join(fixture.dir, 'target-window') + await vi.waitFor(() => expect(existsSync(targetPath)).toBe(true), { + timeout: 15_000, + interval: 100 + }) + const targetWindowId = Number(await readFile(targetPath, 'utf8')) + const main = parseJsonOutput<{ result: ComputerSnapshotResult }>( + ( + await runOrcaCli([ + 'computer', + 'get-app-state', + '--app', + `pid:${fixture.pid}`, + '--no-screenshot', + '--json' + ]) + ).stdout + ).result + expect(main.snapshot.window.title).toBe('orca-e2e-second') + const state = parseJsonOutput<{ result: ComputerSnapshotResult }>( + ( + await runOrcaCli([ + 'computer', + 'get-app-state', + '--app', + `pid:${fixture.pid}`, + '--window-id', + String(targetWindowId), + '--no-screenshot', + '--json' + ]) + ).stdout + ).result + expect(state.snapshot.window.id).toBe(targetWindowId) + expect(state.snapshot.window.title).toBe('orca-e2e-target') + const button = findRoleIndex( + state.snapshot.treeText, + /^\s*(\d+)\s+button Target press(?:,|$)/m + ) + expect(button, state.snapshot.treeText).toBeGreaterThanOrEqual(0) + const pressed = parseJsonOutput<{ result: ComputerActionResult }>( + ( + await runOrcaCli([ + 'computer', + 'click', + '--app', + `pid:${fixture.pid}`, + '--window-id', + String(targetWindowId), + '--element-index', + String(button), + '--no-screenshot', + '--json' + ]) + ).stdout + ).result + expect(pressed.action?.path).toBe('accessibility') + expect(pressed.snapshot.window.id).toBe(targetWindowId) + expect(pressed.snapshot.window.title).toBe('orca-e2e-pressed') + } finally { + await closeWindowFixture(fixture) + } + }) + test('a hung window reports a timeout error and later calls still succeed', async () => { const hung = await launchHungWindowFixture() try {