mirror of
https://github.com/stablyai/orca.git
synced 2026-11-13 09:27:29 +00:00
fix(computer-use): use UI Automation for Windows scroll, expand and slider actions (#41429)
* fix(computer-use): scroll, expand and set sliders on Windows without raising the window * fix(computer-use): wait for Windows scroll and slider writes to settle before verifying * fix(computer-use): reject NaN for Windows slider set-value * test(computer-use): wait on the Windows fixture state with vi.waitFor and a schema * test(computer-use): check activation per action against a holder window in the Windows fixture * test(computer-use): log fixture window events for activation failures * fix(computer-use): report foreground when a Windows app raises itself for a UI Automation action WPF activates its own window for Scroll, ExpandCollapse, RangeValue and SelectionItem calls from any UIA client, so the runtime now compares the foreground window before and after a pattern action instead of assuming background. The Windows e2e checks delivery against the fixture's own activation count, with a holder window in another process. * test(computer-use): skip an unstarted holder when closing the Windows fixture * fix(computer-use): bound Windows pattern scroll paging and tighten slider read-back * fix(computer-use): cap Windows slider read-back tolerance and run the UIA e2e only on CI
This commit is contained in:
Kelvin Amoaba
@@ -54,6 +54,8 @@ on:
|
||||
- 'tests/e2e/computer-linux.e2e.ts'
|
||||
- 'tests/e2e/computer-windows.e2e.ts'
|
||||
- 'tests/e2e/computer-windows-store.e2e.ts'
|
||||
- 'tests/e2e/computer-windows-background.e2e.ts'
|
||||
- 'tests/e2e/helpers/windows-semantic-controls-fixture.*'
|
||||
- 'tests/e2e/helpers/computer-cli-driver.ts'
|
||||
- 'tests/e2e/helpers/computer-driver.ts'
|
||||
- 'tests/e2e/vitest.config.ts'
|
||||
@@ -151,4 +153,4 @@ jobs:
|
||||
- run: pnpm verify:computer-native
|
||||
- run: pnpm build:cli
|
||||
- run: pnpm run build:electron-vite:parallel
|
||||
- run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts tests/e2e/computer-windows-store.e2e.ts
|
||||
- run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts tests/e2e/computer-windows-store.e2e.ts tests/e2e/computer-windows-background.e2e.ts
|
||||
|
||||
@@ -89,7 +89,7 @@ Prefer semantic actions (`click`, `set-value`, `perform-secondary-action`) over
|
||||
|
||||
`hotkey` and `press-key` refuse shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session, such as `Cmd+Q`, `Shift+Cmd+Q`, `Ctrl+Cmd+Q` and `Option+Cmd+Esc` on macOS, `Alt+F4` and `Win+L` on Windows, and `Ctrl+Q`, `Alt+F4`, `Super+L` and `Ctrl+Alt+Delete` on Linux. The error code is `system_shortcut_blocked`. Pass `--allow-system-shortcut` to send one anyway.
|
||||
|
||||
On macOS, actions run in the background: the target app isn't brought forward, your frontmost app and key window stay put, and the pointer doesn't move, so you can keep working while an agent drives another window. Each action result reports `action.delivery` (`background` or `foreground`). Pass `--restore-window` only when an app ignores background input, such as drags in Chromium or Electron apps or games; it brings the app forward and uses the real pointer and keyboard. Popup menus and web `<select>` controls are set with `set-value` instead of a click. On Linux and Windows, clicks, scrolling, drags and keyboard input still bring the window forward.
|
||||
On macOS, actions run in the background: the target app isn't brought forward, your frontmost app and key window stay put, and the pointer doesn't move, so you can keep working while an agent drives another window. Each action result reports `action.delivery` (`background` or `foreground`). Pass `--restore-window` only when an app ignores background input, such as drags in Chromium or Electron apps or games; it brings the app forward and uses the real pointer and keyboard. Popup menus and web `<select>` controls are set with `set-value` instead of a click. On Linux, clicks, scrolling, drags and keyboard input still bring the window forward. On Windows, element clicks, `set-value` (including sliders), element scrolls and secondary actions (`Invoke`, `Toggle`, `Select`, `Expand`, `Collapse`, `ScrollToVisible`) go through UI Automation when the control supports them, without moving the pointer or sending keys. Some apps still bring their own window forward when driven this way (WPF apps do for most of these), so `action.delivery` reports `foreground` whenever the target window came forward. Anything else brings the window forward.
|
||||
|
||||
When accessibility targeting fails, fall back carefully to coordinates:
|
||||
|
||||
|
||||
@@ -51,6 +51,10 @@ public static class OrcaDesktopWin32 {
|
||||
}
|
||||
return Foreground;
|
||||
}
|
||||
public static uint GetWindowThreadProcessId(IntPtr hwnd, out uint processId) {
|
||||
processId = hwnd == new IntPtr(101) ? 4242u : 1u;
|
||||
return 1;
|
||||
}
|
||||
public static bool SetCursorPos(int x, int y) {
|
||||
Calls.Add("move:" + x + "," + y);
|
||||
return true;
|
||||
@@ -74,7 +78,7 @@ $functions = @(
|
||||
"Get-OrcaRequiredNumber", "Get-OrcaPositiveInteger", "Get-OrcaPositiveNumber",
|
||||
"Get-OrcaScreenPoint", "Get-OrcaElementScreenPoint", "Get-OrcaModifierVirtualKey",
|
||||
"Get-OrcaClickModifierVirtualKeys", "Send-OrcaMouseClick", "Send-OrcaDrag", "Invoke-OrcaOperation",
|
||||
"Assert-OrcaInputAllowed"
|
||||
"Assert-OrcaInputAllowed", "Test-OrcaTargetTookForeground"
|
||||
)
|
||||
foreach ($definition in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false)) {
|
||||
if ($definition.Name -in $functions) { . ([scriptblock]::Create($definition.Extent.Text)) }
|
||||
@@ -87,6 +91,7 @@ function Get-OrcaRootElement { $null }
|
||||
function Get-OrcaWindowFrame { @{ x = [OrcaDesktopWin32]::WindowX; y = 200 } }
|
||||
function Find-OrcaElement($RootElement, $Record) { $Record }
|
||||
function Invoke-OrcaPrimaryAction($Element) { $null -ne $Element }
|
||||
function Invoke-OrcaScrollPattern($Element, $RootElement, $Direction, $Pages) { $null -ne $Element }
|
||||
function New-OrcaSnapshot { [pscustomobject]@{} }
|
||||
|
||||
foreach ($case in @(
|
||||
@@ -112,6 +117,7 @@ foreach ($case in @(
|
||||
if ($inputCalls.Count -eq 0) { throw "$($case.tool) did not send pointer input after activation" }
|
||||
if ([OrcaDesktopWin32]::FocusReads -lt 2) { throw "$($case.tool) did not wait for activation" }
|
||||
Assert-TestEqual $inputCalls[0] "move:505,206" "$($case.tool) geometry after delayed activation"
|
||||
Assert-TestEqual $result.action.delivery "foreground" "$($case.tool) pointer delivery"
|
||||
} else {
|
||||
Assert-TestEqual $denied $true "$($case.tool) denied activation"
|
||||
Assert-TestEqual $inputCalls.Count 0 "$($case.tool) denied activation sends no pointer or modifier input"
|
||||
@@ -119,10 +125,18 @@ foreach ($case in @(
|
||||
}
|
||||
}
|
||||
|
||||
[OrcaDesktopWin32]::Reset($false)
|
||||
$result = Invoke-OrcaOperation ([pscustomobject]@{ tool = "click"; app = "Fixture"; element = [pscustomobject]@{ index = 1 } })
|
||||
Assert-TestEqual $result.action.path "accessibility" "pattern click path"
|
||||
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "pattern click bypasses activation and global input"
|
||||
foreach ($case in @(
|
||||
@{ tool = "click" },
|
||||
@{ tool = "scroll"; direction = "down"; pages = 1 }
|
||||
)) {
|
||||
[OrcaDesktopWin32]::Reset($false)
|
||||
$operation = [pscustomobject]@{ app = "Fixture"; element = [pscustomobject]@{ index = 1 } }
|
||||
foreach ($key in $case.Keys) { $operation | Add-Member -NotePropertyName $key -NotePropertyValue $case[$key] }
|
||||
$result = Invoke-OrcaOperation $operation
|
||||
Assert-TestEqual $result.action.path "accessibility" "pattern $($case.tool) path"
|
||||
Assert-TestEqual $result.action.delivery "background" "pattern $($case.tool) delivery"
|
||||
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "pattern $($case.tool) bypasses activation and global input"
|
||||
}
|
||||
foreach ($case in @(
|
||||
@{ tool = "click"; element = [pscustomobject]@{ index = 1 } },
|
||||
@{ tool = "click"; x = 5; y = 6; restoreWindow = $true },
|
||||
@@ -137,4 +151,10 @@ foreach ($case in @(
|
||||
if ($message -notlike "target_elevated:*") { throw "$($case.tool) elevated target expected target_elevated, received '$message'" }
|
||||
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "$($case.tool) elevated target sends no activation or input"
|
||||
}
|
||||
# Why: WPF activates its own window for most patterns; the result must say so rather than claim background.
|
||||
[OrcaDesktopWin32]::Reset($false)
|
||||
function Invoke-OrcaPrimaryAction($Element) { [OrcaDesktopWin32]::Foreground = [IntPtr]101; $true }
|
||||
$result = Invoke-OrcaOperation ([pscustomobject]@{ tool = "click"; app = "Fixture"; element = [pscustomobject]@{ index = 1 } })
|
||||
Assert-TestEqual $result.action.path "accessibility" "self-activating pattern click path"
|
||||
Assert-TestEqual $result.action.delivery "foreground" "self-activating pattern click delivery"
|
||||
Write-Output "windows-pointer-tests-ok"
|
||||
|
||||
@@ -104,6 +104,9 @@ public static class OrcaDesktopWin32 {
|
||||
[DllImport("user32.dll")]
|
||||
public static extern IntPtr GetForegroundWindow();
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
public static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint processId);
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
public static extern bool SetCursorPos(int x, int y);
|
||||
|
||||
@@ -561,6 +564,14 @@ function Restore-OrcaWindow($Process) {
|
||||
[void][OrcaDesktopWin32]::SetForegroundWindow([IntPtr]$Process.MainWindowHandle)
|
||||
}
|
||||
|
||||
function Test-OrcaTargetTookForeground([IntPtr]$Before, $Process) {
|
||||
$after = [OrcaDesktopWin32]::GetForegroundWindow()
|
||||
if ($after -eq $Before) { return $false }
|
||||
$owner = [uint32]0
|
||||
[void][OrcaDesktopWin32]::GetWindowThreadProcessId($after, [ref]$owner)
|
||||
$owner -eq $Process.Id
|
||||
}
|
||||
|
||||
function Test-OrcaWindowFocused([IntPtr]$WindowHandle) {
|
||||
[OrcaDesktopWin32]::GetForegroundWindow() -eq $WindowHandle
|
||||
}
|
||||
@@ -647,15 +658,50 @@ function Get-OrcaValueText($Element) {
|
||||
}
|
||||
}
|
||||
|
||||
function Get-OrcaActions($Element) {
|
||||
function Get-OrcaRangeValueText($Element) {
|
||||
try {
|
||||
$pattern = $Element.GetCurrentPattern([Windows.Automation.RangeValuePattern]::Pattern)
|
||||
([double]$pattern.Current.Value).ToString([Globalization.CultureInfo]::InvariantCulture)
|
||||
} catch {
|
||||
""
|
||||
}
|
||||
}
|
||||
|
||||
function Test-OrcaRangeValueWritable($Element) {
|
||||
try {
|
||||
-not $Element.GetCurrentPattern([Windows.Automation.RangeValuePattern]::Pattern).Current.IsReadOnly
|
||||
} catch {
|
||||
$false
|
||||
}
|
||||
}
|
||||
|
||||
function Get-OrcaExpandCollapseActions($Element) {
|
||||
try {
|
||||
$state = $Element.GetCurrentPattern([Windows.Automation.ExpandCollapsePattern]::Pattern).Current.ExpandCollapseState
|
||||
} catch {
|
||||
return @()
|
||||
}
|
||||
switch ($state) {
|
||||
([Windows.Automation.ExpandCollapseState]::Collapsed) { return @("Expand") }
|
||||
([Windows.Automation.ExpandCollapseState]::Expanded) { return @("Collapse") }
|
||||
([Windows.Automation.ExpandCollapseState]::PartiallyExpanded) { return @("Expand", "Collapse") }
|
||||
}
|
||||
@()
|
||||
}
|
||||
|
||||
function Get-OrcaActions($Element, $Patterns = $null) {
|
||||
if ($null -eq $Patterns) { $Patterns = @($Element.GetSupportedPatterns()) }
|
||||
$actions = New-Object System.Collections.Generic.List[string]
|
||||
foreach ($pattern in $Element.GetSupportedPatterns()) {
|
||||
foreach ($pattern in $Patterns) {
|
||||
$name = [string]$pattern.ProgrammaticName
|
||||
if ($name -like "InvokePatternIdentifiers.Pattern") { $actions.Add("Invoke") }
|
||||
elseif ($name -like "TogglePatternIdentifiers.Pattern") { $actions.Add("Toggle") }
|
||||
elseif ($name -like "SelectionItemPatternIdentifiers.Pattern") { $actions.Add("Select") }
|
||||
elseif ($name -like "ScrollPatternIdentifiers.Pattern") { $actions.Add("Scroll") }
|
||||
elseif ($name -like "ValuePatternIdentifiers.Pattern") { $actions.Add("SetValue") }
|
||||
elseif ($name -like "RangeValuePatternIdentifiers.Pattern") { if (Test-OrcaRangeValueWritable $Element) { $actions.Add("SetValue") } }
|
||||
elseif ($name -like "ExpandCollapsePatternIdentifiers.Pattern") { foreach ($action in @(Get-OrcaExpandCollapseActions $Element)) { $actions.Add($action) } }
|
||||
elseif ($name -like "ScrollItemPatternIdentifiers.Pattern") { $actions.Add("ScrollToVisible") }
|
||||
}
|
||||
@($actions | Select-Object -Unique)
|
||||
}
|
||||
@@ -751,6 +797,11 @@ function Test-OrcaPlainTextSubtree($Element, [int]$MaxDepth = 4) {
|
||||
function New-OrcaElementRecord($Element, [int]$Index, $WindowFrame) {
|
||||
$controlType = try { [string]$Element.Current.ControlType.ProgrammaticName } catch { "" }
|
||||
$nativeWindowHandle = try { [int64]$Element.Current.NativeWindowHandle } catch { 0 }
|
||||
$patterns = @($Element.GetSupportedPatterns())
|
||||
$value = Get-OrcaValueText $Element
|
||||
if ([string]::IsNullOrEmpty($value) -and $patterns -contains [Windows.Automation.RangeValuePattern]::Pattern) {
|
||||
$value = Get-OrcaRangeValueText $Element
|
||||
}
|
||||
[pscustomobject]@{
|
||||
index = $Index
|
||||
runtimeId = @(Get-OrcaRuntimeId $Element)
|
||||
@@ -759,12 +810,12 @@ function New-OrcaElementRecord($Element, [int]$Index, $WindowFrame) {
|
||||
controlType = $controlType
|
||||
localizedControlType = Get-OrcaProperty $Element "LocalizedControlType"
|
||||
className = Get-OrcaProperty $Element "ClassName"
|
||||
value = Get-OrcaValueText $Element
|
||||
value = $value
|
||||
isSelected = Test-OrcaElementSelected $Element
|
||||
isDisabled = Test-OrcaElementDisabled $Element
|
||||
nativeWindowHandle = $nativeWindowHandle
|
||||
frame = Get-OrcaElementFrame $Element $WindowFrame
|
||||
actions = @(Get-OrcaActions $Element)
|
||||
actions = @(Get-OrcaActions $Element $patterns)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1175,7 +1226,8 @@ function Invoke-OrcaPrimaryAction($Element) {
|
||||
|
||||
function Invoke-OrcaNamedAction($Element, [string]$Action) {
|
||||
$wanted = ""
|
||||
if ($null -ne $Action) { $wanted = $Action.Trim().ToLowerInvariant() }
|
||||
# Why: accept the rendered name ("ScrollToVisible") and macOS's spaced form ("scroll to visible").
|
||||
if ($null -ne $Action) { $wanted = ($Action -replace "\s+", "").ToLowerInvariant() }
|
||||
switch ($wanted) {
|
||||
"invoke" {
|
||||
$pattern = $Element.GetCurrentPattern([Windows.Automation.InvokePattern]::Pattern)
|
||||
@@ -1192,21 +1244,128 @@ function Invoke-OrcaNamedAction($Element, [string]$Action) {
|
||||
$pattern.Toggle()
|
||||
return $true
|
||||
}
|
||||
"expand" {
|
||||
$pattern = $Element.GetCurrentPattern([Windows.Automation.ExpandCollapsePattern]::Pattern)
|
||||
$pattern.Expand()
|
||||
return $true
|
||||
}
|
||||
"collapse" {
|
||||
$pattern = $Element.GetCurrentPattern([Windows.Automation.ExpandCollapsePattern]::Pattern)
|
||||
$pattern.Collapse()
|
||||
return $true
|
||||
}
|
||||
"scrolltovisible" {
|
||||
$pattern = $Element.GetCurrentPattern([Windows.Automation.ScrollItemPattern]::Pattern)
|
||||
$pattern.ScrollIntoView()
|
||||
return $true
|
||||
}
|
||||
default {
|
||||
return $false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Test-OrcaRangeValueMatches([string]$Actual, [double]$Expected) {
|
||||
# Why: allow float32 rounding (0.1 reads back 0.100000001), but cap it so a snapped neighbour never verifies at any size.
|
||||
$actualNumber = 0.0
|
||||
[double]::TryParse($Actual, [Globalization.NumberStyles]::Float, [Globalization.CultureInfo]::InvariantCulture, [ref]$actualNumber) -and
|
||||
[Math]::Abs($actualNumber - $Expected) -le [Math]::Min(1e-6 * [Math]::Max(1.0, [Math]::Abs($Expected)), 1e-3)
|
||||
}
|
||||
|
||||
function Set-OrcaElementValue($Element, [string]$Value) {
|
||||
try {
|
||||
$pattern = $Element.GetCurrentPattern([Windows.Automation.ValuePattern]::Pattern)
|
||||
if (-not $pattern.Current.IsReadOnly) {
|
||||
$pattern.SetValue($Value)
|
||||
return $true
|
||||
return [pscustomobject]@{ verification = $null }
|
||||
}
|
||||
} catch {}
|
||||
$false
|
||||
if (-not (Test-OrcaRangeValueWritable $Element)) { return $null }
|
||||
$range = $Element.GetCurrentPattern([Windows.Automation.RangeValuePattern]::Pattern)
|
||||
$culture = [Globalization.CultureInfo]::InvariantCulture
|
||||
$number = 0.0
|
||||
if (-not [double]::TryParse($Value, [Globalization.NumberStyles]::Float, $culture, [ref]$number) -or [double]::IsNaN($number)) {
|
||||
throw "value must be a number for this range control"
|
||||
}
|
||||
$minimum = [double]$range.Current.Minimum
|
||||
$maximum = [double]$range.Current.Maximum
|
||||
if ($number -lt $minimum -or $number -gt $maximum) {
|
||||
throw "value $Value is outside this range control's $($minimum.ToString($culture))..$($maximum.ToString($culture))"
|
||||
}
|
||||
$range.SetValue($number)
|
||||
# Why: Chromium applies the write asynchronously, so poll briefly like macOS's settled readback.
|
||||
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
while ($true) {
|
||||
$actual = Get-OrcaRangeValueText $Element
|
||||
$matched = Test-OrcaRangeValueMatches $actual $number
|
||||
if ($matched -or $stopwatch.ElapsedMilliseconds -ge 500) { break }
|
||||
Start-Sleep -Milliseconds 25
|
||||
}
|
||||
$verification = if ($matched) {
|
||||
[pscustomobject]@{ state = "verified"; property = "value"; expected = $Value; actualPreview = $actual }
|
||||
} elseif ([string]::IsNullOrEmpty($actual)) {
|
||||
# Why: a failed read is not evidence the write failed; only a read-back number can contradict it.
|
||||
[pscustomobject]@{ state = "unverified"; reason = "readback_unsupported"; expected = $Value }
|
||||
} else {
|
||||
[pscustomobject]@{ state = "unverified"; reason = "value_mismatch"; expected = $Value; actualPreview = $actual }
|
||||
}
|
||||
[pscustomobject]@{ verification = $verification }
|
||||
}
|
||||
|
||||
function Get-OrcaScrollPattern($Element, $RootElement, [bool]$Vertical) {
|
||||
$rootId = Get-OrcaRuntimeId $RootElement
|
||||
$current = $Element
|
||||
# Why: an item's ScrollPattern lives on its list or viewport, the same container a wheel at its center would scroll.
|
||||
while ($null -ne $current) {
|
||||
try {
|
||||
$pattern = $current.GetCurrentPattern([Windows.Automation.ScrollPattern]::Pattern)
|
||||
$scrollable = if ($Vertical) { $pattern.Current.VerticallyScrollable } else { $pattern.Current.HorizontallyScrollable }
|
||||
if ($scrollable) { return $pattern }
|
||||
} catch {}
|
||||
if (Test-OrcaSameRuntimeId (Get-OrcaRuntimeId $current) $rootId) { break }
|
||||
$current = try { [Windows.Automation.TreeWalker]::RawViewWalker.GetParent($current) } catch { $null }
|
||||
}
|
||||
$null
|
||||
}
|
||||
|
||||
function Get-OrcaScrollPercent($Pattern, [bool]$Vertical) {
|
||||
try {
|
||||
if ($Vertical) { return [double]$Pattern.Current.VerticalScrollPercent }
|
||||
[double]$Pattern.Current.HorizontalScrollPercent
|
||||
} catch {
|
||||
$null
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-OrcaScrollPattern($Element, $RootElement, [string]$Direction, [double]$Pages) {
|
||||
# Why: ScrollPattern moves in whole pages; fractional pages keep the wheel, as on macOS.
|
||||
if ($null -eq $Element -or $Pages -ne [Math]::Floor($Pages)) { return $false }
|
||||
$vertical = $Direction -in @("up", "down")
|
||||
$pattern = Get-OrcaScrollPattern $Element $RootElement $vertical
|
||||
if ($null -eq $pattern) { return $false }
|
||||
$amount = if ($Direction -in @("down", "right")) { [Windows.Automation.ScrollAmount]::LargeIncrement } else { [Windows.Automation.ScrollAmount]::LargeDecrement }
|
||||
$none = [Windows.Automation.ScrollAmount]::NoAmount
|
||||
# Why: without a readable position the loop can't tell the end of the content, so the wheel handles it.
|
||||
$before = Get-OrcaScrollPercent $pattern $vertical
|
||||
if ($null -eq $before) { return $false }
|
||||
for ($page = 0; $page -lt $Pages; $page++) {
|
||||
try {
|
||||
if ($vertical) { $pattern.Scroll($none, $amount) } else { $pattern.Scroll($amount, $none) }
|
||||
} catch {
|
||||
# Why: a refusal before any page moved falls back to the wheel; after that it is the end of the content.
|
||||
return $page -gt 0
|
||||
}
|
||||
# Why: WPF and Chromium move the viewport after Scroll returns; an unmoved or unreadable percent after the wait is the end.
|
||||
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
||||
while ($true) {
|
||||
$after = Get-OrcaScrollPercent $pattern $vertical
|
||||
if ($after -ne $before -or $stopwatch.ElapsedMilliseconds -ge 500) { break }
|
||||
Start-Sleep -Milliseconds 25
|
||||
}
|
||||
if ($null -eq $after -or $after -eq $before) { break }
|
||||
$before = $after
|
||||
}
|
||||
$true
|
||||
}
|
||||
|
||||
function Get-OrcaRequiredNumber($Value, [string]$Name) {
|
||||
@@ -1479,6 +1638,8 @@ function Invoke-OrcaOperation($Operation) {
|
||||
if ($Operation.tool -in @("type_text", "press_key", "hotkey", "paste_text")) {
|
||||
Assert-OrcaKeyboardFocus $handle $Operation
|
||||
}
|
||||
# Why: pattern paths need an element; reading the foreground only then keeps coordinate input untouched.
|
||||
$foregroundBefore = if ($null -ne $element) { [OrcaDesktopWin32]::GetForegroundWindow() } else { [IntPtr]::Zero }
|
||||
$action = $null
|
||||
|
||||
switch ($Operation.tool) {
|
||||
@@ -1508,7 +1669,8 @@ function Invoke-OrcaOperation($Operation) {
|
||||
$action = [pscustomobject]@{ path = "accessibility"; actionName = $Operation.action; fallbackReason = $null }
|
||||
}
|
||||
"scroll" {
|
||||
$delta = 120 * [int][Math]::Ceiling((Get-OrcaPositiveNumber $Operation.pages "pages"))
|
||||
$pages = Get-OrcaPositiveNumber $Operation.pages "pages"
|
||||
$delta = 120 * [int][Math]::Ceiling($pages)
|
||||
$mouseEvent = $MouseEvents.Wheel
|
||||
if ($Operation.direction -eq "down") {
|
||||
$delta = -1 * $delta
|
||||
@@ -1520,12 +1682,18 @@ function Invoke-OrcaOperation($Operation) {
|
||||
} elseif ($Operation.direction -ne "up") {
|
||||
throw "unsupported scroll direction: $($Operation.direction)"
|
||||
}
|
||||
$windowFrame = Restore-OrcaPointerWindow $process $root
|
||||
$point = Get-OrcaElementScreenPoint $element
|
||||
if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }
|
||||
[void][OrcaDesktopWin32]::SetCursorPos([int]$point.x, [int]$point.y)
|
||||
[OrcaDesktopWin32]::mouse_event($mouseEvent, 0, 0, $delta, [UIntPtr]::Zero)
|
||||
$action = [pscustomobject]@{ path = "synthetic"; actionName = "scroll"; fallbackReason = $null }
|
||||
if (Invoke-OrcaScrollPattern $element $root $Operation.direction $pages) {
|
||||
$action = [pscustomobject]@{ path = "accessibility"; actionName = "scroll"; fallbackReason = $null }
|
||||
} else {
|
||||
# Why: the wheel scrolls whatever is under the pointer, so the window must be on top.
|
||||
$windowFrame = Restore-OrcaPointerWindow $process $root
|
||||
$point = Get-OrcaElementScreenPoint $element
|
||||
if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame }
|
||||
[void][OrcaDesktopWin32]::SetCursorPos([int]$point.x, [int]$point.y)
|
||||
[OrcaDesktopWin32]::mouse_event($mouseEvent, 0, 0, $delta, [UIntPtr]::Zero)
|
||||
$fallbackReason = if ($null -ne $element) { "actionUnsupported" } else { $null }
|
||||
$action = [pscustomobject]@{ path = "synthetic"; actionName = "scroll"; fallbackReason = $fallbackReason }
|
||||
}
|
||||
}
|
||||
"drag" {
|
||||
$windowFrame = Restore-OrcaPointerWindow $process $root
|
||||
@@ -1565,15 +1733,25 @@ function Invoke-OrcaOperation($Operation) {
|
||||
$action = [pscustomobject]@{ path = "clipboard"; actionName = "paste"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "clipboard_paste" } }
|
||||
}
|
||||
"set_value" {
|
||||
if ($null -eq $element -or -not (Set-OrcaElementValue $element ([string]$Operation.value))) {
|
||||
$written = if ($null -ne $element) { Set-OrcaElementValue $element ([string]$Operation.value) } else { $null }
|
||||
if ($null -eq $written) {
|
||||
throw "element value is not settable"
|
||||
}
|
||||
$action = [pscustomobject]@{ path = "accessibility"; actionName = "setValue"; fallbackReason = $null }
|
||||
if ($null -ne $written.verification) {
|
||||
$action | Add-Member -NotePropertyName verification -NotePropertyValue $written.verification
|
||||
}
|
||||
}
|
||||
default {
|
||||
throw "unsupported tool: $($Operation.tool)"
|
||||
}
|
||||
}
|
||||
# Why: input paths raise the target; WPF also activates its own window for most patterns, so those are measured.
|
||||
$delivery = "foreground"
|
||||
if ($action.path -eq "accessibility" -and -not [bool]$Operation.restoreWindow -and -not (Test-OrcaTargetTookForeground $foregroundBefore $process)) {
|
||||
$delivery = "background"
|
||||
}
|
||||
$action | Add-Member -NotePropertyName delivery -NotePropertyValue $delivery
|
||||
|
||||
try {
|
||||
try {
|
||||
|
||||
@@ -148,7 +148,7 @@ Do not pass `--restore-window` by default. Use it only when foreground input is
|
||||
|
||||
Popup menus and web `<select>` controls: set the option with `set-value` on the popup element. Background clicks on them are refused because they open the menu off screen or wedge the app's input.
|
||||
|
||||
On Linux and Windows, clicks, scrolling, drags and keyboard input still bring the target window forward.
|
||||
On Linux, clicks, scrolling, drags and keyboard input still bring the target window forward. On Windows, these go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), element scrolls by whole `--pages`, and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Everything else brings the window forward and reports `foreground`.
|
||||
|
||||
## Screenshots
|
||||
|
||||
|
||||
@@ -1167,7 +1167,7 @@ const ORCA_COMPUTER_USE_MARKDOWN = [
|
||||
"",
|
||||
"Popup menus and web `<select>` controls: set the option with `set-value` on the popup element. Background clicks on them are refused because they open the menu off screen or wedge the app's input.",
|
||||
"",
|
||||
"On Linux and Windows, clicks, scrolling, drags and keyboard input still bring the target window forward.",
|
||||
"On Linux, clicks, scrolling, drags and keyboard input still bring the target window forward. On Windows, these go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), element scrolls by whole `--pages`, and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Everything else brings the window forward and reports `foreground`.",
|
||||
"",
|
||||
"## Screenshots",
|
||||
"",
|
||||
|
||||
@@ -102,7 +102,7 @@ export type ComputerActionMetadata = {
|
||||
fallbackReason?: string | null
|
||||
targetWindowId?: number | null
|
||||
targetWindowIndex?: number | null
|
||||
// Absent from providers that predate background input (Linux, Windows, older macOS helpers).
|
||||
// Absent from providers that predate background input (Linux, older Windows and macOS helpers).
|
||||
delivery?: 'background' | 'foreground'
|
||||
verification?: ComputerActionVerification
|
||||
}
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'vitest'
|
||||
import type { ComputerActionResult, ComputerSnapshotResult } from '../../src/shared/runtime-types'
|
||||
import {
|
||||
ensureOrcaRuntimeLaunched,
|
||||
findRoleIndex,
|
||||
parseJsonOutput,
|
||||
runOrcaCli,
|
||||
stopOrcaRuntime
|
||||
} from './helpers/computer-driver'
|
||||
import {
|
||||
launchWindowsSemanticFixture,
|
||||
type WindowsSemanticFixture,
|
||||
type WindowsSemanticFixtureState
|
||||
} from './helpers/windows-semantic-controls-fixture'
|
||||
|
||||
const isWindows = process.platform === 'win32'
|
||||
const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1'
|
||||
|
||||
let fixture: WindowsSemanticFixture | null = null
|
||||
|
||||
function requireFixture(): WindowsSemanticFixture {
|
||||
if (!fixture) {
|
||||
throw new Error('Windows semantic controls fixture has not been launched')
|
||||
}
|
||||
return fixture
|
||||
}
|
||||
|
||||
async function treeText(app: string): Promise<string> {
|
||||
return parseJsonOutput<{ result: ComputerSnapshotResult }>(
|
||||
(await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json']))
|
||||
.stdout
|
||||
).result.snapshot.treeText
|
||||
}
|
||||
|
||||
async function elementIndex(app: string, line: RegExp): Promise<number> {
|
||||
const text = await treeText(app)
|
||||
const index = findRoleIndex(text, line)
|
||||
expect(index, text).toBeGreaterThanOrEqual(0)
|
||||
return index
|
||||
}
|
||||
|
||||
// Runs a UI Automation action, waits for its effect, and checks action.delivery against whether the window was activated.
|
||||
async function runPatternAction(
|
||||
args: string[],
|
||||
effect: (state: WindowsSemanticFixtureState) => boolean,
|
||||
label: string
|
||||
): Promise<{ result: ComputerActionResult; activated: boolean }> {
|
||||
const { waitForState } = requireFixture()
|
||||
const before = await waitForState((state) => !state.active, 'fixture window inactive')
|
||||
const result = parseJsonOutput<{ result: ComputerActionResult }>(
|
||||
(await runOrcaCli(['computer', ...args, '--no-screenshot', '--json'])).stdout
|
||||
).result
|
||||
expect(result.action?.path).toBe('accessibility')
|
||||
const after = await waitForState(effect, label)
|
||||
const activated = after.activations > before.activations
|
||||
expect(result.action?.delivery, after.events.join('\n')).toBe(
|
||||
activated ? 'foreground' : 'background'
|
||||
)
|
||||
return { result, activated }
|
||||
}
|
||||
|
||||
// The fixture shows real windows and hands foreground back to its holder, so this runs only on CI.
|
||||
describe.skipIf(!isWindows || !e2eOptIn || process.env.CI !== 'true')(
|
||||
'computer-use Windows UI Automation actions',
|
||||
() => {
|
||||
beforeAll(async () => {
|
||||
await ensureOrcaRuntimeLaunched()
|
||||
fixture = await launchWindowsSemanticFixture()
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
await fixture?.close()
|
||||
fixture = null
|
||||
await stopOrcaRuntime()
|
||||
})
|
||||
|
||||
test('scroll pages a list through ScrollPattern and reports whether the window came forward', async () => {
|
||||
const { app } = requireFixture()
|
||||
const item = await elementIndex(app, /^\s*(\d+)\s+list item Semantic item 0(?:,|$)/m)
|
||||
|
||||
await runPatternAction(
|
||||
['scroll', '--app', app, '--element-index', String(item), '--direction', 'down'],
|
||||
(state) => state.listOffset > 0,
|
||||
'list scrolled down'
|
||||
)
|
||||
})
|
||||
|
||||
test('ScrollToVisible brings an off-screen list item into view in the background', async () => {
|
||||
const { app } = requireFixture()
|
||||
const item = await elementIndex(app, /^\s*(\d+)\s+list item Semantic item 150(?:,|$)/m)
|
||||
|
||||
const { activated } = await runPatternAction(
|
||||
[
|
||||
'perform-secondary-action',
|
||||
'--app',
|
||||
app,
|
||||
'--element-index',
|
||||
String(item),
|
||||
'--action',
|
||||
'ScrollToVisible'
|
||||
],
|
||||
(state) => state.listOffset <= 150 && 150 < state.listOffset + state.listViewport,
|
||||
'item 150 in the list viewport'
|
||||
)
|
||||
expect(activated).toBe(false)
|
||||
})
|
||||
|
||||
test('Expand and Collapse drive a tree item through ExpandCollapsePattern', async () => {
|
||||
const { app } = requireFixture()
|
||||
const collapsed = await elementIndex(
|
||||
app,
|
||||
/^\s*(\d+)\s+tree view item Semantic branch,.*Secondary Actions:.*\bExpand\b/m
|
||||
)
|
||||
await runPatternAction(
|
||||
[
|
||||
'perform-secondary-action',
|
||||
'--app',
|
||||
app,
|
||||
'--element-index',
|
||||
String(collapsed),
|
||||
'--action',
|
||||
'Expand'
|
||||
],
|
||||
(state) => state.branchExpanded,
|
||||
'branch expanded'
|
||||
)
|
||||
|
||||
const expanded = await elementIndex(
|
||||
app,
|
||||
/^\s*(\d+)\s+tree view item Semantic branch,.*Secondary Actions:.*\bCollapse\b/m
|
||||
)
|
||||
await runPatternAction(
|
||||
[
|
||||
'perform-secondary-action',
|
||||
'--app',
|
||||
app,
|
||||
'--element-index',
|
||||
String(expanded),
|
||||
'--action',
|
||||
'Collapse'
|
||||
],
|
||||
(state) => !state.branchExpanded,
|
||||
'branch collapsed'
|
||||
)
|
||||
})
|
||||
|
||||
test('set-value moves a slider through RangeValuePattern', async () => {
|
||||
const { app } = requireFixture()
|
||||
const slider = await elementIndex(app, /^\s*(\d+)\s+slider Semantic level(?:,|$)/m)
|
||||
|
||||
const { result } = await runPatternAction(
|
||||
['set-value', '--app', app, '--element-index', String(slider), '--value', '42'],
|
||||
(state) => state.level === 42,
|
||||
'slider at 42'
|
||||
)
|
||||
expect(result.action?.verification).toMatchObject({ state: 'verified', property: 'value' })
|
||||
})
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,95 @@
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$StatePath,
|
||||
[Parameter(Mandatory = $true)][string]$Title,
|
||||
[Parameter(Mandatory = $true)][string]$HolderTitle
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
Add-Type -AssemblyName PresentationFramework
|
||||
Add-Type -TypeDefinition @"
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
public static class OrcaFixtureForeground {
|
||||
[DllImport("user32.dll")] static extern IntPtr GetForegroundWindow();
|
||||
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern int GetWindowText(IntPtr hwnd, StringBuilder text, int count);
|
||||
[DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern IntPtr FindWindow(string className, string title);
|
||||
[DllImport("user32.dll")] static extern bool SetForegroundWindow(IntPtr hwnd);
|
||||
public static bool Give(string title) { return SetForegroundWindow(FindWindow(null, title)); }
|
||||
public static string Title() {
|
||||
var text = new StringBuilder(256);
|
||||
GetWindowText(GetForegroundWindow(), text, text.Capacity);
|
||||
return text.ToString();
|
||||
}
|
||||
}
|
||||
"@
|
||||
|
||||
# Why: ShowActivated=False opens the window behind the foreground one, so any activation by an action is observable.
|
||||
[xml]$xaml = @"
|
||||
<Window xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
|
||||
Title="$Title" Width="420" Height="520" ShowActivated="False" WindowStartupLocation="Manual" Left="40" Top="40">
|
||||
<StackPanel Margin="8">
|
||||
<ListBox Name="Items" Height="160" ScrollViewer.CanContentScroll="True" AutomationProperties.Name="Semantic items">
|
||||
<ListBox.ItemsPanel>
|
||||
<ItemsPanelTemplate><StackPanel /></ItemsPanelTemplate>
|
||||
</ListBox.ItemsPanel>
|
||||
</ListBox>
|
||||
<TreeView Name="Tree" Height="120" AutomationProperties.Name="Semantic tree">
|
||||
<TreeViewItem Name="Branch" Header="Semantic branch">
|
||||
<TreeViewItem Header="Semantic leaf one" />
|
||||
<TreeViewItem Header="Semantic leaf two" />
|
||||
</TreeViewItem>
|
||||
</TreeView>
|
||||
<Slider Name="Level" Minimum="0" Maximum="100" Value="10" AutomationProperties.Name="Semantic level" />
|
||||
</StackPanel>
|
||||
</Window>
|
||||
"@
|
||||
|
||||
$window = [Windows.Markup.XamlReader]::Load((New-Object System.Xml.XmlNodeReader $xaml))
|
||||
$items = $window.FindName("Items")
|
||||
$branch = $window.FindName("Branch")
|
||||
$level = $window.FindName("Level")
|
||||
for ($i = 0; $i -lt 200; $i++) { [void]$items.Items.Add("Semantic item $i") }
|
||||
|
||||
$state = [ordered]@{ ready = $false; activations = 0; active = $false; listOffset = 0.0; listViewport = 0.0; branchExpanded = $false; level = 10.0; events = @() }
|
||||
# Why: a failed activation check prints this log, which says what ran just before the window was activated.
|
||||
$clock = [Diagnostics.Stopwatch]::StartNew()
|
||||
function Add-FixtureEvent([string]$Name) {
|
||||
$state.events = @($state.events + "$($clock.ElapsedMilliseconds) $Name fg=$([OrcaFixtureForeground]::Title())" | Select-Object -Last 40)
|
||||
}
|
||||
function Save-FixtureState {
|
||||
# Why: an exception in a WPF event handler would end the fixture; a reader holding the file only delays one write.
|
||||
try {
|
||||
$temporary = "$StatePath.tmp"
|
||||
($state | ConvertTo-Json -Compress) | Set-Content -LiteralPath $temporary -Encoding UTF8
|
||||
Move-Item -LiteralPath $temporary -Destination $StatePath -Force
|
||||
} catch {}
|
||||
}
|
||||
|
||||
# Why: hand activation back to the holder process later, so each action starts behind another app and Orca still sees what it caused.
|
||||
$handBack = New-Object Windows.Threading.DispatcherTimer
|
||||
$handBack.Interval = [TimeSpan]::FromSeconds(2)
|
||||
$handBack.Add_Tick({ $handBack.Stop(); Add-FixtureEvent "hand back $([OrcaFixtureForeground]::Give($HolderTitle))"; Save-FixtureState })
|
||||
$window.Add_Activated({
|
||||
$state.activations++
|
||||
$state.active = $true
|
||||
Add-FixtureEvent "activated"
|
||||
Save-FixtureState
|
||||
$handBack.Stop()
|
||||
$handBack.Start()
|
||||
})
|
||||
$window.Add_Deactivated({ $state.active = $false; Add-FixtureEvent "deactivated"; Save-FixtureState })
|
||||
$items.AddHandler([Windows.Controls.ScrollViewer]::ScrollChangedEvent, [Windows.Controls.ScrollChangedEventHandler]{
|
||||
param($source, $change)
|
||||
$state.listOffset = $change.VerticalOffset
|
||||
$state.listViewport = $change.ViewportHeight
|
||||
Add-FixtureEvent "scrolled $($change.VerticalOffset)"
|
||||
Save-FixtureState
|
||||
})
|
||||
$branch.Add_Expanded({ $state.branchExpanded = $true; Add-FixtureEvent "expanded"; Save-FixtureState })
|
||||
$branch.Add_Collapsed({ $state.branchExpanded = $false; Add-FixtureEvent "collapsed"; Save-FixtureState })
|
||||
$level.Add_ValueChanged({ $state.level = $level.Value; Add-FixtureEvent "level $($level.Value)"; Save-FixtureState })
|
||||
$window.Add_ContentRendered({ $state.ready = $true; Save-FixtureState })
|
||||
|
||||
# Why: ShowDialog always activates; Application.Run shows the window with ShowActivated honored.
|
||||
[void](New-Object Windows.Application).Run($window)
|
||||
@@ -0,0 +1,116 @@
|
||||
import { execFile, spawn, type ChildProcess } from 'node:child_process'
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { promisify } from 'node:util'
|
||||
import { vi } from 'vitest'
|
||||
import { z } from 'zod'
|
||||
|
||||
const execFileAsync = promisify(execFile)
|
||||
const fixtureScript = join(process.cwd(), 'tests/e2e/helpers/windows-semantic-controls-fixture.ps1')
|
||||
|
||||
const WindowsSemanticFixtureStateSchema = z.object({
|
||||
ready: z.boolean(),
|
||||
activations: z.number(),
|
||||
active: z.boolean(),
|
||||
listOffset: z.number(),
|
||||
listViewport: z.number(),
|
||||
branchExpanded: z.boolean(),
|
||||
level: z.number(),
|
||||
events: z.array(z.string())
|
||||
})
|
||||
|
||||
export type WindowsSemanticFixtureState = z.infer<typeof WindowsSemanticFixtureStateSchema>
|
||||
|
||||
export type WindowsSemanticFixture = {
|
||||
app: string
|
||||
title: string
|
||||
readState: () => Promise<WindowsSemanticFixtureState>
|
||||
waitForState: (
|
||||
predicate: (state: WindowsSemanticFixtureState) => boolean,
|
||||
label: string
|
||||
) => Promise<WindowsSemanticFixtureState>
|
||||
close: () => Promise<void>
|
||||
}
|
||||
|
||||
export async function launchWindowsSemanticFixture(): Promise<WindowsSemanticFixture> {
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'orca-computer-windows-semantic-'))
|
||||
const statePath = join(tempDir, 'state.json')
|
||||
const title = `Orca Semantic Controls ${Date.now()}`
|
||||
const holderTitle = `${title} holder`
|
||||
// Why: a window in another process takes activation back, as the user's own app would hold it.
|
||||
const holder: ChildProcess = spawn(
|
||||
'powershell.exe',
|
||||
[
|
||||
'-NoProfile',
|
||||
'-NonInteractive',
|
||||
'-STA',
|
||||
'-Command',
|
||||
`Add-Type -AssemblyName System.Windows.Forms; $form = New-Object Windows.Forms.Form; $form.Text = '${holderTitle}'; [Windows.Forms.Application]::Run($form)`
|
||||
],
|
||||
{ stdio: 'ignore' }
|
||||
)
|
||||
// Why: no windowsHide; Node's hidden startup flag would also hide the fixture's first WPF window.
|
||||
const child: ChildProcess = spawn(
|
||||
'powershell.exe',
|
||||
[
|
||||
'-NoProfile',
|
||||
'-NonInteractive',
|
||||
'-STA',
|
||||
'-ExecutionPolicy',
|
||||
'Bypass',
|
||||
'-File',
|
||||
fixtureScript,
|
||||
'-StatePath',
|
||||
statePath,
|
||||
'-Title',
|
||||
title,
|
||||
'-HolderTitle',
|
||||
holderTitle
|
||||
],
|
||||
{ stdio: 'ignore' }
|
||||
)
|
||||
if (child.pid === undefined) {
|
||||
throw new Error('failed to start the Windows semantic controls fixture')
|
||||
}
|
||||
const pid = child.pid
|
||||
|
||||
const readState = async (): Promise<WindowsSemanticFixtureState> => {
|
||||
const raw = await readFile(statePath, 'utf8')
|
||||
return WindowsSemanticFixtureStateSchema.parse(JSON.parse(raw.replace(/^/, '')))
|
||||
}
|
||||
const waitForState = async (
|
||||
predicate: (state: WindowsSemanticFixtureState) => boolean,
|
||||
label: string
|
||||
): Promise<WindowsSemanticFixtureState> => {
|
||||
// Why: the fixture replaces the file on every change, so a read can land between writes and throw.
|
||||
return vi.waitFor(
|
||||
async () => {
|
||||
const state = await readState()
|
||||
if (!predicate(state)) {
|
||||
throw new Error(`Fixture state not yet ${label}: ${JSON.stringify(state)}`)
|
||||
}
|
||||
return state
|
||||
},
|
||||
{ timeout: 15_000, interval: 100 }
|
||||
)
|
||||
}
|
||||
const close = async (): Promise<void> => {
|
||||
for (const owned of [pid, holder.pid].filter((value) => value !== undefined)) {
|
||||
try {
|
||||
await execFileAsync('taskkill.exe', ['/PID', String(owned), '/T', '/F'])
|
||||
} catch {
|
||||
// The fixture or holder may already have exited.
|
||||
}
|
||||
}
|
||||
await rm(tempDir, { force: true, recursive: true })
|
||||
}
|
||||
|
||||
try {
|
||||
await waitForState((state) => state.ready, 'window rendered')
|
||||
} catch (error) {
|
||||
await close()
|
||||
throw error
|
||||
}
|
||||
return { app: `pid:${pid}`, title, readState, waitForState, close }
|
||||
}
|
||||
Reference in New Issue
Block a user