diff --git a/.github/workflows/computer-e2e.yml b/.github/workflows/computer-e2e.yml index 51ec40e85fe..d0abf40cae2 100644 --- a/.github/workflows/computer-e2e.yml +++ b/.github/workflows/computer-e2e.yml @@ -54,6 +54,8 @@ on: - 'tests/e2e/computer-linux.e2e.ts' - 'tests/e2e/computer-windows.e2e.ts' - 'tests/e2e/computer-windows-store.e2e.ts' + - 'tests/e2e/computer-windows-background.e2e.ts' + - 'tests/e2e/helpers/windows-semantic-controls-fixture.*' - 'tests/e2e/helpers/computer-cli-driver.ts' - 'tests/e2e/helpers/computer-driver.ts' - 'tests/e2e/vitest.config.ts' @@ -151,4 +153,4 @@ jobs: - run: pnpm verify:computer-native - run: pnpm build:cli - run: pnpm run build:electron-vite:parallel - - run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts tests/e2e/computer-windows-store.e2e.ts + - run: pnpm test:e2e:computer --reporter=verbose tests/e2e/computer-windows.e2e.ts tests/e2e/computer-windows-store.e2e.ts tests/e2e/computer-windows-background.e2e.ts diff --git a/docs/site/content/docs/cli/computer-use.mdx b/docs/site/content/docs/cli/computer-use.mdx index 5f39edea8e5..44ad60afec2 100644 --- a/docs/site/content/docs/cli/computer-use.mdx +++ b/docs/site/content/docs/cli/computer-use.mdx @@ -89,7 +89,7 @@ Prefer semantic actions (`click`, `set-value`, `perform-secondary-action`) over `hotkey` and `press-key` refuse shortcuts that quit or close the app, log out, lock the screen, force-quit or end the session, such as `Cmd+Q`, `Shift+Cmd+Q`, `Ctrl+Cmd+Q` and `Option+Cmd+Esc` on macOS, `Alt+F4` and `Win+L` on Windows, and `Ctrl+Q`, `Alt+F4`, `Super+L` and `Ctrl+Alt+Delete` on Linux. The error code is `system_shortcut_blocked`. Pass `--allow-system-shortcut` to send one anyway. -On macOS, actions run in the background: the target app isn't brought forward, your frontmost app and key window stay put, and the pointer doesn't move, so you can keep working while an agent drives another window. Each action result reports `action.delivery` (`background` or `foreground`). Pass `--restore-window` only when an app ignores background input, such as drags in Chromium or Electron apps or games; it brings the app forward and uses the real pointer and keyboard. Popup menus and web `` controls are set with `set-value` instead of a click. On Linux, clicks, scrolling, drags and keyboard input still bring the window forward. On Windows, element clicks, `set-value` (including sliders), element scrolls and secondary actions (`Invoke`, `Toggle`, `Select`, `Expand`, `Collapse`, `ScrollToVisible`) go through UI Automation when the control supports them, without moving the pointer or sending keys. Some apps still bring their own window forward when driven this way (WPF apps do for most of these), so `action.delivery` reports `foreground` whenever the target window came forward. Anything else brings the window forward. When accessibility targeting fails, fall back carefully to coordinates: diff --git a/native/computer-use-windows/runtime-pointer.test.ps1 b/native/computer-use-windows/runtime-pointer.test.ps1 index 78bc1ba9fd9..e25dc2effe3 100644 --- a/native/computer-use-windows/runtime-pointer.test.ps1 +++ b/native/computer-use-windows/runtime-pointer.test.ps1 @@ -51,6 +51,10 @@ public static class OrcaDesktopWin32 { } return Foreground; } + public static uint GetWindowThreadProcessId(IntPtr hwnd, out uint processId) { + processId = hwnd == new IntPtr(101) ? 4242u : 1u; + return 1; + } public static bool SetCursorPos(int x, int y) { Calls.Add("move:" + x + "," + y); return true; @@ -74,7 +78,7 @@ $functions = @( "Get-OrcaRequiredNumber", "Get-OrcaPositiveInteger", "Get-OrcaPositiveNumber", "Get-OrcaScreenPoint", "Get-OrcaElementScreenPoint", "Get-OrcaModifierVirtualKey", "Get-OrcaClickModifierVirtualKeys", "Send-OrcaMouseClick", "Send-OrcaDrag", "Invoke-OrcaOperation", - "Assert-OrcaInputAllowed" + "Assert-OrcaInputAllowed", "Test-OrcaTargetTookForeground" ) foreach ($definition in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false)) { if ($definition.Name -in $functions) { . ([scriptblock]::Create($definition.Extent.Text)) } @@ -87,6 +91,7 @@ function Get-OrcaRootElement { $null } function Get-OrcaWindowFrame { @{ x = [OrcaDesktopWin32]::WindowX; y = 200 } } function Find-OrcaElement($RootElement, $Record) { $Record } function Invoke-OrcaPrimaryAction($Element) { $null -ne $Element } +function Invoke-OrcaScrollPattern($Element, $RootElement, $Direction, $Pages) { $null -ne $Element } function New-OrcaSnapshot { [pscustomobject]@{} } foreach ($case in @( @@ -112,6 +117,7 @@ foreach ($case in @( if ($inputCalls.Count -eq 0) { throw "$($case.tool) did not send pointer input after activation" } if ([OrcaDesktopWin32]::FocusReads -lt 2) { throw "$($case.tool) did not wait for activation" } Assert-TestEqual $inputCalls[0] "move:505,206" "$($case.tool) geometry after delayed activation" + Assert-TestEqual $result.action.delivery "foreground" "$($case.tool) pointer delivery" } else { Assert-TestEqual $denied $true "$($case.tool) denied activation" Assert-TestEqual $inputCalls.Count 0 "$($case.tool) denied activation sends no pointer or modifier input" @@ -119,10 +125,18 @@ foreach ($case in @( } } -[OrcaDesktopWin32]::Reset($false) -$result = Invoke-OrcaOperation ([pscustomobject]@{ tool = "click"; app = "Fixture"; element = [pscustomobject]@{ index = 1 } }) -Assert-TestEqual $result.action.path "accessibility" "pattern click path" -Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "pattern click bypasses activation and global input" +foreach ($case in @( + @{ tool = "click" }, + @{ tool = "scroll"; direction = "down"; pages = 1 } +)) { + [OrcaDesktopWin32]::Reset($false) + $operation = [pscustomobject]@{ app = "Fixture"; element = [pscustomobject]@{ index = 1 } } + foreach ($key in $case.Keys) { $operation | Add-Member -NotePropertyName $key -NotePropertyValue $case[$key] } + $result = Invoke-OrcaOperation $operation + Assert-TestEqual $result.action.path "accessibility" "pattern $($case.tool) path" + Assert-TestEqual $result.action.delivery "background" "pattern $($case.tool) delivery" + Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "pattern $($case.tool) bypasses activation and global input" +} foreach ($case in @( @{ tool = "click"; element = [pscustomobject]@{ index = 1 } }, @{ tool = "click"; x = 5; y = 6; restoreWindow = $true }, @@ -137,4 +151,10 @@ foreach ($case in @( if ($message -notlike "target_elevated:*") { throw "$($case.tool) elevated target expected target_elevated, received '$message'" } Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "$($case.tool) elevated target sends no activation or input" } +# Why: WPF activates its own window for most patterns; the result must say so rather than claim background. +[OrcaDesktopWin32]::Reset($false) +function Invoke-OrcaPrimaryAction($Element) { [OrcaDesktopWin32]::Foreground = [IntPtr]101; $true } +$result = Invoke-OrcaOperation ([pscustomobject]@{ tool = "click"; app = "Fixture"; element = [pscustomobject]@{ index = 1 } }) +Assert-TestEqual $result.action.path "accessibility" "self-activating pattern click path" +Assert-TestEqual $result.action.delivery "foreground" "self-activating pattern click delivery" Write-Output "windows-pointer-tests-ok" diff --git a/native/computer-use-windows/runtime.ps1 b/native/computer-use-windows/runtime.ps1 index f4e5d1fc62e..afbdfe2ee44 100644 --- a/native/computer-use-windows/runtime.ps1 +++ b/native/computer-use-windows/runtime.ps1 @@ -104,6 +104,9 @@ public static class OrcaDesktopWin32 { [DllImport("user32.dll")] public static extern IntPtr GetForegroundWindow(); + [DllImport("user32.dll")] + public static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint processId); + [DllImport("user32.dll")] public static extern bool SetCursorPos(int x, int y); @@ -561,6 +564,14 @@ function Restore-OrcaWindow($Process) { [void][OrcaDesktopWin32]::SetForegroundWindow([IntPtr]$Process.MainWindowHandle) } +function Test-OrcaTargetTookForeground([IntPtr]$Before, $Process) { + $after = [OrcaDesktopWin32]::GetForegroundWindow() + if ($after -eq $Before) { return $false } + $owner = [uint32]0 + [void][OrcaDesktopWin32]::GetWindowThreadProcessId($after, [ref]$owner) + $owner -eq $Process.Id +} + function Test-OrcaWindowFocused([IntPtr]$WindowHandle) { [OrcaDesktopWin32]::GetForegroundWindow() -eq $WindowHandle } @@ -647,15 +658,50 @@ function Get-OrcaValueText($Element) { } } -function Get-OrcaActions($Element) { +function Get-OrcaRangeValueText($Element) { + try { + $pattern = $Element.GetCurrentPattern([Windows.Automation.RangeValuePattern]::Pattern) + ([double]$pattern.Current.Value).ToString([Globalization.CultureInfo]::InvariantCulture) + } catch { + "" + } +} + +function Test-OrcaRangeValueWritable($Element) { + try { + -not $Element.GetCurrentPattern([Windows.Automation.RangeValuePattern]::Pattern).Current.IsReadOnly + } catch { + $false + } +} + +function Get-OrcaExpandCollapseActions($Element) { + try { + $state = $Element.GetCurrentPattern([Windows.Automation.ExpandCollapsePattern]::Pattern).Current.ExpandCollapseState + } catch { + return @() + } + switch ($state) { + ([Windows.Automation.ExpandCollapseState]::Collapsed) { return @("Expand") } + ([Windows.Automation.ExpandCollapseState]::Expanded) { return @("Collapse") } + ([Windows.Automation.ExpandCollapseState]::PartiallyExpanded) { return @("Expand", "Collapse") } + } + @() +} + +function Get-OrcaActions($Element, $Patterns = $null) { + if ($null -eq $Patterns) { $Patterns = @($Element.GetSupportedPatterns()) } $actions = New-Object System.Collections.Generic.List[string] - foreach ($pattern in $Element.GetSupportedPatterns()) { + foreach ($pattern in $Patterns) { $name = [string]$pattern.ProgrammaticName if ($name -like "InvokePatternIdentifiers.Pattern") { $actions.Add("Invoke") } elseif ($name -like "TogglePatternIdentifiers.Pattern") { $actions.Add("Toggle") } elseif ($name -like "SelectionItemPatternIdentifiers.Pattern") { $actions.Add("Select") } elseif ($name -like "ScrollPatternIdentifiers.Pattern") { $actions.Add("Scroll") } elseif ($name -like "ValuePatternIdentifiers.Pattern") { $actions.Add("SetValue") } + elseif ($name -like "RangeValuePatternIdentifiers.Pattern") { if (Test-OrcaRangeValueWritable $Element) { $actions.Add("SetValue") } } + elseif ($name -like "ExpandCollapsePatternIdentifiers.Pattern") { foreach ($action in @(Get-OrcaExpandCollapseActions $Element)) { $actions.Add($action) } } + elseif ($name -like "ScrollItemPatternIdentifiers.Pattern") { $actions.Add("ScrollToVisible") } } @($actions | Select-Object -Unique) } @@ -751,6 +797,11 @@ function Test-OrcaPlainTextSubtree($Element, [int]$MaxDepth = 4) { function New-OrcaElementRecord($Element, [int]$Index, $WindowFrame) { $controlType = try { [string]$Element.Current.ControlType.ProgrammaticName } catch { "" } $nativeWindowHandle = try { [int64]$Element.Current.NativeWindowHandle } catch { 0 } + $patterns = @($Element.GetSupportedPatterns()) + $value = Get-OrcaValueText $Element + if ([string]::IsNullOrEmpty($value) -and $patterns -contains [Windows.Automation.RangeValuePattern]::Pattern) { + $value = Get-OrcaRangeValueText $Element + } [pscustomobject]@{ index = $Index runtimeId = @(Get-OrcaRuntimeId $Element) @@ -759,12 +810,12 @@ function New-OrcaElementRecord($Element, [int]$Index, $WindowFrame) { controlType = $controlType localizedControlType = Get-OrcaProperty $Element "LocalizedControlType" className = Get-OrcaProperty $Element "ClassName" - value = Get-OrcaValueText $Element + value = $value isSelected = Test-OrcaElementSelected $Element isDisabled = Test-OrcaElementDisabled $Element nativeWindowHandle = $nativeWindowHandle frame = Get-OrcaElementFrame $Element $WindowFrame - actions = @(Get-OrcaActions $Element) + actions = @(Get-OrcaActions $Element $patterns) } } @@ -1175,7 +1226,8 @@ function Invoke-OrcaPrimaryAction($Element) { function Invoke-OrcaNamedAction($Element, [string]$Action) { $wanted = "" - if ($null -ne $Action) { $wanted = $Action.Trim().ToLowerInvariant() } + # Why: accept the rendered name ("ScrollToVisible") and macOS's spaced form ("scroll to visible"). + if ($null -ne $Action) { $wanted = ($Action -replace "\s+", "").ToLowerInvariant() } switch ($wanted) { "invoke" { $pattern = $Element.GetCurrentPattern([Windows.Automation.InvokePattern]::Pattern) @@ -1192,21 +1244,128 @@ function Invoke-OrcaNamedAction($Element, [string]$Action) { $pattern.Toggle() return $true } + "expand" { + $pattern = $Element.GetCurrentPattern([Windows.Automation.ExpandCollapsePattern]::Pattern) + $pattern.Expand() + return $true + } + "collapse" { + $pattern = $Element.GetCurrentPattern([Windows.Automation.ExpandCollapsePattern]::Pattern) + $pattern.Collapse() + return $true + } + "scrolltovisible" { + $pattern = $Element.GetCurrentPattern([Windows.Automation.ScrollItemPattern]::Pattern) + $pattern.ScrollIntoView() + return $true + } default { return $false } } } +function Test-OrcaRangeValueMatches([string]$Actual, [double]$Expected) { + # Why: allow float32 rounding (0.1 reads back 0.100000001), but cap it so a snapped neighbour never verifies at any size. + $actualNumber = 0.0 + [double]::TryParse($Actual, [Globalization.NumberStyles]::Float, [Globalization.CultureInfo]::InvariantCulture, [ref]$actualNumber) -and + [Math]::Abs($actualNumber - $Expected) -le [Math]::Min(1e-6 * [Math]::Max(1.0, [Math]::Abs($Expected)), 1e-3) +} + function Set-OrcaElementValue($Element, [string]$Value) { try { $pattern = $Element.GetCurrentPattern([Windows.Automation.ValuePattern]::Pattern) if (-not $pattern.Current.IsReadOnly) { $pattern.SetValue($Value) - return $true + return [pscustomobject]@{ verification = $null } } } catch {} - $false + if (-not (Test-OrcaRangeValueWritable $Element)) { return $null } + $range = $Element.GetCurrentPattern([Windows.Automation.RangeValuePattern]::Pattern) + $culture = [Globalization.CultureInfo]::InvariantCulture + $number = 0.0 + if (-not [double]::TryParse($Value, [Globalization.NumberStyles]::Float, $culture, [ref]$number) -or [double]::IsNaN($number)) { + throw "value must be a number for this range control" + } + $minimum = [double]$range.Current.Minimum + $maximum = [double]$range.Current.Maximum + if ($number -lt $minimum -or $number -gt $maximum) { + throw "value $Value is outside this range control's $($minimum.ToString($culture))..$($maximum.ToString($culture))" + } + $range.SetValue($number) + # Why: Chromium applies the write asynchronously, so poll briefly like macOS's settled readback. + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + while ($true) { + $actual = Get-OrcaRangeValueText $Element + $matched = Test-OrcaRangeValueMatches $actual $number + if ($matched -or $stopwatch.ElapsedMilliseconds -ge 500) { break } + Start-Sleep -Milliseconds 25 + } + $verification = if ($matched) { + [pscustomobject]@{ state = "verified"; property = "value"; expected = $Value; actualPreview = $actual } + } elseif ([string]::IsNullOrEmpty($actual)) { + # Why: a failed read is not evidence the write failed; only a read-back number can contradict it. + [pscustomobject]@{ state = "unverified"; reason = "readback_unsupported"; expected = $Value } + } else { + [pscustomobject]@{ state = "unverified"; reason = "value_mismatch"; expected = $Value; actualPreview = $actual } + } + [pscustomobject]@{ verification = $verification } +} + +function Get-OrcaScrollPattern($Element, $RootElement, [bool]$Vertical) { + $rootId = Get-OrcaRuntimeId $RootElement + $current = $Element + # Why: an item's ScrollPattern lives on its list or viewport, the same container a wheel at its center would scroll. + while ($null -ne $current) { + try { + $pattern = $current.GetCurrentPattern([Windows.Automation.ScrollPattern]::Pattern) + $scrollable = if ($Vertical) { $pattern.Current.VerticallyScrollable } else { $pattern.Current.HorizontallyScrollable } + if ($scrollable) { return $pattern } + } catch {} + if (Test-OrcaSameRuntimeId (Get-OrcaRuntimeId $current) $rootId) { break } + $current = try { [Windows.Automation.TreeWalker]::RawViewWalker.GetParent($current) } catch { $null } + } + $null +} + +function Get-OrcaScrollPercent($Pattern, [bool]$Vertical) { + try { + if ($Vertical) { return [double]$Pattern.Current.VerticalScrollPercent } + [double]$Pattern.Current.HorizontalScrollPercent + } catch { + $null + } +} + +function Invoke-OrcaScrollPattern($Element, $RootElement, [string]$Direction, [double]$Pages) { + # Why: ScrollPattern moves in whole pages; fractional pages keep the wheel, as on macOS. + if ($null -eq $Element -or $Pages -ne [Math]::Floor($Pages)) { return $false } + $vertical = $Direction -in @("up", "down") + $pattern = Get-OrcaScrollPattern $Element $RootElement $vertical + if ($null -eq $pattern) { return $false } + $amount = if ($Direction -in @("down", "right")) { [Windows.Automation.ScrollAmount]::LargeIncrement } else { [Windows.Automation.ScrollAmount]::LargeDecrement } + $none = [Windows.Automation.ScrollAmount]::NoAmount + # Why: without a readable position the loop can't tell the end of the content, so the wheel handles it. + $before = Get-OrcaScrollPercent $pattern $vertical + if ($null -eq $before) { return $false } + for ($page = 0; $page -lt $Pages; $page++) { + try { + if ($vertical) { $pattern.Scroll($none, $amount) } else { $pattern.Scroll($amount, $none) } + } catch { + # Why: a refusal before any page moved falls back to the wheel; after that it is the end of the content. + return $page -gt 0 + } + # Why: WPF and Chromium move the viewport after Scroll returns; an unmoved or unreadable percent after the wait is the end. + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + while ($true) { + $after = Get-OrcaScrollPercent $pattern $vertical + if ($after -ne $before -or $stopwatch.ElapsedMilliseconds -ge 500) { break } + Start-Sleep -Milliseconds 25 + } + if ($null -eq $after -or $after -eq $before) { break } + $before = $after + } + $true } function Get-OrcaRequiredNumber($Value, [string]$Name) { @@ -1479,6 +1638,8 @@ function Invoke-OrcaOperation($Operation) { if ($Operation.tool -in @("type_text", "press_key", "hotkey", "paste_text")) { Assert-OrcaKeyboardFocus $handle $Operation } + # Why: pattern paths need an element; reading the foreground only then keeps coordinate input untouched. + $foregroundBefore = if ($null -ne $element) { [OrcaDesktopWin32]::GetForegroundWindow() } else { [IntPtr]::Zero } $action = $null switch ($Operation.tool) { @@ -1508,7 +1669,8 @@ function Invoke-OrcaOperation($Operation) { $action = [pscustomobject]@{ path = "accessibility"; actionName = $Operation.action; fallbackReason = $null } } "scroll" { - $delta = 120 * [int][Math]::Ceiling((Get-OrcaPositiveNumber $Operation.pages "pages")) + $pages = Get-OrcaPositiveNumber $Operation.pages "pages" + $delta = 120 * [int][Math]::Ceiling($pages) $mouseEvent = $MouseEvents.Wheel if ($Operation.direction -eq "down") { $delta = -1 * $delta @@ -1520,12 +1682,18 @@ function Invoke-OrcaOperation($Operation) { } elseif ($Operation.direction -ne "up") { throw "unsupported scroll direction: $($Operation.direction)" } - $windowFrame = Restore-OrcaPointerWindow $process $root - $point = Get-OrcaElementScreenPoint $element - if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame } - [void][OrcaDesktopWin32]::SetCursorPos([int]$point.x, [int]$point.y) - [OrcaDesktopWin32]::mouse_event($mouseEvent, 0, 0, $delta, [UIntPtr]::Zero) - $action = [pscustomobject]@{ path = "synthetic"; actionName = "scroll"; fallbackReason = $null } + if (Invoke-OrcaScrollPattern $element $root $Operation.direction $pages) { + $action = [pscustomobject]@{ path = "accessibility"; actionName = "scroll"; fallbackReason = $null } + } else { + # Why: the wheel scrolls whatever is under the pointer, so the window must be on top. + $windowFrame = Restore-OrcaPointerWindow $process $root + $point = Get-OrcaElementScreenPoint $element + if ($null -eq $point) { $point = Get-OrcaScreenPoint $Operation $windowFrame } + [void][OrcaDesktopWin32]::SetCursorPos([int]$point.x, [int]$point.y) + [OrcaDesktopWin32]::mouse_event($mouseEvent, 0, 0, $delta, [UIntPtr]::Zero) + $fallbackReason = if ($null -ne $element) { "actionUnsupported" } else { $null } + $action = [pscustomobject]@{ path = "synthetic"; actionName = "scroll"; fallbackReason = $fallbackReason } + } } "drag" { $windowFrame = Restore-OrcaPointerWindow $process $root @@ -1565,15 +1733,25 @@ function Invoke-OrcaOperation($Operation) { $action = [pscustomobject]@{ path = "clipboard"; actionName = "paste"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "clipboard_paste" } } } "set_value" { - if ($null -eq $element -or -not (Set-OrcaElementValue $element ([string]$Operation.value))) { + $written = if ($null -ne $element) { Set-OrcaElementValue $element ([string]$Operation.value) } else { $null } + if ($null -eq $written) { throw "element value is not settable" } $action = [pscustomobject]@{ path = "accessibility"; actionName = "setValue"; fallbackReason = $null } + if ($null -ne $written.verification) { + $action | Add-Member -NotePropertyName verification -NotePropertyValue $written.verification + } } default { throw "unsupported tool: $($Operation.tool)" } } + # Why: input paths raise the target; WPF also activates its own window for most patterns, so those are measured. + $delivery = "foreground" + if ($action.path -eq "accessibility" -and -not [bool]$Operation.restoreWindow -and -not (Test-OrcaTargetTookForeground $foregroundBefore $process)) { + $delivery = "background" + } + $action | Add-Member -NotePropertyName delivery -NotePropertyValue $delivery try { try { diff --git a/skill-guides/orca-computer-use.md b/skill-guides/orca-computer-use.md index 4a696e5427b..2e84d6b6076 100644 --- a/skill-guides/orca-computer-use.md +++ b/skill-guides/orca-computer-use.md @@ -148,7 +148,7 @@ Do not pass `--restore-window` by default. Use it only when foreground input is Popup menus and web `` controls: set the option with `set-value` on the popup element. Background clicks on them are refused because they open the menu off screen or wedge the app's input.", "", - "On Linux and Windows, clicks, scrolling, drags and keyboard input still bring the target window forward.", + "On Linux, clicks, scrolling, drags and keyboard input still bring the target window forward. On Windows, these go through UI Automation when the control supports them, without moving the pointer or sending keys: element clicks, `set-value` (including sliders), element scrolls by whole `--pages`, and secondary actions such as `Invoke`, `Toggle`, `Select`, `Expand`, `Collapse` and `ScrollToVisible` (the last brings a list or tree item into view). Some apps bring their own window forward when driven this way (WPF apps do for most of these), so read `action.delivery`: `background` means the user's foreground window stayed put, `foreground` means the target came forward. Everything else brings the window forward and reports `foreground`.", "", "## Screenshots", "", diff --git a/src/shared/computer-use-runtime-types.ts b/src/shared/computer-use-runtime-types.ts index c69d54e3756..d5f7e2d8aed 100644 --- a/src/shared/computer-use-runtime-types.ts +++ b/src/shared/computer-use-runtime-types.ts @@ -102,7 +102,7 @@ export type ComputerActionMetadata = { fallbackReason?: string | null targetWindowId?: number | null targetWindowIndex?: number | null - // Absent from providers that predate background input (Linux, Windows, older macOS helpers). + // Absent from providers that predate background input (Linux, older Windows and macOS helpers). delivery?: 'background' | 'foreground' verification?: ComputerActionVerification } diff --git a/tests/e2e/computer-windows-background.e2e.ts b/tests/e2e/computer-windows-background.e2e.ts new file mode 100644 index 00000000000..76759aa6b5a --- /dev/null +++ b/tests/e2e/computer-windows-background.e2e.ts @@ -0,0 +1,159 @@ +import { afterAll, beforeAll, describe, expect, test } from 'vitest' +import type { ComputerActionResult, ComputerSnapshotResult } from '../../src/shared/runtime-types' +import { + ensureOrcaRuntimeLaunched, + findRoleIndex, + parseJsonOutput, + runOrcaCli, + stopOrcaRuntime +} from './helpers/computer-driver' +import { + launchWindowsSemanticFixture, + type WindowsSemanticFixture, + type WindowsSemanticFixtureState +} from './helpers/windows-semantic-controls-fixture' + +const isWindows = process.platform === 'win32' +const e2eOptIn = process.env.ORCA_COMPUTER_E2E === '1' + +let fixture: WindowsSemanticFixture | null = null + +function requireFixture(): WindowsSemanticFixture { + if (!fixture) { + throw new Error('Windows semantic controls fixture has not been launched') + } + return fixture +} + +async function treeText(app: string): Promise { + return parseJsonOutput<{ result: ComputerSnapshotResult }>( + (await runOrcaCli(['computer', 'get-app-state', '--app', app, '--no-screenshot', '--json'])) + .stdout + ).result.snapshot.treeText +} + +async function elementIndex(app: string, line: RegExp): Promise { + const text = await treeText(app) + const index = findRoleIndex(text, line) + expect(index, text).toBeGreaterThanOrEqual(0) + return index +} + +// Runs a UI Automation action, waits for its effect, and checks action.delivery against whether the window was activated. +async function runPatternAction( + args: string[], + effect: (state: WindowsSemanticFixtureState) => boolean, + label: string +): Promise<{ result: ComputerActionResult; activated: boolean }> { + const { waitForState } = requireFixture() + const before = await waitForState((state) => !state.active, 'fixture window inactive') + const result = parseJsonOutput<{ result: ComputerActionResult }>( + (await runOrcaCli(['computer', ...args, '--no-screenshot', '--json'])).stdout + ).result + expect(result.action?.path).toBe('accessibility') + const after = await waitForState(effect, label) + const activated = after.activations > before.activations + expect(result.action?.delivery, after.events.join('\n')).toBe( + activated ? 'foreground' : 'background' + ) + return { result, activated } +} + +// The fixture shows real windows and hands foreground back to its holder, so this runs only on CI. +describe.skipIf(!isWindows || !e2eOptIn || process.env.CI !== 'true')( + 'computer-use Windows UI Automation actions', + () => { + beforeAll(async () => { + await ensureOrcaRuntimeLaunched() + fixture = await launchWindowsSemanticFixture() + }) + + afterAll(async () => { + await fixture?.close() + fixture = null + await stopOrcaRuntime() + }) + + test('scroll pages a list through ScrollPattern and reports whether the window came forward', async () => { + const { app } = requireFixture() + const item = await elementIndex(app, /^\s*(\d+)\s+list item Semantic item 0(?:,|$)/m) + + await runPatternAction( + ['scroll', '--app', app, '--element-index', String(item), '--direction', 'down'], + (state) => state.listOffset > 0, + 'list scrolled down' + ) + }) + + test('ScrollToVisible brings an off-screen list item into view in the background', async () => { + const { app } = requireFixture() + const item = await elementIndex(app, /^\s*(\d+)\s+list item Semantic item 150(?:,|$)/m) + + const { activated } = await runPatternAction( + [ + 'perform-secondary-action', + '--app', + app, + '--element-index', + String(item), + '--action', + 'ScrollToVisible' + ], + (state) => state.listOffset <= 150 && 150 < state.listOffset + state.listViewport, + 'item 150 in the list viewport' + ) + expect(activated).toBe(false) + }) + + test('Expand and Collapse drive a tree item through ExpandCollapsePattern', async () => { + const { app } = requireFixture() + const collapsed = await elementIndex( + app, + /^\s*(\d+)\s+tree view item Semantic branch,.*Secondary Actions:.*\bExpand\b/m + ) + await runPatternAction( + [ + 'perform-secondary-action', + '--app', + app, + '--element-index', + String(collapsed), + '--action', + 'Expand' + ], + (state) => state.branchExpanded, + 'branch expanded' + ) + + const expanded = await elementIndex( + app, + /^\s*(\d+)\s+tree view item Semantic branch,.*Secondary Actions:.*\bCollapse\b/m + ) + await runPatternAction( + [ + 'perform-secondary-action', + '--app', + app, + '--element-index', + String(expanded), + '--action', + 'Collapse' + ], + (state) => !state.branchExpanded, + 'branch collapsed' + ) + }) + + test('set-value moves a slider through RangeValuePattern', async () => { + const { app } = requireFixture() + const slider = await elementIndex(app, /^\s*(\d+)\s+slider Semantic level(?:,|$)/m) + + const { result } = await runPatternAction( + ['set-value', '--app', app, '--element-index', String(slider), '--value', '42'], + (state) => state.level === 42, + 'slider at 42' + ) + expect(result.action?.verification).toMatchObject({ state: 'verified', property: 'value' }) + }) + } +) diff --git a/tests/e2e/helpers/windows-semantic-controls-fixture.ps1 b/tests/e2e/helpers/windows-semantic-controls-fixture.ps1 new file mode 100644 index 00000000000..d0041ebb8eb --- /dev/null +++ b/tests/e2e/helpers/windows-semantic-controls-fixture.ps1 @@ -0,0 +1,95 @@ +param( + [Parameter(Mandatory = $true)][string]$StatePath, + [Parameter(Mandatory = $true)][string]$Title, + [Parameter(Mandatory = $true)][string]$HolderTitle +) + +$ErrorActionPreference = "Stop" +Add-Type -AssemblyName PresentationFramework +Add-Type -TypeDefinition @" +using System; +using System.Runtime.InteropServices; +using System.Text; +public static class OrcaFixtureForeground { + [DllImport("user32.dll")] static extern IntPtr GetForegroundWindow(); + [DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern int GetWindowText(IntPtr hwnd, StringBuilder text, int count); + [DllImport("user32.dll", CharSet = CharSet.Unicode)] static extern IntPtr FindWindow(string className, string title); + [DllImport("user32.dll")] static extern bool SetForegroundWindow(IntPtr hwnd); + public static bool Give(string title) { return SetForegroundWindow(FindWindow(null, title)); } + public static string Title() { + var text = new StringBuilder(256); + GetWindowText(GetForegroundWindow(), text, text.Capacity); + return text.ToString(); + } +} +"@ + +# Why: ShowActivated=False opens the window behind the foreground one, so any activation by an action is observable. +[xml]$xaml = @" + + + + + + + + + + + + + + + + +"@ + +$window = [Windows.Markup.XamlReader]::Load((New-Object System.Xml.XmlNodeReader $xaml)) +$items = $window.FindName("Items") +$branch = $window.FindName("Branch") +$level = $window.FindName("Level") +for ($i = 0; $i -lt 200; $i++) { [void]$items.Items.Add("Semantic item $i") } + +$state = [ordered]@{ ready = $false; activations = 0; active = $false; listOffset = 0.0; listViewport = 0.0; branchExpanded = $false; level = 10.0; events = @() } +# Why: a failed activation check prints this log, which says what ran just before the window was activated. +$clock = [Diagnostics.Stopwatch]::StartNew() +function Add-FixtureEvent([string]$Name) { + $state.events = @($state.events + "$($clock.ElapsedMilliseconds) $Name fg=$([OrcaFixtureForeground]::Title())" | Select-Object -Last 40) +} +function Save-FixtureState { + # Why: an exception in a WPF event handler would end the fixture; a reader holding the file only delays one write. + try { + $temporary = "$StatePath.tmp" + ($state | ConvertTo-Json -Compress) | Set-Content -LiteralPath $temporary -Encoding UTF8 + Move-Item -LiteralPath $temporary -Destination $StatePath -Force + } catch {} +} + +# Why: hand activation back to the holder process later, so each action starts behind another app and Orca still sees what it caused. +$handBack = New-Object Windows.Threading.DispatcherTimer +$handBack.Interval = [TimeSpan]::FromSeconds(2) +$handBack.Add_Tick({ $handBack.Stop(); Add-FixtureEvent "hand back $([OrcaFixtureForeground]::Give($HolderTitle))"; Save-FixtureState }) +$window.Add_Activated({ + $state.activations++ + $state.active = $true + Add-FixtureEvent "activated" + Save-FixtureState + $handBack.Stop() + $handBack.Start() +}) +$window.Add_Deactivated({ $state.active = $false; Add-FixtureEvent "deactivated"; Save-FixtureState }) +$items.AddHandler([Windows.Controls.ScrollViewer]::ScrollChangedEvent, [Windows.Controls.ScrollChangedEventHandler]{ + param($source, $change) + $state.listOffset = $change.VerticalOffset + $state.listViewport = $change.ViewportHeight + Add-FixtureEvent "scrolled $($change.VerticalOffset)" + Save-FixtureState +}) +$branch.Add_Expanded({ $state.branchExpanded = $true; Add-FixtureEvent "expanded"; Save-FixtureState }) +$branch.Add_Collapsed({ $state.branchExpanded = $false; Add-FixtureEvent "collapsed"; Save-FixtureState }) +$level.Add_ValueChanged({ $state.level = $level.Value; Add-FixtureEvent "level $($level.Value)"; Save-FixtureState }) +$window.Add_ContentRendered({ $state.ready = $true; Save-FixtureState }) + +# Why: ShowDialog always activates; Application.Run shows the window with ShowActivated honored. +[void](New-Object Windows.Application).Run($window) diff --git a/tests/e2e/helpers/windows-semantic-controls-fixture.ts b/tests/e2e/helpers/windows-semantic-controls-fixture.ts new file mode 100644 index 00000000000..24f849264ad --- /dev/null +++ b/tests/e2e/helpers/windows-semantic-controls-fixture.ts @@ -0,0 +1,116 @@ +import { execFile, spawn, type ChildProcess } from 'node:child_process' +import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { promisify } from 'node:util' +import { vi } from 'vitest' +import { z } from 'zod' + +const execFileAsync = promisify(execFile) +const fixtureScript = join(process.cwd(), 'tests/e2e/helpers/windows-semantic-controls-fixture.ps1') + +const WindowsSemanticFixtureStateSchema = z.object({ + ready: z.boolean(), + activations: z.number(), + active: z.boolean(), + listOffset: z.number(), + listViewport: z.number(), + branchExpanded: z.boolean(), + level: z.number(), + events: z.array(z.string()) +}) + +export type WindowsSemanticFixtureState = z.infer + +export type WindowsSemanticFixture = { + app: string + title: string + readState: () => Promise + waitForState: ( + predicate: (state: WindowsSemanticFixtureState) => boolean, + label: string + ) => Promise + close: () => Promise +} + +export async function launchWindowsSemanticFixture(): Promise { + const tempDir = await mkdtemp(join(tmpdir(), 'orca-computer-windows-semantic-')) + const statePath = join(tempDir, 'state.json') + const title = `Orca Semantic Controls ${Date.now()}` + const holderTitle = `${title} holder` + // Why: a window in another process takes activation back, as the user's own app would hold it. + const holder: ChildProcess = spawn( + 'powershell.exe', + [ + '-NoProfile', + '-NonInteractive', + '-STA', + '-Command', + `Add-Type -AssemblyName System.Windows.Forms; $form = New-Object Windows.Forms.Form; $form.Text = '${holderTitle}'; [Windows.Forms.Application]::Run($form)` + ], + { stdio: 'ignore' } + ) + // Why: no windowsHide; Node's hidden startup flag would also hide the fixture's first WPF window. + const child: ChildProcess = spawn( + 'powershell.exe', + [ + '-NoProfile', + '-NonInteractive', + '-STA', + '-ExecutionPolicy', + 'Bypass', + '-File', + fixtureScript, + '-StatePath', + statePath, + '-Title', + title, + '-HolderTitle', + holderTitle + ], + { stdio: 'ignore' } + ) + if (child.pid === undefined) { + throw new Error('failed to start the Windows semantic controls fixture') + } + const pid = child.pid + + const readState = async (): Promise => { + const raw = await readFile(statePath, 'utf8') + return WindowsSemanticFixtureStateSchema.parse(JSON.parse(raw.replace(/^/, ''))) + } + const waitForState = async ( + predicate: (state: WindowsSemanticFixtureState) => boolean, + label: string + ): Promise => { + // Why: the fixture replaces the file on every change, so a read can land between writes and throw. + return vi.waitFor( + async () => { + const state = await readState() + if (!predicate(state)) { + throw new Error(`Fixture state not yet ${label}: ${JSON.stringify(state)}`) + } + return state + }, + { timeout: 15_000, interval: 100 } + ) + } + const close = async (): Promise => { + for (const owned of [pid, holder.pid].filter((value) => value !== undefined)) { + try { + await execFileAsync('taskkill.exe', ['/PID', String(owned), '/T', '/F']) + } catch { + // The fixture or holder may already have exited. + } + } + await rm(tempDir, { force: true, recursive: true }) + } + + try { + await waitForState((state) => state.ready, 'window rendered') + } catch (error) { + await close() + throw error + } + return { app: `pid:${pid}`, title, readState, waitForState, close } +}