fix(computer-use): use UI Automation for Windows scroll, expand and slider actions (#41429)

* fix(computer-use): scroll, expand and set sliders on Windows without raising the window

* fix(computer-use): wait for Windows scroll and slider writes to settle before verifying

* fix(computer-use): reject NaN for Windows slider set-value

* test(computer-use): wait on the Windows fixture state with vi.waitFor and a schema

* test(computer-use): check activation per action against a holder window in the Windows fixture

* test(computer-use): log fixture window events for activation failures

* fix(computer-use): report foreground when a Windows app raises itself for a UI Automation action

WPF activates its own window for Scroll, ExpandCollapse, RangeValue and SelectionItem calls from any UIA client, so the runtime now compares the foreground window before and after a pattern action instead of assuming background. The Windows e2e checks delivery against the fixture's own activation count, with a holder window in another process.

* test(computer-use): skip an unstarted holder when closing the Windows fixture

* fix(computer-use): bound Windows pattern scroll paging and tighten slider read-back

* fix(computer-use): cap Windows slider read-back tolerance and run the UIA e2e only on CI
This commit is contained in:
Kelvin Amoaba
2026-11-12 23:17:10 +00:00
committed by GitHub
parent 43df511fad
commit c794f75e4e
10 changed files with 595 additions and 25 deletions
@@ -51,6 +51,10 @@ public static class OrcaDesktopWin32 {
}
return Foreground;
}
public static uint GetWindowThreadProcessId(IntPtr hwnd, out uint processId) {
processId = hwnd == new IntPtr(101) ? 4242u : 1u;
return 1;
}
public static bool SetCursorPos(int x, int y) {
Calls.Add("move:" + x + "," + y);
return true;
@@ -74,7 +78,7 @@ $functions = @(
"Get-OrcaRequiredNumber", "Get-OrcaPositiveInteger", "Get-OrcaPositiveNumber",
"Get-OrcaScreenPoint", "Get-OrcaElementScreenPoint", "Get-OrcaModifierVirtualKey",
"Get-OrcaClickModifierVirtualKeys", "Send-OrcaMouseClick", "Send-OrcaDrag", "Invoke-OrcaOperation",
"Assert-OrcaInputAllowed"
"Assert-OrcaInputAllowed", "Test-OrcaTargetTookForeground"
)
foreach ($definition in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false)) {
if ($definition.Name -in $functions) { . ([scriptblock]::Create($definition.Extent.Text)) }
@@ -87,6 +91,7 @@ function Get-OrcaRootElement { $null }
function Get-OrcaWindowFrame { @{ x = [OrcaDesktopWin32]::WindowX; y = 200 } }
function Find-OrcaElement($RootElement, $Record) { $Record }
function Invoke-OrcaPrimaryAction($Element) { $null -ne $Element }
function Invoke-OrcaScrollPattern($Element, $RootElement, $Direction, $Pages) { $null -ne $Element }
function New-OrcaSnapshot { [pscustomobject]@{} }
foreach ($case in @(
@@ -112,6 +117,7 @@ foreach ($case in @(
if ($inputCalls.Count -eq 0) { throw "$($case.tool) did not send pointer input after activation" }
if ([OrcaDesktopWin32]::FocusReads -lt 2) { throw "$($case.tool) did not wait for activation" }
Assert-TestEqual $inputCalls[0] "move:505,206" "$($case.tool) geometry after delayed activation"
Assert-TestEqual $result.action.delivery "foreground" "$($case.tool) pointer delivery"
} else {
Assert-TestEqual $denied $true "$($case.tool) denied activation"
Assert-TestEqual $inputCalls.Count 0 "$($case.tool) denied activation sends no pointer or modifier input"
@@ -119,10 +125,18 @@ foreach ($case in @(
}
}
[OrcaDesktopWin32]::Reset($false)
$result = Invoke-OrcaOperation ([pscustomobject]@{ tool = "click"; app = "Fixture"; element = [pscustomobject]@{ index = 1 } })
Assert-TestEqual $result.action.path "accessibility" "pattern click path"
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "pattern click bypasses activation and global input"
foreach ($case in @(
@{ tool = "click" },
@{ tool = "scroll"; direction = "down"; pages = 1 }
)) {
[OrcaDesktopWin32]::Reset($false)
$operation = [pscustomobject]@{ app = "Fixture"; element = [pscustomobject]@{ index = 1 } }
foreach ($key in $case.Keys) { $operation | Add-Member -NotePropertyName $key -NotePropertyValue $case[$key] }
$result = Invoke-OrcaOperation $operation
Assert-TestEqual $result.action.path "accessibility" "pattern $($case.tool) path"
Assert-TestEqual $result.action.delivery "background" "pattern $($case.tool) delivery"
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "pattern $($case.tool) bypasses activation and global input"
}
foreach ($case in @(
@{ tool = "click"; element = [pscustomobject]@{ index = 1 } },
@{ tool = "click"; x = 5; y = 6; restoreWindow = $true },
@@ -137,4 +151,10 @@ foreach ($case in @(
if ($message -notlike "target_elevated:*") { throw "$($case.tool) elevated target expected target_elevated, received '$message'" }
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "$($case.tool) elevated target sends no activation or input"
}
# Why: WPF activates its own window for most patterns; the result must say so rather than claim background.
[OrcaDesktopWin32]::Reset($false)
function Invoke-OrcaPrimaryAction($Element) { [OrcaDesktopWin32]::Foreground = [IntPtr]101; $true }
$result = Invoke-OrcaOperation ([pscustomobject]@{ tool = "click"; app = "Fixture"; element = [pscustomobject]@{ index = 1 } })
Assert-TestEqual $result.action.path "accessibility" "self-activating pattern click path"
Assert-TestEqual $result.action.delivery "foreground" "self-activating pattern click delivery"
Write-Output "windows-pointer-tests-ok"