mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
fix(computer-use): use UI Automation for Windows scroll, expand and slider actions (#41429)
* fix(computer-use): scroll, expand and set sliders on Windows without raising the window * fix(computer-use): wait for Windows scroll and slider writes to settle before verifying * fix(computer-use): reject NaN for Windows slider set-value * test(computer-use): wait on the Windows fixture state with vi.waitFor and a schema * test(computer-use): check activation per action against a holder window in the Windows fixture * test(computer-use): log fixture window events for activation failures * fix(computer-use): report foreground when a Windows app raises itself for a UI Automation action WPF activates its own window for Scroll, ExpandCollapse, RangeValue and SelectionItem calls from any UIA client, so the runtime now compares the foreground window before and after a pattern action instead of assuming background. The Windows e2e checks delivery against the fixture's own activation count, with a holder window in another process. * test(computer-use): skip an unstarted holder when closing the Windows fixture * fix(computer-use): bound Windows pattern scroll paging and tighten slider read-back * fix(computer-use): cap Windows slider read-back tolerance and run the UIA e2e only on CI
This commit is contained in:
@@ -51,6 +51,10 @@ public static class OrcaDesktopWin32 {
|
||||
}
|
||||
return Foreground;
|
||||
}
|
||||
public static uint GetWindowThreadProcessId(IntPtr hwnd, out uint processId) {
|
||||
processId = hwnd == new IntPtr(101) ? 4242u : 1u;
|
||||
return 1;
|
||||
}
|
||||
public static bool SetCursorPos(int x, int y) {
|
||||
Calls.Add("move:" + x + "," + y);
|
||||
return true;
|
||||
@@ -74,7 +78,7 @@ $functions = @(
|
||||
"Get-OrcaRequiredNumber", "Get-OrcaPositiveInteger", "Get-OrcaPositiveNumber",
|
||||
"Get-OrcaScreenPoint", "Get-OrcaElementScreenPoint", "Get-OrcaModifierVirtualKey",
|
||||
"Get-OrcaClickModifierVirtualKeys", "Send-OrcaMouseClick", "Send-OrcaDrag", "Invoke-OrcaOperation",
|
||||
"Assert-OrcaInputAllowed"
|
||||
"Assert-OrcaInputAllowed", "Test-OrcaTargetTookForeground"
|
||||
)
|
||||
foreach ($definition in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] }, $false)) {
|
||||
if ($definition.Name -in $functions) { . ([scriptblock]::Create($definition.Extent.Text)) }
|
||||
@@ -87,6 +91,7 @@ function Get-OrcaRootElement { $null }
|
||||
function Get-OrcaWindowFrame { @{ x = [OrcaDesktopWin32]::WindowX; y = 200 } }
|
||||
function Find-OrcaElement($RootElement, $Record) { $Record }
|
||||
function Invoke-OrcaPrimaryAction($Element) { $null -ne $Element }
|
||||
function Invoke-OrcaScrollPattern($Element, $RootElement, $Direction, $Pages) { $null -ne $Element }
|
||||
function New-OrcaSnapshot { [pscustomobject]@{} }
|
||||
|
||||
foreach ($case in @(
|
||||
@@ -112,6 +117,7 @@ foreach ($case in @(
|
||||
if ($inputCalls.Count -eq 0) { throw "$($case.tool) did not send pointer input after activation" }
|
||||
if ([OrcaDesktopWin32]::FocusReads -lt 2) { throw "$($case.tool) did not wait for activation" }
|
||||
Assert-TestEqual $inputCalls[0] "move:505,206" "$($case.tool) geometry after delayed activation"
|
||||
Assert-TestEqual $result.action.delivery "foreground" "$($case.tool) pointer delivery"
|
||||
} else {
|
||||
Assert-TestEqual $denied $true "$($case.tool) denied activation"
|
||||
Assert-TestEqual $inputCalls.Count 0 "$($case.tool) denied activation sends no pointer or modifier input"
|
||||
@@ -119,10 +125,18 @@ foreach ($case in @(
|
||||
}
|
||||
}
|
||||
|
||||
[OrcaDesktopWin32]::Reset($false)
|
||||
$result = Invoke-OrcaOperation ([pscustomobject]@{ tool = "click"; app = "Fixture"; element = [pscustomobject]@{ index = 1 } })
|
||||
Assert-TestEqual $result.action.path "accessibility" "pattern click path"
|
||||
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "pattern click bypasses activation and global input"
|
||||
foreach ($case in @(
|
||||
@{ tool = "click" },
|
||||
@{ tool = "scroll"; direction = "down"; pages = 1 }
|
||||
)) {
|
||||
[OrcaDesktopWin32]::Reset($false)
|
||||
$operation = [pscustomobject]@{ app = "Fixture"; element = [pscustomobject]@{ index = 1 } }
|
||||
foreach ($key in $case.Keys) { $operation | Add-Member -NotePropertyName $key -NotePropertyValue $case[$key] }
|
||||
$result = Invoke-OrcaOperation $operation
|
||||
Assert-TestEqual $result.action.path "accessibility" "pattern $($case.tool) path"
|
||||
Assert-TestEqual $result.action.delivery "background" "pattern $($case.tool) delivery"
|
||||
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "pattern $($case.tool) bypasses activation and global input"
|
||||
}
|
||||
foreach ($case in @(
|
||||
@{ tool = "click"; element = [pscustomobject]@{ index = 1 } },
|
||||
@{ tool = "click"; x = 5; y = 6; restoreWindow = $true },
|
||||
@@ -137,4 +151,10 @@ foreach ($case in @(
|
||||
if ($message -notlike "target_elevated:*") { throw "$($case.tool) elevated target expected target_elevated, received '$message'" }
|
||||
Assert-TestEqual ([OrcaDesktopWin32]::Calls.Count) 0 "$($case.tool) elevated target sends no activation or input"
|
||||
}
|
||||
# Why: WPF activates its own window for most patterns; the result must say so rather than claim background.
|
||||
[OrcaDesktopWin32]::Reset($false)
|
||||
function Invoke-OrcaPrimaryAction($Element) { [OrcaDesktopWin32]::Foreground = [IntPtr]101; $true }
|
||||
$result = Invoke-OrcaOperation ([pscustomobject]@{ tool = "click"; app = "Fixture"; element = [pscustomobject]@{ index = 1 } })
|
||||
Assert-TestEqual $result.action.path "accessibility" "self-activating pattern click path"
|
||||
Assert-TestEqual $result.action.delivery "foreground" "self-activating pattern click delivery"
|
||||
Write-Output "windows-pointer-tests-ok"
|
||||
|
||||
Reference in New Issue
Block a user