mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 11:24:37 +00:00
fix(computer): keep the agent's window id as the target window on Windows (#43162)
The Windows helper checked an agent's --window-id against Process.MainWindowHandle, a z-order guess that moves to a tooltip or another window of the same app. The window id is now the target: a live, visible top-level window of the target process, carried through the tree read, frame, screenshot, raise, keyboard input and the observe after the action. Co-authored-by: toodeceptive <137078450+toodeceptive@users.noreply.github.com>
This commit is contained in:
co-authored by
toodeceptive
parent
9f96bd57e3
commit
98fadae4d8
@@ -104,6 +104,67 @@ public static class OrcaDesktopWin32 {
|
||||
[DllImport("user32.dll")]
|
||||
public static extern uint GetWindowThreadProcessId(IntPtr hwnd, out uint processId);
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
private static extern bool IsWindowVisible(IntPtr hwnd);
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
private static extern IntPtr GetAncestor(IntPtr hwnd, uint flags);
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
private static extern IntPtr GetWindow(IntPtr hwnd, uint command);
|
||||
|
||||
[DllImport("user32.dll", EntryPoint = "GetClassNameW", CharSet = CharSet.Unicode)]
|
||||
private static extern int GetClassName(IntPtr hwnd, System.Text.StringBuilder className, int maxCount);
|
||||
|
||||
[DllImport("user32.dll", EntryPoint = "GetWindowTextW", CharSet = CharSet.Unicode)]
|
||||
private static extern int GetWindowText(IntPtr hwnd, System.Text.StringBuilder text, int maxCount);
|
||||
|
||||
[DllImport("user32.dll", EntryPoint = "GetWindowTextLengthW")]
|
||||
private static extern int GetWindowTextLength(IntPtr hwnd);
|
||||
|
||||
private delegate bool EnumWindowsProc(IntPtr hwnd, IntPtr lParam);
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
private static extern bool EnumWindows(EnumWindowsProc callback, IntPtr lParam);
|
||||
|
||||
// A visible top-level window of the process; tooltips, menus and their shadows are transient popups, not targets.
|
||||
public static bool IsTargetWindow(IntPtr hwnd, int processId) {
|
||||
const uint rootAncestor = 2;
|
||||
if (hwnd == IntPtr.Zero || !IsWindow(hwnd) || !IsWindowVisible(hwnd) || GetAncestor(hwnd, rootAncestor) != hwnd) {
|
||||
return false;
|
||||
}
|
||||
uint ownerProcessId;
|
||||
GetWindowThreadProcessId(hwnd, out ownerProcessId);
|
||||
if (ownerProcessId != (uint)processId) {
|
||||
return false;
|
||||
}
|
||||
var className = new System.Text.StringBuilder(256);
|
||||
GetClassName(hwnd, className, className.Capacity);
|
||||
var name = className.ToString();
|
||||
return name != "tooltips_class32" && name != "#32768" && name != "SysShadow";
|
||||
}
|
||||
|
||||
// Process.MainWindowHandle's rule (first visible unowned window in z-order) without transient popups.
|
||||
public static IntPtr FindMainTargetWindow(int processId) {
|
||||
const uint ownerWindow = 4;
|
||||
var found = IntPtr.Zero;
|
||||
EnumWindows((hwnd, _) => {
|
||||
if (GetWindow(hwnd, ownerWindow) != IntPtr.Zero || !IsTargetWindow(hwnd, processId)) {
|
||||
return true;
|
||||
}
|
||||
found = hwnd;
|
||||
return false;
|
||||
}, IntPtr.Zero);
|
||||
return found;
|
||||
}
|
||||
|
||||
// Reads the caption Windows stores, without sending the target a message, so a hung window cannot block it.
|
||||
public static string GetWindowTitle(IntPtr hwnd) {
|
||||
var title = new System.Text.StringBuilder(GetWindowTextLength(hwnd) + 1);
|
||||
GetWindowText(hwnd, title, title.Capacity);
|
||||
return title.ToString();
|
||||
}
|
||||
|
||||
[DllImport("user32.dll")]
|
||||
public static extern bool SetCursorPos(int x, int y);
|
||||
|
||||
@@ -660,7 +721,8 @@ function Find-OrcaProcess([string]$Query) {
|
||||
$needle = ""
|
||||
if ($null -ne $Query) { $needle = $Query.Trim() }
|
||||
if ([string]::IsNullOrWhiteSpace($needle)) { throw 'appNotFound("")' }
|
||||
if ($needle.StartsWith("pid:", [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||
$byPid = $needle.StartsWith("pid:", [System.StringComparison]::OrdinalIgnoreCase)
|
||||
if ($byPid) {
|
||||
$needle = $needle.Substring(4)
|
||||
}
|
||||
|
||||
@@ -673,6 +735,8 @@ function Find-OrcaProcess([string]$Query) {
|
||||
return $match
|
||||
}
|
||||
}
|
||||
# Why: a pid: query names one process; matching its digits against window titles would pick another app.
|
||||
if ($byPid) { throw "appNotFound(`"$Query`")" }
|
||||
|
||||
$processNeedle = $needle
|
||||
if ($processNeedle.EndsWith(".exe", [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||
@@ -693,8 +757,8 @@ function Find-OrcaProcess([string]$Query) {
|
||||
throw "appNotFound(`"$Query`")"
|
||||
}
|
||||
|
||||
function Assert-OrcaProcessAllowed($Process) {
|
||||
$values = @($Process.ProcessName, $Process.MainWindowTitle) | ForEach-Object { ([string]$_).ToLowerInvariant() }
|
||||
function Assert-OrcaProcessAllowed($Process, [string]$WindowTitle = "") {
|
||||
$values = @($Process.ProcessName, $Process.MainWindowTitle, $WindowTitle) | ForEach-Object { ([string]$_).ToLowerInvariant() }
|
||||
foreach ($fragment in $BlockedAppFragments) {
|
||||
foreach ($value in $values) {
|
||||
if ($value.Contains($fragment)) {
|
||||
@@ -721,14 +785,31 @@ function Test-OrcaBrowserProcess($Process) {
|
||||
$browserProcesses -contains $name
|
||||
}
|
||||
|
||||
function Get-OrcaRootElement($Process) {
|
||||
if ($Process.MainWindowHandle -eq 0) {
|
||||
throw "No top-level UI Automation window is available for $($Process.ProcessName)."
|
||||
# Why: MainWindowHandle is a z-order guess that moves to a tooltip or another window, so the id the agent holds is the window.
|
||||
function Resolve-OrcaTargetWindow($Process, $WindowId, $WindowIndex) {
|
||||
if ($null -ne $WindowIndex -and [int]$WindowIndex -ne 0) {
|
||||
throw "windowNotFound(`"$WindowIndex`")"
|
||||
}
|
||||
if ([OrcaDesktopWin32]::IsWindowHung([IntPtr]$Process.MainWindowHandle, $WindowResponseTimeoutMilliseconds)) {
|
||||
if ($null -ne $WindowId) {
|
||||
$handle = [IntPtr][int64]$WindowId
|
||||
if (-not [OrcaDesktopWin32]::IsTargetWindow($handle, [int]$Process.Id)) {
|
||||
throw "windowNotFound(`"$WindowId`")"
|
||||
}
|
||||
} else {
|
||||
$handle = [OrcaDesktopWin32]::FindMainTargetWindow([int]$Process.Id)
|
||||
if ($handle -eq [IntPtr]::Zero) {
|
||||
throw "No top-level UI Automation window is available for $($Process.ProcessName)."
|
||||
}
|
||||
}
|
||||
Assert-OrcaProcessAllowed $Process ([OrcaDesktopWin32]::GetWindowTitle($handle))
|
||||
$handle
|
||||
}
|
||||
|
||||
function Get-OrcaRootElement($Process, [IntPtr]$WindowHandle) {
|
||||
if ([OrcaDesktopWin32]::IsWindowHung($WindowHandle, $WindowResponseTimeoutMilliseconds)) {
|
||||
throw "window_not_responding: $(Get-OrcaAppName $Process) is not responding; nothing was read from or sent to it"
|
||||
}
|
||||
[Windows.Automation.AutomationElement]::FromHandle([IntPtr]$Process.MainWindowHandle)
|
||||
[Windows.Automation.AutomationElement]::FromHandle($WindowHandle)
|
||||
}
|
||||
|
||||
function Assert-OrcaInputAllowed($Process) {
|
||||
@@ -739,9 +820,9 @@ function Assert-OrcaInputAllowed($Process) {
|
||||
throw "target_elevated: $(Get-OrcaAppName $Process) runs as administrator, so Windows blocks input to it from Orca; no input was sent"
|
||||
}
|
||||
|
||||
function Get-OrcaWindowFrame($Process, $RootElement) {
|
||||
function Get-OrcaWindowFrame([IntPtr]$WindowHandle, $RootElement) {
|
||||
$rect = New-Object OrcaDesktopWin32+RECT
|
||||
if ([OrcaDesktopWin32]::GetWindowRect([IntPtr]$Process.MainWindowHandle, [ref]$rect)) {
|
||||
if ([OrcaDesktopWin32]::GetWindowRect($WindowHandle, [ref]$rect)) {
|
||||
return New-OrcaFrame $rect.Left $rect.Top ($rect.Right - $rect.Left) ($rect.Bottom - $rect.Top)
|
||||
}
|
||||
|
||||
@@ -754,10 +835,6 @@ function Get-OrcaWindowFrame($Process, $RootElement) {
|
||||
$null
|
||||
}
|
||||
|
||||
function Get-OrcaWindowId($Process) {
|
||||
[int64]$Process.MainWindowHandle
|
||||
}
|
||||
|
||||
function Get-OrcaAppName($Process) {
|
||||
if ($Process.ProcessName -eq "ApplicationFrameHost" -and -not [string]::IsNullOrWhiteSpace($Process.MainWindowTitle)) {
|
||||
return [string]$Process.MainWindowTitle
|
||||
@@ -774,15 +851,6 @@ function New-OrcaAppRecord($Process) {
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-OrcaWindowTarget($Process, $WindowId, $WindowIndex) {
|
||||
if ($null -ne $WindowIndex -and [int]$WindowIndex -ne 0) {
|
||||
throw "windowNotFound(`"$WindowIndex`")"
|
||||
}
|
||||
if ($null -ne $WindowId -and [int64]$WindowId -ne (Get-OrcaWindowId $Process)) {
|
||||
throw "windowNotFound(`"$WindowId`")"
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-OrcaWindowPatternActivation($RootElement) {
|
||||
# Why: reapplying the current visual state lets the window's provider activate it when Windows refuses our raise (agent-desktop #651).
|
||||
# Only for maximized windows: re-applying Normal restores a snapped window's pre-snap size.
|
||||
@@ -794,24 +862,22 @@ function Invoke-OrcaWindowPatternActivation($RootElement) {
|
||||
}
|
||||
|
||||
# Brings only the target window to the front; returns $null once it is the foreground window, otherwise why not.
|
||||
function Restore-OrcaWindow($Process, $RootElement) {
|
||||
$handle = [IntPtr]$Process.MainWindowHandle
|
||||
if ($handle -eq [IntPtr]::Zero) { return "it has no top-level window" }
|
||||
if ([OrcaDesktopWin32]::IsWindowCloaked($handle)) {
|
||||
function Restore-OrcaWindow([IntPtr]$Handle, $RootElement) {
|
||||
if ([OrcaDesktopWin32]::IsWindowCloaked($Handle)) {
|
||||
return "it is on another virtual desktop, and raising it would switch the user's desktop"
|
||||
}
|
||||
# Why: SW_RESTORE also un-maximizes, which moves the frame the agent targeted.
|
||||
if ([OrcaDesktopWin32]::IsIconic($handle)) {
|
||||
[void][OrcaDesktopWin32]::ShowWindow($handle, 9)
|
||||
if ([OrcaDesktopWin32]::IsIconic($Handle)) {
|
||||
[void][OrcaDesktopWin32]::ShowWindow($Handle, 9)
|
||||
}
|
||||
if (Test-OrcaWindowFocused $handle) { return $null }
|
||||
if (Test-OrcaWindowFocused $Handle) { return $null }
|
||||
# Why: Windows often refuses a background process's raise; after one refusal try the UIA route, then one more raise.
|
||||
[void][OrcaDesktopWin32]::SetForegroundWindow($handle)
|
||||
if (Wait-OrcaWindowFocused $handle 250) { return $null }
|
||||
[void][OrcaDesktopWin32]::SetForegroundWindow($Handle)
|
||||
if (Wait-OrcaWindowFocused $Handle 250) { return $null }
|
||||
Invoke-OrcaWindowPatternActivation $RootElement
|
||||
if (Wait-OrcaWindowFocused $handle 250) { return $null }
|
||||
[void][OrcaDesktopWin32]::SetForegroundWindow($handle)
|
||||
if (Wait-OrcaWindowFocused $handle 250) { return $null }
|
||||
if (Wait-OrcaWindowFocused $Handle 250) { return $null }
|
||||
[void][OrcaDesktopWin32]::SetForegroundWindow($Handle)
|
||||
if (Wait-OrcaWindowFocused $Handle 250) { return $null }
|
||||
"Windows refused to bring it to the front"
|
||||
}
|
||||
|
||||
@@ -852,12 +918,12 @@ function Assert-OrcaKeyboardFocus([IntPtr]$WindowHandle, $Operation, $RestoreFai
|
||||
}
|
||||
|
||||
# Raises the target for real mouse input, then returns its frame read after the raise (restoring moves it).
|
||||
function Restore-OrcaPointerWindow($Process, $RootElement) {
|
||||
$failure = Restore-OrcaWindow $Process $RootElement
|
||||
function Restore-OrcaPointerWindow($Process, [IntPtr]$WindowHandle, $RootElement) {
|
||||
$failure = Restore-OrcaWindow $WindowHandle $RootElement
|
||||
if ($null -ne $failure) {
|
||||
throw "window_not_focused: restoreWindow was requested but $(Get-OrcaAppName $Process) did not come to the front ($failure); no pointer input was sent"
|
||||
}
|
||||
Get-OrcaWindowFrame $Process $RootElement
|
||||
Get-OrcaWindowFrame $WindowHandle $RootElement
|
||||
}
|
||||
|
||||
function New-OrcaForegroundRefusal($Process, [string]$Need, [string]$Alternative = "", [string]$Takes = "moves the user's pointer") {
|
||||
@@ -1109,7 +1175,7 @@ function Test-OrcaTextPatternReadOnly($Element) {
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-OrcaElementTextInsertion($Element, $Record, $Process, $RootElement, $Operation, [string]$Text) {
|
||||
function Invoke-OrcaElementTextInsertion($Element, $Record, $Process, [IntPtr]$WindowHandle, $RootElement, $Operation, [string]$Text) {
|
||||
if ($null -eq $Element) {
|
||||
throw "stale element: element $($Record.index) is gone; run get-app-state again and use a fresh element index"
|
||||
}
|
||||
@@ -1130,12 +1196,12 @@ function Invoke-OrcaElementTextInsertion($Element, $Record, $Process, $RootEleme
|
||||
if (-not [bool]$Operation.restoreWindow) {
|
||||
throw (New-OrcaForegroundRefusal $Process "element $index accepts text only as keys, which go only to the window in front" "" "takes the user's keyboard")
|
||||
}
|
||||
$failure = Restore-OrcaWindow $Process $RootElement
|
||||
Assert-OrcaKeyboardFocus ([IntPtr]$Process.MainWindowHandle) $Operation $failure $Process
|
||||
$failure = Restore-OrcaWindow $WindowHandle $RootElement
|
||||
Assert-OrcaKeyboardFocus $WindowHandle $Operation $failure $Process
|
||||
try { $Element.SetFocus() } catch {
|
||||
throw "window_not_focused: element $index did not take keyboard focus; no text was sent"
|
||||
}
|
||||
Send-OrcaText ([IntPtr]$Process.MainWindowHandle) $Text
|
||||
Send-OrcaText $WindowHandle $Text
|
||||
[pscustomobject]@{ path = "synthetic"; actionName = "typeText"; fallbackReason = "actionUnsupported"; verification = [pscustomobject]@{ state = "unverified"; reason = "synthetic_input" } }
|
||||
}
|
||||
|
||||
@@ -1602,12 +1668,12 @@ function Get-OrcaScreenshot($ScreenshotRequest, $WindowFrame, [IntPtr]$WindowHan
|
||||
|
||||
function New-OrcaSnapshot([string]$Query, $ScreenshotRequest, $WindowId = $null, $WindowIndex = $null, [bool]$RestoreWindow = $false, $TreeRoot = $null) {
|
||||
$process = Find-OrcaProcess $Query
|
||||
Assert-OrcaWindowTarget $process $WindowId $WindowIndex
|
||||
$handle = Resolve-OrcaTargetWindow $process $WindowId $WindowIndex
|
||||
# Root first: its hang probe must run before ShowWindow, which blocks on a hung window.
|
||||
$root = Get-OrcaRootElement $process
|
||||
$root = Get-OrcaRootElement $process $handle
|
||||
# Why: observation works behind other windows, so a refused raise still returns the snapshot.
|
||||
if ($RestoreWindow) { [void](Restore-OrcaWindow $process $root) }
|
||||
$windowFrame = Get-OrcaWindowFrame $process $root
|
||||
if ($RestoreWindow) { [void](Restore-OrcaWindow $handle $root) }
|
||||
$windowFrame = Get-OrcaWindowFrame $handle $root
|
||||
$walkRoot = $root
|
||||
if ($null -ne $TreeRoot) {
|
||||
$walkRoot = Find-OrcaElement $root $TreeRoot
|
||||
@@ -1616,13 +1682,13 @@ function New-OrcaSnapshot([string]$Query, $ScreenshotRequest, $WindowId = $null,
|
||||
}
|
||||
}
|
||||
$tree = Render-OrcaTree $walkRoot $windowFrame (Test-OrcaBrowserProcess $process)
|
||||
$screenshot = Get-OrcaScreenshot $ScreenshotRequest $windowFrame ([IntPtr]$process.MainWindowHandle)
|
||||
$screenshot = Get-OrcaScreenshot $ScreenshotRequest $windowFrame $handle
|
||||
|
||||
[pscustomobject]@{
|
||||
snapshotId = [guid]::NewGuid().ToString()
|
||||
app = New-OrcaAppRecord $process
|
||||
windowTitle = $process.MainWindowTitle
|
||||
windowId = Get-OrcaWindowId $process
|
||||
windowTitle = [OrcaDesktopWin32]::GetWindowTitle($handle)
|
||||
windowId = [int64]$handle
|
||||
windowBounds = $windowFrame
|
||||
screenshotBase64 = if ($null -ne $screenshot) { $screenshot.base64 } else { $null }
|
||||
screenshotFormat = if ($null -ne $screenshot) { $screenshot.format } else { $null }
|
||||
@@ -1648,8 +1714,9 @@ function Get-OrcaAppList {
|
||||
|
||||
function Get-OrcaWindowList([string]$Query) {
|
||||
$process = Find-OrcaProcess $Query
|
||||
$root = Get-OrcaRootElement $process
|
||||
$windowFrame = Get-OrcaWindowFrame $process $root
|
||||
$handle = Resolve-OrcaTargetWindow $process $null $null
|
||||
$root = Get-OrcaRootElement $process $handle
|
||||
$windowFrame = Get-OrcaWindowFrame $handle $root
|
||||
$x = $null
|
||||
$y = $null
|
||||
$width = 0
|
||||
@@ -1666,8 +1733,8 @@ function Get-OrcaWindowList([string]$Query) {
|
||||
windows = @([pscustomobject]@{
|
||||
index = 0
|
||||
app = $app
|
||||
id = Get-OrcaWindowId $process
|
||||
title = $process.MainWindowTitle
|
||||
id = [int64]$handle
|
||||
title = [OrcaDesktopWin32]::GetWindowTitle($handle)
|
||||
x = $x
|
||||
y = $y
|
||||
width = $width
|
||||
@@ -1675,7 +1742,7 @@ function Get-OrcaWindowList([string]$Query) {
|
||||
isMinimized = $false
|
||||
isOffscreen = $false
|
||||
screenIndex = $null
|
||||
platform = [pscustomobject]@{ backend = "uia"; nativeWindowHandle = Get-OrcaWindowId $process }
|
||||
platform = [pscustomobject]@{ backend = "uia"; nativeWindowHandle = [int64]$handle }
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2480,17 +2547,16 @@ function Invoke-OrcaOperation($Operation) {
|
||||
|
||||
$process = Find-OrcaProcess $Operation.app
|
||||
Assert-OrcaInputAllowed $process
|
||||
Assert-OrcaWindowTarget $process $Operation.windowId $Operation.windowIndex
|
||||
$root = Get-OrcaRootElement $process
|
||||
$handle = Resolve-OrcaTargetWindow $process $Operation.windowId $Operation.windowIndex
|
||||
$root = Get-OrcaRootElement $process $handle
|
||||
$restoreWindow = [bool]$Operation.restoreWindow
|
||||
$windowFrame = if ($null -ne $Operation.windowBounds) { $Operation.windowBounds } else { Get-OrcaWindowFrame $process $root }
|
||||
$windowFrame = if ($null -ne $Operation.windowBounds) { $Operation.windowBounds } else { Get-OrcaWindowFrame $handle $root }
|
||||
$element = Find-OrcaElement $root $Operation.element
|
||||
$fromElement = Find-OrcaElement $root $Operation.fromElement
|
||||
$toElement = Find-OrcaElement $root $Operation.toElement
|
||||
$handle = [IntPtr]$process.MainWindowHandle
|
||||
$typesIntoElement = $Operation.tool -eq "type_text" -and $null -ne $Operation.element
|
||||
if ($Operation.tool -in @("type_text", "press_key", "hotkey", "paste_text") -and -not $typesIntoElement) {
|
||||
$restoreFailure = if ($restoreWindow) { Restore-OrcaWindow $process $root } else { $null }
|
||||
$restoreFailure = if ($restoreWindow) { Restore-OrcaWindow $handle $root } else { $null }
|
||||
Assert-OrcaKeyboardFocus $handle $Operation $restoreFailure $process
|
||||
}
|
||||
$foregroundBefore = [OrcaDesktopWin32]::GetForegroundWindow()
|
||||
@@ -2527,7 +2593,7 @@ function Invoke-OrcaOperation($Operation) {
|
||||
if (-not $restoreWindow) {
|
||||
throw (New-OrcaForegroundRefusal $process $pointerNeed $alternative)
|
||||
}
|
||||
$windowFrame = Restore-OrcaPointerWindow $process $root
|
||||
$windowFrame = Restore-OrcaPointerWindow $process $handle $root
|
||||
$point = Get-OrcaPointerPoint $element $Operation.element $Operation.x $Operation.y "x" "y" $windowFrame
|
||||
$cursor = Get-OrcaCursorPosition
|
||||
try {
|
||||
@@ -2574,7 +2640,7 @@ function Invoke-OrcaOperation($Operation) {
|
||||
throw (New-OrcaForegroundRefusal $process "nothing under $where scrolls $($Operation.direction) through UI Automation, so this scroll needs the real mouse wheel")
|
||||
}
|
||||
# Why: the wheel scrolls whatever is under the pointer, so the window must be on top.
|
||||
$windowFrame = Restore-OrcaPointerWindow $process $root
|
||||
$windowFrame = Restore-OrcaPointerWindow $process $handle $root
|
||||
$point = Get-OrcaPointerPoint $element $Operation.element $Operation.x $Operation.y "x" "y" $windowFrame
|
||||
$cursor = Get-OrcaCursorPosition
|
||||
try {
|
||||
@@ -2596,7 +2662,7 @@ function Invoke-OrcaOperation($Operation) {
|
||||
if (-not $restoreWindow) {
|
||||
throw (New-OrcaForegroundRefusal $process "a drag needs the real mouse, because UI Automation has no drag")
|
||||
}
|
||||
$windowFrame = Restore-OrcaPointerWindow $process $root
|
||||
$windowFrame = Restore-OrcaPointerWindow $process $handle $root
|
||||
$from = Get-OrcaPointerPoint $fromElement $Operation.fromElement $Operation.from_x $Operation.from_y "from_x" "from_y" $windowFrame
|
||||
$to = Get-OrcaPointerPoint $toElement $Operation.toElement $Operation.to_x $Operation.to_y "to_x" "to_y" $windowFrame
|
||||
$cursor = Get-OrcaCursorPosition
|
||||
@@ -2610,7 +2676,7 @@ function Invoke-OrcaOperation($Operation) {
|
||||
"type_text" {
|
||||
$text = Get-OrcaRequiredString $Operation.text "text"
|
||||
if ($typesIntoElement) {
|
||||
$action = Invoke-OrcaElementTextInsertion $element $Operation.element $process $root $Operation $text
|
||||
$action = Invoke-OrcaElementTextInsertion $element $Operation.element $process $handle $root $Operation $text
|
||||
} else {
|
||||
Send-OrcaText $handle $text
|
||||
$action = [pscustomobject]@{ path = "synthetic"; actionName = "typeText"; fallbackReason = $null; verification = [pscustomobject]@{ state = "unverified"; reason = "synthetic_input" } }
|
||||
@@ -2647,14 +2713,16 @@ function Invoke-OrcaOperation($Operation) {
|
||||
|
||||
try {
|
||||
try {
|
||||
$snapshot = New-OrcaSnapshot $Operation.app $screenshotRequest $Operation.windowId $Operation.windowIndex
|
||||
# Why: the observe reads the process and window the input went to, even if the action retitled it.
|
||||
$snapshot = New-OrcaSnapshot "pid:$($process.Id)" $screenshotRequest ([int64]$handle) $null
|
||||
} catch {
|
||||
if ($_.Exception.Message -like "window_not_responding:*") { throw }
|
||||
if ($null -eq $Operation.windowId -and $null -eq $Operation.windowIndex) { throw }
|
||||
# Why: only a target window the action closed may hand the observe to the app's main window.
|
||||
if ([OrcaDesktopWin32]::IsTargetWindow($handle, [int]$process.Id)) { throw }
|
||||
if ($null -eq $action.verification) {
|
||||
$action | Add-Member -NotePropertyName verification -NotePropertyValue ([pscustomobject]@{ state = "unverified"; reason = "window_changed" })
|
||||
}
|
||||
$snapshot = New-OrcaSnapshot $Operation.app $screenshotRequest $null $null
|
||||
$snapshot = New-OrcaSnapshot "pid:$($process.Id)" $screenshotRequest $null $null
|
||||
}
|
||||
} catch {
|
||||
# Why: the input was already sent, so the pre-action "nothing was sent" wording would invite a duplicate retry.
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { existsSync } from 'node:fs'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { afterAll, beforeAll, describe, expect, test } from 'vitest'
|
||||
import { join } from 'node:path'
|
||||
import { afterAll, beforeAll, describe, expect, test, vi } from 'vitest'
|
||||
import type {
|
||||
ComputerActionResult,
|
||||
ComputerListAppsResult,
|
||||
@@ -178,6 +180,90 @@ describe.skipIf(!isWindows || !e2eOptIn)('computer-use Windows e2e (Notepad)', (
|
||||
}
|
||||
})
|
||||
|
||||
test('a window id keeps its window when another window of the app becomes the main one', async () => {
|
||||
// A topmost second window comes first in z-order, so Process.MainWindowHandle moves to it.
|
||||
const fixture = await launchWindowFixture(
|
||||
[
|
||||
"$form.Text = 'orca-e2e-target'",
|
||||
'$button = New-Object System.Windows.Forms.Button',
|
||||
"$button.Text = 'Target press'",
|
||||
"$button.Add_Click({ $form.Text = 'orca-e2e-pressed' })",
|
||||
'$form.Controls.Add($button)',
|
||||
"$targetPath = Join-Path $PSScriptRoot 'target-window'"
|
||||
].join('\n'),
|
||||
[
|
||||
'$second = New-Object System.Windows.Forms.Form',
|
||||
"$second.Text = 'orca-e2e-second'",
|
||||
'$second.TopMost = $true',
|
||||
'$second.Show()',
|
||||
'[IO.File]::WriteAllText($targetPath, [string][int64]$form.Handle)'
|
||||
].join('; ')
|
||||
)
|
||||
try {
|
||||
const targetPath = join(fixture.dir, 'target-window')
|
||||
await vi.waitFor(() => expect(existsSync(targetPath)).toBe(true), {
|
||||
timeout: 15_000,
|
||||
interval: 100
|
||||
})
|
||||
const targetWindowId = Number(await readFile(targetPath, 'utf8'))
|
||||
const main = parseJsonOutput<{ result: ComputerSnapshotResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'get-app-state',
|
||||
'--app',
|
||||
`pid:${fixture.pid}`,
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
).result
|
||||
expect(main.snapshot.window.title).toBe('orca-e2e-second')
|
||||
const state = parseJsonOutput<{ result: ComputerSnapshotResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'get-app-state',
|
||||
'--app',
|
||||
`pid:${fixture.pid}`,
|
||||
'--window-id',
|
||||
String(targetWindowId),
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
).result
|
||||
expect(state.snapshot.window.id).toBe(targetWindowId)
|
||||
expect(state.snapshot.window.title).toBe('orca-e2e-target')
|
||||
const button = findRoleIndex(
|
||||
state.snapshot.treeText,
|
||||
/^\s*(\d+)\s+button Target press(?:,|$)/m
|
||||
)
|
||||
expect(button, state.snapshot.treeText).toBeGreaterThanOrEqual(0)
|
||||
const pressed = parseJsonOutput<{ result: ComputerActionResult }>(
|
||||
(
|
||||
await runOrcaCli([
|
||||
'computer',
|
||||
'click',
|
||||
'--app',
|
||||
`pid:${fixture.pid}`,
|
||||
'--window-id',
|
||||
String(targetWindowId),
|
||||
'--element-index',
|
||||
String(button),
|
||||
'--no-screenshot',
|
||||
'--json'
|
||||
])
|
||||
).stdout
|
||||
).result
|
||||
expect(pressed.action?.path).toBe('accessibility')
|
||||
expect(pressed.snapshot.window.id).toBe(targetWindowId)
|
||||
expect(pressed.snapshot.window.title).toBe('orca-e2e-pressed')
|
||||
} finally {
|
||||
await closeWindowFixture(fixture)
|
||||
}
|
||||
})
|
||||
|
||||
test('a hung window reports a timeout error and later calls still succeed', async () => {
|
||||
const hung = await launchHungWindowFixture()
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user