diff --git a/config/scripts/verify-computer-native.mjs b/config/scripts/verify-computer-native.mjs index 91c27e03a5b..ba23a49cdd0 100644 --- a/config/scripts/verify-computer-native.mjs +++ b/config/scripts/verify-computer-native.mjs @@ -27,6 +27,12 @@ const checks = [ ], enabled: true }, + { + name: 'Linux snapshot renderer tests', + command: 'python3', + args: ['native/computer-use-linux/runtime_render_test.py'], + enabled: true + }, { name: 'native provider argument guardrails', run: verifyNativeArgumentGuardrails, @@ -69,6 +75,19 @@ const checks = [ run: verifyWindowsProviderHandshake, enabled: process.platform === 'win32' }, + { + name: 'Windows snapshot renderer tests', + command: 'powershell.exe', + args: [ + '-NoLogo', + '-NoProfile', + '-ExecutionPolicy', + 'Bypass', + '-File', + 'native/computer-use-windows/runtime-render.test.ps1' + ], + enabled: process.platform === 'win32' + }, { name: 'macOS helper app bundle and signature', run: verifyMacOSHelperApp, diff --git a/native/computer-use-linux/runtime.py b/native/computer-use-linux/runtime.py index baeed544bd6..ac53f6cf65f 100644 --- a/native/computer-use-linux/runtime.py +++ b/native/computer-use-linux/runtime.py @@ -408,8 +408,11 @@ def render_accessibility_tree(root, window_rect, root_path, compact_browser_tabs item = record(node, len(records), path, window_rect) child_items = list(children(node)) role_key = (item["controlType"] or "").lower() - summary_values = text_snippets(node, limit=8, max_depth=4) - generic_summary = " ".join(summary_values) if role_key in {"panel", "filler", "unknown", "section"} and not item["name"] and not item["value"] and len(summary_values) >= 2 and is_plain_text_subtree(node) else None + generic_summary = None + if role_key in {"panel", "filler", "unknown", "section"} and not item["name"] and not item["value"]: + summary_values = text_snippets(node, limit=8, max_depth=4) + if len(summary_values) >= 2 and is_plain_text_subtree(node): + generic_summary = " ".join(summary_values) if should_elide(item, len(child_items), generic_summary): for child_index, child in child_items: walk(child, depth, path + [child_index]) diff --git a/native/computer-use-linux/runtime_render_test.py b/native/computer-use-linux/runtime_render_test.py new file mode 100644 index 00000000000..83a600c7d2d --- /dev/null +++ b/native/computer-use-linux/runtime_render_test.py @@ -0,0 +1,198 @@ +import importlib.util +import sys +import types +import unittest +from pathlib import Path + + +class FakeStateType: + PROTECTED = 1 + SELECTED = 2 + + +class FakeText: + @staticmethod + def get_character_count(node): + return len(node.value) + + @staticmethod + def get_text(node, start, end): + return node.value[start:end] + + +def load_runtime(): + gi = types.ModuleType("gi") + repository = types.ModuleType("gi.repository") + gi.require_version = lambda *_: None + repository.Atspi = types.SimpleNamespace(StateType=FakeStateType, Text=FakeText) + repository.Gdk = types.SimpleNamespace() + repository.GdkPixbuf = types.SimpleNamespace() + gi.repository = repository + sys.modules["gi"] = gi + sys.modules["gi.repository"] = repository + + path = Path(__file__).with_name("runtime.py") + spec = importlib.util.spec_from_file_location("orca_linux_runtime_test", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +runtime = load_runtime() + + +class FakeAccessible: + def __init__(self, role, name="", value="", children=(), counter=None): + self.role = role + self.name = name + self.value = value + self.children = list(children) + self.counter = counter or {"child_reads": 0} + for child in self.children: + child.use_counter(self.counter) + + def use_counter(self, counter): + self.counter = counter + for child in self.children: + child.use_counter(counter) + + def get_child_count(self): + self.counter["child_reads"] += 1 + return len(self.children) + + def get_child_at_index(self, index): + self.counter["child_reads"] += 1 + return self.children[index] + + def get_role_name(self): + return self.role + + def get_name(self): + return self.name + + def get_accessible_id(self): + return "" + + def get_toolkit_name(self): + return "fake" + + def get_component_iface(self): + return None + + def get_state_set(self): + return None + + def get_n_actions(self): + return 0 + + def is_text(self): + return bool(self.value) + + def get_text_iface(self): + return self if self.value else None + + def get_value_iface(self): + return None + + +class FailingChildrenAccessible(FakeAccessible): + def get_child_count(self): + raise RuntimeError("defunct node") + + +def render(root): + return runtime.render_accessibility_tree(root, None, [0]) + + +class RuntimeRenderTest(unittest.TestCase): + def test_named_non_generic_node_skips_unused_summary_walk(self): + root = FakeAccessible("button", "Save", children=[FakeAccessible("text", "unused")]) + + records, lines, truncation = render(root) + + self.assertEqual([record["name"] for record in records], ["Save"]) + self.assertEqual(lines, ["0 button Save"]) + self.assertEqual(root.counter["child_reads"], 2) + self.assertFalse(truncation["truncated"]) + + def test_named_generic_node_skips_unused_summary_walk(self): + root = FakeAccessible("section", "Details", children=[FakeAccessible("text", "body")]) + + records, lines, _ = render(root) + + self.assertEqual([record["name"] for record in records], ["Details", "body"]) + self.assertEqual(lines, ["0 section Details", "\t1 text body"]) + self.assertEqual(root.counter["child_reads"], 3) + + def test_unnamed_generic_node_keeps_plain_text_summary(self): + root = FakeAccessible( + "section", + children=[FakeAccessible("text", "Alpha"), FakeAccessible("text", "Beta")], + ) + + records, lines, _ = render(root) + + self.assertEqual(len(records), 1) + self.assertEqual(lines, ["0 section, Text: Alpha Beta"]) + self.assertEqual(root.counter["child_reads"], 13) + + def test_row_keeps_its_specific_summary_walk(self): + root = FakeAccessible( + "row", + "Invoice", + children=[FakeAccessible("text", "Alpha"), FakeAccessible("text", "Beta")], + ) + + records, lines, _ = render(root) + + self.assertEqual([record["name"] for record in records], ["Invoice", "Alpha", "Beta"]) + self.assertEqual(lines, ["0 row Invoice, Text: Alpha Beta", "\t1 text Alpha", "\t2 text Beta"]) + self.assertEqual(root.counter["child_reads"], 10) + + def test_empty_generic_wrapper_keeps_elision_path_and_depth(self): + root = FakeAccessible("section", children=[FakeAccessible("button", "Continue")]) + + records, lines, _ = render(root) + + self.assertEqual([record["runtimeId"] for record in records], [[0, 0]]) + self.assertEqual(lines, ["0 button Continue"]) + + def test_child_failure_keeps_fail_soft_row_output(self): + root = FailingChildrenAccessible("row", "Invoice") + + records, lines, truncation = render(root) + + self.assertEqual([record["name"] for record in records], ["Invoice"]) + self.assertEqual(lines, ["0 row Invoice"]) + self.assertFalse(truncation["truncated"]) + + def test_node_limit_keeps_exact_prefix_and_truncation(self): + root = FakeAccessible( + "document", + "Results", + children=[FakeAccessible("text", f"Item {index}") for index in range(runtime.MAX_NODES)], + ) + + records, _, truncation = render(root) + + self.assertEqual(len(records), runtime.MAX_NODES) + self.assertEqual(records[-1]["name"], f"Item {runtime.MAX_NODES - 2}") + self.assertTrue(truncation["truncated"]) + self.assertFalse(truncation["maxDepthReached"]) + + def test_depth_limit_keeps_exact_prefix_and_flag(self): + root = FakeAccessible("document", "Depth 65") + for depth in range(runtime.MAX_DEPTH, -1, -1): + root = FakeAccessible("document", f"Depth {depth}", children=[root]) + + records, _, truncation = render(root) + + self.assertEqual(len(records), runtime.MAX_DEPTH + 1) + self.assertEqual(records[-1]["name"], f"Depth {runtime.MAX_DEPTH}") + self.assertTrue(truncation["truncated"]) + self.assertTrue(truncation["maxDepthReached"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/native/computer-use-windows/runtime-render.test.ps1 b/native/computer-use-windows/runtime-render.test.ps1 new file mode 100644 index 00000000000..3e52b1a203e --- /dev/null +++ b/native/computer-use-windows/runtime-render.test.ps1 @@ -0,0 +1,224 @@ +$ErrorActionPreference = "Stop" + +function Assert-TestEqual($Actual, $Expected, [string]$Label) { + if ($Actual -ne $Expected) { + throw "$Label expected '$Expected', received '$Actual'" + } +} + +Add-Type -TypeDefinition @" +using System.Collections; +using System.Collections.Generic; + +public sealed class OrcaRenderTestCounter { + public int FindAll; +} + +public sealed class OrcaRenderTestBounds { + public bool IsEmpty = true; + public double X; + public double Y; + public double Width; + public double Height; +} + +public sealed class OrcaRenderTestControlType { + public string ProgrammaticName = ""; +} + +public sealed class OrcaRenderTestCurrent { + public string AutomationId = ""; + public OrcaRenderTestBounds BoundingRectangle = new OrcaRenderTestBounds(); + public string ClassName = "fake"; + public OrcaRenderTestControlType ControlType = new OrcaRenderTestControlType(); + public bool IsPassword; + public string LocalizedControlType = ""; + public string Name = ""; + public long NativeWindowHandle; +} + +public sealed class OrcaRenderTestPatternCurrent { + public bool IsSelected; + public string Value = ""; +} + +public sealed class OrcaRenderTestPattern { + public OrcaRenderTestPatternCurrent Current = new OrcaRenderTestPatternCurrent(); +} + +public sealed class OrcaRenderTestCollection : IEnumerable { + private readonly OrcaRenderTestElement[] values; + + public OrcaRenderTestCollection(OrcaRenderTestElement[] values) { + this.values = values; + } + + public int Count { + get { return values.Length; } + } + + public OrcaRenderTestElement Item(int index) { + return values[index]; + } + + public IEnumerator GetEnumerator() { + return values.GetEnumerator(); + } +} + +public sealed class OrcaRenderTestElement { + public readonly List Children = + new List(); + public OrcaRenderTestCounter Counter = new OrcaRenderTestCounter(); + public readonly OrcaRenderTestCurrent Current = new OrcaRenderTestCurrent(); + public bool FailFindAll; + public int RuntimeIdValue; + public string ValueText = ""; + + public OrcaRenderTestCollection FindAll(object scope, object condition) { + Counter.FindAll++; + if (FailFindAll) { + throw new System.InvalidOperationException("defunct node"); + } + return new OrcaRenderTestCollection(Children.ToArray()); + } + + public OrcaRenderTestPattern GetCurrentPattern(object pattern) { + OrcaRenderTestPattern result = new OrcaRenderTestPattern(); + result.Current.Value = ValueText; + return result; + } + + public int[] GetRuntimeId() { + return new int[] { RuntimeIdValue }; + } + + public object[] GetSupportedPatterns() { + return new object[0]; + } +} +"@ + +function New-TestCounter { + New-Object -TypeName OrcaRenderTestCounter +} + +function New-TestElement { + param( + [string]$Role, + [string]$Name = "", + [string]$Value = "", + [object[]]$Children = @(), + $Counter = $(New-TestCounter), + [switch]$FailFindAll, + [int]$RuntimeId = 1 + ) + $element = New-Object -TypeName OrcaRenderTestElement + $element.Counter = $Counter + $element.FailFindAll = [bool]$FailFindAll + $element.RuntimeIdValue = $RuntimeId + $element.ValueText = $Value + $element.Current.ControlType.ProgrammaticName = $Role + $element.Current.LocalizedControlType = $Role + $element.Current.Name = $Name + foreach ($child in @($Children)) { + [void]$element.Children.Add($child) + } + $element +} + +$operationPath = Join-Path ([IO.Path]::GetTempPath()) ("orca-runtime-render-test-" + [guid]::NewGuid() + ".json") +try { + Set-Content -LiteralPath $operationPath -Encoding UTF8 -Value '{"tool":"handshake"}' + $runtimeOutput = . (Join-Path $PSScriptRoot "runtime.ps1") -OperationPath $operationPath + $handshake = $runtimeOutput | ConvertFrom-Json + Assert-TestEqual $handshake.ok $true "runtime handshake" + + $counter = New-TestCounter + $leaf = New-TestElement -Role "text" -Name "unused" -Counter $counter -RuntimeId 2 + $root = New-TestElement -Role "button" -Name "Save" -Children @($leaf) -Counter $counter + $tree = Render-OrcaTree $root $null + Assert-TestEqual $tree.elements.Count 1 "named control record count" + Assert-TestEqual ([string]$tree.lines[0]) "0 button Save" "named control line" + Assert-TestEqual $counter.findAll 1 "named control child enumeration" + + $counter = New-TestCounter + $leaf = New-TestElement -Role "text" -Name "body" -Counter $counter -RuntimeId 2 + $root = New-TestElement -Role "group" -Name "Details" -Children @($leaf) -Counter $counter + $tree = Render-OrcaTree $root $null + Assert-TestEqual (($tree.elements | ForEach-Object { $_.name }) -join "|") "Details|body" "named generic records" + Assert-TestEqual (@($tree.lines) -join "|") "0 group Details|`t1 text body" "named generic lines" + Assert-TestEqual $counter.findAll 2 "named generic child enumeration" + + $counter = New-TestCounter + $alpha = New-TestElement -Role "text" -Name "Alpha" -Counter $counter -RuntimeId 2 + $beta = New-TestElement -Role "text" -Name "Beta" -Counter $counter -RuntimeId 3 + $root = New-TestElement -Role "group" -Children @($alpha, $beta) -Counter $counter + $tree = Render-OrcaTree $root $null + Assert-TestEqual $tree.elements.Count 1 "anonymous generic record count" + Assert-TestEqual ([string]$tree.lines[0]) "0 group, Text: Alpha Beta" "anonymous generic summary" + Assert-TestEqual $counter.findAll 7 "anonymous generic child enumeration" + + $counter = New-TestCounter + $alpha = New-TestElement -Role "text" -Name "Alpha" -Counter $counter -RuntimeId 2 + $beta = New-TestElement -Role "text" -Name "Beta" -Counter $counter -RuntimeId 3 + $root = New-TestElement -Role "row" -Name "Invoice" -Children @($alpha, $beta) -Counter $counter + $tree = Render-OrcaTree $root $null + Assert-TestEqual (($tree.elements | ForEach-Object { $_.name }) -join "|") "Invoice|Alpha|Beta" "row records" + Assert-TestEqual ([string]$tree.lines[0]) "0 row Invoice, Text: Alpha Beta" "row summary" + Assert-TestEqual $counter.findAll 6 "row child enumeration" + + $counter = New-TestCounter + $button = New-TestElement -Role "button" -Name "Continue" -Counter $counter -RuntimeId 2 + $root = New-TestElement -Role "group" -Children @($button) -Counter $counter + $tree = Render-OrcaTree $root $null + Assert-TestEqual $tree.elements.Count 1 "elided wrapper record count" + Assert-TestEqual ([string]$tree.lines[0]) "0 button Continue" "elided wrapper line" + Assert-TestEqual $counter.findAll 4 "elided wrapper child enumeration" + + $counter = New-TestCounter + $root = New-TestElement -Role "row" -Name "Invoice" -Counter $counter -FailFindAll + $tree = Render-OrcaTree $root $null + Assert-TestEqual $tree.elements.Count 1 "failed child read record count" + Assert-TestEqual ([string]$tree.lines[0]) "0 row Invoice" "failed child read line" + Assert-TestEqual $counter.findAll 2 "failed child read retries" + + $originalMaxNodes = $MaxNodes + try { + $MaxNodes = 3 + $counter = New-TestCounter + $children = @() + for ($index = 0; $index -lt 3; $index++) { + $children += New-TestElement -Role "text" -Name "Item $index" -Counter $counter -RuntimeId ($index + 2) + } + $root = New-TestElement -Role "document" -Name "Results" -Children $children -Counter $counter + $tree = Render-OrcaTree $root $null + Assert-TestEqual $tree.elements.Count 3 "node limit record count" + Assert-TestEqual ([string]$tree.elements[-1].name) "Item 1" "node limit prefix" + Assert-TestEqual $tree.truncation.truncated $true "node limit truncation" + Assert-TestEqual $tree.truncation.maxDepthReached $false "node limit depth flag" + } finally { + $MaxNodes = $originalMaxNodes + } + + $originalMaxDepth = $MaxDepth + try { + $MaxDepth = 2 + $counter = New-TestCounter + $root = New-TestElement -Role "document" -Name "Depth 3" -Counter $counter -RuntimeId 4 + for ($depth = 2; $depth -ge 0; $depth--) { + $root = New-TestElement -Role "document" -Name "Depth $depth" -Children @($root) -Counter $counter -RuntimeId ($depth + 1) + } + $tree = Render-OrcaTree $root $null + Assert-TestEqual $tree.elements.Count 3 "depth limit record count" + Assert-TestEqual ([string]$tree.elements[-1].name) "Depth 2" "depth limit prefix" + Assert-TestEqual $tree.truncation.truncated $true "depth limit truncation" + Assert-TestEqual $tree.truncation.maxDepthReached $true "depth limit flag" + } finally { + $MaxDepth = $originalMaxDepth + } + + Write-Output "windows-snapshot-render-tests-ok" +} finally { + Remove-Item -LiteralPath $operationPath -Force -ErrorAction SilentlyContinue +} diff --git a/native/computer-use-windows/runtime.ps1 b/native/computer-use-windows/runtime.ps1 index 39ab77e1704..2aa581901e6 100644 --- a/native/computer-use-windows/runtime.ps1 +++ b/native/computer-use-windows/runtime.ps1 @@ -561,10 +561,12 @@ function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs = $title = if ([string]::IsNullOrWhiteSpace($record.name)) { $record.automationId } else { $record.name } $role = if ([string]::IsNullOrWhiteSpace($record.localizedControlType)) { $record.controlType } else { $record.localizedControlType } $roleKey = $role.ToLowerInvariant() - $snippets = @(Get-OrcaTextSnippets $Node 8 4) $genericSummary = $null - if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $snippets.Count -ge 2 -and (Test-OrcaPlainTextSubtree $Node)) { - $genericSummary = ($snippets -join " ") + if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value)) { + $snippets = @(Get-OrcaTextSnippets $Node 8 4) + if ($snippets.Count -ge 2 -and (Test-OrcaPlainTextSubtree $Node)) { + $genericSummary = ($snippets -join " ") + } } if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $meaningfulActions.Count -eq 0 -and $null -eq $genericSummary -and $children.Count -le 1) { for ($i = 0; $i -lt $children.Count; $i++) {