perf(computer-use): skip unused accessibility summaries (#13559)

config/scripts/verify-computer-native.mjs

@@ -27,6 +27,12 @@ const checks = [
     ],
     enabled: true
   },
+  {
+    name: 'Linux snapshot renderer tests',
+    command: 'python3',
+    args: ['native/computer-use-linux/runtime_render_test.py'],
+    enabled: true
+  },
   {
     name: 'native provider argument guardrails',
     run: verifyNativeArgumentGuardrails,
@@ -69,6 +75,19 @@ const checks = [
     run: verifyWindowsProviderHandshake,
     enabled: process.platform === 'win32'
   },
+  {
+    name: 'Windows snapshot renderer tests',
+    command: 'powershell.exe',
+    args: [
+      '-NoLogo',
+      '-NoProfile',
+      '-ExecutionPolicy',
+      'Bypass',
+      '-File',
+      'native/computer-use-windows/runtime-render.test.ps1'
+    ],
+    enabled: process.platform === 'win32'
+  },
   {
     name: 'macOS helper app bundle and signature',
     run: verifyMacOSHelperApp,

native/computer-use-linux/runtime.py

@@ -408,8 +408,11 @@ def render_accessibility_tree(root, window_rect, root_path, compact_browser_tabs
         item = record(node, len(records), path, window_rect)
         child_items = list(children(node))
         role_key = (item["controlType"] or "").lower()
-        summary_values = text_snippets(node, limit=8, max_depth=4)
-        generic_summary = " ".join(summary_values) if role_key in {"panel", "filler", "unknown", "section"} and not item["name"] and not item["value"] and len(summary_values) >= 2 and is_plain_text_subtree(node) else None
+        generic_summary = None
+        if role_key in {"panel", "filler", "unknown", "section"} and not item["name"] and not item["value"]:
+            summary_values = text_snippets(node, limit=8, max_depth=4)
+            if len(summary_values) >= 2 and is_plain_text_subtree(node):
+                generic_summary = " ".join(summary_values)
         if should_elide(item, len(child_items), generic_summary):
             for child_index, child in child_items:
                 walk(child, depth, path + [child_index])

native/computer-use-linux/runtime_render_test.py

@@ -0,0 +1,198 @@
+import importlib.util
+import sys
+import types
+import unittest
+from pathlib import Path
+
+
+class FakeStateType:
+    PROTECTED = 1
+    SELECTED = 2
+
+
+class FakeText:
+    @staticmethod
+    def get_character_count(node):
+        return len(node.value)
+
+    @staticmethod
+    def get_text(node, start, end):
+        return node.value[start:end]
+
+
+def load_runtime():
+    gi = types.ModuleType("gi")
+    repository = types.ModuleType("gi.repository")
+    gi.require_version = lambda *_: None
+    repository.Atspi = types.SimpleNamespace(StateType=FakeStateType, Text=FakeText)
+    repository.Gdk = types.SimpleNamespace()
+    repository.GdkPixbuf = types.SimpleNamespace()
+    gi.repository = repository
+    sys.modules["gi"] = gi
+    sys.modules["gi.repository"] = repository
+
+    path = Path(__file__).with_name("runtime.py")
+    spec = importlib.util.spec_from_file_location("orca_linux_runtime_test", path)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+runtime = load_runtime()
+
+
+class FakeAccessible:
+    def __init__(self, role, name="", value="", children=(), counter=None):
+        self.role = role
+        self.name = name
+        self.value = value
+        self.children = list(children)
+        self.counter = counter or {"child_reads": 0}
+        for child in self.children:
+            child.use_counter(self.counter)
+
+    def use_counter(self, counter):
+        self.counter = counter
+        for child in self.children:
+            child.use_counter(counter)
+
+    def get_child_count(self):
+        self.counter["child_reads"] += 1
+        return len(self.children)
+
+    def get_child_at_index(self, index):
+        self.counter["child_reads"] += 1
+        return self.children[index]
+
+    def get_role_name(self):
+        return self.role
+
+    def get_name(self):
+        return self.name
+
+    def get_accessible_id(self):
+        return ""
+
+    def get_toolkit_name(self):
+        return "fake"
+
+    def get_component_iface(self):
+        return None
+
+    def get_state_set(self):
+        return None
+
+    def get_n_actions(self):
+        return 0
+
+    def is_text(self):
+        return bool(self.value)
+
+    def get_text_iface(self):
+        return self if self.value else None
+
+    def get_value_iface(self):
+        return None
+
+
+class FailingChildrenAccessible(FakeAccessible):
+    def get_child_count(self):
+        raise RuntimeError("defunct node")
+
+
+def render(root):
+    return runtime.render_accessibility_tree(root, None, [0])
+
+
+class RuntimeRenderTest(unittest.TestCase):
+    def test_named_non_generic_node_skips_unused_summary_walk(self):
+        root = FakeAccessible("button", "Save", children=[FakeAccessible("text", "unused")])
+
+        records, lines, truncation = render(root)
+
+        self.assertEqual([record["name"] for record in records], ["Save"])
+        self.assertEqual(lines, ["0 button Save"])
+        self.assertEqual(root.counter["child_reads"], 2)
+        self.assertFalse(truncation["truncated"])
+
+    def test_named_generic_node_skips_unused_summary_walk(self):
+        root = FakeAccessible("section", "Details", children=[FakeAccessible("text", "body")])
+
+        records, lines, _ = render(root)
+
+        self.assertEqual([record["name"] for record in records], ["Details", "body"])
+        self.assertEqual(lines, ["0 section Details", "\t1 text body"])
+        self.assertEqual(root.counter["child_reads"], 3)
+
+    def test_unnamed_generic_node_keeps_plain_text_summary(self):
+        root = FakeAccessible(
+            "section",
+            children=[FakeAccessible("text", "Alpha"), FakeAccessible("text", "Beta")],
+        )
+
+        records, lines, _ = render(root)
+
+        self.assertEqual(len(records), 1)
+        self.assertEqual(lines, ["0 section, Text: Alpha Beta"])
+        self.assertEqual(root.counter["child_reads"], 13)
+
+    def test_row_keeps_its_specific_summary_walk(self):
+        root = FakeAccessible(
+            "row",
+            "Invoice",
+            children=[FakeAccessible("text", "Alpha"), FakeAccessible("text", "Beta")],
+        )
+
+        records, lines, _ = render(root)
+
+        self.assertEqual([record["name"] for record in records], ["Invoice", "Alpha", "Beta"])
+        self.assertEqual(lines, ["0 row Invoice, Text: Alpha Beta", "\t1 text Alpha", "\t2 text Beta"])
+        self.assertEqual(root.counter["child_reads"], 10)
+
+    def test_empty_generic_wrapper_keeps_elision_path_and_depth(self):
+        root = FakeAccessible("section", children=[FakeAccessible("button", "Continue")])
+
+        records, lines, _ = render(root)
+
+        self.assertEqual([record["runtimeId"] for record in records], [[0, 0]])
+        self.assertEqual(lines, ["0 button Continue"])
+
+    def test_child_failure_keeps_fail_soft_row_output(self):
+        root = FailingChildrenAccessible("row", "Invoice")
+
+        records, lines, truncation = render(root)
+
+        self.assertEqual([record["name"] for record in records], ["Invoice"])
+        self.assertEqual(lines, ["0 row Invoice"])
+        self.assertFalse(truncation["truncated"])
+
+    def test_node_limit_keeps_exact_prefix_and_truncation(self):
+        root = FakeAccessible(
+            "document",
+            "Results",
+            children=[FakeAccessible("text", f"Item {index}") for index in range(runtime.MAX_NODES)],
+        )
+
+        records, _, truncation = render(root)
+
+        self.assertEqual(len(records), runtime.MAX_NODES)
+        self.assertEqual(records[-1]["name"], f"Item {runtime.MAX_NODES - 2}")
+        self.assertTrue(truncation["truncated"])
+        self.assertFalse(truncation["maxDepthReached"])
+
+    def test_depth_limit_keeps_exact_prefix_and_flag(self):
+        root = FakeAccessible("document", "Depth 65")
+        for depth in range(runtime.MAX_DEPTH, -1, -1):
+            root = FakeAccessible("document", f"Depth {depth}", children=[root])
+
+        records, _, truncation = render(root)
+
+        self.assertEqual(len(records), runtime.MAX_DEPTH + 1)
+        self.assertEqual(records[-1]["name"], f"Depth {runtime.MAX_DEPTH}")
+        self.assertTrue(truncation["truncated"])
+        self.assertTrue(truncation["maxDepthReached"])
+
+
+if __name__ == "__main__":
+    unittest.main()

native/computer-use-windows/runtime-render.test.ps1

@@ -0,0 +1,224 @@
+$ErrorActionPreference = "Stop"
+
+function Assert-TestEqual($Actual, $Expected, [string]$Label) {
+    if ($Actual -ne $Expected) {
+        throw "$Label expected '$Expected', received '$Actual'"
+    }
+}
+
+Add-Type -TypeDefinition @"
+using System.Collections;
+using System.Collections.Generic;
+
+public sealed class OrcaRenderTestCounter {
+    public int FindAll;
+}
+
+public sealed class OrcaRenderTestBounds {
+    public bool IsEmpty = true;
+    public double X;
+    public double Y;
+    public double Width;
+    public double Height;
+}
+
+public sealed class OrcaRenderTestControlType {
+    public string ProgrammaticName = "";
+}
+
+public sealed class OrcaRenderTestCurrent {
+    public string AutomationId = "";
+    public OrcaRenderTestBounds BoundingRectangle = new OrcaRenderTestBounds();
+    public string ClassName = "fake";
+    public OrcaRenderTestControlType ControlType = new OrcaRenderTestControlType();
+    public bool IsPassword;
+    public string LocalizedControlType = "";
+    public string Name = "";
+    public long NativeWindowHandle;
+}
+
+public sealed class OrcaRenderTestPatternCurrent {
+    public bool IsSelected;
+    public string Value = "";
+}
+
+public sealed class OrcaRenderTestPattern {
+    public OrcaRenderTestPatternCurrent Current = new OrcaRenderTestPatternCurrent();
+}
+
+public sealed class OrcaRenderTestCollection : IEnumerable {
+    private readonly OrcaRenderTestElement[] values;
+
+    public OrcaRenderTestCollection(OrcaRenderTestElement[] values) {
+        this.values = values;
+    }
+
+    public int Count {
+        get { return values.Length; }
+    }
+
+    public OrcaRenderTestElement Item(int index) {
+        return values[index];
+    }
+
+    public IEnumerator GetEnumerator() {
+        return values.GetEnumerator();
+    }
+}
+
+public sealed class OrcaRenderTestElement {
+    public readonly List<OrcaRenderTestElement> Children =
+        new List<OrcaRenderTestElement>();
+    public OrcaRenderTestCounter Counter = new OrcaRenderTestCounter();
+    public readonly OrcaRenderTestCurrent Current = new OrcaRenderTestCurrent();
+    public bool FailFindAll;
+    public int RuntimeIdValue;
+    public string ValueText = "";
+
+    public OrcaRenderTestCollection FindAll(object scope, object condition) {
+        Counter.FindAll++;
+        if (FailFindAll) {
+            throw new System.InvalidOperationException("defunct node");
+        }
+        return new OrcaRenderTestCollection(Children.ToArray());
+    }
+
+    public OrcaRenderTestPattern GetCurrentPattern(object pattern) {
+        OrcaRenderTestPattern result = new OrcaRenderTestPattern();
+        result.Current.Value = ValueText;
+        return result;
+    }
+
+    public int[] GetRuntimeId() {
+        return new int[] { RuntimeIdValue };
+    }
+
+    public object[] GetSupportedPatterns() {
+        return new object[0];
+    }
+}
+"@
+
+function New-TestCounter {
+    New-Object -TypeName OrcaRenderTestCounter
+}
+
+function New-TestElement {
+    param(
+        [string]$Role,
+        [string]$Name = "",
+        [string]$Value = "",
+        [object[]]$Children = @(),
+        $Counter = $(New-TestCounter),
+        [switch]$FailFindAll,
+        [int]$RuntimeId = 1
+    )
+    $element = New-Object -TypeName OrcaRenderTestElement
+    $element.Counter = $Counter
+    $element.FailFindAll = [bool]$FailFindAll
+    $element.RuntimeIdValue = $RuntimeId
+    $element.ValueText = $Value
+    $element.Current.ControlType.ProgrammaticName = $Role
+    $element.Current.LocalizedControlType = $Role
+    $element.Current.Name = $Name
+    foreach ($child in @($Children)) {
+        [void]$element.Children.Add($child)
+    }
+    $element
+}
+
+$operationPath = Join-Path ([IO.Path]::GetTempPath()) ("orca-runtime-render-test-" + [guid]::NewGuid() + ".json")
+try {
+    Set-Content -LiteralPath $operationPath -Encoding UTF8 -Value '{"tool":"handshake"}'
+    $runtimeOutput = . (Join-Path $PSScriptRoot "runtime.ps1") -OperationPath $operationPath
+    $handshake = $runtimeOutput | ConvertFrom-Json
+    Assert-TestEqual $handshake.ok $true "runtime handshake"
+
+    $counter = New-TestCounter
+    $leaf = New-TestElement -Role "text" -Name "unused" -Counter $counter -RuntimeId 2
+    $root = New-TestElement -Role "button" -Name "Save" -Children @($leaf) -Counter $counter
+    $tree = Render-OrcaTree $root $null
+    Assert-TestEqual $tree.elements.Count 1 "named control record count"
+    Assert-TestEqual ([string]$tree.lines[0]) "0 button Save" "named control line"
+    Assert-TestEqual $counter.findAll 1 "named control child enumeration"
+
+    $counter = New-TestCounter
+    $leaf = New-TestElement -Role "text" -Name "body" -Counter $counter -RuntimeId 2
+    $root = New-TestElement -Role "group" -Name "Details" -Children @($leaf) -Counter $counter
+    $tree = Render-OrcaTree $root $null
+    Assert-TestEqual (($tree.elements | ForEach-Object { $_.name }) -join "|") "Details|body" "named generic records"
+    Assert-TestEqual (@($tree.lines) -join "|") "0 group Details|`t1 text body" "named generic lines"
+    Assert-TestEqual $counter.findAll 2 "named generic child enumeration"
+
+    $counter = New-TestCounter
+    $alpha = New-TestElement -Role "text" -Name "Alpha" -Counter $counter -RuntimeId 2
+    $beta = New-TestElement -Role "text" -Name "Beta" -Counter $counter -RuntimeId 3
+    $root = New-TestElement -Role "group" -Children @($alpha, $beta) -Counter $counter
+    $tree = Render-OrcaTree $root $null
+    Assert-TestEqual $tree.elements.Count 1 "anonymous generic record count"
+    Assert-TestEqual ([string]$tree.lines[0]) "0 group, Text: Alpha Beta" "anonymous generic summary"
+    Assert-TestEqual $counter.findAll 7 "anonymous generic child enumeration"
+
+    $counter = New-TestCounter
+    $alpha = New-TestElement -Role "text" -Name "Alpha" -Counter $counter -RuntimeId 2
+    $beta = New-TestElement -Role "text" -Name "Beta" -Counter $counter -RuntimeId 3
+    $root = New-TestElement -Role "row" -Name "Invoice" -Children @($alpha, $beta) -Counter $counter
+    $tree = Render-OrcaTree $root $null
+    Assert-TestEqual (($tree.elements | ForEach-Object { $_.name }) -join "|") "Invoice|Alpha|Beta" "row records"
+    Assert-TestEqual ([string]$tree.lines[0]) "0 row Invoice, Text: Alpha Beta" "row summary"
+    Assert-TestEqual $counter.findAll 6 "row child enumeration"
+
+    $counter = New-TestCounter
+    $button = New-TestElement -Role "button" -Name "Continue" -Counter $counter -RuntimeId 2
+    $root = New-TestElement -Role "group" -Children @($button) -Counter $counter
+    $tree = Render-OrcaTree $root $null
+    Assert-TestEqual $tree.elements.Count 1 "elided wrapper record count"
+    Assert-TestEqual ([string]$tree.lines[0]) "0 button Continue" "elided wrapper line"
+    Assert-TestEqual $counter.findAll 4 "elided wrapper child enumeration"
+
+    $counter = New-TestCounter
+    $root = New-TestElement -Role "row" -Name "Invoice" -Counter $counter -FailFindAll
+    $tree = Render-OrcaTree $root $null
+    Assert-TestEqual $tree.elements.Count 1 "failed child read record count"
+    Assert-TestEqual ([string]$tree.lines[0]) "0 row Invoice" "failed child read line"
+    Assert-TestEqual $counter.findAll 2 "failed child read retries"
+
+    $originalMaxNodes = $MaxNodes
+    try {
+        $MaxNodes = 3
+        $counter = New-TestCounter
+        $children = @()
+        for ($index = 0; $index -lt 3; $index++) {
+            $children += New-TestElement -Role "text" -Name "Item $index" -Counter $counter -RuntimeId ($index + 2)
+        }
+        $root = New-TestElement -Role "document" -Name "Results" -Children $children -Counter $counter
+        $tree = Render-OrcaTree $root $null
+        Assert-TestEqual $tree.elements.Count 3 "node limit record count"
+        Assert-TestEqual ([string]$tree.elements[-1].name) "Item 1" "node limit prefix"
+        Assert-TestEqual $tree.truncation.truncated $true "node limit truncation"
+        Assert-TestEqual $tree.truncation.maxDepthReached $false "node limit depth flag"
+    } finally {
+        $MaxNodes = $originalMaxNodes
+    }
+
+    $originalMaxDepth = $MaxDepth
+    try {
+        $MaxDepth = 2
+        $counter = New-TestCounter
+        $root = New-TestElement -Role "document" -Name "Depth 3" -Counter $counter -RuntimeId 4
+        for ($depth = 2; $depth -ge 0; $depth--) {
+            $root = New-TestElement -Role "document" -Name "Depth $depth" -Children @($root) -Counter $counter -RuntimeId ($depth + 1)
+        }
+        $tree = Render-OrcaTree $root $null
+        Assert-TestEqual $tree.elements.Count 3 "depth limit record count"
+        Assert-TestEqual ([string]$tree.elements[-1].name) "Depth 2" "depth limit prefix"
+        Assert-TestEqual $tree.truncation.truncated $true "depth limit truncation"
+        Assert-TestEqual $tree.truncation.maxDepthReached $true "depth limit flag"
+    } finally {
+        $MaxDepth = $originalMaxDepth
+    }
+
+    Write-Output "windows-snapshot-render-tests-ok"
+} finally {
+    Remove-Item -LiteralPath $operationPath -Force -ErrorAction SilentlyContinue
+}

native/computer-use-windows/runtime.ps1

@@ -561,10 +561,12 @@ function Render-OrcaTree($RootElement, $WindowFrame, [bool]$CompactBrowserTabs =
         $title = if ([string]::IsNullOrWhiteSpace($record.name)) { $record.automationId } else { $record.name }
         $role = if ([string]::IsNullOrWhiteSpace($record.localizedControlType)) { $record.controlType } else { $record.localizedControlType }
         $roleKey = $role.ToLowerInvariant()
-        $snippets = @(Get-OrcaTextSnippets $Node 8 4)
         $genericSummary = $null
-        if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $snippets.Count -ge 2 -and (Test-OrcaPlainTextSubtree $Node)) {
-            $genericSummary = ($snippets -join " ")
+        if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value)) {
+            $snippets = @(Get-OrcaTextSnippets $Node 8 4)
+            if ($snippets.Count -ge 2 -and (Test-OrcaPlainTextSubtree $Node)) {
+                $genericSummary = ($snippets -join " ")
+            }
         }
         if (($roleKey -in @("pane", "group", "custom", "unknown")) -and [string]::IsNullOrWhiteSpace($title) -and [string]::IsNullOrWhiteSpace($record.value) -and $meaningfulActions.Count -eq 0 -and $null -eq $genericSummary -and $children.Count -le 1) {
             for ($i = 0; $i -lt $children.Count; $i++) {