From 596e5f52d6353341b671965ac211def7cb21913a Mon Sep 17 00:00:00 2001
From: Kelvin Amoaba <97001695+AmoabaKelvin@users.noreply.github.com>
Date: Fri, 30 Oct 2026 04:20:18 +0000
Subject: [PATCH] test(computer): measure independently verified tasks (#36216)
* test(computer): measure independently verified tasks
* ci(computer): fit benchmark to repository guards
* fix(computer): stabilize benchmark fixtures and CI
---
.github/workflows/ci-closed-pr-caches.yml | 1 +
.github/workflows/computer-task-benchmark.yml | 158 ++++++++
config/benchmarks/computer-use/README.md | 78 ++++
config/benchmarks/computer-use/bench.py | 378 ++++++++++++++++++
.../benchmarks/computer-use/electron/main.js | 81 ++++
config/benchmarks/computer-use/fixture_gtk.py | 125 ++++++
.../benchmarks/computer-use/fixture_win.ps1 | 124 ++++++
config/benchmarks/computer-use/obs_win.py | 309 ++++++++++++++
config/benchmarks/computer-use/obs_x11.py | 263 ++++++++++++
.../benchmarks/computer-use/process_scope.py | 103 +++++
config/benchmarks/computer-use/providers.py | 275 +++++++++++++
config/benchmarks/computer-use/sentinel.py | 57 +++
.../computer-use/test_trial_contract.py | 111 +++++
.../benchmarks/computer-use/trial_contract.py | 124 ++++++
14 files changed, 2187 insertions(+)
create mode 100644 .github/workflows/computer-task-benchmark.yml
create mode 100644 config/benchmarks/computer-use/README.md
create mode 100644 config/benchmarks/computer-use/bench.py
create mode 100644 config/benchmarks/computer-use/electron/main.js
create mode 100644 config/benchmarks/computer-use/fixture_gtk.py
create mode 100644 config/benchmarks/computer-use/fixture_win.ps1
create mode 100644 config/benchmarks/computer-use/obs_win.py
create mode 100644 config/benchmarks/computer-use/obs_x11.py
create mode 100644 config/benchmarks/computer-use/process_scope.py
create mode 100644 config/benchmarks/computer-use/providers.py
create mode 100644 config/benchmarks/computer-use/sentinel.py
create mode 100644 config/benchmarks/computer-use/test_trial_contract.py
create mode 100644 config/benchmarks/computer-use/trial_contract.py
diff --git a/.github/workflows/ci-closed-pr-caches.yml b/.github/workflows/ci-closed-pr-caches.yml
index a2091211188..37dc522417f 100644
--- a/.github/workflows/ci-closed-pr-caches.yml
+++ b/.github/workflows/ci-closed-pr-caches.yml
@@ -28,6 +28,7 @@ jobs:
['ssh-hostile-hosts', 'ssh-hostile-hosts.yml'],
['mobile', 'mobile.yml'],
['computer-e2e', 'computer-e2e.yml'],
+ ['computer-task-benchmark', 'computer-task-benchmark.yml'],
['computer-macos-native', 'computer-macos-native.yml'],
['wsl-orcad-e2e', 'wsl-orcad-e2e.yml'],
['performance-contracts', 'performance-contracts.yml'],
diff --git a/.github/workflows/computer-task-benchmark.yml b/.github/workflows/computer-task-benchmark.yml
new file mode 100644
index 00000000000..baec1d44e87
--- /dev/null
+++ b/.github/workflows/computer-task-benchmark.yml
@@ -0,0 +1,158 @@
+name: Computer-use task benchmark
+
+on:
+ pull_request:
+ types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
+ paths:
+ - 'config/benchmarks/computer-use/**'
+ - 'native/computer-use-linux/**'
+ - 'native/computer-use-windows/**'
+ - '.github/workflows/computer-task-benchmark.yml'
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: computer-task-benchmark-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+env:
+ ORCA_BACKGROUND_LAUNCH: '1'
+ CUA_TAG: cua-driver-rs-v0.38.0
+ CUA_DRIVER_RS_TELEMETRY_ENABLED: '0'
+ DO_NOT_TRACK: '1'
+ CUA_DRIVER_RS_UPDATE_CHECK: '0'
+ ELECTRON_VERSION: '43.8.1'
+ BASELINE_SHA: 030c88deecf3adc212d22f57b085e4bd5c6e48e6
+
+jobs:
+ contract:
+ if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [ubuntu-latest, windows-latest, macos-latest]
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@v6
+ with:
+ python-version: '3.12'
+ - run: python -m unittest discover -s config/benchmarks/computer-use -p 'test_*.py'
+
+ linux:
+ if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
+ runs-on: ubuntu-latest
+ timeout-minutes: 40
+ strategy:
+ fail-fast: false
+ matrix:
+ target: [fixture, electron]
+ env:
+ BENCH_TARGET: ${{ matrix.target }}
+ ORCA_COMPUTER_BENCH_ISOLATED: '1'
+ ACCESSIBILITY_ENABLED: '1'
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ persist-credentials: false
+ ref: ${{ github.event.pull_request.head.sha || github.sha }}
+ - name: Fetch immutable baseline
+ run: git fetch --no-tags --depth=1 origin "$BASELINE_SHA"
+ - name: Install isolated desktop and accessibility libraries
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y --no-install-recommends xvfb openbox dbus-x11 at-spi2-core python3-gi \
+ gir1.2-atspi-2.0 gir1.2-gtk-3.0 python3-tk xdotool xclip ripgrep libnss3 libgbm1 libasound2t64
+ - name: Download and verify pinned cua-driver
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ mkdir -p "$RUNNER_TEMP/cua-dl" "$RUNNER_TEMP/cua"
+ cd "$RUNNER_TEMP/cua-dl"
+ asset='cua-driver-rs-0.38.0-linux-x86_64-binary.tar.gz'
+ gh release download "$CUA_TAG" --repo trycua/cua --pattern checksums.txt --pattern "$asset"
+ rg " $asset\$" checksums.txt | sha256sum -c -
+ tar -xzf "$asset" -C "$RUNNER_TEMP/cua"
+ - name: Install Electron
+ if: matrix.target == 'electron'
+ run: |
+ npm install --no-save --no-package-lock --prefix "$RUNNER_TEMP/electron" "electron@$ELECTRON_VERSION"
+ node "$RUNNER_TEMP/electron/node_modules/electron/install.js"
+ - name: Run task comparison on Xvfb
+ run: |
+ export CANDIDATE_SHA="$(git rev-parse HEAD)"
+ xvfb-run -a --server-args='-screen 0 1280x1024x24' dbus-run-session -- bash -euc '
+ openbox >"$RUNNER_TEMP/openbox.log" 2>&1 &
+ /usr/bin/python3 config/benchmarks/computer-use/bench.py --target "$BENCH_TARGET" \
+ --baseline "$BASELINE_SHA" --candidate "$CANDIDATE_SHA" \
+ --cua "$RUNNER_TEMP/cua/cua-driver" \
+ --electron "$RUNNER_TEMP/electron/node_modules/electron/dist/electron" \
+ --out "out/task-linux-$BENCH_TARGET" --n 3 --budget 1800
+ '
+ - if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: computer-task-linux-${{ matrix.target }}
+ path: out/task-linux-${{ matrix.target }}
+ if-no-files-found: warn
+
+ windows:
+ if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
+ runs-on: windows-latest
+ timeout-minutes: 40
+ strategy:
+ fail-fast: false
+ matrix:
+ target: [fixture, electron]
+ env:
+ BENCH_TARGET: ${{ matrix.target }}
+ ORCA_COMPUTER_BENCH_ISOLATED: '1'
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ persist-credentials: false
+ ref: ${{ github.event.pull_request.head.sha || github.sha }}
+ - uses: actions/setup-python@v6
+ with:
+ python-version: '3.12'
+ - name: Fetch immutable baseline
+ run: git fetch --no-tags --depth=1 origin $env:BASELINE_SHA
+ - name: Download and verify pinned cua-driver
+ env:
+ GH_TOKEN: ${{ github.token }}
+ shell: pwsh
+ run: |
+ $download = Join-Path $env:RUNNER_TEMP 'cua-dl'
+ $destination = Join-Path $env:RUNNER_TEMP 'cua'
+ New-Item -ItemType Directory -Force $download, $destination | Out-Null
+ $asset = 'cua-driver-rs-0.38.0-windows-x86_64-binary.zip'
+ gh release download $env:CUA_TAG --repo trycua/cua --pattern checksums.txt --pattern $asset --dir $download
+ $matches = @(Select-String -Path (Join-Path $download 'checksums.txt') -Pattern " $([regex]::Escape($asset))$")
+ if ($matches.Count -ne 1) { throw 'expected exactly one checksum' }
+ $expected = $matches[0].Line.Split(' ')[0]
+ $actual = (Get-FileHash (Join-Path $download $asset) -Algorithm SHA256).Hash.ToLower()
+ if ($expected -ne $actual) { throw 'cua-driver checksum mismatch' }
+ Expand-Archive (Join-Path $download $asset) -DestinationPath $destination
+ - name: Install Electron
+ if: matrix.target == 'electron'
+ run: |
+ npm install --no-save --no-package-lock --prefix "$env:RUNNER_TEMP\electron" "electron@$env:ELECTRON_VERSION"
+ node "$env:RUNNER_TEMP\electron\node_modules\electron\install.js"
+ - name: Run task comparison on disposable Windows desktop
+ run: |
+ $candidate = git rev-parse HEAD
+ python config/benchmarks/computer-use/bench.py --target $env:BENCH_TARGET `
+ --baseline $env:BASELINE_SHA --candidate $candidate `
+ --cua "$env:RUNNER_TEMP\cua\cua-driver.exe" `
+ --electron "$env:RUNNER_TEMP\electron\node_modules\electron\dist\electron.exe" `
+ --out "out/task-windows-$env:BENCH_TARGET" --n 3 --budget 1800
+ - if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: computer-task-windows-${{ matrix.target }}
+ path: out/task-windows-${{ matrix.target }}
+ if-no-files-found: warn
diff --git a/config/benchmarks/computer-use/README.md b/config/benchmarks/computer-use/README.md
new file mode 100644
index 00000000000..f58895cb13d
--- /dev/null
+++ b/config/benchmarks/computer-use/README.md
@@ -0,0 +1,78 @@
+# Computer-use task benchmark
+
+Compares Orca's direct Linux AT-SPI and Windows UI Automation providers with
+cua-driver 0.38.0. This does not measure the CLI, sidecar, model, or complete agent
+workflows. It does not install cua-driver in Orca.
+
+Run the `Computer-use task benchmark` GitHub Actions workflow on a branch. Native
+input runs only on its disposable Windows runner or Linux Xvfb desktop. The runner
+refuses to run on the developer's desktop. Keep `ORCA_BACKGROUND_LAUNCH=1`.
+
+The first matrix covers GTK/WinForms and Electron fixtures. Editor applications,
+Wayland, macOS and additional Windows control frameworks need separate coverage.
+Do not generalize these results to every application on a platform.
+
+## Measurement contract
+
+- Baseline and candidate runtimes come from full commit SHAs, copied before the
+ run. Artifacts include source hashes, cua executable hash/version, image,
+ seed, complete trial plan, responses, snapshots and independent fixture states.
+- Each individual trial launches a fresh target, sentinel and provider. Process
+ groups on Linux and jobs on Windows contain children, including timed-out calls.
+ The provider is terminated and reaped before the terminal oracle read. No retry
+ follows an uncertain mutation.
+- Reset carries a unique nonce bound to the fixture instance and PID. The fixture
+ acknowledges after applying text, selection, logical focus, counters, combo,
+ scroll and menu state. Sentinel foreground and stacking are independently checked.
+- Invalid preparation executes no provider operation, is excluded from the scored
+ denominator, and makes the run incomplete with nonzero exit status. A missing or
+ ambiguous control after valid setup is a scored coverage failure.
+- Strict selectors require a unique matching role and label. References are fresh
+ before the task and after every step. There is no label-only or role-only fallback.
+- Completion requires exact independent final state, including unchanged decoy text
+ and exactly one click/submit. Raw text is never cleaned. API errors, missing final
+ observations and observed interference remain separate from task completion.
+- The observer starts before provider initialization and initial selector snapshot,
+ and stays active through all steps, reads, shutdown and settling. No sentinel
+ refocusing happens during measurement. Idle and deliberate-steal controls must pass.
+- Complete tasks run without simulated user typing. Separate click and scroll probes
+ type `q` into the sentinel. Event counters preserve evidence even if agent input
+ selects or deletes text. Observations sample roughly every 4 ms; they cannot prove
+ the absence of shorter focus/pointer/clipboard changes. X11 clipboard observations
+ detect ownership changes, not all changes made by the same owner.
+- Engine order rotates across rounds from a recorded seed. Use a multiple of three
+ rounds for balanced positions. Default/background and explicitly requested
+ foreground profiles are separate trials. Foreground can intentionally interfere.
+- Comparable per-step time includes the action and an additional full no-screenshot
+ observation for every provider. Raw action time is diagnostic because Orca already
+ returns a snapshot with mutations. Task time includes provider startup, discovery
+ and all action/read pairs. Medians include completed tasks only; failed-attempt
+ durations and sample counts are separate. Diagnostic serialization and disk I/O
+ happen after timing. Final independent readback after shutdown is an upper
+ bound on effect time, recorded separately.
+- Fatal errors, invalid setup, failed observers, unobserved completion, budget
+ exhaustion or missing planned cells leave JSON evidence and exit nonzero. A valid
+ observed task failure remains a benchmark result, not an infrastructure failure.
+
+Tasks cover explicit text targeting with another field logically focused, Unicode
+entry plus submit, select-all replacement plus submit, and color choice plus one
+click. Interference probes remain separate from task latency comparisons.
+
+Explicit-target typing is a native capability probe. Orca's public CLI/RPC currently
+accepts typing at the focused field and does not expose an element target for text.
+Failure of this probe identifies a capability to consider adding; it does not prove
+a violation of the current public typing contract. Click-then-type tasks exercise
+the existing sequence, although this benchmark still bypasses public transports.
+
+## Headless contract checks
+
+```sh
+ORCA_BACKGROUND_LAUNCH=1 python3 -m unittest discover -s config/benchmarks/computer-use -p 'test_*.py'
+```
+
+These checks protect the historical false-success cases and process-tree cleanup.
+They do not import desktop observers or generate native input.
+
+Transports, fixtures and observers were adapted from the historical
+`bench/computer-cua` prototype at `68a68ccf0f16c0a7f952d55adfe21850247b8ee9`.
+Historical checkouts and result artifacts remain unchanged.
diff --git a/config/benchmarks/computer-use/bench.py b/config/benchmarks/computer-use/bench.py
new file mode 100644
index 00000000000..dd1b814e2fb
--- /dev/null
+++ b/config/benchmarks/computer-use/bench.py
@@ -0,0 +1,378 @@
+#!/usr/bin/env python3
+"""Compare direct native providers on disposable, independently observed task fixtures."""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import platform
+import re
+import subprocess
+import sys
+import time
+import uuid
+
+from process_scope import ProcessScope
+from trial_contract import (DiscoveryError, MARKER, SELECTORS, TASKS, acknowledged, prepared,
+ schedule, select_unique, summarize, verdict)
+
+HERE = Path(__file__).resolve().parent
+REPO = HERE.parents[2]
+WIN = sys.platform == "win32"
+
+
+def read_json(path):
+ try:
+ return json.loads(Path(path).read_text(encoding="utf-8-sig"))
+ except (OSError, ValueError):
+ return None
+
+
+def write_json(path, value):
+ temporary = Path(str(path) + ".tmp")
+ temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8")
+ temporary.replace(path)
+
+
+def wait_for(predicate, timeout=10):
+ end = time.monotonic() + timeout
+ while time.monotonic() < end:
+ value = predicate()
+ if value:
+ return value
+ time.sleep(0.02)
+ raise TimeoutError("independent setup/observation did not become ready")
+
+
+def pin_source(ref, output):
+ if not re.fullmatch(r"[0-9a-f]{40}", ref):
+ raise ValueError("source must be a full commit SHA")
+ hashes = {}
+ for relative in ("native/computer-use-linux/runtime.py", "native/computer-use-windows/runtime.ps1"):
+ data = subprocess.check_output(["git", "show", f"{ref}:{relative}"], cwd=REPO)
+ destination = output / relative
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ destination.write_bytes(data)
+ hashes[relative] = hashlib.sha256(data).hexdigest()
+ return {"sha": ref, "files": hashes}
+
+
+class Trial:
+ def __init__(self, args, directory, log):
+ self.args, self.directory, self.log = args, directory, log
+ self.instance = uuid.uuid4().hex
+ directory.mkdir()
+ self.targets = ProcessScope(str(directory))
+ self.engines = ProcessScope(str(directory))
+ self.sampler = self.typer = None
+ self.state_path = directory / "state.json"
+ self.sentinel_path = directory / "sentinel.json"
+ self.nonce = None
+
+ def spawn_target(self, command, **extra):
+ env = {**os.environ, "BENCH_INSTANCE": self.instance, "BENCH_STATE_DIR": str(self.directory)}
+ return self.targets.spawn(command, stdout=self.log, stderr=self.log, env=env, **extra)
+
+ def window(self, pid, title):
+ return wait_for(lambda: obs.find_window(pid=pid, title=title) if WIN
+ else obs.find_client(pid, title), 30)
+
+ def state(self):
+ state = read_json(self.state_path)
+ if state and state.get("instance") == self.instance and state.get("pid") == self.pid:
+ return state
+ return None
+
+ def launch(self):
+ if self.args.target == "electron":
+ flags = ["--force-renderer-accessibility"] + ([] if WIN else ["--no-sandbox"])
+ proc = self.spawn_target([self.args.electron, *flags, str(HERE / "electron/main.js")])
+ title = "BenchElectron"
+ elif WIN:
+ proc = self.spawn_target(["powershell.exe", "-NoLogo", "-NoProfile", "-STA", "-File",
+ str(HERE / "fixture_win.ps1"), "-StateDir", str(self.directory)], hidden=True)
+ title = "BenchTarget"
+ else:
+ proc = self.spawn_target([sys.executable, str(HERE / "fixture_gtk.py"), str(self.directory)])
+ title = "BenchTarget"
+ self.pid = proc.pid
+ self.target = self.window(proc.pid, title)
+ obs.place(self.target, 60, 60, 760, 560)
+ wait_for(self.state)
+ proc = self.spawn_target([sys.executable, str(HERE / "sentinel.py"),
+ str(self.sentinel_path), "880x660+20+20"], hidden=True)
+ self.sentinel_pid = proc.pid
+ self.sentinel = self.window(proc.pid, "BenchSentinel")
+ wait_for(lambda: read_json(self.sentinel_path))
+ # Tk's initial focus callback must finish before calibration or measurement.
+ time.sleep(0.2)
+
+ def front(self):
+ if WIN:
+ return obs.foreground() == self.sentinel and not obs.owned_above(self.sentinel, self.pid)
+ display = obs.main_display()
+ stacking = display.stacking()
+ windows = display.windows_of(self.pid)
+ return (display.active() == self.sentinel and display.focus() in display.descendants(self.sentinel)
+ and self.sentinel in stacking and self.target in stacking and windows
+ and all(stacking.index(window) < stacking.index(self.sentinel) for window in windows if window in stacking))
+
+ def prepare(self, task):
+ self.nonce = uuid.uuid4().hex
+ write_json(self.directory / "cmd", {"instance": self.instance, "nonce": self.nonce, "task": task})
+ wait_for(lambda: prepared(self.state() or {}, task, self.instance, self.pid, self.nonce), 5)
+ obs.force_foreground(self.sentinel)
+ wait_for(self.front, 5)
+ if WIN:
+ obs.u.SetCursorPos(2, 2)
+ else:
+ obs.xdotool("mousemove", 2, 2)
+ wait_for(lambda: prepared(self.state() or {}, task, self.instance, self.pid, self.nonce), 5)
+
+ def sample(self):
+ self.sampler = obs.Sampler(self.sentinel, self.target, self.pid)
+ self.sampler.start()
+
+ def stop_sampling(self):
+ if self.sampler:
+ sampler, self.sampler = self.sampler, None
+ result = sampler.stop()
+ if result["n_samples"] < 3:
+ raise RuntimeError("observer did not collect enough samples")
+ return result
+ return {}
+
+ def calibrate(self):
+ self.prepare("click_interference")
+ self.sample()
+ self.typer = obs.Typer()
+ self.typer.start()
+ time.sleep(0.65)
+ self.typer.stop()
+ sent = self.typer.sent
+ self.typer = None
+ if sent < 3:
+ raise RuntimeError("simulated user input control failed")
+ wait_for(lambda: (read_json(self.sentinel_path) or {}).get("q", 0) == sent, 2)
+ idle = self.stop_sampling()
+ if any(idle[key] for key in ("sentinel_lost_focus", "target_raised", "pointer_moved", "clipboard_changed")):
+ raise RuntimeError("idle observer control failed")
+ self.sample()
+ obs.force_foreground(self.target)
+ time.sleep(0.15)
+ steal = self.stop_sampling()
+ if not all(steal[key] for key in ("sentinel_lost_focus", "target_activated", "target_raised")):
+ raise RuntimeError("intentional focus-steal observer control failed")
+ return {"idle": idle, "steal": steal, "keys_sent": sent}
+
+ def engine(self, name, sources):
+ from providers import Cua, CuaEngine, OrcaEngine, OrcaLinux, OrcaWin, cua_sc
+ if name == "cua":
+ client = Cua(self.args.cua, str(self.directory / "cua-home"), self.log, self.engines)
+ response = client.call("list_windows", {"pid": self.pid})
+ if response.get("isError") or "_rpc_error" in response:
+ raise RuntimeError("cua list_windows failed: " + str(response))
+ windows = cua_sc(response).get("windows") or []
+ matches = [window for window in windows if window.get("window_id") is not None
+ and window.get("title") in ("BenchTarget", "BenchElectron")]
+ if len(matches) != 1:
+ raise DiscoveryError(f"cua list_windows matched {len(matches)} fixture windows")
+ return CuaEngine(client, self.pid, matches[0]["window_id"])
+ return OrcaEngine((OrcaWin if WIN else OrcaLinux)(str(sources / name), self.log, self.engines), self.pid)
+
+ def run(self, cell, sources):
+ row = {**cell, "prepared": False, "api_errors": [], "action_read_ms": [], "steps": []}
+ state = None
+ task_started = None
+ snapshots, responses = [], []
+ try:
+ self.launch()
+ self.prepare(cell["task"])
+ row["initial_state"] = self.state()
+ row["prepared"] = True
+ self.sample()
+ row["sentinel_before"] = read_json(self.sentinel_path)
+ if cell["task"].endswith("_interference"):
+ self.typer = obs.Typer()
+ self.typer.start()
+ task_started = time.perf_counter()
+ engine = self.engine(cell["engine"], sources)
+ acquire_started = time.perf_counter()
+ elements = engine.snapshot()
+ row["acquisition_ms"] = (time.perf_counter() - acquire_started) * 1000
+ snapshots.append(engine.last_snapshot)
+ for action, selector in TASKS[cell["task"]]:
+ element = select_unique(elements, SELECTORS[selector])
+ started = time.perf_counter()
+ result = engine.act(action, element, cell["profile"] == "foreground", MARKER)
+ row["steps"].append({"action": action, "result": result,
+ "action_ms": (time.perf_counter() - started) * 1000})
+ if result.get("error"):
+ row["api_errors"].append(result)
+ responses.append(engine.last_response)
+ # Identical extra full no-screenshot read for all providers; reuse fresh refs next step.
+ elements = engine.snapshot()
+ row["action_read_ms"].append((time.perf_counter() - started) * 1000)
+ snapshots.append(engine.last_snapshot)
+ if result.get("error"):
+ break
+ except DiscoveryError as error:
+ row["discovery_error"] = str(error)
+ except Exception as error:
+ row["api_errors"].append(repr(error))
+ finally:
+ if task_started is not None:
+ row["attempt_ms"] = (time.perf_counter() - task_started) * 1000
+ try:
+ # No late operation can mutate the fixture after the terminal oracle starts.
+ self.engines.close()
+ if self.typer:
+ self.typer.stop()
+ row["keys_sent"] = self.typer.sent
+ self.typer = None
+ if row["prepared"]:
+ if task_started is not None:
+ row["elapsed_to_quiescent_ms"] = (time.perf_counter() - task_started) * 1000
+ before = (self.state() or {}).get("seq", -1)
+ time.sleep(0.4)
+ def fresh_state():
+ value = self.state()
+ return value if value and value.get("seq", -1) > before and acknowledged(
+ value, self.instance, self.pid, self.nonce) else None
+ state = wait_for(fresh_state, 2)
+ if task_started is not None:
+ row["terminal_readback_ms"] = (time.perf_counter() - task_started) * 1000
+ row["sentinel_after"] = read_json(self.sentinel_path)
+ sentinel_before = row["sentinel_before"]
+ sentinel_after = row["sentinel_after"]
+ if not all(value and value.get("pid") == self.sentinel_pid
+ for value in (sentinel_before, sentinel_after)):
+ raise RuntimeError("missing independent sentinel counters")
+ if cell["task"].endswith("_interference") and row.get("keys_sent", 0) == 0:
+ row["input_observer_error"] = "no simulated user input was delivered"
+ row["keys_received"] = sentinel_after.get("q", 0) - sentinel_before.get("q", 0)
+ row["keys_lost"] = max(0, row.get("keys_sent", 0) - row["keys_received"])
+ row["agent_text_in_sentinel"] = sentinel_after.get("other", 0) - sentinel_before.get("other", 0)
+ row["target_q_characters"] = state["text"].count("q") + state["decoyText"].count("q")
+ except Exception as error:
+ row["terminal_observation_error"] = repr(error)
+ try:
+ row["observation"] = self.stop_sampling()
+ except Exception as error:
+ row["observer_error"] = repr(error)
+ row["final_state"] = state
+ row["verdict"] = verdict(row, state)
+ for index, snapshot in enumerate(snapshots):
+ write_json(self.directory / f"snapshot-{index}.json", snapshot)
+ for index, response in enumerate(responses):
+ write_json(self.directory / f"response-{index}.json", response)
+ return row
+
+ def close(self):
+ if self.typer:
+ self.typer.stop()
+ try:
+ self.engines.close()
+ finally:
+ try:
+ self.stop_sampling()
+ finally:
+ self.targets.close()
+
+
+def markdown(meta, summary):
+ lines = [f"## {meta['os']} / {meta['target']}", "",
+ f"Run complete: {meta['complete']}. Cells: {meta['executed']}/{meta['planned']}. Seed: {meta['seed']}.", "",
+ "| task | profile | engine | completed/scored | invalid | unobserved | API errors | task ms | action + read ms | focus lost | raised |",
+ "|---|---|---|---|---|---|---|---|---|---|---|"]
+ for row in summary:
+ lines.append(f"| {row['task']} | {row['profile']} | {row['engine']} | {row['completed']}/{row['scored']} | "
+ f"{row['invalid']} | {row['unobserved']} | {row['api_errors']} | {row['task_ms']} | "
+ f"{row['action_read_ms']} | {row['sentinel_lost_focus']} | {row['target_raised']} |")
+ if meta.get("fatal"):
+ lines.extend(["", f"Incomplete: `{meta['fatal']}`"])
+ return "\n".join(lines) + "\n"
+
+
+def main():
+ if (os.environ.get("ORCA_COMPUTER_BENCH_ISOLATED") != "1" or os.environ.get("GITHUB_ACTIONS") != "true"
+ or os.environ.get("ORCA_BACKGROUND_LAUNCH") != "1" or sys.platform not in ("linux", "win32")):
+ raise SystemExit("Native input benchmark requires the isolated Linux/Windows GitHub Actions desktop")
+ global obs
+ if WIN:
+ import obs_win as obs
+ else:
+ import obs_x11 as obs
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--target", choices=["fixture", "electron"], required=True)
+ parser.add_argument("--cua", required=True)
+ parser.add_argument("--electron")
+ parser.add_argument("--out", type=Path, required=True)
+ parser.add_argument("--baseline", default="030c88deecf3adc212d22f57b085e4bd5c6e48e6")
+ parser.add_argument("--candidate", required=True)
+ parser.add_argument("--n", type=int, default=3)
+ parser.add_argument("--seed", type=int, default=417)
+ parser.add_argument("--budget", type=int, default=1800)
+ args = parser.parse_args()
+ if args.n < 1 or (args.target == "electron" and not args.electron):
+ parser.error("positive --n and an Electron executable for Electron targets are required")
+ args.out = args.out.resolve()
+ args.out.mkdir(parents=True, exist_ok=False)
+ sources = args.out / "sources"
+ cells = []
+ for cell in schedule(["baseline", "candidate", "cua"], TASKS, args.n, args.seed):
+ profiles = ["background", "foreground"] if not cell["task"].endswith("_interference") else ["background"]
+ cells.extend({**cell, "profile": profile} for profile in profiles)
+ meta = {"os": platform.platform(), "target": args.target, "seed": args.seed, "n": args.n,
+ "python": sys.version, "image": os.environ.get("ImageVersion"), "planned": len(cells),
+ "complete": False, "executed": 0, "sources": {}}
+ rows = []
+ started = time.monotonic()
+ try:
+ meta["sources"] = {name: pin_source(sha, sources / name)
+ for name, sha in (("baseline", args.baseline), ("candidate", args.candidate))}
+ meta["cua_sha256"] = hashlib.sha256(Path(args.cua).read_bytes()).hexdigest()
+ meta["cua_version"] = subprocess.check_output([args.cua, "--version"], text=True, timeout=15).strip()
+ if "0.38.0" not in meta["cua_version"]:
+ raise RuntimeError("expected pinned cua-driver 0.38.0")
+ with (args.out / "engines.log").open("ab") as log:
+ control = Trial(args, args.out / "controls", log)
+ try:
+ control.launch()
+ meta["calibration"] = control.calibrate()
+ finally:
+ control.close()
+ for index, cell in enumerate(cells):
+ if time.monotonic() - started >= args.budget:
+ raise TimeoutError("budget exhausted; planned cells remain")
+ trial = Trial(args, args.out / f"trial-{index:03d}", log)
+ try:
+ row = trial.run(cell, sources)
+ rows.append(row)
+ finally:
+ trial.close()
+ meta["executed"] = len(rows)
+ write_json(args.out / "results.json", {"meta": meta, "rows": rows})
+ print(f"{index + 1}/{len(cells)} {cell}: {row['verdict']}", flush=True)
+ meta["complete"] = (len(rows) == len(cells) and all(row["prepared"] and not row.get("observer_error")
+ and not row.get("terminal_observation_error") and not row.get("input_observer_error") and row["verdict"] != "unobserved" for row in rows))
+ except Exception as error:
+ import traceback
+ traceback.print_exc()
+ meta["fatal"] = repr(error)
+ finally:
+ meta["executed"] = len(rows)
+ meta["elapsed_seconds"] = time.monotonic() - started
+ summary = summarize(rows)
+ write_json(args.out / "results.json", {"meta": meta, "summary": summary, "rows": rows, "plan": cells})
+ report = markdown(meta, summary)
+ (args.out / "summary.md").write_text(report, encoding="utf-8")
+ print(report)
+ if os.environ.get("GITHUB_STEP_SUMMARY"):
+ with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as stream:
+ stream.write(report)
+ return 0 if meta["complete"] else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/config/benchmarks/computer-use/electron/main.js b/config/benchmarks/computer-use/electron/main.js
new file mode 100644
index 00000000000..5da2090d86f
--- /dev/null
+++ b/config/benchmarks/computer-use/electron/main.js
@@ -0,0 +1,81 @@
+// Disposable native-provider fixture, launched only on an isolated desktop.
+const { app, BrowserWindow, ipcMain } = require('electron')
+const fs = require('node:fs')
+const path = require('node:path')
+const dir = process.env.BENCH_STATE_DIR
+const instance = process.env.BENCH_INSTANCE
+process.on('uncaughtException', (error) => {
+ console.error(error)
+ app.exit(1)
+})
+if (process.env.ORCA_COMPUTER_BENCH_ISOLATED !== '1') {
+ process.exit(1)
+}
+app.whenReady().then(() => {
+ const w = new BrowserWindow({
+ x: 60,
+ y: 60,
+ width: 760,
+ height: 560,
+ title: 'BenchElectron',
+ webPreferences: { nodeIntegration: true, contextIsolation: false, backgroundThrottling: false }
+ })
+ ipcMain.on('st', (_event, state) => {
+ const data = JSON.stringify({ ...state, instance, activeForm: w.isFocused(), pid: process.pid })
+ try {
+ fs.writeFileSync(path.join(dir, 'state.json.tmp'), data)
+ fs.renameSync(path.join(dir, 'state.json.tmp'), path.join(dir, 'state.json'))
+ } catch (error) {
+ // Windows readers can briefly deny replacement; the next state tick retries.
+ if (!['EACCES', 'EPERM', 'EBUSY'].includes(error.code)) {
+ throw error
+ }
+ }
+ })
+ w.loadURL(
+ `data:text/html,${encodeURIComponent(`BenchElectron
+
+
+
+
+ `)}`
+ )
+ setInterval(() => {
+ const cmdPath = path.join(dir, 'cmd')
+ if (!fs.existsSync(cmdPath)) {
+ return
+ }
+ const raw = fs.readFileSync(cmdPath, 'utf8').trim()
+ if (raw) {
+ const cmd = JSON.parse(raw)
+ fs.unlinkSync(cmdPath)
+ if (cmd.instance === instance) {
+ w.webContents.send('cmd', cmd)
+ }
+ }
+ }, 30)
+})
diff --git a/config/benchmarks/computer-use/fixture_gtk.py b/config/benchmarks/computer-use/fixture_gtk.py
new file mode 100644
index 00000000000..978aad9fb0c
--- /dev/null
+++ b/config/benchmarks/computer-use/fixture_gtk.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""GTK3 target for the cua-vs-Orca background bench. Writes state.json every 30 ms; obeys cmd."""
+import json
+import os
+import sys
+
+if os.environ.get("ORCA_COMPUTER_BENCH_ISOLATED") != "1" or os.environ.get("GITHUB_ACTIONS") != "true":
+ raise SystemExit("fixture requires isolated CI desktop")
+
+import gi
+
+gi.require_version("Gtk", "3.0")
+from gi.repository import Gdk, GLib, Gtk # noqa: E402
+
+state_dir = sys.argv[1]
+STATE = os.path.join(state_dir, "state.json")
+CMD = os.path.join(state_dir, "cmd")
+instance = os.environ["BENCH_INSTANCE"]
+ack = None
+sequence = 0
+s = {"count": 0, "returns": 0, "menuHits": 0, "last": None}
+
+
+def named(widget, name):
+ widget.get_accessible().set_name(name)
+ return widget
+
+
+Gtk.Settings.get_default().set_property("gtk-entry-select-on-focus", False)
+win = Gtk.Window(title="BenchTarget")
+win.set_default_size(760, 560)
+win.move(60, 60)
+win.connect("destroy", Gtk.main_quit)
+accel = Gtk.AccelGroup()
+win.add_accel_group(accel)
+box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=6)
+win.add(box)
+
+menubar = Gtk.MenuBar()
+edit = Gtk.MenuItem(label="Edit")
+sub = Gtk.Menu()
+sub.set_accel_group(accel)
+bump = Gtk.MenuItem(label="Bump")
+bump.add_accelerator("activate", accel, ord("b"), Gdk.ModifierType.CONTROL_MASK, Gtk.AccelFlags.VISIBLE)
+bump.connect("activate", lambda _w: s.__setitem__("menuHits", s["menuHits"] + 1))
+sub.append(bump)
+edit.set_submenu(sub)
+menubar.append(edit)
+box.pack_start(menubar, False, False, 0)
+
+button = Gtk.Button(label="Increment")
+button.connect("clicked", lambda _b: s.__setitem__("count", s["count"] + 1))
+check = Gtk.CheckButton(label="Check me")
+field = named(Gtk.Entry(), "field")
+field.connect("activate", lambda _e: s.__setitem__("returns", s["returns"] + 1))
+decoy = named(Gtk.Entry(), "decoy")
+combo = named(Gtk.ComboBoxText(), "Color")
+for item in ("Red", "Green", "Blue"):
+ combo.append_text(item)
+combo.set_active(0)
+scroll = named(Gtk.ScrolledWindow(), "list")
+scroll.set_size_request(300, 250)
+rows = Gtk.ListBox()
+for i in range(1, 301):
+ rows.add(Gtk.Label(label=f"Row {i}"))
+scroll.add(rows)
+for widget in (button, check, field, decoy, combo):
+ box.pack_start(widget, False, False, 0)
+box.pack_start(scroll, True, True, 0)
+
+
+def command(c):
+ global ack
+ if c["instance"] != instance:
+ return
+ s["count"] = s["returns"] = s["menuHits"] = 0
+ check.set_active(False)
+ sub.popdown()
+ combo.set_active(0)
+ scroll.get_vadjustment().set_value(0)
+ field.set_text("hello" if c["task"] == "replace_submit" else "")
+ field.set_position(len(field.get_text()))
+ field.select_region(len(field.get_text()), len(field.get_text()))
+ decoy.set_text("untouched")
+ decoy.set_position(9)
+ decoy.select_region(9, 9)
+ win.set_focus(decoy if c["task"] == "targeted_type" else field)
+ ack = c["nonce"]
+
+
+def tick():
+ global sequence
+ sequence += 1
+ try:
+ with open(CMD) as f:
+ c = f.read().strip()
+ if c:
+ open(CMD, "w").close()
+ command(json.loads(c))
+ except (OSError, ValueError):
+ pass
+ bounds = field.get_selection_bounds()
+ sel = list(bounds) if bounds else [field.get_position(), field.get_position()]
+ state = {
+ "instance": instance, "ack": ack, "seq": sequence, "decoyText": decoy.get_text(),
+ "logicalFocus": "field" if win.get_focus() is field else "decoy" if win.get_focus() is decoy else "other",
+ "count": s["count"], "check": check.get_active(), "text": field.get_text(), "returns": s["returns"],
+ "selStart": sel[0], "selLen": sel[1] - sel[0], "combo": combo.get_active_text(),
+ "top": scroll.get_vadjustment().get_value(), "menuHits": s["menuHits"], "dropOpen": sub.get_visible(),
+ "activeForm": win.is_active(), "fieldFocused": win.get_focus() is field, "pid": os.getpid(),
+ }
+ data = json.dumps(state)
+ if data != s["last"]:
+ tmp = STATE + ".tmp"
+ with open(tmp, "w") as f:
+ f.write(data)
+ os.replace(tmp, STATE)
+ s["last"] = data
+ return True
+
+
+win.show_all()
+win.set_focus(field)
+GLib.timeout_add(30, tick)
+Gtk.main()
diff --git a/config/benchmarks/computer-use/fixture_win.ps1 b/config/benchmarks/computer-use/fixture_win.ps1
new file mode 100644
index 00000000000..f321fa5b6a6
--- /dev/null
+++ b/config/benchmarks/computer-use/fixture_win.ps1
@@ -0,0 +1,124 @@
+# WinForms target for the cua-vs-Orca background bench. Writes its state to state.json every 30 ms
+# and acknowledges each JSON command after applying it.
+param([string]$StateDir)
+$ErrorActionPreference = "Stop"
+if ($env:ORCA_COMPUTER_BENCH_ISOLATED -ne "1" -or $env:GITHUB_ACTIONS -ne "true") { throw "fixture requires isolated CI desktop" }
+Add-Type -AssemblyName System.Windows.Forms
+Add-Type -AssemblyName System.Drawing
+[System.Windows.Forms.Application]::EnableVisualStyles()
+
+$statePath = Join-Path $StateDir "state.json"
+$cmdPath = Join-Path $StateDir "cmd"
+$instance = $env:BENCH_INSTANCE
+$script:ack = $null
+$script:sequence = 0
+$script:count = 0
+$script:returns = 0
+$script:menuHits = 0
+$script:last = ""
+
+$form = New-Object System.Windows.Forms.Form
+$form.Text = "BenchTarget"
+$form.StartPosition = "Manual"
+$form.Location = New-Object System.Drawing.Point(60, 60)
+$form.Size = New-Object System.Drawing.Size(760, 560)
+
+$menu = New-Object System.Windows.Forms.MenuStrip
+$edit = New-Object System.Windows.Forms.ToolStripMenuItem("Edit")
+$bump = New-Object System.Windows.Forms.ToolStripMenuItem("Bump")
+$bump.ShortcutKeys = [System.Windows.Forms.Keys]::Control -bor [System.Windows.Forms.Keys]::B
+$bump.Add_Click({ $script:menuHits++ })
+[void]$edit.DropDownItems.Add($bump)
+[void]$menu.Items.Add($edit)
+$form.MainMenuStrip = $menu
+$form.Controls.Add($menu)
+
+$button = New-Object System.Windows.Forms.Button
+$button.Text = "Increment"
+$button.Location = New-Object System.Drawing.Point(20, 40)
+$button.Add_Click({ $script:count++ })
+$form.Controls.Add($button)
+
+$check = New-Object System.Windows.Forms.CheckBox
+$check.Text = "Check me"
+$check.Location = New-Object System.Drawing.Point(20, 80)
+$form.Controls.Add($check)
+
+$field = New-Object System.Windows.Forms.TextBox
+$field.Name = "field"
+$field.AccessibleName = "field"
+$field.Location = New-Object System.Drawing.Point(20, 120)
+$field.Width = 300
+$field.Add_KeyDown({
+ param($sender, $e)
+ if ($e.KeyCode -eq [System.Windows.Forms.Keys]::Enter) { $script:returns++; $e.SuppressKeyPress = $true }
+})
+$form.Controls.Add($field)
+
+$decoy = New-Object System.Windows.Forms.TextBox
+$decoy.Name = "decoy"
+$decoy.AccessibleName = "decoy"
+$decoy.Location = New-Object System.Drawing.Point(350, 120)
+$decoy.Width = 300
+$form.Controls.Add($decoy)
+
+$combo = New-Object System.Windows.Forms.ComboBox
+$combo.DropDownStyle = "DropDownList"
+$combo.AccessibleName = "Color"
+[void]$combo.Items.AddRange(@("Red", "Green", "Blue"))
+$combo.SelectedIndex = 0
+$combo.Location = New-Object System.Drawing.Point(20, 160)
+$form.Controls.Add($combo)
+
+$list = New-Object System.Windows.Forms.ListBox
+$list.AccessibleName = "list"
+$list.Location = New-Object System.Drawing.Point(20, 200)
+$list.Size = New-Object System.Drawing.Size(300, 250)
+foreach ($i in 1..300) { [void]$list.Items.Add("Row $i") }
+$form.Controls.Add($list)
+
+function Invoke-BenchCommand($c) {
+ if ($c.instance -ne $instance) { return }
+ $script:count = 0; $script:returns = 0; $script:menuHits = 0
+ $check.Checked = $false
+ $edit.HideDropDown()
+ $combo.SelectedIndex = 0
+ $combo.DroppedDown = $false
+ $list.TopIndex = 0
+ $field.Text = if ($c.task -eq 'replace_submit') { 'hello' } else { '' }
+ $field.SelectionStart = $field.Text.Length
+ $field.SelectionLength = 0
+ $decoy.Text = 'untouched'
+ $decoy.SelectionStart = $decoy.Text.Length
+ $decoy.SelectionLength = 0
+ $form.ActiveControl = if ($c.task -eq 'targeted_type') { $decoy } else { $field }
+ $script:ack = $c.nonce
+}
+
+$timer = New-Object System.Windows.Forms.Timer
+$timer.Interval = 30
+$timer.Add_Tick({
+ $script:sequence++
+ try {
+ if (Test-Path $cmdPath) {
+ $c = ([IO.File]::ReadAllText($cmdPath)).Trim()
+ if ($c) { [IO.File]::WriteAllText($cmdPath, ""); Invoke-BenchCommand ($c | ConvertFrom-Json) }
+ }
+ } catch {}
+ $state = [ordered]@{
+ instance = $instance; ack = $script:ack; seq = $script:sequence
+ decoyText = $decoy.Text
+ logicalFocus = $(if ($form.ActiveControl -eq $field) { 'field' } elseif ($form.ActiveControl -eq $decoy) { 'decoy' } else { 'other' })
+ count = $script:count; check = $check.Checked; text = $field.Text; returns = $script:returns
+ selStart = $field.SelectionStart; selLen = $field.SelectionLength; combo = [string]$combo.SelectedItem
+ top = $list.TopIndex; menuHits = $script:menuHits; dropOpen = ($edit.DropDown.Visible -or $combo.DroppedDown)
+ activeForm = ($null -ne [System.Windows.Forms.Form]::ActiveForm); fieldFocused = $field.Focused
+ pid = $PID
+ }
+ $json = $state | ConvertTo-Json -Compress
+ if ($json -ne $script:last) {
+ try { [IO.File]::WriteAllText($statePath, $json); $script:last = $json } catch {}
+ }
+})
+$form.Add_Shown({ $form.ActiveControl = $field; $timer.Start() })
+[System.Windows.Forms.Application]::Run($form)
diff --git a/config/benchmarks/computer-use/obs_win.py b/config/benchmarks/computer-use/obs_win.py
new file mode 100644
index 00000000000..17d9b9e579d
--- /dev/null
+++ b/config/benchmarks/computer-use/obs_win.py
@@ -0,0 +1,309 @@
+"""Windows host observation (foreground, z-order, cursor, clipboard) and the user's typer, via user32."""
+import ctypes
+import ctypes.wintypes as W
+import threading
+import time
+
+try:
+ ctypes.windll.shcore.SetProcessDpiAwareness(2)
+except Exception:
+ ctypes.windll.user32.SetProcessDPIAware()
+
+u = ctypes.WinDLL("user32", use_last_error=True)
+k32 = ctypes.WinDLL("kernel32", use_last_error=True)
+
+GW_HWNDPREV = 3
+WM_GETTEXT, WM_GETTEXTLENGTH, WM_SETTEXT = 0x000D, 0x000E, 0x000C
+EM_GETSEL, EM_SETSEL, EM_GETFIRSTVISIBLELINE, EM_LINESCROLL = 0x00B0, 0x00B1, 0x00CE, 0x00B6
+WM_VSCROLL, SB_TOP = 0x0115, 6
+SMTO_ABORTIFHUNG = 0x2
+SWP_NOACTIVATE, SWP_NOZORDER = 0x0010, 0x0004
+SPI_GETFOREGROUNDLOCKTIMEOUT = 0x2000
+
+ULONG_PTR = ctypes.c_size_t
+
+
+class KEYBDINPUT(ctypes.Structure):
+ _fields_ = [("wVk", W.WORD), ("wScan", W.WORD), ("dwFlags", W.DWORD), ("time", W.DWORD), ("dwExtraInfo", ULONG_PTR)]
+
+
+class MOUSEINPUT(ctypes.Structure):
+ _fields_ = [("dx", W.LONG), ("dy", W.LONG), ("mouseData", W.DWORD), ("dwFlags", W.DWORD), ("time", W.DWORD), ("dwExtraInfo", ULONG_PTR)]
+
+
+class _U(ctypes.Union):
+ _fields_ = [("ki", KEYBDINPUT), ("mi", MOUSEINPUT)]
+
+
+class INPUT(ctypes.Structure):
+ _fields_ = [("type", W.DWORD), ("u", _U)]
+
+
+WNDENUMPROC = ctypes.WINFUNCTYPE(W.BOOL, W.HWND, W.LPARAM)
+u.EnumWindows.argtypes = [WNDENUMPROC, W.LPARAM]
+u.GetWindowThreadProcessId.argtypes = [W.HWND, ctypes.POINTER(W.DWORD)]
+u.GetWindowThreadProcessId.restype = W.DWORD
+u.GetForegroundWindow.restype = W.HWND
+u.GetWindow.restype = W.HWND
+u.GetWindow.argtypes = [W.HWND, W.UINT]
+u.FindWindowExW.restype = W.HWND
+u.FindWindowExW.argtypes = [W.HWND, W.HWND, W.LPCWSTR, W.LPCWSTR]
+u.SendMessageTimeoutW.argtypes = [W.HWND, W.UINT, W.WPARAM, W.LPARAM, W.UINT, W.UINT, ctypes.POINTER(ULONG_PTR)]
+u.SendMessageTimeoutW.restype = W.LPARAM
+u.SetWindowPos.argtypes = [W.HWND, W.HWND, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, W.UINT]
+u.SendInput.argtypes = [W.UINT, ctypes.POINTER(INPUT), ctypes.c_int]
+u.AttachThreadInput.argtypes = [W.DWORD, W.DWORD, W.BOOL]
+u.SetForegroundWindow.argtypes = [W.HWND]
+u.BringWindowToTop.argtypes = [W.HWND]
+u.IsWindowVisible.argtypes = [W.HWND]
+u.GetClassNameW.argtypes = [W.HWND, W.LPWSTR, ctypes.c_int]
+u.GetWindowTextW.argtypes = [W.HWND, W.LPWSTR, ctypes.c_int]
+u.GetWindowRect.argtypes = [W.HWND, ctypes.POINTER(W.RECT)]
+
+
+def hval(h):
+ return int(h or 0)
+
+
+def pid_of(hwnd):
+ p = W.DWORD(0)
+ u.GetWindowThreadProcessId(hwnd, ctypes.byref(p))
+ return p.value
+
+
+def title_of(hwnd):
+ b = ctypes.create_unicode_buffer(512)
+ u.GetWindowTextW(hwnd, b, 512)
+ return b.value
+
+
+def class_of(hwnd):
+ b = ctypes.create_unicode_buffer(256)
+ u.GetClassNameW(hwnd, b, 256)
+ return b.value
+
+
+def top_windows():
+ out = []
+
+ def cb(h, _l):
+ out.append(hval(h))
+ return True
+
+ u.EnumWindows(WNDENUMPROC(cb), 0)
+ return out
+
+
+def find_window(pid=None, title=None, cls=None, visible=True):
+ for h in top_windows():
+ if visible and not u.IsWindowVisible(h):
+ continue
+ if pid is not None and pid_of(h) != pid:
+ continue
+ if title is not None and title not in title_of(h):
+ continue
+ if cls is not None and class_of(h) != cls:
+ continue
+ return h
+ return 0
+
+
+def describe(h):
+ r = W.RECT()
+ u.GetWindowRect(h, ctypes.byref(r))
+ return {"hwnd": h, "pid": pid_of(h), "title": title_of(h), "class": class_of(h), "rect": [r.left, r.top, r.right, r.bottom]}
+
+
+def foreground():
+ return hval(u.GetForegroundWindow())
+
+
+def cursor():
+ p = W.POINT()
+ u.GetCursorPos(ctypes.byref(p))
+ return (p.x, p.y)
+
+
+def clip_seq():
+ return u.GetClipboardSequenceNumber()
+
+
+def lock_timeout():
+ v = W.DWORD(0)
+ u.SystemParametersInfoW(SPI_GETFOREGROUNDLOCKTIMEOUT, 0, ctypes.byref(v), 0)
+ return v.value
+
+
+def session_id():
+ s = W.DWORD(0)
+ k32.ProcessIdToSessionId(k32.GetCurrentProcessId(), ctypes.byref(s))
+ return s.value
+
+
+def owned_above(sentinel, target_pid, limit=2000):
+ """True when a visible top-level window of target_pid sits above the sentinel in z-order."""
+ h = u.GetWindow(sentinel, GW_HWNDPREV)
+ n = 0
+ while h and n < limit:
+ if u.IsWindowVisible(h) and pid_of(h) == target_pid:
+ return True
+ h = u.GetWindow(h, GW_HWNDPREV)
+ n += 1
+ return False
+
+
+def place(hwnd, x, y, w, h):
+ u.SetWindowPos(hwnd, 0, x, y, w, h, SWP_NOACTIVATE | SWP_NOZORDER)
+
+
+def force_foreground(hwnd):
+ """Bench setup only: re-front the sentinel between trials (AttachThreadInput, no keystrokes)."""
+ if foreground() == hwnd:
+ return True
+ cur = k32.GetCurrentThreadId()
+ fg = foreground()
+ threads = {u.GetWindowThreadProcessId(fg, None) if fg else 0, u.GetWindowThreadProcessId(hwnd, None)} - {0, cur}
+ attached = [t for t in threads if u.AttachThreadInput(cur, t, True)]
+ u.BringWindowToTop(hwnd)
+ u.SetForegroundWindow(hwnd)
+ for t in attached:
+ u.AttachThreadInput(cur, t, False)
+ for _ in range(25):
+ if foreground() == hwnd:
+ return True
+ time.sleep(0.02)
+ return foreground() == hwnd
+
+
+def send(hwnd, msg, wp=0, lp=0, timeout=1000):
+ res = ULONG_PTR(0)
+ ok = u.SendMessageTimeoutW(hwnd, msg, wp, lp, SMTO_ABORTIFHUNG, timeout, ctypes.byref(res))
+ return res.value if ok else None
+
+
+def edit_child(top):
+ """Notepad's editor: classic Edit, or RichEditD2DPT in the WinUI Notepad."""
+ for cls in ("Edit", "RichEditD2DPT"):
+ h = hval(u.FindWindowExW(top, None, cls, None))
+ if h:
+ return h
+ stack = [top]
+ while stack:
+ p = stack.pop()
+ c = hval(u.FindWindowExW(p, None, None, None))
+ while c:
+ if class_of(c) in ("Edit", "RichEditD2DPT"):
+ return c
+ stack.append(c)
+ c = hval(u.FindWindowExW(p, c, None, None))
+ return 0
+
+
+def edit_text(h):
+ n = send(h, WM_GETTEXTLENGTH) or 0
+ buf = ctypes.create_unicode_buffer(n + 2)
+ send(h, WM_GETTEXT, n + 1, ctypes.addressof(buf))
+ return buf.value
+
+
+def edit_set(h, text, caret=None):
+ buf = ctypes.create_unicode_buffer(text)
+ send(h, WM_SETTEXT, 0, ctypes.addressof(buf))
+ c = len(text) if caret is None else caret
+ send(h, EM_SETSEL, c, c)
+ send(h, WM_VSCROLL, SB_TOP, 0)
+
+
+def edit_sel(h):
+ v = send(h, EM_GETSEL) or 0
+ return (v & 0xFFFF, (v >> 16) & 0xFFFF)
+
+
+def edit_first_line(h):
+ return send(h, EM_GETFIRSTVISIBLELINE) or 0
+
+
+class Typer:
+ """Simulated user: one 'q' every 200 ms through SendInput (goes to whatever has keyboard focus)."""
+
+ def __init__(self, period=0.2):
+ self.period = period
+ self.sent = 0
+ self.run = False
+ self.vk = 0x51
+ self.scan = u.MapVirtualKeyW(self.vk, 0)
+
+ def _key(self, up):
+ i = INPUT(type=1)
+ i.u.ki = KEYBDINPUT(self.vk, self.scan, 0x0002 if up else 0, 0, 0)
+ return i
+
+ def _loop(self):
+ while self.run:
+ arr = (INPUT * 2)(self._key(False), self._key(True))
+ if u.SendInput(2, arr, ctypes.sizeof(INPUT)) == 2:
+ self.sent += 1
+ time.sleep(self.period)
+
+ def start(self):
+ self.run = True
+ self.thread = threading.Thread(target=self._loop, daemon=True)
+ self.thread.start()
+
+ def stop(self):
+ self.run = False
+ self.thread.join()
+ time.sleep(0.1)
+
+
+class Sampler:
+ """Samples every ~4 ms: foreground window, target-above-sentinel, cursor, clipboard sequence."""
+
+ def __init__(self, sentinel, target, target_pid, own_pids=()):
+ self.s, self.t, self.tpid = sentinel, target, target_pid
+ self.own = set(own_pids)
+
+ def _sample(self):
+ fg = foreground()
+ return (time.perf_counter(), fg, pid_of(fg) if fg else 0, owned_above(self.s, self.tpid), cursor(), clip_seq())
+
+ def start(self):
+ self.base = self._sample()
+ self.samples = [self.base]
+ self.error = None
+ self.run = True
+ self.th = threading.Thread(target=self._loop, daemon=True)
+ self.th.start()
+
+ def _loop(self):
+ try:
+ while self.run:
+ self.samples.append(self._sample())
+ time.sleep(0.004)
+ except Exception as error:
+ self.error = error
+
+ def stop(self):
+ self.run = False
+ self.th.join()
+ if self.error:
+ raise RuntimeError("observer sampling failed") from self.error
+ self.samples.append(self._sample())
+ b = self.base
+ ss = self.samples
+ lost = [x for x in ss if x[1] != self.s]
+ return {
+ "sentinel_lost_focus": bool(lost),
+ "lost_ms": round(_dur(ss, lambda x: x[1] != self.s) * 1000, 1),
+ "target_activated": any(x[2] == self.tpid for x in ss),
+ "target_raised": any(x[3] for x in ss),
+ "pointer_moved": any(abs(x[4][0] - b[4][0]) > 1 or abs(x[4][1] - b[4][1]) > 1 for x in ss),
+ "pointer_end": list(ss[-1][4]), "pointer_start": list(b[4]),
+ "clipboard_changed": any(x[5] != b[5] for x in ss),
+ "fg_pids": sorted({x[2] for x in ss}),
+ "n_samples": len(ss),
+ }
+
+
+def _dur(ss, pred):
+ return sum(b[0] - a[0] for a, b in zip(ss, ss[1:]) if pred(a))
diff --git a/config/benchmarks/computer-use/obs_x11.py b/config/benchmarks/computer-use/obs_x11.py
new file mode 100644
index 00000000000..2c4954017b8
--- /dev/null
+++ b/config/benchmarks/computer-use/obs_x11.py
@@ -0,0 +1,263 @@
+"""X11 host observation (active window, input focus, stacking, pointer, selections) and the user's typer."""
+import ctypes
+import ctypes.util
+import subprocess
+import threading
+import time
+
+X = ctypes.cdll.LoadLibrary(ctypes.util.find_library("X11") or "libX11.so.6")
+XT = ctypes.cdll.LoadLibrary(ctypes.util.find_library("Xtst") or "libXtst.so.6")
+Window = ctypes.c_ulong
+Atom = ctypes.c_ulong
+P = ctypes.POINTER
+vp = ctypes.c_void_p
+
+X.XOpenDisplay.restype = vp
+X.XOpenDisplay.argtypes = [ctypes.c_char_p]
+X.XInitThreads.restype = ctypes.c_int
+X.XDefaultRootWindow.restype = Window
+X.XDefaultRootWindow.argtypes = [vp]
+X.XInternAtom.restype = Atom
+X.XInternAtom.argtypes = [vp, ctypes.c_char_p, ctypes.c_int]
+X.XGetWindowProperty.argtypes = [vp, Window, Atom, ctypes.c_long, ctypes.c_long, ctypes.c_int, Atom,
+ P(Atom), P(ctypes.c_int), P(ctypes.c_ulong), P(ctypes.c_ulong), P(vp)]
+X.XFree.argtypes = [vp]
+X.XGetInputFocus.argtypes = [vp, P(Window), P(ctypes.c_int)]
+X.XQueryPointer.argtypes = [vp, Window, P(Window), P(Window), P(ctypes.c_int), P(ctypes.c_int),
+ P(ctypes.c_int), P(ctypes.c_int), P(ctypes.c_uint)]
+X.XGetSelectionOwner.restype = Window
+X.XGetSelectionOwner.argtypes = [vp, Atom]
+X.XQueryTree.argtypes = [vp, Window, P(Window), P(Window), P(P(Window)), P(ctypes.c_uint)]
+X.XStringToKeysym.restype = ctypes.c_ulong
+X.XStringToKeysym.argtypes = [ctypes.c_char_p]
+X.XKeysymToKeycode.restype = ctypes.c_ubyte
+X.XKeysymToKeycode.argtypes = [vp, ctypes.c_ulong]
+X.XFlush.argtypes = [vp]
+X.XCloseDisplay.argtypes = [vp]
+XT.XTestFakeKeyEvent.argtypes = [vp, ctypes.c_uint, ctypes.c_int, ctypes.c_ulong]
+
+X.XInitThreads()
+ERRH = ctypes.CFUNCTYPE(ctypes.c_int, vp, vp)(lambda _d, _e: 0)
+X.XSetErrorHandler(ERRH)
+
+
+class Display:
+ def __init__(self):
+ self.d = X.XOpenDisplay(None)
+ if not self.d:
+ raise RuntimeError("cannot open X display")
+ self.root = X.XDefaultRootWindow(self.d)
+ self.atoms = {}
+
+ def atom(self, name):
+ if name not in self.atoms:
+ self.atoms[name] = X.XInternAtom(self.d, name.encode(), 0)
+ return self.atoms[name]
+
+ def prop_longs(self, win, name):
+ at, fmt, n, after, data = Atom(), ctypes.c_int(), ctypes.c_ulong(), ctypes.c_ulong(), vp()
+ rc = X.XGetWindowProperty(self.d, win, self.atom(name), 0, 4096, 0, 0, ctypes.byref(at), ctypes.byref(fmt),
+ ctypes.byref(n), ctypes.byref(after), ctypes.byref(data))
+ if rc != 0 or not data.value:
+ return []
+ try:
+ if fmt.value != 32:
+ return []
+ arr = ctypes.cast(data, P(ctypes.c_ulong))
+ return [arr[i] for i in range(n.value)]
+ finally:
+ X.XFree(data)
+
+ def active(self):
+ v = self.prop_longs(self.root, "_NET_ACTIVE_WINDOW")
+ return v[0] if v else 0
+
+ def stacking(self):
+ return self.prop_longs(self.root, "_NET_CLIENT_LIST_STACKING")
+
+ def clients(self):
+ return self.prop_longs(self.root, "_NET_CLIENT_LIST")
+
+ def pid_of(self, w):
+ v = self.prop_longs(w, "_NET_WM_PID")
+ return v[0] if v else 0
+
+ def focus(self):
+ w, r = Window(), ctypes.c_int()
+ X.XGetInputFocus(self.d, ctypes.byref(w), ctypes.byref(r))
+ return w.value
+
+ def pointer(self):
+ r, c = Window(), Window()
+ rx, ry, wx, wy = ctypes.c_int(), ctypes.c_int(), ctypes.c_int(), ctypes.c_int()
+ m = ctypes.c_uint()
+ X.XQueryPointer(self.d, self.root, ctypes.byref(r), ctypes.byref(c), ctypes.byref(rx), ctypes.byref(ry),
+ ctypes.byref(wx), ctypes.byref(wy), ctypes.byref(m))
+ return (rx.value, ry.value)
+
+ def owner(self, sel):
+ return X.XGetSelectionOwner(self.d, self.atom(sel))
+
+ def descendants(self, w):
+ out, stack = {w}, [w]
+ while stack:
+ p = stack.pop()
+ r, par, kids, n = Window(), Window(), P(Window)(), ctypes.c_uint()
+ if not X.XQueryTree(self.d, p, ctypes.byref(r), ctypes.byref(par), ctypes.byref(kids), ctypes.byref(n)):
+ continue
+ for i in range(n.value):
+ out.add(kids[i])
+ stack.append(kids[i])
+ if kids:
+ X.XFree(kids)
+ return out
+
+ def windows_of(self, pid):
+ return [w for w in self.clients() if self.pid_of(w) == pid]
+
+ def close(self):
+ if self.d:
+ X.XCloseDisplay(self.d)
+ self.d = None
+
+
+_MAIN = []
+
+
+def main_display():
+ """One connection for the bench's main thread; Xvfb allows only ~256 clients."""
+ if not _MAIN:
+ _MAIN.append(Display())
+ return _MAIN[0]
+
+
+def find_client(pid, title=None):
+ """Managed client window of pid (by _NET_WM_PID), else by exact title (Tk may omit _NET_WM_PID)."""
+ d = main_display()
+ ws = d.windows_of(pid)
+ if ws:
+ return ws[0]
+ if title:
+ clients = set(d.clients())
+ out = xdotool("search", "--name", f"^{title}$").stdout.split()
+ for w in out:
+ if int(w) in clients:
+ return int(w)
+ return 0
+
+
+def xdotool(*args):
+ return subprocess.run(["xdotool", *map(str, args)], capture_output=True, text=True, timeout=10)
+
+
+def window_name(w):
+ return xdotool("getwindowname", w).stdout.strip()
+
+
+def place(w, x, y, width, height):
+ xdotool("windowsize", w, width, height)
+ xdotool("windowmove", w, x, y)
+
+
+def force_foreground(w):
+ """Bench setup only: re-front the sentinel between trials."""
+ d = main_display()
+ if d.active() == w:
+ return True
+ xdotool("windowactivate", "--sync", w)
+ xdotool("windowraise", w)
+ for _ in range(25):
+ if d.active() == w:
+ return True
+ time.sleep(0.02)
+ return d.active() == w
+
+
+class Typer:
+ """Simulated user: one 'q' every 200 ms through XTest (goes to the X input focus)."""
+
+ def __init__(self, period=0.2):
+ self.period = period
+ self.sent = 0
+ self.run = False
+
+ def _loop(self):
+ d = Display()
+ kc = X.XKeysymToKeycode(d.d, X.XStringToKeysym(b"q"))
+ while self.run:
+ XT.XTestFakeKeyEvent(d.d, kc, 1, 0)
+ XT.XTestFakeKeyEvent(d.d, kc, 0, 0)
+ X.XFlush(d.d)
+ self.sent += 1
+ time.sleep(self.period)
+ d.close()
+
+ def start(self):
+ self.run = True
+ self.thread = threading.Thread(target=self._loop, daemon=True)
+ self.thread.start()
+
+ def stop(self):
+ self.run = False
+ self.thread.join()
+ time.sleep(0.1)
+
+
+class Sampler:
+ """Samples every ~4 ms: active window, input focus, stacking, pointer, CLIPBOARD/PRIMARY owners."""
+
+ def __init__(self, sentinel, target, target_pid, own_pids=()):
+ self.s, self.t, self.tpid = sentinel, target, target_pid
+ self.s_set = main_display().descendants(sentinel)
+
+ def _sample(self, d):
+ st = d.stacking()
+ act = d.active()
+ foc = d.focus()
+ tw = set(d.windows_of(self.tpid)) or {self.t}
+ raised = False
+ if self.s in st:
+ si = st.index(self.s)
+ raised = any(st.index(w) > si for w in tw if w in st)
+ return (time.perf_counter(), act, foc, raised, d.pointer(), d.owner("CLIPBOARD"), d.owner("PRIMARY"), act in tw)
+
+ def start(self):
+ self.d = main_display()
+ self.base = self._sample(self.d)
+ self.samples = [self.base]
+ self.error = None
+ self.run = True
+ self.th = threading.Thread(target=self._loop, daemon=True)
+ self.th.start()
+
+ def _loop(self):
+ d = Display()
+ try:
+ while self.run:
+ self.samples.append(self._sample(d))
+ time.sleep(0.004)
+ except Exception as error:
+ self.error = error
+ finally:
+ d.close()
+
+ def stop(self):
+ self.run = False
+ self.th.join()
+ if self.error:
+ raise RuntimeError("observer sampling failed") from self.error
+ self.samples.append(self._sample(self.d))
+ b, ss = self.base, self.samples
+ lost = lambda x: x[1] != self.s or (x[2] not in self.s_set)
+ return {
+ "sentinel_lost_focus": any(lost(x) for x in ss),
+ "lost_ms": round(sum(q[0] - p[0] for p, q in zip(ss, ss[1:]) if lost(p)) * 1000, 1),
+ "target_activated": any(x[7] for x in ss),
+ "target_raised": any(x[3] for x in ss),
+ "pointer_moved": any(abs(x[4][0] - b[4][0]) > 1 or abs(x[4][1] - b[4][1]) > 1 for x in ss),
+ "pointer_end": list(ss[-1][4]), "pointer_start": list(b[4]),
+ "clipboard_changed": any(x[5] != b[5] for x in ss),
+ "primary_changed": any(x[6] != b[6] for x in ss),
+ "focus_windows": sorted({hex(x[2]) for x in ss})[:6],
+ "n_samples": len(ss),
+ }
diff --git a/config/benchmarks/computer-use/process_scope.py b/config/benchmarks/computer-use/process_scope.py
new file mode 100644
index 00000000000..577a702766d
--- /dev/null
+++ b/config/benchmarks/computer-use/process_scope.py
@@ -0,0 +1,103 @@
+"""Contain each trial's process tree, including operations that time out."""
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+
+WIN = sys.platform == "win32"
+
+if WIN:
+ import ctypes
+ from ctypes import wintypes as W
+
+ class BasicLimit(ctypes.Structure):
+ _fields_ = [("processTime", ctypes.c_longlong), ("jobTime", ctypes.c_longlong),
+ ("flags", W.DWORD), ("minWorkingSet", ctypes.c_size_t),
+ ("maxWorkingSet", ctypes.c_size_t), ("activeProcesses", W.DWORD),
+ ("affinity", ctypes.c_size_t), ("priority", W.DWORD), ("scheduling", W.DWORD)]
+
+ class IoCounters(ctypes.Structure):
+ _fields_ = [(name, ctypes.c_ulonglong) for name in
+ ("readOps", "writeOps", "otherOps", "readBytes", "writeBytes", "otherBytes")]
+
+ class ExtendedLimit(ctypes.Structure):
+ _fields_ = [("basic", BasicLimit), ("io", IoCounters), ("processMemory", ctypes.c_size_t),
+ ("jobMemory", ctypes.c_size_t), ("peakProcess", ctypes.c_size_t), ("peakJob", ctypes.c_size_t)]
+
+ kernel = ctypes.WinDLL("kernel32", use_last_error=True)
+ kernel.CreateJobObjectW.argtypes = [ctypes.c_void_p, W.LPCWSTR]
+ kernel.CreateJobObjectW.restype = W.HANDLE
+ kernel.SetInformationJobObject.argtypes = [W.HANDLE, ctypes.c_int, ctypes.c_void_p, W.DWORD]
+ kernel.AssignProcessToJobObject.argtypes = [W.HANDLE, W.HANDLE]
+ kernel.TerminateJobObject.argtypes = [W.HANDLE, W.UINT]
+ kernel.CloseHandle.argtypes = [W.HANDLE]
+ kernel.QueryInformationJobObject.argtypes = [W.HANDLE, ctypes.c_int, ctypes.c_void_p, W.DWORD, ctypes.c_void_p]
+ nt = ctypes.WinDLL("ntdll")
+ nt.NtResumeProcess.argtypes = [W.HANDLE]
+ nt.NtResumeProcess.restype = ctypes.c_long
+
+ class Accounting(ctypes.Structure):
+ _fields_ = [("user", ctypes.c_longlong), ("kernel", ctypes.c_longlong),
+ ("periodUser", ctypes.c_longlong), ("periodKernel", ctypes.c_longlong),
+ ("faults", W.DWORD), ("total", W.DWORD), ("active", W.DWORD), ("terminated", W.DWORD)]
+
+
+class ProcessScope:
+ def __init__(self, directory=None):
+ self.directory = directory or tempfile.mkdtemp(prefix="orca-computer-trial-")
+ self.procs = []
+ self.job = None
+ if WIN:
+ self.job = kernel.CreateJobObjectW(None, None)
+ limits = ExtendedLimit()
+ limits.basic.flags = 0x2000 # Closing the job kills descendants too.
+ if not self.job or not kernel.SetInformationJobObject(self.job, 9, ctypes.byref(limits), ctypes.sizeof(limits)):
+ raise ctypes.WinError(ctypes.get_last_error())
+
+ def spawn(self, args, hidden=False, **kwargs):
+ if WIN:
+ kwargs["creationflags"] = 0x4 | (0x08000000 if hidden else 0)
+ else:
+ kwargs["start_new_session"] = True
+ proc = subprocess.Popen(args, **kwargs)
+ self.procs.append(proc)
+ if WIN:
+ # Suspend until assigned so children cannot escape the trial job.
+ if not kernel.AssignProcessToJobObject(self.job, int(proc._handle)):
+ proc.kill()
+ proc.wait()
+ raise ctypes.WinError(ctypes.get_last_error())
+ if nt.NtResumeProcess(int(proc._handle)) != 0:
+ raise RuntimeError("failed to resume contained process")
+ return proc
+
+ def close(self):
+ import time
+ if WIN and self.job:
+ if not kernel.TerminateJobObject(self.job, 1):
+ raise ctypes.WinError(ctypes.get_last_error())
+ end = time.monotonic() + 10
+ while True:
+ info = Accounting()
+ if not kernel.QueryInformationJobObject(self.job, 1, ctypes.byref(info), ctypes.sizeof(info), None):
+ raise ctypes.WinError(ctypes.get_last_error())
+ if info.active == 0:
+ break
+ if time.monotonic() >= end:
+ raise TimeoutError("trial job did not become quiescent")
+ time.sleep(0.02)
+ kernel.CloseHandle(self.job)
+ self.job = None
+ elif not WIN:
+ for proc in self.procs:
+ try:
+ os.killpg(proc.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ for proc in self.procs:
+ proc.wait(timeout=10)
+ for stream in (proc.stdin, proc.stdout):
+ if stream:
+ stream.close()
+ self.procs.clear()
diff --git a/config/benchmarks/computer-use/providers.py b/config/benchmarks/computer-use/providers.py
new file mode 100644
index 00000000000..cc4544f8830
--- /dev/null
+++ b/config/benchmarks/computer-use/providers.py
@@ -0,0 +1,275 @@
+"""Direct native-provider transports, adapted from bench/computer-cua at 68a68ccf0f16."""
+import json
+import os
+import queue
+import re
+import sys
+import subprocess
+import threading
+import time
+
+WIN = sys.platform == "win32"
+
+
+def now():
+ return time.perf_counter() * 1000
+
+
+def md_rows(text, snapshot_id):
+ out = []
+ for line in (text or "").splitlines():
+ match = re.match(r'\s*-\s*\[(\d+)\]\s*(.*)', line)
+ if match:
+ body = match.group(2)
+ role = re.split(r'["\[(]', body, maxsplit=1)[0].strip()
+ out.append({"role": role, "label": body[len(role):],
+ "element_token": f"{snapshot_id}:{match.group(1)}"})
+ return out
+
+
+class Cua:
+ """cua-driver over MCP stdio, `mcp --direct` (the process owns the runtime on Windows/Linux)."""
+
+ def __init__(self, exe, home, logf, processes):
+ env = dict(os.environ)
+ env.update({
+ "CUA_DRIVER_RS_TELEMETRY_ENABLED": "0", "CUA_TELEMETRY_ENABLED": "0", "DO_NOT_TRACK": "1",
+ "CUA_DRIVER_RS_UPDATE_CHECK": "0", "CUA_DRIVER_RS_HOME": home, "CUA_HOME": home,
+ "CUA_DRIVER_TELEMETRY_HOME": home,
+ # Disposable CI desktop, as in cua's own live-smoke workflow: no interactive approvals.
+ "CUA_DRIVER_PERMISSION_MODE": "unrestricted", "CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS": "1",
+ })
+ os.makedirs(home, exist_ok=True)
+ t0 = now()
+ self.p = processes.spawn([exe, "mcp", "--direct", "--no-overlay"], stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, stderr=logf, env=env)
+ self.pid = self.p.pid
+ self.q = {}
+ self.cv = threading.Condition()
+ self.nid = 0
+ threading.Thread(target=self._reader, daemon=True).start()
+ init = self.rpc("initialize", {"protocolVersion": "2025-06-18", "capabilities": {},
+ "clientInfo": {"name": "orca-bench", "version": "0"}}, timeout=60)
+ self._send({"jsonrpc": "2.0", "method": "notifications/initialized"})
+ self.startup_ms = round(now() - t0, 1)
+ self.server_info = (init or {}).get("serverInfo")
+
+ def _reader(self):
+ for line in self.p.stdout:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ m = json.loads(line.decode("utf-8", "replace"))
+ except ValueError:
+ continue
+ if "id" in m:
+ with self.cv:
+ self.q[m["id"]] = (m, len(line))
+ self.cv.notify_all()
+
+ def _send(self, obj):
+ self.p.stdin.write((json.dumps(obj) + "\n").encode())
+ self.p.stdin.flush()
+
+ def rpc(self, method, params, timeout=30):
+ with self.cv:
+ self.nid += 1
+ i = self.nid
+ self._send({"jsonrpc": "2.0", "id": i, "method": method, "params": params})
+ end = time.time() + timeout
+ with self.cv:
+ while i not in self.q:
+ left = end - time.time()
+ if left <= 0:
+ raise TimeoutError(method)
+ self.cv.wait(left)
+ m, n = self.q.pop(i)
+ self.last_raw = n
+ if "error" in m:
+ return {"_rpc_error": m["error"]}
+ return m["result"]
+
+ def call(self, tool, args, timeout=30):
+ t0 = now()
+ r = self.rpc("tools/call", {"name": tool, "arguments": args}, timeout)
+ r["_ms"] = now() - t0
+ r["_raw"] = self.last_raw
+ return r
+
+
+
+def cua_text(r):
+ return "\n".join(c.get("text", "") for c in r.get("content") or [] if c.get("type") == "text")
+
+
+def cua_sc(r):
+ return r.get("structuredContent") or {}
+
+
+class OrcaWin:
+ """runtime.ps1 -Serve over NDJSON, as DesktopScriptRuntimeHost runs it."""
+
+ def __init__(self, root, logf, processes):
+ script = os.path.join(root, "native", "computer-use-windows", "runtime.ps1")
+ t0 = now()
+ self.p = processes.spawn(["powershell.exe", "-NoLogo", "-NoProfile", "-NonInteractive",
+ "-File", script, "-Serve"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=logf,
+ hidden=True)
+ self.pid = self.p.pid
+ self.lines = queue.Queue()
+ threading.Thread(target=self._reader, daemon=True).start()
+ ready = self.lines.get(timeout=120)
+ if '"ready"' not in ready:
+ raise RuntimeError("runtime.ps1 did not announce readiness: " + ready[:200])
+ self.startup_ms = round(now() - t0, 1)
+ self.rid = 0
+
+ def _reader(self):
+ for line in self.p.stdout:
+ self.lines.put(line.decode("utf-8", "replace").strip())
+
+ def op(self, req, timeout=30):
+ self.rid += 1
+ req = {**req, "requestId": self.rid}
+ t0 = now()
+ self.p.stdin.write((json.dumps(req) + "\n").encode("utf-8"))
+ self.p.stdin.flush()
+ end = time.time() + timeout
+ while True:
+ line = self.lines.get(timeout=max(0.01, end - time.time()))
+ try:
+ m = json.loads(line)
+ except ValueError:
+ continue
+ if m.get("requestId") == self.rid:
+ return m, now() - t0, len(line)
+
+
+
+class OrcaLinux:
+ """One subprocess per operation, matching the Linux production transport."""
+
+ def __init__(self, root, logf, processes):
+ self.script = os.path.join(root, "native", "computer-use-linux", "runtime.py")
+ self.logf, self.processes = logf, processes
+ self.path = os.path.join(processes.directory, "operation.json")
+
+ def op(self, req, timeout=30):
+ with open(self.path, "w", encoding="utf-8") as stream:
+ json.dump(req, stream)
+ started = now()
+ proc = self.processes.spawn([sys.executable, self.script, self.path],
+ stdout=-1, stderr=self.logf)
+ raw, _ = proc.communicate(timeout=timeout)
+ text = raw.decode("utf-8", "replace")
+ try:
+ result = json.loads(text)
+ except ValueError:
+ raise RuntimeError("invalid native response: " + text[:300])
+ return result, now() - started, len(raw)
+
+
+
+class CuaEngine:
+ name = "cua"
+
+ def __init__(self, c, pid, wid):
+ self.c, self.pid, self.wid = c, pid, wid
+ self.elements = []
+
+ def snapshot(self):
+ r = self.c.call("get_window_state", {"pid": self.pid, "window_id": self.wid, "include_screenshot": False, "full_output": True})
+ s = cua_sc(r)
+ if r.get("isError") or "_rpc_error" in r:
+ raise RuntimeError(cua_text(r) or str(r))
+ self.elements = s.get("elements") or md_rows(s.get("tree_markdown") or cua_text(r), s.get("snapshot_id"))
+ self.last_snapshot = r
+ return self.elements
+
+ def act(self, action, element, escalate, text=""):
+ base = {"pid": self.pid, "window_id": self.wid}
+ tok = element.get("element_token")
+ if not tok:
+ raise RuntimeError("selected cua element has no token")
+ if action == "click":
+ tool, args = "click", {**base, "element_token": tok}
+ elif action == "type":
+ tool, args = "type_text", {**base, "element_token": tok, "text": text}
+ elif action == "return":
+ tool, args = "press_key", {**base, "element_token": tok, "key": "return"}
+ elif action == "ctrl_a":
+ tool, args = "hotkey", {**base, "element_token": tok, "keys": ["ctrl", "a"]}
+ elif action == "scroll":
+ tool, args = "scroll", {**base, "element_token": tok, "direction": "down", "amount": 3}
+ elif action == "choose":
+ tool, args = "set_value", {**base, "element_token": tok, "value": "Blue"}
+ if escalate and tool in ("click", "type_text", "press_key", "hotkey", "scroll"):
+ args["delivery_mode"] = "foreground"
+ if "element_token" in args and not args["element_token"]:
+ return {"ms": 0, "reported": "no_element", "detail": "selector found no element", "error": True}
+ r = self.c.call(tool, args)
+ self.last_response = r
+ if "_rpc_error" in r:
+ return {"ms": r["_ms"], "reported": "rpc_error", "detail": json.dumps(r["_rpc_error"])[:300], "error": True}
+ s = cua_sc(r)
+ err = bool(r.get("isError"))
+ code = s.get("code") or (s["error"].get("code") if isinstance(s.get("error"), dict) else None)
+ effect = s.get("effect")
+ reported = effect or (("error:" + str(code)) if err else "ok(no effect field)")
+ if err and effect:
+ reported = f"{effect}(isError)"
+ return {"ms": r["_ms"], "reported": reported, "error": err, "route": s.get("route"),
+ "delivery": (s.get("delivery") or {}).get("mode") if isinstance(s.get("delivery"), dict) else s.get("delivery"),
+ "code": code, "detail": (s.get("summary") or cua_text(r))[:300]}
+
+
+
+class OrcaEngine:
+ name = "orca"
+
+ def __init__(self, o, pid):
+ self.o, self.app = o, f"pid:{pid}"
+ self.target = {}
+ self.elements = []
+
+ def snapshot(self):
+ r, ms, n = self.o.op({"tool": "get_app_state", "app": self.app, "noScreenshot": True})
+ if not r.get("ok"):
+ raise RuntimeError("orca get_app_state: " + str(r.get("error"))[:300])
+ snap = r["snapshot"]
+ self.elements = snap.get("elements") or []
+ # Mirror desktopActionWindowTarget: Windows targets by id, Linux by index.
+ self.target = {"windowId": snap.get("windowId")} if snap.get("windowId") is not None else {"windowIndex": snap.get("windowIndex")}
+ self.last_snapshot = snap
+ return self.elements
+
+ def _req(self, tool, element=None, escalate=False, **extra):
+ req = {"tool": tool, "app": self.app, "element": element, "windowBounds": None, **self.target,
+ "noScreenshot": True, "restoreWindow": bool(escalate)}
+ req.update(extra)
+ return req
+
+ def act(self, action, el, escalate, text=""):
+ if action == "click":
+ req = self._req("click", el, escalate)
+ elif action == "type":
+ req = self._req("type_text", el, escalate, text=text)
+ elif action == "return":
+ req = self._req("press_key", el, escalate, key="return")
+ elif action == "ctrl_a":
+ req = self._req("hotkey", el, escalate, key="ctrl+a")
+ elif action == "scroll":
+ req = self._req("scroll", el, escalate, direction="down", pages=1)
+ elif action == "choose":
+ req = self._req("set_value", el, escalate, value="Blue")
+ r, ms, _ = self.o.op(req)
+ self.last_response = r
+ return self._result(r, ms)
+
+ def _result(self, r, ms):
+ if not r.get("ok"):
+ return {"ms": ms, "reported": "error", "error": True, "detail": str(r.get("error"))[:300]}
+ a = r.get("action") or {}
+ v = (a.get("verification") or {}).get("state")
+ return {"ms": ms, "reported": v or "ok(no verification)", "error": False, "route": a.get("path"), "detail": json.dumps(a)[:300]}
diff --git a/config/benchmarks/computer-use/sentinel.py b/config/benchmarks/computer-use/sentinel.py
new file mode 100644
index 00000000000..bbf7027ab36
--- /dev/null
+++ b/config/benchmarks/computer-use/sentinel.py
@@ -0,0 +1,57 @@
+"""The "user": a frontmost Tk window that receives one 'q' every 200 ms from the bench typer.
+
+Writes cumulative counts to : q = 'q' keystrokes that reached it, other = any other
+characters (agent input that leaked into the user's window).
+"""
+import json
+import os
+import sys
+
+if os.environ.get("ORCA_COMPUTER_BENCH_ISOLATED") != "1" or os.environ.get("GITHUB_ACTIONS") != "true":
+ raise SystemExit("fixture requires isolated CI desktop")
+import tkinter as tk
+
+out, geometry = sys.argv[1], sys.argv[2]
+root = tk.Tk()
+root.title("BenchSentinel")
+root.geometry(geometry)
+text = tk.Text(root, font=("Courier", 10))
+text.pack(fill="both", expand=True)
+base = {"q": 0, "other": 0}
+ptr = {"clicks": 0, "wheels": 0}
+root.bind_all("", lambda e: ptr.__setitem__("wheels" if e.num in (4, 5) else "clicks", ptr["wheels" if e.num in (4, 5) else "clicks"] + 1))
+root.bind_all("", lambda e: ptr.__setitem__("wheels", ptr["wheels"] + 1))
+last = [None]
+
+
+def key(event):
+ if event.char == 'q':
+ base['q'] += 1
+ elif event.char:
+ base['other'] += 1
+
+
+root.bind_all('', key)
+
+
+def tick():
+ try:
+ focus = str(root.focus_get())
+ except Exception:
+ focus = "?"
+ data = json.dumps({"q": base["q"], "other": base["other"], "focus": focus, "pid": os.getpid(), **ptr})
+ if data != last[0]:
+ tmp = out + ".tmp"
+ try:
+ with open(tmp, "w") as f:
+ f.write(data)
+ os.replace(tmp, out)
+ last[0] = data
+ except OSError:
+ pass
+ root.after(20, tick)
+
+
+root.after(100, lambda: (root.lift(), root.focus_force(), text.focus_set()))
+root.after(20, tick)
+root.mainloop()
diff --git a/config/benchmarks/computer-use/test_trial_contract.py b/config/benchmarks/computer-use/test_trial_contract.py
new file mode 100644
index 00000000000..096dca0b41e
--- /dev/null
+++ b/config/benchmarks/computer-use/test_trial_contract.py
@@ -0,0 +1,111 @@
+"""Regressions for false success claims and unsafe trial reuse in the original benchmark."""
+from pathlib import Path
+import sys
+import tempfile
+import time
+import unittest
+
+from process_scope import ProcessScope
+from trial_contract import (DECOY, DiscoveryError, MARKER, TASKS, acknowledged, completed,
+ initial_state, prepared, schedule, select_unique, summarize, verdict)
+
+
+class TaskContractTests(unittest.TestCase):
+ def test_reset_requires_current_command_on_current_fixture(self):
+ state = {**initial_state("targeted_type"), "instance": "new", "pid": 19, "ack": "current"}
+ self.assertTrue(prepared(state, "targeted_type", "new", 19, "current"))
+ for key, value in (("ack", "previous"), ("instance", "old"), ("pid", 20),
+ ("logicalFocus", "field"), ("selLen", 1), ("top", 1), ("dropOpen", True)):
+ with self.subTest(key=key):
+ self.assertFalse(prepared({**state, key: value}, "targeted_type", "new", 19, "current"))
+ self.assertFalse(acknowledged(state, "new", 19, "previous"))
+
+ def test_selectors_never_substitute_another_control(self):
+ selector = (r"button", r"\bincrement\b")
+ desired = {"role": "button", "label": "Increment"}
+ self.assertEqual(select_unique([{"role": "button", "label": "Delete"}, desired], selector), desired)
+ for elements in ([{"role": "button", "label": "Delete"}],
+ [{"role": "pane", "label": "Increment"}], [desired, dict(desired)], []):
+ with self.subTest(elements=elements), self.assertRaises(DiscoveryError):
+ select_unique(elements, selector)
+
+ def test_completion_requires_exact_raw_text_and_unchanged_decoy(self):
+ final = {**initial_state("targeted_type"), "text": MARKER}
+ self.assertTrue(completed(final, "targeted_type"))
+ for text in ("q" + MARKER, MARKER + "q", "Orca qcafé Ω", ""):
+ with self.subTest(text=text):
+ self.assertFalse(completed({**final, "text": text}, "targeted_type"))
+ self.assertFalse(completed({**final, "decoyText": DECOY + MARKER}, "targeted_type"))
+
+ def test_exactly_once_actions_reject_intermediate_and_duplicate_effects(self):
+ for task, key in (("choose_and_click", "count"), ("type_submit", "returns"), ("replace_submit", "returns")):
+ final = {**initial_state(task), key: 1}
+ if task == "choose_and_click":
+ final["combo"] = "Blue"
+ else:
+ final["text"] = MARKER
+ self.assertTrue(completed(final, task))
+ for value in (0, 2):
+ with self.subTest(task=task, value=value):
+ self.assertFalse(completed({**final, key: value}, task))
+
+ def test_observed_completion_is_independent_of_api_errors(self):
+ row = {"prepared": True, "task": "targeted_type", "api_errors": ["timeout"]}
+ self.assertEqual(verdict(row, {**initial_state(row["task"]), "text": MARKER}), "completed")
+ self.assertEqual(verdict(row, initial_state(row["task"])), "failed")
+ self.assertEqual(verdict(row, None), "unobserved")
+ self.assertEqual(verdict({**row, "prepared": False}, None), "invalid_setup")
+
+ def test_invalid_setup_does_not_inflate_scored_completion(self):
+ base = {"engine": "candidate", "task": "targeted_type", "profile": "background"}
+ result = summarize([{**base, "verdict": "invalid_setup"},
+ {**base, "verdict": "failed", "discovery_error": "no field"},
+ {**base, "verdict": "unobserved"},
+ {**base, "verdict": "completed", "api_errors": ["late error"]}])[0]
+ self.assertEqual((result["planned"], result["scored"], result["invalid"], result["completed"]), (4, 3, 1, 1))
+ self.assertEqual((result["discovery_failures"], result["unobserved"], result["api_errors"]), (1, 1, 1))
+
+ def test_partial_failures_cannot_make_task_latency_look_fast(self):
+ base = {"engine": "candidate", "task": "type_submit", "profile": "background"}
+ result = summarize([{**base, "verdict": "completed", "attempt_ms": 400, "action_read_ms": [100, 150]},
+ {**base, "verdict": "failed", "attempt_ms": 2, "action_read_ms": [1]},
+ {**base, "verdict": "unobserved", "attempt_ms": 30000}])[0]
+ self.assertEqual(result["task_ms"], 400)
+ self.assertEqual(result["action_read_ms"], 125)
+ self.assertEqual((result["timed_completions"], result["timed_failures"]), (1, 2))
+ self.assertEqual(result["failed_attempt_ms"], 15001)
+
+ def test_engine_order_is_reproducible_and_rotates_each_round(self):
+ cells = schedule(["baseline", "candidate", "cua"], TASKS, 3, 417)
+ self.assertEqual(cells, schedule(["baseline", "candidate", "cua"], TASKS, 3, 417))
+ self.assertEqual(len(cells), len(TASKS) * 9)
+ for task in TASKS:
+ positions = [[cell["engine"] for cell in cells if cell["task"] == task and cell["round"] == index]
+ for index in range(3)]
+ for index in range(3):
+ self.assertEqual({order[index] for order in positions}, {"baseline", "candidate", "cua"})
+
+ def test_process_scope_stops_descendant_writes_before_returning(self):
+ with tempfile.TemporaryDirectory() as directory:
+ heartbeat = Path(directory) / "heartbeat"
+ child = "import pathlib,time; p=pathlib.Path(" + repr(str(heartbeat)) + "); "
+ child += "\nwhile True:\n p.write_text(str(time.monotonic())); time.sleep(0.02)"
+ parent = "import subprocess,sys,time; subprocess.Popen([sys.executable,'-c'," + repr(child) + "]); time.sleep(60)"
+ scope = ProcessScope(directory)
+ try:
+ proc = scope.spawn([sys.executable, "-c", parent], hidden=True)
+ end = time.monotonic() + 10
+ while not heartbeat.exists() and time.monotonic() < end:
+ time.sleep(0.01)
+ self.assertTrue(heartbeat.exists(), "descendant never started")
+ scope.close()
+ self.assertIsNotNone(proc.poll())
+ final = heartbeat.read_bytes()
+ time.sleep(0.12)
+ self.assertEqual(heartbeat.read_bytes(), final)
+ finally:
+ scope.close()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/config/benchmarks/computer-use/trial_contract.py b/config/benchmarks/computer-use/trial_contract.py
new file mode 100644
index 00000000000..d7172625ec5
--- /dev/null
+++ b/config/benchmarks/computer-use/trial_contract.py
@@ -0,0 +1,124 @@
+"""Task definitions and independently observed benchmark verdicts. No desktop imports."""
+import random
+import re
+import statistics
+
+MARKER = "Orca café Ω"
+DECOY = "untouched"
+SELECTORS = {
+ "field": (r"\b(edit|text|entry|textbox)\b", r"\bfield\b"),
+ "button": (r"\b(button|push button)\b", r"\bincrement\b"),
+ "color": (r"combo|popup|list ?box|menu ?button", r"\bcolor\b"),
+ "scroll": (r"list|scroll|document|web", r"\blist\b|\bbenchelectron\b"),
+}
+TASKS = {
+ "targeted_type": [("type", "field")],
+ "type_submit": [("click", "field"), ("type", "field"), ("return", "field")],
+ "replace_submit": [("click", "field"), ("ctrl_a", "field"), ("type", "field"), ("return", "field")],
+ "choose_and_click": [("choose", "color"), ("click", "button")],
+ "click_interference": [("click", "button")],
+ "scroll_interference": [("scroll", "scroll")],
+}
+ROLE_KEYS = ("role", "control_type", "role_description", "localized_role", "subrole", "controlType", "localizedControlType")
+LABEL_KEYS = ("label", "name", "title", "description", "automation_id", "automationId", "help", "placeholder")
+
+
+class DiscoveryError(Exception):
+ pass
+
+
+def select_unique(elements, selector):
+ role, label = selector
+ matches = [element for element in elements
+ if re.search(role, " ".join(str(element.get(key) or "") for key in ROLE_KEYS), re.I)
+ and any(re.search(label, str(element.get(key) or ""), re.I) for key in LABEL_KEYS)]
+ if len(matches) != 1:
+ raise DiscoveryError(f"selector {selector!r} matched {len(matches)} elements")
+ return matches[0]
+
+
+def initial_state(task):
+ text = "hello" if task == "replace_submit" else ""
+ return {"text": text, "decoyText": DECOY, "count": 0, "returns": 0, "combo": "Red", "top": 0,
+ "check": False, "menuHits": 0, "dropOpen": False, "selStart": len(text), "selLen": 0,
+ "logicalFocus": "decoy" if task == "targeted_type" else "field"}
+
+
+def acknowledged(state, instance, pid, nonce):
+ return (state.get("instance") == instance and state.get("pid") == pid
+ and state.get("ack") == nonce)
+
+
+def prepared(state, task, instance, pid, nonce):
+ return acknowledged(state, instance, pid, nonce) and all(
+ state.get(key) == value for key, value in initial_state(task).items())
+
+
+def completed(state, task):
+ expected = initial_state(task)
+ for key in ("selStart", "selLen", "logicalFocus"):
+ del expected[key]
+ if task in ("targeted_type", "type_submit", "replace_submit"):
+ expected["text"] = MARKER
+ if task in ("type_submit", "replace_submit"):
+ expected["returns"] = 1
+ if task in ("choose_and_click", "click_interference"):
+ expected["count"] = 1
+ if task == "choose_and_click":
+ expected["combo"] = "Blue"
+ if task == "scroll_interference":
+ del expected["top"]
+ if not isinstance(state.get("top"), (int, float)) or state["top"] <= 0:
+ return False
+ return all(state.get(key) == value for key, value in expected.items())
+
+
+def verdict(row, state):
+ if not row.get("prepared"):
+ return "invalid_setup"
+ if state is None:
+ return "unobserved"
+ return "completed" if completed(state, row["task"]) else "failed"
+
+
+def schedule(engines, tasks, rounds, seed):
+ rng = random.Random(seed)
+ order = list(engines)
+ rng.shuffle(order)
+ cells = []
+ for round_index in range(rounds):
+ task_order = list(tasks)
+ rng.shuffle(task_order)
+ for task in task_order:
+ for index in range(len(order)):
+ cells.append({"engine": order[(index + round_index) % len(order)],
+ "task": task, "round": round_index})
+ return cells
+
+
+def summarize(rows):
+ groups = {}
+ for row in rows:
+ groups.setdefault((row["engine"], row["task"], row["profile"]), []).append(row)
+ summary = []
+ for (engine, task, profile), group in sorted(groups.items()):
+ valid = [row for row in group if row["verdict"] != "invalid_setup"]
+ successful = [row for row in valid if row["verdict"] == "completed"]
+ times = [row["attempt_ms"] for row in successful if row.get("attempt_ms") is not None]
+ failures = [row["attempt_ms"] for row in valid if row["verdict"] != "completed" and row.get("attempt_ms") is not None]
+ paired = [sample for row in successful for sample in row.get("action_read_ms", [])]
+ summary.append({"engine": engine, "task": task, "profile": profile, "planned": len(group),
+ "scored": len(valid), "invalid": len(group) - len(valid),
+ "completed": sum(row["verdict"] == "completed" for row in valid),
+ "unobserved": sum(row["verdict"] == "unobserved" for row in valid),
+ "api_errors": sum(bool(row.get("api_errors")) for row in valid),
+ **{key: sum(row.get(key, 0) for row in valid) for key in
+ ("keys_sent", "keys_received", "keys_lost", "agent_text_in_sentinel", "target_q_characters")},
+ "discovery_failures": sum(bool(row.get("discovery_error")) for row in valid),
+ "task_ms": round(statistics.median(times), 1) if times else None,
+ "failed_attempt_ms": round(statistics.median(failures), 1) if failures else None,
+ "timed_completions": len(times), "timed_failures": len(failures),
+ "action_read_ms": round(statistics.median(paired), 1) if paired else None,
+ **{key: sum(bool(row.get("observation", {}).get(key)) for row in valid)
+ for key in ("sentinel_lost_focus", "target_raised", "pointer_moved", "clipboard_changed")}})
+ return summary