mirror of
https://github.com/stablyai/orca.git
synced 2026-10-31 00:57:45 +00:00
test(computer): measure independently verified tasks (#36216)
* test(computer): measure independently verified tasks * ci(computer): fit benchmark to repository guards * fix(computer): stabilize benchmark fixtures and CI
This commit is contained in:
Kelvin Amoaba
@@ -28,6 +28,7 @@ jobs:
|
||||
['ssh-hostile-hosts', 'ssh-hostile-hosts.yml'],
|
||||
['mobile', 'mobile.yml'],
|
||||
['computer-e2e', 'computer-e2e.yml'],
|
||||
['computer-task-benchmark', 'computer-task-benchmark.yml'],
|
||||
['computer-macos-native', 'computer-macos-native.yml'],
|
||||
['wsl-orcad-e2e', 'wsl-orcad-e2e.yml'],
|
||||
['performance-contracts', 'performance-contracts.yml'],
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
name: Computer-use task benchmark
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened, ready_for_review, converted_to_draft]
|
||||
paths:
|
||||
- 'config/benchmarks/computer-use/**'
|
||||
- 'native/computer-use-linux/**'
|
||||
- 'native/computer-use-windows/**'
|
||||
- '.github/workflows/computer-task-benchmark.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: computer-task-benchmark-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
ORCA_BACKGROUND_LAUNCH: '1'
|
||||
CUA_TAG: cua-driver-rs-v0.38.0
|
||||
CUA_DRIVER_RS_TELEMETRY_ENABLED: '0'
|
||||
DO_NOT_TRACK: '1'
|
||||
CUA_DRIVER_RS_UPDATE_CHECK: '0'
|
||||
ELECTRON_VERSION: '43.8.1'
|
||||
BASELINE_SHA: 030c88deecf3adc212d22f57b085e4bd5c6e48e6
|
||||
|
||||
jobs:
|
||||
contract:
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-latest, windows-latest, macos-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: python -m unittest discover -s config/benchmarks/computer-use -p 'test_*.py'
|
||||
|
||||
linux:
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 40
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
target: [fixture, electron]
|
||||
env:
|
||||
BENCH_TARGET: ${{ matrix.target }}
|
||||
ORCA_COMPUTER_BENCH_ISOLATED: '1'
|
||||
ACCESSIBILITY_ENABLED: '1'
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
- name: Fetch immutable baseline
|
||||
run: git fetch --no-tags --depth=1 origin "$BASELINE_SHA"
|
||||
- name: Install isolated desktop and accessibility libraries
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends xvfb openbox dbus-x11 at-spi2-core python3-gi \
|
||||
gir1.2-atspi-2.0 gir1.2-gtk-3.0 python3-tk xdotool xclip ripgrep libnss3 libgbm1 libasound2t64
|
||||
- name: Download and verify pinned cua-driver
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/cua-dl" "$RUNNER_TEMP/cua"
|
||||
cd "$RUNNER_TEMP/cua-dl"
|
||||
asset='cua-driver-rs-0.38.0-linux-x86_64-binary.tar.gz'
|
||||
gh release download "$CUA_TAG" --repo trycua/cua --pattern checksums.txt --pattern "$asset"
|
||||
rg " $asset\$" checksums.txt | sha256sum -c -
|
||||
tar -xzf "$asset" -C "$RUNNER_TEMP/cua"
|
||||
- name: Install Electron
|
||||
if: matrix.target == 'electron'
|
||||
run: |
|
||||
npm install --no-save --no-package-lock --prefix "$RUNNER_TEMP/electron" "electron@$ELECTRON_VERSION"
|
||||
node "$RUNNER_TEMP/electron/node_modules/electron/install.js"
|
||||
- name: Run task comparison on Xvfb
|
||||
run: |
|
||||
export CANDIDATE_SHA="$(git rev-parse HEAD)"
|
||||
xvfb-run -a --server-args='-screen 0 1280x1024x24' dbus-run-session -- bash -euc '
|
||||
openbox >"$RUNNER_TEMP/openbox.log" 2>&1 &
|
||||
/usr/bin/python3 config/benchmarks/computer-use/bench.py --target "$BENCH_TARGET" \
|
||||
--baseline "$BASELINE_SHA" --candidate "$CANDIDATE_SHA" \
|
||||
--cua "$RUNNER_TEMP/cua/cua-driver" \
|
||||
--electron "$RUNNER_TEMP/electron/node_modules/electron/dist/electron" \
|
||||
--out "out/task-linux-$BENCH_TARGET" --n 3 --budget 1800
|
||||
'
|
||||
- if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: computer-task-linux-${{ matrix.target }}
|
||||
path: out/task-linux-${{ matrix.target }}
|
||||
if-no-files-found: warn
|
||||
|
||||
windows:
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
|
||||
runs-on: windows-latest
|
||||
timeout-minutes: 40
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
target: [fixture, electron]
|
||||
env:
|
||||
BENCH_TARGET: ${{ matrix.target }}
|
||||
ORCA_COMPUTER_BENCH_ISOLATED: '1'
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- name: Fetch immutable baseline
|
||||
run: git fetch --no-tags --depth=1 origin $env:BASELINE_SHA
|
||||
- name: Download and verify pinned cua-driver
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
shell: pwsh
|
||||
run: |
|
||||
$download = Join-Path $env:RUNNER_TEMP 'cua-dl'
|
||||
$destination = Join-Path $env:RUNNER_TEMP 'cua'
|
||||
New-Item -ItemType Directory -Force $download, $destination | Out-Null
|
||||
$asset = 'cua-driver-rs-0.38.0-windows-x86_64-binary.zip'
|
||||
gh release download $env:CUA_TAG --repo trycua/cua --pattern checksums.txt --pattern $asset --dir $download
|
||||
$matches = @(Select-String -Path (Join-Path $download 'checksums.txt') -Pattern " $([regex]::Escape($asset))$")
|
||||
if ($matches.Count -ne 1) { throw 'expected exactly one checksum' }
|
||||
$expected = $matches[0].Line.Split(' ')[0]
|
||||
$actual = (Get-FileHash (Join-Path $download $asset) -Algorithm SHA256).Hash.ToLower()
|
||||
if ($expected -ne $actual) { throw 'cua-driver checksum mismatch' }
|
||||
Expand-Archive (Join-Path $download $asset) -DestinationPath $destination
|
||||
- name: Install Electron
|
||||
if: matrix.target == 'electron'
|
||||
run: |
|
||||
npm install --no-save --no-package-lock --prefix "$env:RUNNER_TEMP\electron" "electron@$env:ELECTRON_VERSION"
|
||||
node "$env:RUNNER_TEMP\electron\node_modules\electron\install.js"
|
||||
- name: Run task comparison on disposable Windows desktop
|
||||
run: |
|
||||
$candidate = git rev-parse HEAD
|
||||
python config/benchmarks/computer-use/bench.py --target $env:BENCH_TARGET `
|
||||
--baseline $env:BASELINE_SHA --candidate $candidate `
|
||||
--cua "$env:RUNNER_TEMP\cua\cua-driver.exe" `
|
||||
--electron "$env:RUNNER_TEMP\electron\node_modules\electron\dist\electron.exe" `
|
||||
--out "out/task-windows-$env:BENCH_TARGET" --n 3 --budget 1800
|
||||
- if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: computer-task-windows-${{ matrix.target }}
|
||||
path: out/task-windows-${{ matrix.target }}
|
||||
if-no-files-found: warn
|
||||
@@ -0,0 +1,78 @@
|
||||
# Computer-use task benchmark
|
||||
|
||||
Compares Orca's direct Linux AT-SPI and Windows UI Automation providers with
|
||||
cua-driver 0.38.0. This does not measure the CLI, sidecar, model, or complete agent
|
||||
workflows. It does not install cua-driver in Orca.
|
||||
|
||||
Run the `Computer-use task benchmark` GitHub Actions workflow on a branch. Native
|
||||
input runs only on its disposable Windows runner or Linux Xvfb desktop. The runner
|
||||
refuses to run on the developer's desktop. Keep `ORCA_BACKGROUND_LAUNCH=1`.
|
||||
|
||||
The first matrix covers GTK/WinForms and Electron fixtures. Editor applications,
|
||||
Wayland, macOS and additional Windows control frameworks need separate coverage.
|
||||
Do not generalize these results to every application on a platform.
|
||||
|
||||
## Measurement contract
|
||||
|
||||
- Baseline and candidate runtimes come from full commit SHAs, copied before the
|
||||
run. Artifacts include source hashes, cua executable hash/version, image,
|
||||
seed, complete trial plan, responses, snapshots and independent fixture states.
|
||||
- Each individual trial launches a fresh target, sentinel and provider. Process
|
||||
groups on Linux and jobs on Windows contain children, including timed-out calls.
|
||||
The provider is terminated and reaped before the terminal oracle read. No retry
|
||||
follows an uncertain mutation.
|
||||
- Reset carries a unique nonce bound to the fixture instance and PID. The fixture
|
||||
acknowledges after applying text, selection, logical focus, counters, combo,
|
||||
scroll and menu state. Sentinel foreground and stacking are independently checked.
|
||||
- Invalid preparation executes no provider operation, is excluded from the scored
|
||||
denominator, and makes the run incomplete with nonzero exit status. A missing or
|
||||
ambiguous control after valid setup is a scored coverage failure.
|
||||
- Strict selectors require a unique matching role and label. References are fresh
|
||||
before the task and after every step. There is no label-only or role-only fallback.
|
||||
- Completion requires exact independent final state, including unchanged decoy text
|
||||
and exactly one click/submit. Raw text is never cleaned. API errors, missing final
|
||||
observations and observed interference remain separate from task completion.
|
||||
- The observer starts before provider initialization and initial selector snapshot,
|
||||
and stays active through all steps, reads, shutdown and settling. No sentinel
|
||||
refocusing happens during measurement. Idle and deliberate-steal controls must pass.
|
||||
- Complete tasks run without simulated user typing. Separate click and scroll probes
|
||||
type `q` into the sentinel. Event counters preserve evidence even if agent input
|
||||
selects or deletes text. Observations sample roughly every 4 ms; they cannot prove
|
||||
the absence of shorter focus/pointer/clipboard changes. X11 clipboard observations
|
||||
detect ownership changes, not all changes made by the same owner.
|
||||
- Engine order rotates across rounds from a recorded seed. Use a multiple of three
|
||||
rounds for balanced positions. Default/background and explicitly requested
|
||||
foreground profiles are separate trials. Foreground can intentionally interfere.
|
||||
- Comparable per-step time includes the action and an additional full no-screenshot
|
||||
observation for every provider. Raw action time is diagnostic because Orca already
|
||||
returns a snapshot with mutations. Task time includes provider startup, discovery
|
||||
and all action/read pairs. Medians include completed tasks only; failed-attempt
|
||||
durations and sample counts are separate. Diagnostic serialization and disk I/O
|
||||
happen after timing. Final independent readback after shutdown is an upper
|
||||
bound on effect time, recorded separately.
|
||||
- Fatal errors, invalid setup, failed observers, unobserved completion, budget
|
||||
exhaustion or missing planned cells leave JSON evidence and exit nonzero. A valid
|
||||
observed task failure remains a benchmark result, not an infrastructure failure.
|
||||
|
||||
Tasks cover explicit text targeting with another field logically focused, Unicode
|
||||
entry plus submit, select-all replacement plus submit, and color choice plus one
|
||||
click. Interference probes remain separate from task latency comparisons.
|
||||
|
||||
Explicit-target typing is a native capability probe. Orca's public CLI/RPC currently
|
||||
accepts typing at the focused field and does not expose an element target for text.
|
||||
Failure of this probe identifies a capability to consider adding; it does not prove
|
||||
a violation of the current public typing contract. Click-then-type tasks exercise
|
||||
the existing sequence, although this benchmark still bypasses public transports.
|
||||
|
||||
## Headless contract checks
|
||||
|
||||
```sh
|
||||
ORCA_BACKGROUND_LAUNCH=1 python3 -m unittest discover -s config/benchmarks/computer-use -p 'test_*.py'
|
||||
```
|
||||
|
||||
These checks protect the historical false-success cases and process-tree cleanup.
|
||||
They do not import desktop observers or generate native input.
|
||||
|
||||
Transports, fixtures and observers were adapted from the historical
|
||||
`bench/computer-cua` prototype at `68a68ccf0f16c0a7f952d55adfe21850247b8ee9`.
|
||||
Historical checkouts and result artifacts remain unchanged.
|
||||
@@ -0,0 +1,378 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compare direct native providers on disposable, independently observed task fixtures."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from process_scope import ProcessScope
|
||||
from trial_contract import (DiscoveryError, MARKER, SELECTORS, TASKS, acknowledged, prepared,
|
||||
schedule, select_unique, summarize, verdict)
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
REPO = HERE.parents[2]
|
||||
WIN = sys.platform == "win32"
|
||||
|
||||
|
||||
def read_json(path):
|
||||
try:
|
||||
return json.loads(Path(path).read_text(encoding="utf-8-sig"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def write_json(path, value):
|
||||
temporary = Path(str(path) + ".tmp")
|
||||
temporary.write_text(json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def wait_for(predicate, timeout=10):
|
||||
end = time.monotonic() + timeout
|
||||
while time.monotonic() < end:
|
||||
value = predicate()
|
||||
if value:
|
||||
return value
|
||||
time.sleep(0.02)
|
||||
raise TimeoutError("independent setup/observation did not become ready")
|
||||
|
||||
|
||||
def pin_source(ref, output):
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", ref):
|
||||
raise ValueError("source must be a full commit SHA")
|
||||
hashes = {}
|
||||
for relative in ("native/computer-use-linux/runtime.py", "native/computer-use-windows/runtime.ps1"):
|
||||
data = subprocess.check_output(["git", "show", f"{ref}:{relative}"], cwd=REPO)
|
||||
destination = output / relative
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_bytes(data)
|
||||
hashes[relative] = hashlib.sha256(data).hexdigest()
|
||||
return {"sha": ref, "files": hashes}
|
||||
|
||||
|
||||
class Trial:
|
||||
def __init__(self, args, directory, log):
|
||||
self.args, self.directory, self.log = args, directory, log
|
||||
self.instance = uuid.uuid4().hex
|
||||
directory.mkdir()
|
||||
self.targets = ProcessScope(str(directory))
|
||||
self.engines = ProcessScope(str(directory))
|
||||
self.sampler = self.typer = None
|
||||
self.state_path = directory / "state.json"
|
||||
self.sentinel_path = directory / "sentinel.json"
|
||||
self.nonce = None
|
||||
|
||||
def spawn_target(self, command, **extra):
|
||||
env = {**os.environ, "BENCH_INSTANCE": self.instance, "BENCH_STATE_DIR": str(self.directory)}
|
||||
return self.targets.spawn(command, stdout=self.log, stderr=self.log, env=env, **extra)
|
||||
|
||||
def window(self, pid, title):
|
||||
return wait_for(lambda: obs.find_window(pid=pid, title=title) if WIN
|
||||
else obs.find_client(pid, title), 30)
|
||||
|
||||
def state(self):
|
||||
state = read_json(self.state_path)
|
||||
if state and state.get("instance") == self.instance and state.get("pid") == self.pid:
|
||||
return state
|
||||
return None
|
||||
|
||||
def launch(self):
|
||||
if self.args.target == "electron":
|
||||
flags = ["--force-renderer-accessibility"] + ([] if WIN else ["--no-sandbox"])
|
||||
proc = self.spawn_target([self.args.electron, *flags, str(HERE / "electron/main.js")])
|
||||
title = "BenchElectron"
|
||||
elif WIN:
|
||||
proc = self.spawn_target(["powershell.exe", "-NoLogo", "-NoProfile", "-STA", "-File",
|
||||
str(HERE / "fixture_win.ps1"), "-StateDir", str(self.directory)], hidden=True)
|
||||
title = "BenchTarget"
|
||||
else:
|
||||
proc = self.spawn_target([sys.executable, str(HERE / "fixture_gtk.py"), str(self.directory)])
|
||||
title = "BenchTarget"
|
||||
self.pid = proc.pid
|
||||
self.target = self.window(proc.pid, title)
|
||||
obs.place(self.target, 60, 60, 760, 560)
|
||||
wait_for(self.state)
|
||||
proc = self.spawn_target([sys.executable, str(HERE / "sentinel.py"),
|
||||
str(self.sentinel_path), "880x660+20+20"], hidden=True)
|
||||
self.sentinel_pid = proc.pid
|
||||
self.sentinel = self.window(proc.pid, "BenchSentinel")
|
||||
wait_for(lambda: read_json(self.sentinel_path))
|
||||
# Tk's initial focus callback must finish before calibration or measurement.
|
||||
time.sleep(0.2)
|
||||
|
||||
def front(self):
|
||||
if WIN:
|
||||
return obs.foreground() == self.sentinel and not obs.owned_above(self.sentinel, self.pid)
|
||||
display = obs.main_display()
|
||||
stacking = display.stacking()
|
||||
windows = display.windows_of(self.pid)
|
||||
return (display.active() == self.sentinel and display.focus() in display.descendants(self.sentinel)
|
||||
and self.sentinel in stacking and self.target in stacking and windows
|
||||
and all(stacking.index(window) < stacking.index(self.sentinel) for window in windows if window in stacking))
|
||||
|
||||
def prepare(self, task):
|
||||
self.nonce = uuid.uuid4().hex
|
||||
write_json(self.directory / "cmd", {"instance": self.instance, "nonce": self.nonce, "task": task})
|
||||
wait_for(lambda: prepared(self.state() or {}, task, self.instance, self.pid, self.nonce), 5)
|
||||
obs.force_foreground(self.sentinel)
|
||||
wait_for(self.front, 5)
|
||||
if WIN:
|
||||
obs.u.SetCursorPos(2, 2)
|
||||
else:
|
||||
obs.xdotool("mousemove", 2, 2)
|
||||
wait_for(lambda: prepared(self.state() or {}, task, self.instance, self.pid, self.nonce), 5)
|
||||
|
||||
def sample(self):
|
||||
self.sampler = obs.Sampler(self.sentinel, self.target, self.pid)
|
||||
self.sampler.start()
|
||||
|
||||
def stop_sampling(self):
|
||||
if self.sampler:
|
||||
sampler, self.sampler = self.sampler, None
|
||||
result = sampler.stop()
|
||||
if result["n_samples"] < 3:
|
||||
raise RuntimeError("observer did not collect enough samples")
|
||||
return result
|
||||
return {}
|
||||
|
||||
def calibrate(self):
|
||||
self.prepare("click_interference")
|
||||
self.sample()
|
||||
self.typer = obs.Typer()
|
||||
self.typer.start()
|
||||
time.sleep(0.65)
|
||||
self.typer.stop()
|
||||
sent = self.typer.sent
|
||||
self.typer = None
|
||||
if sent < 3:
|
||||
raise RuntimeError("simulated user input control failed")
|
||||
wait_for(lambda: (read_json(self.sentinel_path) or {}).get("q", 0) == sent, 2)
|
||||
idle = self.stop_sampling()
|
||||
if any(idle[key] for key in ("sentinel_lost_focus", "target_raised", "pointer_moved", "clipboard_changed")):
|
||||
raise RuntimeError("idle observer control failed")
|
||||
self.sample()
|
||||
obs.force_foreground(self.target)
|
||||
time.sleep(0.15)
|
||||
steal = self.stop_sampling()
|
||||
if not all(steal[key] for key in ("sentinel_lost_focus", "target_activated", "target_raised")):
|
||||
raise RuntimeError("intentional focus-steal observer control failed")
|
||||
return {"idle": idle, "steal": steal, "keys_sent": sent}
|
||||
|
||||
def engine(self, name, sources):
|
||||
from providers import Cua, CuaEngine, OrcaEngine, OrcaLinux, OrcaWin, cua_sc
|
||||
if name == "cua":
|
||||
client = Cua(self.args.cua, str(self.directory / "cua-home"), self.log, self.engines)
|
||||
response = client.call("list_windows", {"pid": self.pid})
|
||||
if response.get("isError") or "_rpc_error" in response:
|
||||
raise RuntimeError("cua list_windows failed: " + str(response))
|
||||
windows = cua_sc(response).get("windows") or []
|
||||
matches = [window for window in windows if window.get("window_id") is not None
|
||||
and window.get("title") in ("BenchTarget", "BenchElectron")]
|
||||
if len(matches) != 1:
|
||||
raise DiscoveryError(f"cua list_windows matched {len(matches)} fixture windows")
|
||||
return CuaEngine(client, self.pid, matches[0]["window_id"])
|
||||
return OrcaEngine((OrcaWin if WIN else OrcaLinux)(str(sources / name), self.log, self.engines), self.pid)
|
||||
|
||||
def run(self, cell, sources):
|
||||
row = {**cell, "prepared": False, "api_errors": [], "action_read_ms": [], "steps": []}
|
||||
state = None
|
||||
task_started = None
|
||||
snapshots, responses = [], []
|
||||
try:
|
||||
self.launch()
|
||||
self.prepare(cell["task"])
|
||||
row["initial_state"] = self.state()
|
||||
row["prepared"] = True
|
||||
self.sample()
|
||||
row["sentinel_before"] = read_json(self.sentinel_path)
|
||||
if cell["task"].endswith("_interference"):
|
||||
self.typer = obs.Typer()
|
||||
self.typer.start()
|
||||
task_started = time.perf_counter()
|
||||
engine = self.engine(cell["engine"], sources)
|
||||
acquire_started = time.perf_counter()
|
||||
elements = engine.snapshot()
|
||||
row["acquisition_ms"] = (time.perf_counter() - acquire_started) * 1000
|
||||
snapshots.append(engine.last_snapshot)
|
||||
for action, selector in TASKS[cell["task"]]:
|
||||
element = select_unique(elements, SELECTORS[selector])
|
||||
started = time.perf_counter()
|
||||
result = engine.act(action, element, cell["profile"] == "foreground", MARKER)
|
||||
row["steps"].append({"action": action, "result": result,
|
||||
"action_ms": (time.perf_counter() - started) * 1000})
|
||||
if result.get("error"):
|
||||
row["api_errors"].append(result)
|
||||
responses.append(engine.last_response)
|
||||
# Identical extra full no-screenshot read for all providers; reuse fresh refs next step.
|
||||
elements = engine.snapshot()
|
||||
row["action_read_ms"].append((time.perf_counter() - started) * 1000)
|
||||
snapshots.append(engine.last_snapshot)
|
||||
if result.get("error"):
|
||||
break
|
||||
except DiscoveryError as error:
|
||||
row["discovery_error"] = str(error)
|
||||
except Exception as error:
|
||||
row["api_errors"].append(repr(error))
|
||||
finally:
|
||||
if task_started is not None:
|
||||
row["attempt_ms"] = (time.perf_counter() - task_started) * 1000
|
||||
try:
|
||||
# No late operation can mutate the fixture after the terminal oracle starts.
|
||||
self.engines.close()
|
||||
if self.typer:
|
||||
self.typer.stop()
|
||||
row["keys_sent"] = self.typer.sent
|
||||
self.typer = None
|
||||
if row["prepared"]:
|
||||
if task_started is not None:
|
||||
row["elapsed_to_quiescent_ms"] = (time.perf_counter() - task_started) * 1000
|
||||
before = (self.state() or {}).get("seq", -1)
|
||||
time.sleep(0.4)
|
||||
def fresh_state():
|
||||
value = self.state()
|
||||
return value if value and value.get("seq", -1) > before and acknowledged(
|
||||
value, self.instance, self.pid, self.nonce) else None
|
||||
state = wait_for(fresh_state, 2)
|
||||
if task_started is not None:
|
||||
row["terminal_readback_ms"] = (time.perf_counter() - task_started) * 1000
|
||||
row["sentinel_after"] = read_json(self.sentinel_path)
|
||||
sentinel_before = row["sentinel_before"]
|
||||
sentinel_after = row["sentinel_after"]
|
||||
if not all(value and value.get("pid") == self.sentinel_pid
|
||||
for value in (sentinel_before, sentinel_after)):
|
||||
raise RuntimeError("missing independent sentinel counters")
|
||||
if cell["task"].endswith("_interference") and row.get("keys_sent", 0) == 0:
|
||||
row["input_observer_error"] = "no simulated user input was delivered"
|
||||
row["keys_received"] = sentinel_after.get("q", 0) - sentinel_before.get("q", 0)
|
||||
row["keys_lost"] = max(0, row.get("keys_sent", 0) - row["keys_received"])
|
||||
row["agent_text_in_sentinel"] = sentinel_after.get("other", 0) - sentinel_before.get("other", 0)
|
||||
row["target_q_characters"] = state["text"].count("q") + state["decoyText"].count("q")
|
||||
except Exception as error:
|
||||
row["terminal_observation_error"] = repr(error)
|
||||
try:
|
||||
row["observation"] = self.stop_sampling()
|
||||
except Exception as error:
|
||||
row["observer_error"] = repr(error)
|
||||
row["final_state"] = state
|
||||
row["verdict"] = verdict(row, state)
|
||||
for index, snapshot in enumerate(snapshots):
|
||||
write_json(self.directory / f"snapshot-{index}.json", snapshot)
|
||||
for index, response in enumerate(responses):
|
||||
write_json(self.directory / f"response-{index}.json", response)
|
||||
return row
|
||||
|
||||
def close(self):
|
||||
if self.typer:
|
||||
self.typer.stop()
|
||||
try:
|
||||
self.engines.close()
|
||||
finally:
|
||||
try:
|
||||
self.stop_sampling()
|
||||
finally:
|
||||
self.targets.close()
|
||||
|
||||
|
||||
def markdown(meta, summary):
|
||||
lines = [f"## {meta['os']} / {meta['target']}", "",
|
||||
f"Run complete: {meta['complete']}. Cells: {meta['executed']}/{meta['planned']}. Seed: {meta['seed']}.", "",
|
||||
"| task | profile | engine | completed/scored | invalid | unobserved | API errors | task ms | action + read ms | focus lost | raised |",
|
||||
"|---|---|---|---|---|---|---|---|---|---|---|"]
|
||||
for row in summary:
|
||||
lines.append(f"| {row['task']} | {row['profile']} | {row['engine']} | {row['completed']}/{row['scored']} | "
|
||||
f"{row['invalid']} | {row['unobserved']} | {row['api_errors']} | {row['task_ms']} | "
|
||||
f"{row['action_read_ms']} | {row['sentinel_lost_focus']} | {row['target_raised']} |")
|
||||
if meta.get("fatal"):
|
||||
lines.extend(["", f"Incomplete: `{meta['fatal']}`"])
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def main():
|
||||
if (os.environ.get("ORCA_COMPUTER_BENCH_ISOLATED") != "1" or os.environ.get("GITHUB_ACTIONS") != "true"
|
||||
or os.environ.get("ORCA_BACKGROUND_LAUNCH") != "1" or sys.platform not in ("linux", "win32")):
|
||||
raise SystemExit("Native input benchmark requires the isolated Linux/Windows GitHub Actions desktop")
|
||||
global obs
|
||||
if WIN:
|
||||
import obs_win as obs
|
||||
else:
|
||||
import obs_x11 as obs
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--target", choices=["fixture", "electron"], required=True)
|
||||
parser.add_argument("--cua", required=True)
|
||||
parser.add_argument("--electron")
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
parser.add_argument("--baseline", default="030c88deecf3adc212d22f57b085e4bd5c6e48e6")
|
||||
parser.add_argument("--candidate", required=True)
|
||||
parser.add_argument("--n", type=int, default=3)
|
||||
parser.add_argument("--seed", type=int, default=417)
|
||||
parser.add_argument("--budget", type=int, default=1800)
|
||||
args = parser.parse_args()
|
||||
if args.n < 1 or (args.target == "electron" and not args.electron):
|
||||
parser.error("positive --n and an Electron executable for Electron targets are required")
|
||||
args.out = args.out.resolve()
|
||||
args.out.mkdir(parents=True, exist_ok=False)
|
||||
sources = args.out / "sources"
|
||||
cells = []
|
||||
for cell in schedule(["baseline", "candidate", "cua"], TASKS, args.n, args.seed):
|
||||
profiles = ["background", "foreground"] if not cell["task"].endswith("_interference") else ["background"]
|
||||
cells.extend({**cell, "profile": profile} for profile in profiles)
|
||||
meta = {"os": platform.platform(), "target": args.target, "seed": args.seed, "n": args.n,
|
||||
"python": sys.version, "image": os.environ.get("ImageVersion"), "planned": len(cells),
|
||||
"complete": False, "executed": 0, "sources": {}}
|
||||
rows = []
|
||||
started = time.monotonic()
|
||||
try:
|
||||
meta["sources"] = {name: pin_source(sha, sources / name)
|
||||
for name, sha in (("baseline", args.baseline), ("candidate", args.candidate))}
|
||||
meta["cua_sha256"] = hashlib.sha256(Path(args.cua).read_bytes()).hexdigest()
|
||||
meta["cua_version"] = subprocess.check_output([args.cua, "--version"], text=True, timeout=15).strip()
|
||||
if "0.38.0" not in meta["cua_version"]:
|
||||
raise RuntimeError("expected pinned cua-driver 0.38.0")
|
||||
with (args.out / "engines.log").open("ab") as log:
|
||||
control = Trial(args, args.out / "controls", log)
|
||||
try:
|
||||
control.launch()
|
||||
meta["calibration"] = control.calibrate()
|
||||
finally:
|
||||
control.close()
|
||||
for index, cell in enumerate(cells):
|
||||
if time.monotonic() - started >= args.budget:
|
||||
raise TimeoutError("budget exhausted; planned cells remain")
|
||||
trial = Trial(args, args.out / f"trial-{index:03d}", log)
|
||||
try:
|
||||
row = trial.run(cell, sources)
|
||||
rows.append(row)
|
||||
finally:
|
||||
trial.close()
|
||||
meta["executed"] = len(rows)
|
||||
write_json(args.out / "results.json", {"meta": meta, "rows": rows})
|
||||
print(f"{index + 1}/{len(cells)} {cell}: {row['verdict']}", flush=True)
|
||||
meta["complete"] = (len(rows) == len(cells) and all(row["prepared"] and not row.get("observer_error")
|
||||
and not row.get("terminal_observation_error") and not row.get("input_observer_error") and row["verdict"] != "unobserved" for row in rows))
|
||||
except Exception as error:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
meta["fatal"] = repr(error)
|
||||
finally:
|
||||
meta["executed"] = len(rows)
|
||||
meta["elapsed_seconds"] = time.monotonic() - started
|
||||
summary = summarize(rows)
|
||||
write_json(args.out / "results.json", {"meta": meta, "summary": summary, "rows": rows, "plan": cells})
|
||||
report = markdown(meta, summary)
|
||||
(args.out / "summary.md").write_text(report, encoding="utf-8")
|
||||
print(report)
|
||||
if os.environ.get("GITHUB_STEP_SUMMARY"):
|
||||
with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as stream:
|
||||
stream.write(report)
|
||||
return 0 if meta["complete"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,81 @@
|
||||
// Disposable native-provider fixture, launched only on an isolated desktop.
|
||||
const { app, BrowserWindow, ipcMain } = require('electron')
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const dir = process.env.BENCH_STATE_DIR
|
||||
const instance = process.env.BENCH_INSTANCE
|
||||
process.on('uncaughtException', (error) => {
|
||||
console.error(error)
|
||||
app.exit(1)
|
||||
})
|
||||
if (process.env.ORCA_COMPUTER_BENCH_ISOLATED !== '1') {
|
||||
process.exit(1)
|
||||
}
|
||||
app.whenReady().then(() => {
|
||||
const w = new BrowserWindow({
|
||||
x: 60,
|
||||
y: 60,
|
||||
width: 760,
|
||||
height: 560,
|
||||
title: 'BenchElectron',
|
||||
webPreferences: { nodeIntegration: true, contextIsolation: false, backgroundThrottling: false }
|
||||
})
|
||||
ipcMain.on('st', (_event, state) => {
|
||||
const data = JSON.stringify({ ...state, instance, activeForm: w.isFocused(), pid: process.pid })
|
||||
try {
|
||||
fs.writeFileSync(path.join(dir, 'state.json.tmp'), data)
|
||||
fs.renameSync(path.join(dir, 'state.json.tmp'), path.join(dir, 'state.json'))
|
||||
} catch (error) {
|
||||
// Windows readers can briefly deny replacement; the next state tick retries.
|
||||
if (!['EACCES', 'EPERM', 'EBUSY'].includes(error.code)) {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
})
|
||||
w.loadURL(
|
||||
`data:text/html,${encodeURIComponent(`<html><head><title>BenchElectron</title></head>
|
||||
<body style="font:14px sans-serif;height:4000px">
|
||||
<input id=field aria-label="field"><input id=decoy aria-label="decoy">
|
||||
<button id=button>Increment</button>
|
||||
<select id=color aria-label="Color"><option>Red</option><option>Green</option><option>Blue</option></select>
|
||||
<script>
|
||||
const { ipcRenderer } = require('electron');
|
||||
const field = document.getElementById('field'), decoy = document.getElementById('decoy');
|
||||
const color = document.getElementById('color');
|
||||
let count = 0, returns = 0, ack = null, seq = 0;
|
||||
document.getElementById('button').onclick = () => count++;
|
||||
field.addEventListener('keydown', event => { if (event.key === 'Enter') returns++ });
|
||||
function report() {
|
||||
ipcRenderer.send('st', { count, returns, ack, seq: ++seq, text: field.value, decoyText: decoy.value,
|
||||
top: window.scrollY, combo: color.value, selStart: field.selectionStart,
|
||||
selLen: field.selectionEnd - field.selectionStart, check: false, menuHits: 0, dropOpen: false,
|
||||
logicalFocus: document.activeElement === field ? 'field' : document.activeElement === decoy ? 'decoy' : 'other' });
|
||||
}
|
||||
ipcRenderer.on('cmd', (_event, cmd) => {
|
||||
count = returns = 0;
|
||||
field.value = cmd.task === 'replace_submit' ? 'hello' : '';
|
||||
field.setSelectionRange(field.value.length, field.value.length);
|
||||
decoy.value = 'untouched'; decoy.setSelectionRange(9, 9);
|
||||
color.selectedIndex = 0; color.blur(); window.scrollTo(0, 0);
|
||||
(cmd.task === 'targeted_type' ? decoy : field).focus();
|
||||
ack = cmd.nonce;
|
||||
report();
|
||||
});
|
||||
setInterval(report, 30);
|
||||
</script></body></html>`)}`
|
||||
)
|
||||
setInterval(() => {
|
||||
const cmdPath = path.join(dir, 'cmd')
|
||||
if (!fs.existsSync(cmdPath)) {
|
||||
return
|
||||
}
|
||||
const raw = fs.readFileSync(cmdPath, 'utf8').trim()
|
||||
if (raw) {
|
||||
const cmd = JSON.parse(raw)
|
||||
fs.unlinkSync(cmdPath)
|
||||
if (cmd.instance === instance) {
|
||||
w.webContents.send('cmd', cmd)
|
||||
}
|
||||
}
|
||||
}, 30)
|
||||
})
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""GTK3 target for the cua-vs-Orca background bench. Writes state.json every 30 ms; obeys cmd."""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
if os.environ.get("ORCA_COMPUTER_BENCH_ISOLATED") != "1" or os.environ.get("GITHUB_ACTIONS") != "true":
|
||||
raise SystemExit("fixture requires isolated CI desktop")
|
||||
|
||||
import gi
|
||||
|
||||
gi.require_version("Gtk", "3.0")
|
||||
from gi.repository import Gdk, GLib, Gtk # noqa: E402
|
||||
|
||||
state_dir = sys.argv[1]
|
||||
STATE = os.path.join(state_dir, "state.json")
|
||||
CMD = os.path.join(state_dir, "cmd")
|
||||
instance = os.environ["BENCH_INSTANCE"]
|
||||
ack = None
|
||||
sequence = 0
|
||||
s = {"count": 0, "returns": 0, "menuHits": 0, "last": None}
|
||||
|
||||
|
||||
def named(widget, name):
|
||||
widget.get_accessible().set_name(name)
|
||||
return widget
|
||||
|
||||
|
||||
Gtk.Settings.get_default().set_property("gtk-entry-select-on-focus", False)
|
||||
win = Gtk.Window(title="BenchTarget")
|
||||
win.set_default_size(760, 560)
|
||||
win.move(60, 60)
|
||||
win.connect("destroy", Gtk.main_quit)
|
||||
accel = Gtk.AccelGroup()
|
||||
win.add_accel_group(accel)
|
||||
box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=6)
|
||||
win.add(box)
|
||||
|
||||
menubar = Gtk.MenuBar()
|
||||
edit = Gtk.MenuItem(label="Edit")
|
||||
sub = Gtk.Menu()
|
||||
sub.set_accel_group(accel)
|
||||
bump = Gtk.MenuItem(label="Bump")
|
||||
bump.add_accelerator("activate", accel, ord("b"), Gdk.ModifierType.CONTROL_MASK, Gtk.AccelFlags.VISIBLE)
|
||||
bump.connect("activate", lambda _w: s.__setitem__("menuHits", s["menuHits"] + 1))
|
||||
sub.append(bump)
|
||||
edit.set_submenu(sub)
|
||||
menubar.append(edit)
|
||||
box.pack_start(menubar, False, False, 0)
|
||||
|
||||
button = Gtk.Button(label="Increment")
|
||||
button.connect("clicked", lambda _b: s.__setitem__("count", s["count"] + 1))
|
||||
check = Gtk.CheckButton(label="Check me")
|
||||
field = named(Gtk.Entry(), "field")
|
||||
field.connect("activate", lambda _e: s.__setitem__("returns", s["returns"] + 1))
|
||||
decoy = named(Gtk.Entry(), "decoy")
|
||||
combo = named(Gtk.ComboBoxText(), "Color")
|
||||
for item in ("Red", "Green", "Blue"):
|
||||
combo.append_text(item)
|
||||
combo.set_active(0)
|
||||
scroll = named(Gtk.ScrolledWindow(), "list")
|
||||
scroll.set_size_request(300, 250)
|
||||
rows = Gtk.ListBox()
|
||||
for i in range(1, 301):
|
||||
rows.add(Gtk.Label(label=f"Row {i}"))
|
||||
scroll.add(rows)
|
||||
for widget in (button, check, field, decoy, combo):
|
||||
box.pack_start(widget, False, False, 0)
|
||||
box.pack_start(scroll, True, True, 0)
|
||||
|
||||
|
||||
def command(c):
|
||||
global ack
|
||||
if c["instance"] != instance:
|
||||
return
|
||||
s["count"] = s["returns"] = s["menuHits"] = 0
|
||||
check.set_active(False)
|
||||
sub.popdown()
|
||||
combo.set_active(0)
|
||||
scroll.get_vadjustment().set_value(0)
|
||||
field.set_text("hello" if c["task"] == "replace_submit" else "")
|
||||
field.set_position(len(field.get_text()))
|
||||
field.select_region(len(field.get_text()), len(field.get_text()))
|
||||
decoy.set_text("untouched")
|
||||
decoy.set_position(9)
|
||||
decoy.select_region(9, 9)
|
||||
win.set_focus(decoy if c["task"] == "targeted_type" else field)
|
||||
ack = c["nonce"]
|
||||
|
||||
|
||||
def tick():
|
||||
global sequence
|
||||
sequence += 1
|
||||
try:
|
||||
with open(CMD) as f:
|
||||
c = f.read().strip()
|
||||
if c:
|
||||
open(CMD, "w").close()
|
||||
command(json.loads(c))
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
bounds = field.get_selection_bounds()
|
||||
sel = list(bounds) if bounds else [field.get_position(), field.get_position()]
|
||||
state = {
|
||||
"instance": instance, "ack": ack, "seq": sequence, "decoyText": decoy.get_text(),
|
||||
"logicalFocus": "field" if win.get_focus() is field else "decoy" if win.get_focus() is decoy else "other",
|
||||
"count": s["count"], "check": check.get_active(), "text": field.get_text(), "returns": s["returns"],
|
||||
"selStart": sel[0], "selLen": sel[1] - sel[0], "combo": combo.get_active_text(),
|
||||
"top": scroll.get_vadjustment().get_value(), "menuHits": s["menuHits"], "dropOpen": sub.get_visible(),
|
||||
"activeForm": win.is_active(), "fieldFocused": win.get_focus() is field, "pid": os.getpid(),
|
||||
}
|
||||
data = json.dumps(state)
|
||||
if data != s["last"]:
|
||||
tmp = STATE + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
f.write(data)
|
||||
os.replace(tmp, STATE)
|
||||
s["last"] = data
|
||||
return True
|
||||
|
||||
|
||||
win.show_all()
|
||||
win.set_focus(field)
|
||||
GLib.timeout_add(30, tick)
|
||||
Gtk.main()
|
||||
@@ -0,0 +1,124 @@
|
||||
# WinForms target for the cua-vs-Orca background bench. Writes its state to state.json every 30 ms
|
||||
# and acknowledges each JSON command after applying it.
|
||||
param([string]$StateDir)
|
||||
$ErrorActionPreference = "Stop"
|
||||
if ($env:ORCA_COMPUTER_BENCH_ISOLATED -ne "1" -or $env:GITHUB_ACTIONS -ne "true") { throw "fixture requires isolated CI desktop" }
|
||||
Add-Type -AssemblyName System.Windows.Forms
|
||||
Add-Type -AssemblyName System.Drawing
|
||||
[System.Windows.Forms.Application]::EnableVisualStyles()
|
||||
|
||||
$statePath = Join-Path $StateDir "state.json"
|
||||
$cmdPath = Join-Path $StateDir "cmd"
|
||||
$instance = $env:BENCH_INSTANCE
|
||||
$script:ack = $null
|
||||
$script:sequence = 0
|
||||
$script:count = 0
|
||||
$script:returns = 0
|
||||
$script:menuHits = 0
|
||||
$script:last = ""
|
||||
|
||||
$form = New-Object System.Windows.Forms.Form
|
||||
$form.Text = "BenchTarget"
|
||||
$form.StartPosition = "Manual"
|
||||
$form.Location = New-Object System.Drawing.Point(60, 60)
|
||||
$form.Size = New-Object System.Drawing.Size(760, 560)
|
||||
|
||||
$menu = New-Object System.Windows.Forms.MenuStrip
|
||||
$edit = New-Object System.Windows.Forms.ToolStripMenuItem("Edit")
|
||||
$bump = New-Object System.Windows.Forms.ToolStripMenuItem("Bump")
|
||||
$bump.ShortcutKeys = [System.Windows.Forms.Keys]::Control -bor [System.Windows.Forms.Keys]::B
|
||||
$bump.Add_Click({ $script:menuHits++ })
|
||||
[void]$edit.DropDownItems.Add($bump)
|
||||
[void]$menu.Items.Add($edit)
|
||||
$form.MainMenuStrip = $menu
|
||||
$form.Controls.Add($menu)
|
||||
|
||||
$button = New-Object System.Windows.Forms.Button
|
||||
$button.Text = "Increment"
|
||||
$button.Location = New-Object System.Drawing.Point(20, 40)
|
||||
$button.Add_Click({ $script:count++ })
|
||||
$form.Controls.Add($button)
|
||||
|
||||
$check = New-Object System.Windows.Forms.CheckBox
|
||||
$check.Text = "Check me"
|
||||
$check.Location = New-Object System.Drawing.Point(20, 80)
|
||||
$form.Controls.Add($check)
|
||||
|
||||
$field = New-Object System.Windows.Forms.TextBox
|
||||
$field.Name = "field"
|
||||
$field.AccessibleName = "field"
|
||||
$field.Location = New-Object System.Drawing.Point(20, 120)
|
||||
$field.Width = 300
|
||||
$field.Add_KeyDown({
|
||||
param($sender, $e)
|
||||
if ($e.KeyCode -eq [System.Windows.Forms.Keys]::Enter) { $script:returns++; $e.SuppressKeyPress = $true }
|
||||
})
|
||||
$form.Controls.Add($field)
|
||||
|
||||
$decoy = New-Object System.Windows.Forms.TextBox
|
||||
$decoy.Name = "decoy"
|
||||
$decoy.AccessibleName = "decoy"
|
||||
$decoy.Location = New-Object System.Drawing.Point(350, 120)
|
||||
$decoy.Width = 300
|
||||
$form.Controls.Add($decoy)
|
||||
|
||||
$combo = New-Object System.Windows.Forms.ComboBox
|
||||
$combo.DropDownStyle = "DropDownList"
|
||||
$combo.AccessibleName = "Color"
|
||||
[void]$combo.Items.AddRange(@("Red", "Green", "Blue"))
|
||||
$combo.SelectedIndex = 0
|
||||
$combo.Location = New-Object System.Drawing.Point(20, 160)
|
||||
$form.Controls.Add($combo)
|
||||
|
||||
$list = New-Object System.Windows.Forms.ListBox
|
||||
$list.AccessibleName = "list"
|
||||
$list.Location = New-Object System.Drawing.Point(20, 200)
|
||||
$list.Size = New-Object System.Drawing.Size(300, 250)
|
||||
foreach ($i in 1..300) { [void]$list.Items.Add("Row $i") }
|
||||
$form.Controls.Add($list)
|
||||
|
||||
function Invoke-BenchCommand($c) {
|
||||
if ($c.instance -ne $instance) { return }
|
||||
$script:count = 0; $script:returns = 0; $script:menuHits = 0
|
||||
$check.Checked = $false
|
||||
$edit.HideDropDown()
|
||||
$combo.SelectedIndex = 0
|
||||
$combo.DroppedDown = $false
|
||||
$list.TopIndex = 0
|
||||
$field.Text = if ($c.task -eq 'replace_submit') { 'hello' } else { '' }
|
||||
$field.SelectionStart = $field.Text.Length
|
||||
$field.SelectionLength = 0
|
||||
$decoy.Text = 'untouched'
|
||||
$decoy.SelectionStart = $decoy.Text.Length
|
||||
$decoy.SelectionLength = 0
|
||||
$form.ActiveControl = if ($c.task -eq 'targeted_type') { $decoy } else { $field }
|
||||
$script:ack = $c.nonce
|
||||
}
|
||||
|
||||
$timer = New-Object System.Windows.Forms.Timer
|
||||
$timer.Interval = 30
|
||||
$timer.Add_Tick({
|
||||
$script:sequence++
|
||||
try {
|
||||
if (Test-Path $cmdPath) {
|
||||
$c = ([IO.File]::ReadAllText($cmdPath)).Trim()
|
||||
if ($c) { [IO.File]::WriteAllText($cmdPath, ""); Invoke-BenchCommand ($c | ConvertFrom-Json) }
|
||||
}
|
||||
} catch {}
|
||||
$state = [ordered]@{
|
||||
instance = $instance; ack = $script:ack; seq = $script:sequence
|
||||
decoyText = $decoy.Text
|
||||
logicalFocus = $(if ($form.ActiveControl -eq $field) { 'field' } elseif ($form.ActiveControl -eq $decoy) { 'decoy' } else { 'other' })
|
||||
count = $script:count; check = $check.Checked; text = $field.Text; returns = $script:returns
|
||||
selStart = $field.SelectionStart; selLen = $field.SelectionLength; combo = [string]$combo.SelectedItem
|
||||
top = $list.TopIndex; menuHits = $script:menuHits; dropOpen = ($edit.DropDown.Visible -or $combo.DroppedDown)
|
||||
activeForm = ($null -ne [System.Windows.Forms.Form]::ActiveForm); fieldFocused = $field.Focused
|
||||
pid = $PID
|
||||
}
|
||||
$json = $state | ConvertTo-Json -Compress
|
||||
if ($json -ne $script:last) {
|
||||
try { [IO.File]::WriteAllText($statePath, $json); $script:last = $json } catch {}
|
||||
}
|
||||
})
|
||||
$form.Add_Shown({ $form.ActiveControl = $field; $timer.Start() })
|
||||
[System.Windows.Forms.Application]::Run($form)
|
||||
@@ -0,0 +1,309 @@
|
||||
"""Windows host observation (foreground, z-order, cursor, clipboard) and the user's typer, via user32."""
|
||||
import ctypes
|
||||
import ctypes.wintypes as W
|
||||
import threading
|
||||
import time
|
||||
|
||||
try:
|
||||
ctypes.windll.shcore.SetProcessDpiAwareness(2)
|
||||
except Exception:
|
||||
ctypes.windll.user32.SetProcessDPIAware()
|
||||
|
||||
u = ctypes.WinDLL("user32", use_last_error=True)
|
||||
k32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
|
||||
GW_HWNDPREV = 3
|
||||
WM_GETTEXT, WM_GETTEXTLENGTH, WM_SETTEXT = 0x000D, 0x000E, 0x000C
|
||||
EM_GETSEL, EM_SETSEL, EM_GETFIRSTVISIBLELINE, EM_LINESCROLL = 0x00B0, 0x00B1, 0x00CE, 0x00B6
|
||||
WM_VSCROLL, SB_TOP = 0x0115, 6
|
||||
SMTO_ABORTIFHUNG = 0x2
|
||||
SWP_NOACTIVATE, SWP_NOZORDER = 0x0010, 0x0004
|
||||
SPI_GETFOREGROUNDLOCKTIMEOUT = 0x2000
|
||||
|
||||
ULONG_PTR = ctypes.c_size_t
|
||||
|
||||
|
||||
class KEYBDINPUT(ctypes.Structure):
|
||||
_fields_ = [("wVk", W.WORD), ("wScan", W.WORD), ("dwFlags", W.DWORD), ("time", W.DWORD), ("dwExtraInfo", ULONG_PTR)]
|
||||
|
||||
|
||||
class MOUSEINPUT(ctypes.Structure):
|
||||
_fields_ = [("dx", W.LONG), ("dy", W.LONG), ("mouseData", W.DWORD), ("dwFlags", W.DWORD), ("time", W.DWORD), ("dwExtraInfo", ULONG_PTR)]
|
||||
|
||||
|
||||
class _U(ctypes.Union):
|
||||
_fields_ = [("ki", KEYBDINPUT), ("mi", MOUSEINPUT)]
|
||||
|
||||
|
||||
class INPUT(ctypes.Structure):
|
||||
_fields_ = [("type", W.DWORD), ("u", _U)]
|
||||
|
||||
|
||||
WNDENUMPROC = ctypes.WINFUNCTYPE(W.BOOL, W.HWND, W.LPARAM)
|
||||
u.EnumWindows.argtypes = [WNDENUMPROC, W.LPARAM]
|
||||
u.GetWindowThreadProcessId.argtypes = [W.HWND, ctypes.POINTER(W.DWORD)]
|
||||
u.GetWindowThreadProcessId.restype = W.DWORD
|
||||
u.GetForegroundWindow.restype = W.HWND
|
||||
u.GetWindow.restype = W.HWND
|
||||
u.GetWindow.argtypes = [W.HWND, W.UINT]
|
||||
u.FindWindowExW.restype = W.HWND
|
||||
u.FindWindowExW.argtypes = [W.HWND, W.HWND, W.LPCWSTR, W.LPCWSTR]
|
||||
u.SendMessageTimeoutW.argtypes = [W.HWND, W.UINT, W.WPARAM, W.LPARAM, W.UINT, W.UINT, ctypes.POINTER(ULONG_PTR)]
|
||||
u.SendMessageTimeoutW.restype = W.LPARAM
|
||||
u.SetWindowPos.argtypes = [W.HWND, W.HWND, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, W.UINT]
|
||||
u.SendInput.argtypes = [W.UINT, ctypes.POINTER(INPUT), ctypes.c_int]
|
||||
u.AttachThreadInput.argtypes = [W.DWORD, W.DWORD, W.BOOL]
|
||||
u.SetForegroundWindow.argtypes = [W.HWND]
|
||||
u.BringWindowToTop.argtypes = [W.HWND]
|
||||
u.IsWindowVisible.argtypes = [W.HWND]
|
||||
u.GetClassNameW.argtypes = [W.HWND, W.LPWSTR, ctypes.c_int]
|
||||
u.GetWindowTextW.argtypes = [W.HWND, W.LPWSTR, ctypes.c_int]
|
||||
u.GetWindowRect.argtypes = [W.HWND, ctypes.POINTER(W.RECT)]
|
||||
|
||||
|
||||
def hval(h):
|
||||
return int(h or 0)
|
||||
|
||||
|
||||
def pid_of(hwnd):
|
||||
p = W.DWORD(0)
|
||||
u.GetWindowThreadProcessId(hwnd, ctypes.byref(p))
|
||||
return p.value
|
||||
|
||||
|
||||
def title_of(hwnd):
|
||||
b = ctypes.create_unicode_buffer(512)
|
||||
u.GetWindowTextW(hwnd, b, 512)
|
||||
return b.value
|
||||
|
||||
|
||||
def class_of(hwnd):
|
||||
b = ctypes.create_unicode_buffer(256)
|
||||
u.GetClassNameW(hwnd, b, 256)
|
||||
return b.value
|
||||
|
||||
|
||||
def top_windows():
|
||||
out = []
|
||||
|
||||
def cb(h, _l):
|
||||
out.append(hval(h))
|
||||
return True
|
||||
|
||||
u.EnumWindows(WNDENUMPROC(cb), 0)
|
||||
return out
|
||||
|
||||
|
||||
def find_window(pid=None, title=None, cls=None, visible=True):
|
||||
for h in top_windows():
|
||||
if visible and not u.IsWindowVisible(h):
|
||||
continue
|
||||
if pid is not None and pid_of(h) != pid:
|
||||
continue
|
||||
if title is not None and title not in title_of(h):
|
||||
continue
|
||||
if cls is not None and class_of(h) != cls:
|
||||
continue
|
||||
return h
|
||||
return 0
|
||||
|
||||
|
||||
def describe(h):
|
||||
r = W.RECT()
|
||||
u.GetWindowRect(h, ctypes.byref(r))
|
||||
return {"hwnd": h, "pid": pid_of(h), "title": title_of(h), "class": class_of(h), "rect": [r.left, r.top, r.right, r.bottom]}
|
||||
|
||||
|
||||
def foreground():
|
||||
return hval(u.GetForegroundWindow())
|
||||
|
||||
|
||||
def cursor():
|
||||
p = W.POINT()
|
||||
u.GetCursorPos(ctypes.byref(p))
|
||||
return (p.x, p.y)
|
||||
|
||||
|
||||
def clip_seq():
|
||||
return u.GetClipboardSequenceNumber()
|
||||
|
||||
|
||||
def lock_timeout():
|
||||
v = W.DWORD(0)
|
||||
u.SystemParametersInfoW(SPI_GETFOREGROUNDLOCKTIMEOUT, 0, ctypes.byref(v), 0)
|
||||
return v.value
|
||||
|
||||
|
||||
def session_id():
|
||||
s = W.DWORD(0)
|
||||
k32.ProcessIdToSessionId(k32.GetCurrentProcessId(), ctypes.byref(s))
|
||||
return s.value
|
||||
|
||||
|
||||
def owned_above(sentinel, target_pid, limit=2000):
|
||||
"""True when a visible top-level window of target_pid sits above the sentinel in z-order."""
|
||||
h = u.GetWindow(sentinel, GW_HWNDPREV)
|
||||
n = 0
|
||||
while h and n < limit:
|
||||
if u.IsWindowVisible(h) and pid_of(h) == target_pid:
|
||||
return True
|
||||
h = u.GetWindow(h, GW_HWNDPREV)
|
||||
n += 1
|
||||
return False
|
||||
|
||||
|
||||
def place(hwnd, x, y, w, h):
|
||||
u.SetWindowPos(hwnd, 0, x, y, w, h, SWP_NOACTIVATE | SWP_NOZORDER)
|
||||
|
||||
|
||||
def force_foreground(hwnd):
|
||||
"""Bench setup only: re-front the sentinel between trials (AttachThreadInput, no keystrokes)."""
|
||||
if foreground() == hwnd:
|
||||
return True
|
||||
cur = k32.GetCurrentThreadId()
|
||||
fg = foreground()
|
||||
threads = {u.GetWindowThreadProcessId(fg, None) if fg else 0, u.GetWindowThreadProcessId(hwnd, None)} - {0, cur}
|
||||
attached = [t for t in threads if u.AttachThreadInput(cur, t, True)]
|
||||
u.BringWindowToTop(hwnd)
|
||||
u.SetForegroundWindow(hwnd)
|
||||
for t in attached:
|
||||
u.AttachThreadInput(cur, t, False)
|
||||
for _ in range(25):
|
||||
if foreground() == hwnd:
|
||||
return True
|
||||
time.sleep(0.02)
|
||||
return foreground() == hwnd
|
||||
|
||||
|
||||
def send(hwnd, msg, wp=0, lp=0, timeout=1000):
|
||||
res = ULONG_PTR(0)
|
||||
ok = u.SendMessageTimeoutW(hwnd, msg, wp, lp, SMTO_ABORTIFHUNG, timeout, ctypes.byref(res))
|
||||
return res.value if ok else None
|
||||
|
||||
|
||||
def edit_child(top):
|
||||
"""Notepad's editor: classic Edit, or RichEditD2DPT in the WinUI Notepad."""
|
||||
for cls in ("Edit", "RichEditD2DPT"):
|
||||
h = hval(u.FindWindowExW(top, None, cls, None))
|
||||
if h:
|
||||
return h
|
||||
stack = [top]
|
||||
while stack:
|
||||
p = stack.pop()
|
||||
c = hval(u.FindWindowExW(p, None, None, None))
|
||||
while c:
|
||||
if class_of(c) in ("Edit", "RichEditD2DPT"):
|
||||
return c
|
||||
stack.append(c)
|
||||
c = hval(u.FindWindowExW(p, c, None, None))
|
||||
return 0
|
||||
|
||||
|
||||
def edit_text(h):
|
||||
n = send(h, WM_GETTEXTLENGTH) or 0
|
||||
buf = ctypes.create_unicode_buffer(n + 2)
|
||||
send(h, WM_GETTEXT, n + 1, ctypes.addressof(buf))
|
||||
return buf.value
|
||||
|
||||
|
||||
def edit_set(h, text, caret=None):
|
||||
buf = ctypes.create_unicode_buffer(text)
|
||||
send(h, WM_SETTEXT, 0, ctypes.addressof(buf))
|
||||
c = len(text) if caret is None else caret
|
||||
send(h, EM_SETSEL, c, c)
|
||||
send(h, WM_VSCROLL, SB_TOP, 0)
|
||||
|
||||
|
||||
def edit_sel(h):
|
||||
v = send(h, EM_GETSEL) or 0
|
||||
return (v & 0xFFFF, (v >> 16) & 0xFFFF)
|
||||
|
||||
|
||||
def edit_first_line(h):
|
||||
return send(h, EM_GETFIRSTVISIBLELINE) or 0
|
||||
|
||||
|
||||
class Typer:
|
||||
"""Simulated user: one 'q' every 200 ms through SendInput (goes to whatever has keyboard focus)."""
|
||||
|
||||
def __init__(self, period=0.2):
|
||||
self.period = period
|
||||
self.sent = 0
|
||||
self.run = False
|
||||
self.vk = 0x51
|
||||
self.scan = u.MapVirtualKeyW(self.vk, 0)
|
||||
|
||||
def _key(self, up):
|
||||
i = INPUT(type=1)
|
||||
i.u.ki = KEYBDINPUT(self.vk, self.scan, 0x0002 if up else 0, 0, 0)
|
||||
return i
|
||||
|
||||
def _loop(self):
|
||||
while self.run:
|
||||
arr = (INPUT * 2)(self._key(False), self._key(True))
|
||||
if u.SendInput(2, arr, ctypes.sizeof(INPUT)) == 2:
|
||||
self.sent += 1
|
||||
time.sleep(self.period)
|
||||
|
||||
def start(self):
|
||||
self.run = True
|
||||
self.thread = threading.Thread(target=self._loop, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def stop(self):
|
||||
self.run = False
|
||||
self.thread.join()
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
class Sampler:
|
||||
"""Samples every ~4 ms: foreground window, target-above-sentinel, cursor, clipboard sequence."""
|
||||
|
||||
def __init__(self, sentinel, target, target_pid, own_pids=()):
|
||||
self.s, self.t, self.tpid = sentinel, target, target_pid
|
||||
self.own = set(own_pids)
|
||||
|
||||
def _sample(self):
|
||||
fg = foreground()
|
||||
return (time.perf_counter(), fg, pid_of(fg) if fg else 0, owned_above(self.s, self.tpid), cursor(), clip_seq())
|
||||
|
||||
def start(self):
|
||||
self.base = self._sample()
|
||||
self.samples = [self.base]
|
||||
self.error = None
|
||||
self.run = True
|
||||
self.th = threading.Thread(target=self._loop, daemon=True)
|
||||
self.th.start()
|
||||
|
||||
def _loop(self):
|
||||
try:
|
||||
while self.run:
|
||||
self.samples.append(self._sample())
|
||||
time.sleep(0.004)
|
||||
except Exception as error:
|
||||
self.error = error
|
||||
|
||||
def stop(self):
|
||||
self.run = False
|
||||
self.th.join()
|
||||
if self.error:
|
||||
raise RuntimeError("observer sampling failed") from self.error
|
||||
self.samples.append(self._sample())
|
||||
b = self.base
|
||||
ss = self.samples
|
||||
lost = [x for x in ss if x[1] != self.s]
|
||||
return {
|
||||
"sentinel_lost_focus": bool(lost),
|
||||
"lost_ms": round(_dur(ss, lambda x: x[1] != self.s) * 1000, 1),
|
||||
"target_activated": any(x[2] == self.tpid for x in ss),
|
||||
"target_raised": any(x[3] for x in ss),
|
||||
"pointer_moved": any(abs(x[4][0] - b[4][0]) > 1 or abs(x[4][1] - b[4][1]) > 1 for x in ss),
|
||||
"pointer_end": list(ss[-1][4]), "pointer_start": list(b[4]),
|
||||
"clipboard_changed": any(x[5] != b[5] for x in ss),
|
||||
"fg_pids": sorted({x[2] for x in ss}),
|
||||
"n_samples": len(ss),
|
||||
}
|
||||
|
||||
|
||||
def _dur(ss, pred):
|
||||
return sum(b[0] - a[0] for a, b in zip(ss, ss[1:]) if pred(a))
|
||||
@@ -0,0 +1,263 @@
|
||||
"""X11 host observation (active window, input focus, stacking, pointer, selections) and the user's typer."""
|
||||
import ctypes
|
||||
import ctypes.util
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
X = ctypes.cdll.LoadLibrary(ctypes.util.find_library("X11") or "libX11.so.6")
|
||||
XT = ctypes.cdll.LoadLibrary(ctypes.util.find_library("Xtst") or "libXtst.so.6")
|
||||
Window = ctypes.c_ulong
|
||||
Atom = ctypes.c_ulong
|
||||
P = ctypes.POINTER
|
||||
vp = ctypes.c_void_p
|
||||
|
||||
X.XOpenDisplay.restype = vp
|
||||
X.XOpenDisplay.argtypes = [ctypes.c_char_p]
|
||||
X.XInitThreads.restype = ctypes.c_int
|
||||
X.XDefaultRootWindow.restype = Window
|
||||
X.XDefaultRootWindow.argtypes = [vp]
|
||||
X.XInternAtom.restype = Atom
|
||||
X.XInternAtom.argtypes = [vp, ctypes.c_char_p, ctypes.c_int]
|
||||
X.XGetWindowProperty.argtypes = [vp, Window, Atom, ctypes.c_long, ctypes.c_long, ctypes.c_int, Atom,
|
||||
P(Atom), P(ctypes.c_int), P(ctypes.c_ulong), P(ctypes.c_ulong), P(vp)]
|
||||
X.XFree.argtypes = [vp]
|
||||
X.XGetInputFocus.argtypes = [vp, P(Window), P(ctypes.c_int)]
|
||||
X.XQueryPointer.argtypes = [vp, Window, P(Window), P(Window), P(ctypes.c_int), P(ctypes.c_int),
|
||||
P(ctypes.c_int), P(ctypes.c_int), P(ctypes.c_uint)]
|
||||
X.XGetSelectionOwner.restype = Window
|
||||
X.XGetSelectionOwner.argtypes = [vp, Atom]
|
||||
X.XQueryTree.argtypes = [vp, Window, P(Window), P(Window), P(P(Window)), P(ctypes.c_uint)]
|
||||
X.XStringToKeysym.restype = ctypes.c_ulong
|
||||
X.XStringToKeysym.argtypes = [ctypes.c_char_p]
|
||||
X.XKeysymToKeycode.restype = ctypes.c_ubyte
|
||||
X.XKeysymToKeycode.argtypes = [vp, ctypes.c_ulong]
|
||||
X.XFlush.argtypes = [vp]
|
||||
X.XCloseDisplay.argtypes = [vp]
|
||||
XT.XTestFakeKeyEvent.argtypes = [vp, ctypes.c_uint, ctypes.c_int, ctypes.c_ulong]
|
||||
|
||||
X.XInitThreads()
|
||||
ERRH = ctypes.CFUNCTYPE(ctypes.c_int, vp, vp)(lambda _d, _e: 0)
|
||||
X.XSetErrorHandler(ERRH)
|
||||
|
||||
|
||||
class Display:
|
||||
def __init__(self):
|
||||
self.d = X.XOpenDisplay(None)
|
||||
if not self.d:
|
||||
raise RuntimeError("cannot open X display")
|
||||
self.root = X.XDefaultRootWindow(self.d)
|
||||
self.atoms = {}
|
||||
|
||||
def atom(self, name):
|
||||
if name not in self.atoms:
|
||||
self.atoms[name] = X.XInternAtom(self.d, name.encode(), 0)
|
||||
return self.atoms[name]
|
||||
|
||||
def prop_longs(self, win, name):
|
||||
at, fmt, n, after, data = Atom(), ctypes.c_int(), ctypes.c_ulong(), ctypes.c_ulong(), vp()
|
||||
rc = X.XGetWindowProperty(self.d, win, self.atom(name), 0, 4096, 0, 0, ctypes.byref(at), ctypes.byref(fmt),
|
||||
ctypes.byref(n), ctypes.byref(after), ctypes.byref(data))
|
||||
if rc != 0 or not data.value:
|
||||
return []
|
||||
try:
|
||||
if fmt.value != 32:
|
||||
return []
|
||||
arr = ctypes.cast(data, P(ctypes.c_ulong))
|
||||
return [arr[i] for i in range(n.value)]
|
||||
finally:
|
||||
X.XFree(data)
|
||||
|
||||
def active(self):
|
||||
v = self.prop_longs(self.root, "_NET_ACTIVE_WINDOW")
|
||||
return v[0] if v else 0
|
||||
|
||||
def stacking(self):
|
||||
return self.prop_longs(self.root, "_NET_CLIENT_LIST_STACKING")
|
||||
|
||||
def clients(self):
|
||||
return self.prop_longs(self.root, "_NET_CLIENT_LIST")
|
||||
|
||||
def pid_of(self, w):
|
||||
v = self.prop_longs(w, "_NET_WM_PID")
|
||||
return v[0] if v else 0
|
||||
|
||||
def focus(self):
|
||||
w, r = Window(), ctypes.c_int()
|
||||
X.XGetInputFocus(self.d, ctypes.byref(w), ctypes.byref(r))
|
||||
return w.value
|
||||
|
||||
def pointer(self):
|
||||
r, c = Window(), Window()
|
||||
rx, ry, wx, wy = ctypes.c_int(), ctypes.c_int(), ctypes.c_int(), ctypes.c_int()
|
||||
m = ctypes.c_uint()
|
||||
X.XQueryPointer(self.d, self.root, ctypes.byref(r), ctypes.byref(c), ctypes.byref(rx), ctypes.byref(ry),
|
||||
ctypes.byref(wx), ctypes.byref(wy), ctypes.byref(m))
|
||||
return (rx.value, ry.value)
|
||||
|
||||
def owner(self, sel):
|
||||
return X.XGetSelectionOwner(self.d, self.atom(sel))
|
||||
|
||||
def descendants(self, w):
|
||||
out, stack = {w}, [w]
|
||||
while stack:
|
||||
p = stack.pop()
|
||||
r, par, kids, n = Window(), Window(), P(Window)(), ctypes.c_uint()
|
||||
if not X.XQueryTree(self.d, p, ctypes.byref(r), ctypes.byref(par), ctypes.byref(kids), ctypes.byref(n)):
|
||||
continue
|
||||
for i in range(n.value):
|
||||
out.add(kids[i])
|
||||
stack.append(kids[i])
|
||||
if kids:
|
||||
X.XFree(kids)
|
||||
return out
|
||||
|
||||
def windows_of(self, pid):
|
||||
return [w for w in self.clients() if self.pid_of(w) == pid]
|
||||
|
||||
def close(self):
|
||||
if self.d:
|
||||
X.XCloseDisplay(self.d)
|
||||
self.d = None
|
||||
|
||||
|
||||
_MAIN = []
|
||||
|
||||
|
||||
def main_display():
|
||||
"""One connection for the bench's main thread; Xvfb allows only ~256 clients."""
|
||||
if not _MAIN:
|
||||
_MAIN.append(Display())
|
||||
return _MAIN[0]
|
||||
|
||||
|
||||
def find_client(pid, title=None):
|
||||
"""Managed client window of pid (by _NET_WM_PID), else by exact title (Tk may omit _NET_WM_PID)."""
|
||||
d = main_display()
|
||||
ws = d.windows_of(pid)
|
||||
if ws:
|
||||
return ws[0]
|
||||
if title:
|
||||
clients = set(d.clients())
|
||||
out = xdotool("search", "--name", f"^{title}$").stdout.split()
|
||||
for w in out:
|
||||
if int(w) in clients:
|
||||
return int(w)
|
||||
return 0
|
||||
|
||||
|
||||
def xdotool(*args):
|
||||
return subprocess.run(["xdotool", *map(str, args)], capture_output=True, text=True, timeout=10)
|
||||
|
||||
|
||||
def window_name(w):
|
||||
return xdotool("getwindowname", w).stdout.strip()
|
||||
|
||||
|
||||
def place(w, x, y, width, height):
|
||||
xdotool("windowsize", w, width, height)
|
||||
xdotool("windowmove", w, x, y)
|
||||
|
||||
|
||||
def force_foreground(w):
|
||||
"""Bench setup only: re-front the sentinel between trials."""
|
||||
d = main_display()
|
||||
if d.active() == w:
|
||||
return True
|
||||
xdotool("windowactivate", "--sync", w)
|
||||
xdotool("windowraise", w)
|
||||
for _ in range(25):
|
||||
if d.active() == w:
|
||||
return True
|
||||
time.sleep(0.02)
|
||||
return d.active() == w
|
||||
|
||||
|
||||
class Typer:
|
||||
"""Simulated user: one 'q' every 200 ms through XTest (goes to the X input focus)."""
|
||||
|
||||
def __init__(self, period=0.2):
|
||||
self.period = period
|
||||
self.sent = 0
|
||||
self.run = False
|
||||
|
||||
def _loop(self):
|
||||
d = Display()
|
||||
kc = X.XKeysymToKeycode(d.d, X.XStringToKeysym(b"q"))
|
||||
while self.run:
|
||||
XT.XTestFakeKeyEvent(d.d, kc, 1, 0)
|
||||
XT.XTestFakeKeyEvent(d.d, kc, 0, 0)
|
||||
X.XFlush(d.d)
|
||||
self.sent += 1
|
||||
time.sleep(self.period)
|
||||
d.close()
|
||||
|
||||
def start(self):
|
||||
self.run = True
|
||||
self.thread = threading.Thread(target=self._loop, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def stop(self):
|
||||
self.run = False
|
||||
self.thread.join()
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
class Sampler:
|
||||
"""Samples every ~4 ms: active window, input focus, stacking, pointer, CLIPBOARD/PRIMARY owners."""
|
||||
|
||||
def __init__(self, sentinel, target, target_pid, own_pids=()):
|
||||
self.s, self.t, self.tpid = sentinel, target, target_pid
|
||||
self.s_set = main_display().descendants(sentinel)
|
||||
|
||||
def _sample(self, d):
|
||||
st = d.stacking()
|
||||
act = d.active()
|
||||
foc = d.focus()
|
||||
tw = set(d.windows_of(self.tpid)) or {self.t}
|
||||
raised = False
|
||||
if self.s in st:
|
||||
si = st.index(self.s)
|
||||
raised = any(st.index(w) > si for w in tw if w in st)
|
||||
return (time.perf_counter(), act, foc, raised, d.pointer(), d.owner("CLIPBOARD"), d.owner("PRIMARY"), act in tw)
|
||||
|
||||
def start(self):
|
||||
self.d = main_display()
|
||||
self.base = self._sample(self.d)
|
||||
self.samples = [self.base]
|
||||
self.error = None
|
||||
self.run = True
|
||||
self.th = threading.Thread(target=self._loop, daemon=True)
|
||||
self.th.start()
|
||||
|
||||
def _loop(self):
|
||||
d = Display()
|
||||
try:
|
||||
while self.run:
|
||||
self.samples.append(self._sample(d))
|
||||
time.sleep(0.004)
|
||||
except Exception as error:
|
||||
self.error = error
|
||||
finally:
|
||||
d.close()
|
||||
|
||||
def stop(self):
|
||||
self.run = False
|
||||
self.th.join()
|
||||
if self.error:
|
||||
raise RuntimeError("observer sampling failed") from self.error
|
||||
self.samples.append(self._sample(self.d))
|
||||
b, ss = self.base, self.samples
|
||||
lost = lambda x: x[1] != self.s or (x[2] not in self.s_set)
|
||||
return {
|
||||
"sentinel_lost_focus": any(lost(x) for x in ss),
|
||||
"lost_ms": round(sum(q[0] - p[0] for p, q in zip(ss, ss[1:]) if lost(p)) * 1000, 1),
|
||||
"target_activated": any(x[7] for x in ss),
|
||||
"target_raised": any(x[3] for x in ss),
|
||||
"pointer_moved": any(abs(x[4][0] - b[4][0]) > 1 or abs(x[4][1] - b[4][1]) > 1 for x in ss),
|
||||
"pointer_end": list(ss[-1][4]), "pointer_start": list(b[4]),
|
||||
"clipboard_changed": any(x[5] != b[5] for x in ss),
|
||||
"primary_changed": any(x[6] != b[6] for x in ss),
|
||||
"focus_windows": sorted({hex(x[2]) for x in ss})[:6],
|
||||
"n_samples": len(ss),
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Contain each trial's process tree, including operations that time out."""
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
WIN = sys.platform == "win32"
|
||||
|
||||
if WIN:
|
||||
import ctypes
|
||||
from ctypes import wintypes as W
|
||||
|
||||
class BasicLimit(ctypes.Structure):
|
||||
_fields_ = [("processTime", ctypes.c_longlong), ("jobTime", ctypes.c_longlong),
|
||||
("flags", W.DWORD), ("minWorkingSet", ctypes.c_size_t),
|
||||
("maxWorkingSet", ctypes.c_size_t), ("activeProcesses", W.DWORD),
|
||||
("affinity", ctypes.c_size_t), ("priority", W.DWORD), ("scheduling", W.DWORD)]
|
||||
|
||||
class IoCounters(ctypes.Structure):
|
||||
_fields_ = [(name, ctypes.c_ulonglong) for name in
|
||||
("readOps", "writeOps", "otherOps", "readBytes", "writeBytes", "otherBytes")]
|
||||
|
||||
class ExtendedLimit(ctypes.Structure):
|
||||
_fields_ = [("basic", BasicLimit), ("io", IoCounters), ("processMemory", ctypes.c_size_t),
|
||||
("jobMemory", ctypes.c_size_t), ("peakProcess", ctypes.c_size_t), ("peakJob", ctypes.c_size_t)]
|
||||
|
||||
kernel = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
kernel.CreateJobObjectW.argtypes = [ctypes.c_void_p, W.LPCWSTR]
|
||||
kernel.CreateJobObjectW.restype = W.HANDLE
|
||||
kernel.SetInformationJobObject.argtypes = [W.HANDLE, ctypes.c_int, ctypes.c_void_p, W.DWORD]
|
||||
kernel.AssignProcessToJobObject.argtypes = [W.HANDLE, W.HANDLE]
|
||||
kernel.TerminateJobObject.argtypes = [W.HANDLE, W.UINT]
|
||||
kernel.CloseHandle.argtypes = [W.HANDLE]
|
||||
kernel.QueryInformationJobObject.argtypes = [W.HANDLE, ctypes.c_int, ctypes.c_void_p, W.DWORD, ctypes.c_void_p]
|
||||
nt = ctypes.WinDLL("ntdll")
|
||||
nt.NtResumeProcess.argtypes = [W.HANDLE]
|
||||
nt.NtResumeProcess.restype = ctypes.c_long
|
||||
|
||||
class Accounting(ctypes.Structure):
|
||||
_fields_ = [("user", ctypes.c_longlong), ("kernel", ctypes.c_longlong),
|
||||
("periodUser", ctypes.c_longlong), ("periodKernel", ctypes.c_longlong),
|
||||
("faults", W.DWORD), ("total", W.DWORD), ("active", W.DWORD), ("terminated", W.DWORD)]
|
||||
|
||||
|
||||
class ProcessScope:
|
||||
def __init__(self, directory=None):
|
||||
self.directory = directory or tempfile.mkdtemp(prefix="orca-computer-trial-")
|
||||
self.procs = []
|
||||
self.job = None
|
||||
if WIN:
|
||||
self.job = kernel.CreateJobObjectW(None, None)
|
||||
limits = ExtendedLimit()
|
||||
limits.basic.flags = 0x2000 # Closing the job kills descendants too.
|
||||
if not self.job or not kernel.SetInformationJobObject(self.job, 9, ctypes.byref(limits), ctypes.sizeof(limits)):
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
|
||||
def spawn(self, args, hidden=False, **kwargs):
|
||||
if WIN:
|
||||
kwargs["creationflags"] = 0x4 | (0x08000000 if hidden else 0)
|
||||
else:
|
||||
kwargs["start_new_session"] = True
|
||||
proc = subprocess.Popen(args, **kwargs)
|
||||
self.procs.append(proc)
|
||||
if WIN:
|
||||
# Suspend until assigned so children cannot escape the trial job.
|
||||
if not kernel.AssignProcessToJobObject(self.job, int(proc._handle)):
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
if nt.NtResumeProcess(int(proc._handle)) != 0:
|
||||
raise RuntimeError("failed to resume contained process")
|
||||
return proc
|
||||
|
||||
def close(self):
|
||||
import time
|
||||
if WIN and self.job:
|
||||
if not kernel.TerminateJobObject(self.job, 1):
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
end = time.monotonic() + 10
|
||||
while True:
|
||||
info = Accounting()
|
||||
if not kernel.QueryInformationJobObject(self.job, 1, ctypes.byref(info), ctypes.sizeof(info), None):
|
||||
raise ctypes.WinError(ctypes.get_last_error())
|
||||
if info.active == 0:
|
||||
break
|
||||
if time.monotonic() >= end:
|
||||
raise TimeoutError("trial job did not become quiescent")
|
||||
time.sleep(0.02)
|
||||
kernel.CloseHandle(self.job)
|
||||
self.job = None
|
||||
elif not WIN:
|
||||
for proc in self.procs:
|
||||
try:
|
||||
os.killpg(proc.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
for proc in self.procs:
|
||||
proc.wait(timeout=10)
|
||||
for stream in (proc.stdin, proc.stdout):
|
||||
if stream:
|
||||
stream.close()
|
||||
self.procs.clear()
|
||||
@@ -0,0 +1,275 @@
|
||||
"""Direct native-provider transports, adapted from bench/computer-cua at 68a68ccf0f16."""
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
import sys
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
WIN = sys.platform == "win32"
|
||||
|
||||
|
||||
def now():
|
||||
return time.perf_counter() * 1000
|
||||
|
||||
|
||||
def md_rows(text, snapshot_id):
|
||||
out = []
|
||||
for line in (text or "").splitlines():
|
||||
match = re.match(r'\s*-\s*\[(\d+)\]\s*(.*)', line)
|
||||
if match:
|
||||
body = match.group(2)
|
||||
role = re.split(r'["\[(]', body, maxsplit=1)[0].strip()
|
||||
out.append({"role": role, "label": body[len(role):],
|
||||
"element_token": f"{snapshot_id}:{match.group(1)}"})
|
||||
return out
|
||||
|
||||
|
||||
class Cua:
|
||||
"""cua-driver over MCP stdio, `mcp --direct` (the process owns the runtime on Windows/Linux)."""
|
||||
|
||||
def __init__(self, exe, home, logf, processes):
|
||||
env = dict(os.environ)
|
||||
env.update({
|
||||
"CUA_DRIVER_RS_TELEMETRY_ENABLED": "0", "CUA_TELEMETRY_ENABLED": "0", "DO_NOT_TRACK": "1",
|
||||
"CUA_DRIVER_RS_UPDATE_CHECK": "0", "CUA_DRIVER_RS_HOME": home, "CUA_HOME": home,
|
||||
"CUA_DRIVER_TELEMETRY_HOME": home,
|
||||
# Disposable CI desktop, as in cua's own live-smoke workflow: no interactive approvals.
|
||||
"CUA_DRIVER_PERMISSION_MODE": "unrestricted", "CUA_DRIVER_DANGEROUSLY_BYPASS_APPROVALS": "1",
|
||||
})
|
||||
os.makedirs(home, exist_ok=True)
|
||||
t0 = now()
|
||||
self.p = processes.spawn([exe, "mcp", "--direct", "--no-overlay"], stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE, stderr=logf, env=env)
|
||||
self.pid = self.p.pid
|
||||
self.q = {}
|
||||
self.cv = threading.Condition()
|
||||
self.nid = 0
|
||||
threading.Thread(target=self._reader, daemon=True).start()
|
||||
init = self.rpc("initialize", {"protocolVersion": "2025-06-18", "capabilities": {},
|
||||
"clientInfo": {"name": "orca-bench", "version": "0"}}, timeout=60)
|
||||
self._send({"jsonrpc": "2.0", "method": "notifications/initialized"})
|
||||
self.startup_ms = round(now() - t0, 1)
|
||||
self.server_info = (init or {}).get("serverInfo")
|
||||
|
||||
def _reader(self):
|
||||
for line in self.p.stdout:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
m = json.loads(line.decode("utf-8", "replace"))
|
||||
except ValueError:
|
||||
continue
|
||||
if "id" in m:
|
||||
with self.cv:
|
||||
self.q[m["id"]] = (m, len(line))
|
||||
self.cv.notify_all()
|
||||
|
||||
def _send(self, obj):
|
||||
self.p.stdin.write((json.dumps(obj) + "\n").encode())
|
||||
self.p.stdin.flush()
|
||||
|
||||
def rpc(self, method, params, timeout=30):
|
||||
with self.cv:
|
||||
self.nid += 1
|
||||
i = self.nid
|
||||
self._send({"jsonrpc": "2.0", "id": i, "method": method, "params": params})
|
||||
end = time.time() + timeout
|
||||
with self.cv:
|
||||
while i not in self.q:
|
||||
left = end - time.time()
|
||||
if left <= 0:
|
||||
raise TimeoutError(method)
|
||||
self.cv.wait(left)
|
||||
m, n = self.q.pop(i)
|
||||
self.last_raw = n
|
||||
if "error" in m:
|
||||
return {"_rpc_error": m["error"]}
|
||||
return m["result"]
|
||||
|
||||
def call(self, tool, args, timeout=30):
|
||||
t0 = now()
|
||||
r = self.rpc("tools/call", {"name": tool, "arguments": args}, timeout)
|
||||
r["_ms"] = now() - t0
|
||||
r["_raw"] = self.last_raw
|
||||
return r
|
||||
|
||||
|
||||
|
||||
def cua_text(r):
|
||||
return "\n".join(c.get("text", "") for c in r.get("content") or [] if c.get("type") == "text")
|
||||
|
||||
|
||||
def cua_sc(r):
|
||||
return r.get("structuredContent") or {}
|
||||
|
||||
|
||||
class OrcaWin:
|
||||
"""runtime.ps1 -Serve over NDJSON, as DesktopScriptRuntimeHost runs it."""
|
||||
|
||||
def __init__(self, root, logf, processes):
|
||||
script = os.path.join(root, "native", "computer-use-windows", "runtime.ps1")
|
||||
t0 = now()
|
||||
self.p = processes.spawn(["powershell.exe", "-NoLogo", "-NoProfile", "-NonInteractive",
|
||||
"-File", script, "-Serve"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=logf,
|
||||
hidden=True)
|
||||
self.pid = self.p.pid
|
||||
self.lines = queue.Queue()
|
||||
threading.Thread(target=self._reader, daemon=True).start()
|
||||
ready = self.lines.get(timeout=120)
|
||||
if '"ready"' not in ready:
|
||||
raise RuntimeError("runtime.ps1 did not announce readiness: " + ready[:200])
|
||||
self.startup_ms = round(now() - t0, 1)
|
||||
self.rid = 0
|
||||
|
||||
def _reader(self):
|
||||
for line in self.p.stdout:
|
||||
self.lines.put(line.decode("utf-8", "replace").strip())
|
||||
|
||||
def op(self, req, timeout=30):
|
||||
self.rid += 1
|
||||
req = {**req, "requestId": self.rid}
|
||||
t0 = now()
|
||||
self.p.stdin.write((json.dumps(req) + "\n").encode("utf-8"))
|
||||
self.p.stdin.flush()
|
||||
end = time.time() + timeout
|
||||
while True:
|
||||
line = self.lines.get(timeout=max(0.01, end - time.time()))
|
||||
try:
|
||||
m = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if m.get("requestId") == self.rid:
|
||||
return m, now() - t0, len(line)
|
||||
|
||||
|
||||
|
||||
class OrcaLinux:
|
||||
"""One subprocess per operation, matching the Linux production transport."""
|
||||
|
||||
def __init__(self, root, logf, processes):
|
||||
self.script = os.path.join(root, "native", "computer-use-linux", "runtime.py")
|
||||
self.logf, self.processes = logf, processes
|
||||
self.path = os.path.join(processes.directory, "operation.json")
|
||||
|
||||
def op(self, req, timeout=30):
|
||||
with open(self.path, "w", encoding="utf-8") as stream:
|
||||
json.dump(req, stream)
|
||||
started = now()
|
||||
proc = self.processes.spawn([sys.executable, self.script, self.path],
|
||||
stdout=-1, stderr=self.logf)
|
||||
raw, _ = proc.communicate(timeout=timeout)
|
||||
text = raw.decode("utf-8", "replace")
|
||||
try:
|
||||
result = json.loads(text)
|
||||
except ValueError:
|
||||
raise RuntimeError("invalid native response: " + text[:300])
|
||||
return result, now() - started, len(raw)
|
||||
|
||||
|
||||
|
||||
class CuaEngine:
|
||||
name = "cua"
|
||||
|
||||
def __init__(self, c, pid, wid):
|
||||
self.c, self.pid, self.wid = c, pid, wid
|
||||
self.elements = []
|
||||
|
||||
def snapshot(self):
|
||||
r = self.c.call("get_window_state", {"pid": self.pid, "window_id": self.wid, "include_screenshot": False, "full_output": True})
|
||||
s = cua_sc(r)
|
||||
if r.get("isError") or "_rpc_error" in r:
|
||||
raise RuntimeError(cua_text(r) or str(r))
|
||||
self.elements = s.get("elements") or md_rows(s.get("tree_markdown") or cua_text(r), s.get("snapshot_id"))
|
||||
self.last_snapshot = r
|
||||
return self.elements
|
||||
|
||||
def act(self, action, element, escalate, text=""):
|
||||
base = {"pid": self.pid, "window_id": self.wid}
|
||||
tok = element.get("element_token")
|
||||
if not tok:
|
||||
raise RuntimeError("selected cua element has no token")
|
||||
if action == "click":
|
||||
tool, args = "click", {**base, "element_token": tok}
|
||||
elif action == "type":
|
||||
tool, args = "type_text", {**base, "element_token": tok, "text": text}
|
||||
elif action == "return":
|
||||
tool, args = "press_key", {**base, "element_token": tok, "key": "return"}
|
||||
elif action == "ctrl_a":
|
||||
tool, args = "hotkey", {**base, "element_token": tok, "keys": ["ctrl", "a"]}
|
||||
elif action == "scroll":
|
||||
tool, args = "scroll", {**base, "element_token": tok, "direction": "down", "amount": 3}
|
||||
elif action == "choose":
|
||||
tool, args = "set_value", {**base, "element_token": tok, "value": "Blue"}
|
||||
if escalate and tool in ("click", "type_text", "press_key", "hotkey", "scroll"):
|
||||
args["delivery_mode"] = "foreground"
|
||||
if "element_token" in args and not args["element_token"]:
|
||||
return {"ms": 0, "reported": "no_element", "detail": "selector found no element", "error": True}
|
||||
r = self.c.call(tool, args)
|
||||
self.last_response = r
|
||||
if "_rpc_error" in r:
|
||||
return {"ms": r["_ms"], "reported": "rpc_error", "detail": json.dumps(r["_rpc_error"])[:300], "error": True}
|
||||
s = cua_sc(r)
|
||||
err = bool(r.get("isError"))
|
||||
code = s.get("code") or (s["error"].get("code") if isinstance(s.get("error"), dict) else None)
|
||||
effect = s.get("effect")
|
||||
reported = effect or (("error:" + str(code)) if err else "ok(no effect field)")
|
||||
if err and effect:
|
||||
reported = f"{effect}(isError)"
|
||||
return {"ms": r["_ms"], "reported": reported, "error": err, "route": s.get("route"),
|
||||
"delivery": (s.get("delivery") or {}).get("mode") if isinstance(s.get("delivery"), dict) else s.get("delivery"),
|
||||
"code": code, "detail": (s.get("summary") or cua_text(r))[:300]}
|
||||
|
||||
|
||||
|
||||
class OrcaEngine:
|
||||
name = "orca"
|
||||
|
||||
def __init__(self, o, pid):
|
||||
self.o, self.app = o, f"pid:{pid}"
|
||||
self.target = {}
|
||||
self.elements = []
|
||||
|
||||
def snapshot(self):
|
||||
r, ms, n = self.o.op({"tool": "get_app_state", "app": self.app, "noScreenshot": True})
|
||||
if not r.get("ok"):
|
||||
raise RuntimeError("orca get_app_state: " + str(r.get("error"))[:300])
|
||||
snap = r["snapshot"]
|
||||
self.elements = snap.get("elements") or []
|
||||
# Mirror desktopActionWindowTarget: Windows targets by id, Linux by index.
|
||||
self.target = {"windowId": snap.get("windowId")} if snap.get("windowId") is not None else {"windowIndex": snap.get("windowIndex")}
|
||||
self.last_snapshot = snap
|
||||
return self.elements
|
||||
|
||||
def _req(self, tool, element=None, escalate=False, **extra):
|
||||
req = {"tool": tool, "app": self.app, "element": element, "windowBounds": None, **self.target,
|
||||
"noScreenshot": True, "restoreWindow": bool(escalate)}
|
||||
req.update(extra)
|
||||
return req
|
||||
|
||||
def act(self, action, el, escalate, text=""):
|
||||
if action == "click":
|
||||
req = self._req("click", el, escalate)
|
||||
elif action == "type":
|
||||
req = self._req("type_text", el, escalate, text=text)
|
||||
elif action == "return":
|
||||
req = self._req("press_key", el, escalate, key="return")
|
||||
elif action == "ctrl_a":
|
||||
req = self._req("hotkey", el, escalate, key="ctrl+a")
|
||||
elif action == "scroll":
|
||||
req = self._req("scroll", el, escalate, direction="down", pages=1)
|
||||
elif action == "choose":
|
||||
req = self._req("set_value", el, escalate, value="Blue")
|
||||
r, ms, _ = self.o.op(req)
|
||||
self.last_response = r
|
||||
return self._result(r, ms)
|
||||
|
||||
def _result(self, r, ms):
|
||||
if not r.get("ok"):
|
||||
return {"ms": ms, "reported": "error", "error": True, "detail": str(r.get("error"))[:300]}
|
||||
a = r.get("action") or {}
|
||||
v = (a.get("verification") or {}).get("state")
|
||||
return {"ms": ms, "reported": v or "ok(no verification)", "error": False, "route": a.get("path"), "detail": json.dumps(a)[:300]}
|
||||
@@ -0,0 +1,57 @@
|
||||
"""The "user": a frontmost Tk window that receives one 'q' every 200 ms from the bench typer.
|
||||
|
||||
Writes cumulative counts to <out>: q = 'q' keystrokes that reached it, other = any other
|
||||
characters (agent input that leaked into the user's window).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
if os.environ.get("ORCA_COMPUTER_BENCH_ISOLATED") != "1" or os.environ.get("GITHUB_ACTIONS") != "true":
|
||||
raise SystemExit("fixture requires isolated CI desktop")
|
||||
import tkinter as tk
|
||||
|
||||
out, geometry = sys.argv[1], sys.argv[2]
|
||||
root = tk.Tk()
|
||||
root.title("BenchSentinel")
|
||||
root.geometry(geometry)
|
||||
text = tk.Text(root, font=("Courier", 10))
|
||||
text.pack(fill="both", expand=True)
|
||||
base = {"q": 0, "other": 0}
|
||||
ptr = {"clicks": 0, "wheels": 0}
|
||||
root.bind_all("<ButtonPress>", lambda e: ptr.__setitem__("wheels" if e.num in (4, 5) else "clicks", ptr["wheels" if e.num in (4, 5) else "clicks"] + 1))
|
||||
root.bind_all("<MouseWheel>", lambda e: ptr.__setitem__("wheels", ptr["wheels"] + 1))
|
||||
last = [None]
|
||||
|
||||
|
||||
def key(event):
|
||||
if event.char == 'q':
|
||||
base['q'] += 1
|
||||
elif event.char:
|
||||
base['other'] += 1
|
||||
|
||||
|
||||
root.bind_all('<KeyPress>', key)
|
||||
|
||||
|
||||
def tick():
|
||||
try:
|
||||
focus = str(root.focus_get())
|
||||
except Exception:
|
||||
focus = "?"
|
||||
data = json.dumps({"q": base["q"], "other": base["other"], "focus": focus, "pid": os.getpid(), **ptr})
|
||||
if data != last[0]:
|
||||
tmp = out + ".tmp"
|
||||
try:
|
||||
with open(tmp, "w") as f:
|
||||
f.write(data)
|
||||
os.replace(tmp, out)
|
||||
last[0] = data
|
||||
except OSError:
|
||||
pass
|
||||
root.after(20, tick)
|
||||
|
||||
|
||||
root.after(100, lambda: (root.lift(), root.focus_force(), text.focus_set()))
|
||||
root.after(20, tick)
|
||||
root.mainloop()
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Regressions for false success claims and unsafe trial reuse in the original benchmark."""
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from process_scope import ProcessScope
|
||||
from trial_contract import (DECOY, DiscoveryError, MARKER, TASKS, acknowledged, completed,
|
||||
initial_state, prepared, schedule, select_unique, summarize, verdict)
|
||||
|
||||
|
||||
class TaskContractTests(unittest.TestCase):
|
||||
def test_reset_requires_current_command_on_current_fixture(self):
|
||||
state = {**initial_state("targeted_type"), "instance": "new", "pid": 19, "ack": "current"}
|
||||
self.assertTrue(prepared(state, "targeted_type", "new", 19, "current"))
|
||||
for key, value in (("ack", "previous"), ("instance", "old"), ("pid", 20),
|
||||
("logicalFocus", "field"), ("selLen", 1), ("top", 1), ("dropOpen", True)):
|
||||
with self.subTest(key=key):
|
||||
self.assertFalse(prepared({**state, key: value}, "targeted_type", "new", 19, "current"))
|
||||
self.assertFalse(acknowledged(state, "new", 19, "previous"))
|
||||
|
||||
def test_selectors_never_substitute_another_control(self):
|
||||
selector = (r"button", r"\bincrement\b")
|
||||
desired = {"role": "button", "label": "Increment"}
|
||||
self.assertEqual(select_unique([{"role": "button", "label": "Delete"}, desired], selector), desired)
|
||||
for elements in ([{"role": "button", "label": "Delete"}],
|
||||
[{"role": "pane", "label": "Increment"}], [desired, dict(desired)], []):
|
||||
with self.subTest(elements=elements), self.assertRaises(DiscoveryError):
|
||||
select_unique(elements, selector)
|
||||
|
||||
def test_completion_requires_exact_raw_text_and_unchanged_decoy(self):
|
||||
final = {**initial_state("targeted_type"), "text": MARKER}
|
||||
self.assertTrue(completed(final, "targeted_type"))
|
||||
for text in ("q" + MARKER, MARKER + "q", "Orca qcafé Ω", ""):
|
||||
with self.subTest(text=text):
|
||||
self.assertFalse(completed({**final, "text": text}, "targeted_type"))
|
||||
self.assertFalse(completed({**final, "decoyText": DECOY + MARKER}, "targeted_type"))
|
||||
|
||||
def test_exactly_once_actions_reject_intermediate_and_duplicate_effects(self):
|
||||
for task, key in (("choose_and_click", "count"), ("type_submit", "returns"), ("replace_submit", "returns")):
|
||||
final = {**initial_state(task), key: 1}
|
||||
if task == "choose_and_click":
|
||||
final["combo"] = "Blue"
|
||||
else:
|
||||
final["text"] = MARKER
|
||||
self.assertTrue(completed(final, task))
|
||||
for value in (0, 2):
|
||||
with self.subTest(task=task, value=value):
|
||||
self.assertFalse(completed({**final, key: value}, task))
|
||||
|
||||
def test_observed_completion_is_independent_of_api_errors(self):
|
||||
row = {"prepared": True, "task": "targeted_type", "api_errors": ["timeout"]}
|
||||
self.assertEqual(verdict(row, {**initial_state(row["task"]), "text": MARKER}), "completed")
|
||||
self.assertEqual(verdict(row, initial_state(row["task"])), "failed")
|
||||
self.assertEqual(verdict(row, None), "unobserved")
|
||||
self.assertEqual(verdict({**row, "prepared": False}, None), "invalid_setup")
|
||||
|
||||
def test_invalid_setup_does_not_inflate_scored_completion(self):
|
||||
base = {"engine": "candidate", "task": "targeted_type", "profile": "background"}
|
||||
result = summarize([{**base, "verdict": "invalid_setup"},
|
||||
{**base, "verdict": "failed", "discovery_error": "no field"},
|
||||
{**base, "verdict": "unobserved"},
|
||||
{**base, "verdict": "completed", "api_errors": ["late error"]}])[0]
|
||||
self.assertEqual((result["planned"], result["scored"], result["invalid"], result["completed"]), (4, 3, 1, 1))
|
||||
self.assertEqual((result["discovery_failures"], result["unobserved"], result["api_errors"]), (1, 1, 1))
|
||||
|
||||
def test_partial_failures_cannot_make_task_latency_look_fast(self):
|
||||
base = {"engine": "candidate", "task": "type_submit", "profile": "background"}
|
||||
result = summarize([{**base, "verdict": "completed", "attempt_ms": 400, "action_read_ms": [100, 150]},
|
||||
{**base, "verdict": "failed", "attempt_ms": 2, "action_read_ms": [1]},
|
||||
{**base, "verdict": "unobserved", "attempt_ms": 30000}])[0]
|
||||
self.assertEqual(result["task_ms"], 400)
|
||||
self.assertEqual(result["action_read_ms"], 125)
|
||||
self.assertEqual((result["timed_completions"], result["timed_failures"]), (1, 2))
|
||||
self.assertEqual(result["failed_attempt_ms"], 15001)
|
||||
|
||||
def test_engine_order_is_reproducible_and_rotates_each_round(self):
|
||||
cells = schedule(["baseline", "candidate", "cua"], TASKS, 3, 417)
|
||||
self.assertEqual(cells, schedule(["baseline", "candidate", "cua"], TASKS, 3, 417))
|
||||
self.assertEqual(len(cells), len(TASKS) * 9)
|
||||
for task in TASKS:
|
||||
positions = [[cell["engine"] for cell in cells if cell["task"] == task and cell["round"] == index]
|
||||
for index in range(3)]
|
||||
for index in range(3):
|
||||
self.assertEqual({order[index] for order in positions}, {"baseline", "candidate", "cua"})
|
||||
|
||||
def test_process_scope_stops_descendant_writes_before_returning(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
heartbeat = Path(directory) / "heartbeat"
|
||||
child = "import pathlib,time; p=pathlib.Path(" + repr(str(heartbeat)) + "); "
|
||||
child += "\nwhile True:\n p.write_text(str(time.monotonic())); time.sleep(0.02)"
|
||||
parent = "import subprocess,sys,time; subprocess.Popen([sys.executable,'-c'," + repr(child) + "]); time.sleep(60)"
|
||||
scope = ProcessScope(directory)
|
||||
try:
|
||||
proc = scope.spawn([sys.executable, "-c", parent], hidden=True)
|
||||
end = time.monotonic() + 10
|
||||
while not heartbeat.exists() and time.monotonic() < end:
|
||||
time.sleep(0.01)
|
||||
self.assertTrue(heartbeat.exists(), "descendant never started")
|
||||
scope.close()
|
||||
self.assertIsNotNone(proc.poll())
|
||||
final = heartbeat.read_bytes()
|
||||
time.sleep(0.12)
|
||||
self.assertEqual(heartbeat.read_bytes(), final)
|
||||
finally:
|
||||
scope.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Task definitions and independently observed benchmark verdicts. No desktop imports."""
|
||||
import random
|
||||
import re
|
||||
import statistics
|
||||
|
||||
MARKER = "Orca café Ω"
|
||||
DECOY = "untouched"
|
||||
SELECTORS = {
|
||||
"field": (r"\b(edit|text|entry|textbox)\b", r"\bfield\b"),
|
||||
"button": (r"\b(button|push button)\b", r"\bincrement\b"),
|
||||
"color": (r"combo|popup|list ?box|menu ?button", r"\bcolor\b"),
|
||||
"scroll": (r"list|scroll|document|web", r"\blist\b|\bbenchelectron\b"),
|
||||
}
|
||||
TASKS = {
|
||||
"targeted_type": [("type", "field")],
|
||||
"type_submit": [("click", "field"), ("type", "field"), ("return", "field")],
|
||||
"replace_submit": [("click", "field"), ("ctrl_a", "field"), ("type", "field"), ("return", "field")],
|
||||
"choose_and_click": [("choose", "color"), ("click", "button")],
|
||||
"click_interference": [("click", "button")],
|
||||
"scroll_interference": [("scroll", "scroll")],
|
||||
}
|
||||
ROLE_KEYS = ("role", "control_type", "role_description", "localized_role", "subrole", "controlType", "localizedControlType")
|
||||
LABEL_KEYS = ("label", "name", "title", "description", "automation_id", "automationId", "help", "placeholder")
|
||||
|
||||
|
||||
class DiscoveryError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def select_unique(elements, selector):
|
||||
role, label = selector
|
||||
matches = [element for element in elements
|
||||
if re.search(role, " ".join(str(element.get(key) or "") for key in ROLE_KEYS), re.I)
|
||||
and any(re.search(label, str(element.get(key) or ""), re.I) for key in LABEL_KEYS)]
|
||||
if len(matches) != 1:
|
||||
raise DiscoveryError(f"selector {selector!r} matched {len(matches)} elements")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def initial_state(task):
|
||||
text = "hello" if task == "replace_submit" else ""
|
||||
return {"text": text, "decoyText": DECOY, "count": 0, "returns": 0, "combo": "Red", "top": 0,
|
||||
"check": False, "menuHits": 0, "dropOpen": False, "selStart": len(text), "selLen": 0,
|
||||
"logicalFocus": "decoy" if task == "targeted_type" else "field"}
|
||||
|
||||
|
||||
def acknowledged(state, instance, pid, nonce):
|
||||
return (state.get("instance") == instance and state.get("pid") == pid
|
||||
and state.get("ack") == nonce)
|
||||
|
||||
|
||||
def prepared(state, task, instance, pid, nonce):
|
||||
return acknowledged(state, instance, pid, nonce) and all(
|
||||
state.get(key) == value for key, value in initial_state(task).items())
|
||||
|
||||
|
||||
def completed(state, task):
|
||||
expected = initial_state(task)
|
||||
for key in ("selStart", "selLen", "logicalFocus"):
|
||||
del expected[key]
|
||||
if task in ("targeted_type", "type_submit", "replace_submit"):
|
||||
expected["text"] = MARKER
|
||||
if task in ("type_submit", "replace_submit"):
|
||||
expected["returns"] = 1
|
||||
if task in ("choose_and_click", "click_interference"):
|
||||
expected["count"] = 1
|
||||
if task == "choose_and_click":
|
||||
expected["combo"] = "Blue"
|
||||
if task == "scroll_interference":
|
||||
del expected["top"]
|
||||
if not isinstance(state.get("top"), (int, float)) or state["top"] <= 0:
|
||||
return False
|
||||
return all(state.get(key) == value for key, value in expected.items())
|
||||
|
||||
|
||||
def verdict(row, state):
|
||||
if not row.get("prepared"):
|
||||
return "invalid_setup"
|
||||
if state is None:
|
||||
return "unobserved"
|
||||
return "completed" if completed(state, row["task"]) else "failed"
|
||||
|
||||
|
||||
def schedule(engines, tasks, rounds, seed):
|
||||
rng = random.Random(seed)
|
||||
order = list(engines)
|
||||
rng.shuffle(order)
|
||||
cells = []
|
||||
for round_index in range(rounds):
|
||||
task_order = list(tasks)
|
||||
rng.shuffle(task_order)
|
||||
for task in task_order:
|
||||
for index in range(len(order)):
|
||||
cells.append({"engine": order[(index + round_index) % len(order)],
|
||||
"task": task, "round": round_index})
|
||||
return cells
|
||||
|
||||
|
||||
def summarize(rows):
|
||||
groups = {}
|
||||
for row in rows:
|
||||
groups.setdefault((row["engine"], row["task"], row["profile"]), []).append(row)
|
||||
summary = []
|
||||
for (engine, task, profile), group in sorted(groups.items()):
|
||||
valid = [row for row in group if row["verdict"] != "invalid_setup"]
|
||||
successful = [row for row in valid if row["verdict"] == "completed"]
|
||||
times = [row["attempt_ms"] for row in successful if row.get("attempt_ms") is not None]
|
||||
failures = [row["attempt_ms"] for row in valid if row["verdict"] != "completed" and row.get("attempt_ms") is not None]
|
||||
paired = [sample for row in successful for sample in row.get("action_read_ms", [])]
|
||||
summary.append({"engine": engine, "task": task, "profile": profile, "planned": len(group),
|
||||
"scored": len(valid), "invalid": len(group) - len(valid),
|
||||
"completed": sum(row["verdict"] == "completed" for row in valid),
|
||||
"unobserved": sum(row["verdict"] == "unobserved" for row in valid),
|
||||
"api_errors": sum(bool(row.get("api_errors")) for row in valid),
|
||||
**{key: sum(row.get(key, 0) for row in valid) for key in
|
||||
("keys_sent", "keys_received", "keys_lost", "agent_text_in_sentinel", "target_q_characters")},
|
||||
"discovery_failures": sum(bool(row.get("discovery_error")) for row in valid),
|
||||
"task_ms": round(statistics.median(times), 1) if times else None,
|
||||
"failed_attempt_ms": round(statistics.median(failures), 1) if failures else None,
|
||||
"timed_completions": len(times), "timed_failures": len(failures),
|
||||
"action_read_ms": round(statistics.median(paired), 1) if paired else None,
|
||||
**{key: sum(bool(row.get("observation", {}).get(key)) for row in valid)
|
||||
for key in ("sentinel_lost_focus", "target_raised", "pointer_moved", "clipboard_changed")}})
|
||||
return summary
|
||||
Reference in New Issue
Block a user