mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
* [comment audit] Align legacy failure-read counts with cached display * [comment audit] Condense shared provider lifecycle and browser recovery comments [skip ci] * [comment audit] Condense shared generation, path and preview comments [skip ci] * [comment audit] Condense host, file and Git contract rationale [skip ci] * [comment audit] Tighten Git lifecycle and provider contract comments [skip ci] * [comment audit] Clarify shared host authority and launch isolation [skip ci] * [comment audit] simplify local access mobile and native chat contracts [skip ci] * [comment audit] simplify native chat presentation and state contracts [skip ci] * [comment audit] simplify native runtime and provider contracts [skip ci] * [comment audit] simplify migration and orchestration contracts [skip ci] * [comment audit] simplify plugin trust and process capture rationale [skip ci] * [comment audit] simplify terminal delivery and remote compatibility rationale [skip ci] * [comment audit] clarify remote recovery and launch wire rationale [skip ci] * [comment audit] shorten shared RPC compatibility and runtime evidence comments [skip ci] * [comment audit] simplify secure storage shell startup and sidebar rationale [skip ci] * [comment audit] clarify sidebar, skill and SSH invariants [skip ci] * [comment audit] condense structured chat delivery and timing rationale [skip ci] * [comment audit] condense terminal replay and telemetry invariants [skip ci] * [comment audit] clarify launch permission and terminal delivery rationale [skip ci] * [comment audit] clarify workspace ownership and layout contracts [skip ci] * [comment audit] wait for fitted and painted SSH terminal links * [comment audit] clarify layout persistence and session ownership [skip ci] * [comment audit] clarify host routing and workspace lifecycle contracts [skip ci] * [comment audit] clarify startup persistence and app lifetime comments [skip ci] * [comment audit] clarify recovery and activity boundaries [skip ci] * [comment audit] condense activity and artifact invariants [skip ci] * [comment audit] condense automation authority contracts [skip ci] * [comment audit] simplify automation scheduling and history contracts [skip ci] * [comment audit] clarify browser chrome and annotation lifecycles [skip ci] * [comment audit] clarify browser routing and retained guest ownership [skip ci] * [comment audit] clarify browser recovery and document ownership comments [skip ci] * [comment audit] restore hosted runners required by native IME fixtures * [comment audit] shorten preview, palette and dashboard invariants [skip ci] * [comment audit] clarify terminal preview, dashboard and retention comments [skip ci] * [comment audit] clarify diff-note recovery and editor lifecycle comments [skip ci] * [comment audit] shorten rich-text and combined-diff ownership comments [skip ci] * [comment audit] shorten autosave, draft recovery and model ownership comments [skip ci] * [comment audit] shorten save-queue, notebook and Markdown constraints [skip ci] * [comment audit] shorten Monaco, PDF and rich Markdown safeguards [skip ci] * [comment audit] clarify rich Markdown source and selection safeguards [skip ci] * [comment audit] clarify editor loading and emulator stream lifetimes [skip ci] * [comment audit] trim emulator and feature tour narration [skip ci] * [comment audit] clarify floating workspace and GitHub cache ownership [skip ci] * [comment audit] trim provider controls and Linear filter explanations [skip ci] * [comment audit] clarify pairing and chat resume lifetimes [skip ci] * [comment audit] clarify chat delivery and transcript lifetimes [skip ci] * [comment audit] clarify session options and host-owned status [skip ci] * [comment audit] clarify draft durability and live-status precedence [skip ci] * [comment audit] Trim chat send and model ownership explanations [skip ci] * [comment audit] Shorten transcript ownership and send recovery comments [skip ci] * [comment audit] Simplify native chat composer and transcript hook comments [skip ci] * [comment audit] Shorten native chat scroll and structured session explanations [skip ci] * [comment audit] Trim workspace picker and onboarding commentary [skip ci] * [comment audit] Simplify pet rendering, review cache and quick open comments [skip ci] * [comment audit] Trim sidebar and session history explanations [skip ci] * [comment audit] Condense session resume and review checks comments [skip ci] * [comment audit] Shorten checks state and file explorer refresh comments [skip ci] * [comment audit] Trim plugin bridge and source-control action rationale [skip ci] * [comment audit] Trim source-control listing and review rationale [skip ci] * [comment audit] Shorten sidebar lifecycle and explorer safeguards [skip ci] * [comment audit] Condense Settings account and appearance rationale [skip ci] * [comment audit] Trim pairing and repository Settings explanations [skip ci] * [comment audit] Condense Settings search SSH and terminal safeguards [skip ci] * [comment audit] Shorten Settings voice and provider ownership comments [skip ci] * [comment audit] Condense Settings draft persistence and host-state rationale [skip ci] * [comment audit] Trim Settings hooks and sidebar setup rationale [skip ci] * [comment audit] Condense sidebar deletion and board-filter safeguards [skip ci] * [comment audit] Condense worktree cards, host ownership and Markdown contracts [skip ci] * [comment audit] Condense sidebar drag, folder and filtering contracts [skip ci] * [comment audit] Condense sidebar parity, sleep and host ownership contracts [skip ci] * [comment audit] Prune focus-read baseline after comment cleanup * [comment audit] Clarify sidebar visibility, board drops and agent evidence [skip ci] * [comment audit] Tighten sidebar listing, drag and reveal contracts [skip ci] * [comment audit] Clarify viewport geometry and skill update safeguards [skip ci] * [comment audit] Tighten skill inventory, usage and host status comments [skip ci] * [comment audit] Clarify usage ownership and resource kill safeguards [skip ci] * [comment audit] Tighten tab selection, rename and search contracts [skip ci] * [comment audit] Tighten tab-strip geometry and drag ownership comments [skip ci] * [comment audit] Tighten Tasks recovery, pagination and mutation comments [skip ci] * [comment audit] Tighten provider loading and terminal recovery comments [skip ci] * [comment audit] Tighten completion, cadence and PTY ownership comments [skip ci] * [comment audit] Clarify terminal restore and parked-pane boundaries [skip ci] * [comment audit] Tighten PTY startup and replay explanations [skip ci] * [comment audit] Clarify PTY identity and delivery ownership [skip ci] * [comment audit] Shorten remote reconnect and input replay rationale [skip ci] * [comment audit] Condense terminal parking clipboard and file link comments [skip ci] * [comment audit] Clarify IME ownership and trim trace history [skip ci] * [comment audit] Shorten pane recovery and parked watcher explanations [skip ci] * [comment audit] Condense snapshot replay and visibility recovery rationale [skip ci] * [comment audit] Shorten terminal parking and keyboard ownership rationale [skip ci] * [comment audit] Trim renderer lifecycle and menu rationale [skip ci] * [comment audit] Condense workspace cleanup ownership and settlement notes [skip ci] * [comment audit] Clarify renderer hook evidence and source ownership [skip ci] * [comment audit] Tighten IPC ownership and reconnect explanations [skip ci] * [comment audit] Trim renderer focus, paste and attention comments [skip ci] * [comment audit] Shorten scroll, locale and agent lifecycle comments [skip ci] * [comment audit] Trim agent launch, resume and ownership commentary [skip ci] * [comment audit] Shorten account, browser and editor comments [skip ci] * [comment audit] Clarify host-mirror waits, previews and input ownership [skip ci] * [comment audit] Condense launch recovery and editor tokenization comments [skip ci] * [comment audit] Condense workspace selection and terminal pane rationale [skip ci] * [comment audit] Condense terminal fitting and rendering recovery comments [skip ci] * [comment audit] Condense terminal input and host routing explanations [skip ci] * [comment audit] Condense ownership and sleeping-session recovery rationale [skip ci] * [comment audit] Condense structured launch and terminal link explanations [skip ci] * [comment audit] Condense terminal ownership and workspace file routing rationale [skip ci] * [comment audit] Condense session persistence and workspace activation rationale [skip ci] * [comment audit] Condense workspace routing and browser ownership rationale [skip ci] * [comment audit] Condense mirror recovery and layout authority rationale [skip ci] * [comment audit] Condense runtime transport and file safety rationale [skip ci] * [comment audit] Condense runtime graph and structured session rationale [skip ci] * [comment audit] Condense browser creation and remote tab recovery rationale [skip ci] * [comment audit] Condense snapshot authority and publisher recovery rationale [skip ci] * [comment audit] Condense frame, startup and provider cache rationale [skip ci] * [comment audit] Condense host twin and agent lifecycle rationale [skip ci] * [comment audit] Condense browser authority and editor cleanup rationale [skip ci] * [comment audit] Condense editor buffer identity and review cache rationale [skip ci] * [comment audit] Condense provider lifetime and repository ownership rationale [skip ci] * [comment audit] Condense reconnect, hydration and terminal retirement rationale [skip ci] * [comment audit] Condense recovery, reconciliation and persistence rationale [skip ci] * [comment audit] Condense cleanup ownership and listing admission rationale [skip ci] * [comment audit] Condense teardown ordering and terminal ownership rationale [skip ci] * [comment audit] Condense recovery ledger and web host-boundary rationale [skip ci] * [comment audit] Condense web reconnect and credential lifecycle rationale [skip ci] * [comment audit] Condense cloud rollout and assignment safety rationale [skip ci] * [comment audit] Condense relay session and database recovery rationale [skip ci] * [comment audit] Condense relay placement and protocol compatibility rationale [skip ci] * [comment audit] Condense build and packaging constraints [skip ci] * [comment audit] Condense CI gates and validation isolation rationale [skip ci] * [comment audit] Condense catalog identity and browser validation rationale [skip ci] * [comment audit] Condense mobile browser and frame-budget rationale [skip ci] * [comment audit] Shorten mobile route and browser test explanations [skip ci] * [comment audit] Condense editor and session seam test rationale [skip ci] * [comment audit] Trim session and terminal closure history [skip ci] * [comment audit] Shorten terminal browser and census explanations [skip ci] * [comment audit] Preserve root-only E2E diagnostic baseline * [comment audit] Condense mobile seam and HTML preview test rationale [skip ci] * [comment audit] Condense preview instrumentation and render harness rationale [skip ci] * [comment audit] Condense bundle contracts and native-build constraints [skip ci] * [comment audit] Shorten native packaging and CI routing rationale [skip ci] * [comment audit] Condense release tooling and benchmark constraints [skip ci] * [comment audit] Condense packaging and platform gate rationale [skip ci] * [comment audit] Shorten mobile route and history rationale [skip ci] * [comment audit] Condense mobile browser and component rationale [skip ci] * [comment audit] Shorten drawer and workspace-form rationale [skip ci] * [comment audit] Trim mobile review and editor-document explanations [skip ci] * [comment audit] Condense editor lifecycle and Markdown explanations [skip ci] * [comment audit] Trim mobile file, diagnostics and dictation commentary [skip ci] * [comment audit] Simplify mobile host-list and workspace-view explanations [skip ci] * [comment audit] Clarify mobile shell lifetime and bridge delivery constraints [skip ci] * [comment audit] Condense bridge stream, native request and audio rationale [skip ci] * [comment audit] Reduce bridge schema and page-client commentary [skip ci] * [comment audit] Tighten mobile page bridge contracts [skip ci] * [comment audit] Clarify mobile cache and session constraints [skip ci] * [comment audit] Condense mobile route and lifecycle rationale [skip ci] * [comment audit] Tighten mobile shell hook timing rationale [skip ci] * [comment audit] Clarify route handoff and notification compatibility [skip ci] * [comment audit] Clarify push registration and dictation cleanup constraints [skip ci] * [comment audit] Condense keyboard geometry and native media ownership rationale [skip ci] * [comment audit] Tighten mobile chat scroll and composer concurrency rationale [skip ci] * [comment audit] Condense mobile reply compatibility and upload ownership rationale [skip ci] * [comment audit] Condense optimistic echo and scroll ownership rationale [skip ci] * [comment audit] Tighten stream replay and queued session ownership rationale [skip ci] * [comment audit] Condense mobile composer and reply-schema constraints [skip ci] * [comment audit] Clarify mobile transcript recovery and session ownership [skip ci] * [comment audit] Clarify mobile queue and source-control compatibility [skip ci] * [comment audit] Condense storage and task launch constraints [skip ci] * [comment audit] Clarify task schemas and terminal document ownership [skip ci] * [comment audit] Tighten terminal input and viewport rationale [skip ci] * [comment audit] Shorten terminal and RPC recorder contracts [skip ci] * [comment audit] Condense recorder engine and connection contracts [skip ci] * [comment audit] Shorten host credential and relay recovery comments [skip ci] * [comment audit] Tighten pairing recovery and bundle transport rationale [skip ci] * [comment audit] Clarify RPC delivery and workspace catalog boundaries [skip ci] * [comment audit] Condense catalog and relay entry contracts [skip ci] * [comment audit] Shorten shutdown and SSH relay delivery contracts [skip ci] * [comment audit] Condense WSL relay and E2E regression rationale [skip ci] * [comment audit] Shorten cross-version harness and renderer fixture explanations [skip ci] * [comment audit] Condense paired-host and E2E lifecycle rationale [skip ci] * [comment audit] Shorten layout oracle and regression explanations [skip ci] * [comment audit] Condense paired recovery and ownership regression contracts [skip ci] * [comment audit] Shorten restart, sidebar and SSH lifecycle contracts [skip ci] * [comment audit] Condense terminal rendering and IME regression rationale [skip ci] * [comment audit] Shorten terminal recovery and benchmark contracts [skip ci] * [comment audit] Condense tooling and Windows lifecycle proof rationale [skip ci] * [comment audit] Condense workflow routing and release rationale [skip ci] * [comment audit] Condense mobile security and installer contracts [skip ci] * [comment audit] Shorten native rationale and stylesheet workarounds [skip ci] * [comment audit] Shorten Windows visibility probe rationale [skip ci] * [comment audit] Condense embedded release and signing rationale [skip ci] * [comment audit] Retain current main test behavior in final reconciliation [skip ci] * [comment audit] Complete final comment review across current source inventory * [comment audit] Handle consolidated audit reports before final CI * [comment audit] Preserve credential storage census detail [skip ci] * [comment audit] Scale broad changed E2E selections with verified timing shards * [comment audit] Align E2E assertions with browser close and settled session metadata * [comment audit] Match existing E2Es to current resource and windowing contracts * [comment audit] Observe the shared server outage notice in reconnect coverage * [comment audit] Make existing renderer probes measure settled tab and PTY work * [comment audit] Align palette E2E selector with task search * [comment audit] Refresh remaining palette search selectors * [comment audit] Keep selector correction within selected audit specs * [comment audit] Await emoji fixture settings inside renderer evaluation * [comment audit] Cancel fixture fit retries between terminal tests --------- Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
552 lines
18 KiB
JavaScript
552 lines
18 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* Cross-terminal pipeline benchmark. Run INSIDE the terminal under test
|
|
* (Orca pane, iTerm2, Ghostty, Terminal.app, VS Code, ...) — it measures the
|
|
* full byte path of whatever terminal hosts it: PTY -> (daemon/ptyHost ->
|
|
* IPC ->) parser -> response.
|
|
*
|
|
* Metrics per run:
|
|
* 1. dsr-idle — DSR (ESC[6n) round-trip latency at rest. The reply is
|
|
* produced only after the terminal's parser reaches the
|
|
* query, so this tracks the input/echo pipeline without
|
|
* needing OS-level keystroke injection.
|
|
* 2. throughput — wall time to stream each fixture, ended by a DSR
|
|
* fence. The fence matters: xterm.js-class terminals
|
|
* ingest at wire speed into an internal queue and parse
|
|
* later, so socket drain alone would flatter them.
|
|
* 3. dsr-under-load — DSR latency sampled while a paced (default 1 MB/s)
|
|
* agent-TUI stream plays. This is the "typing while an
|
|
* agent floods output" complaint, quantified.
|
|
*
|
|
* Usage (run in EACH terminal being compared, same machine, on AC power):
|
|
* node tests/tools/benchmarks/terminal-pipeline-bench.mjs --label m2max-2026-07-02
|
|
* [--size-mb 10] [--iterations 5] [--dsr-count 200] [--skip-load]
|
|
* [--fixtures ascii-log,cjk-emoji,agent-tui,styles-stress]
|
|
*
|
|
* Aggregate results from all terminals into one comparison table:
|
|
* node tests/tools/benchmarks/terminal-pipeline-bench.mjs report [--label <filter>]
|
|
*
|
|
* Protocol notes: keep hands off the keyboard during a run (stdin is parsed
|
|
* for DSR replies), use a comparable window size everywhere, avoid tmux/screen
|
|
* (they proxy the queries and would be the thing measured). `styles-stress` is
|
|
* deliberately pathological (every cell restyled); read it as a ceiling probe,
|
|
* not a realistic workload. Complementary manual metric: Typometer for true
|
|
* keypress->pixel latency — this probe stops at the parser reply.
|
|
* Results: tests/tools/benchmarks/results/terminal-pipeline-<label>-<timestamp>.json
|
|
*/
|
|
import { mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'
|
|
import os from 'node:os'
|
|
import { join, resolve } from 'node:path'
|
|
import { performance } from 'node:perf_hooks'
|
|
|
|
const scriptDir = import.meta.dirname
|
|
const resultsDir = resolve(scriptDir, 'results')
|
|
|
|
const ESC = '\x1b'
|
|
const DSR_QUERY = `${ESC}[6n`
|
|
// oxlint-disable-next-line no-control-regex -- the ESC byte is the payload: this parses the terminal's cursor-position reply
|
|
const DSR_REPLY_RE = /\x1b\[(\d+);(\d+)R/
|
|
const CHUNK_BYTES = 64 * 1024
|
|
// Overridable: a dev-mode Electron terminal can hold >15s of parse backlog at
|
|
// a fence, which is a measurement (slow), not a hang — don't die on it.
|
|
let dsrTimeoutMs = 15_000
|
|
|
|
function parseArgs(argv) {
|
|
const args = {
|
|
mode: 'run',
|
|
label: 'run',
|
|
sizeMb: 10,
|
|
iterations: 5,
|
|
dsrCount: 200,
|
|
skipLoad: false,
|
|
forceNonTty: false,
|
|
fixtures: ['ascii-log', 'cjk-emoji', 'agent-tui', 'styles-stress']
|
|
}
|
|
for (let i = 2; i < argv.length; i++) {
|
|
const next = () => argv[++i]
|
|
switch (argv[i]) {
|
|
case 'report':
|
|
args.mode = 'report'
|
|
break
|
|
case '--label':
|
|
args.label = next()
|
|
break
|
|
case '--size-mb':
|
|
args.sizeMb = Number(next())
|
|
break
|
|
case '--iterations':
|
|
args.iterations = Number(next())
|
|
break
|
|
case '--dsr-count':
|
|
args.dsrCount = Number(next())
|
|
break
|
|
case '--fixtures':
|
|
args.fixtures = next().split(',')
|
|
break
|
|
case '--dsr-timeout-ms':
|
|
dsrTimeoutMs = Number(next())
|
|
break
|
|
case '--skip-load':
|
|
args.skipLoad = true
|
|
break
|
|
// For CI smoke tests of the script itself; numbers from a pipe are
|
|
// meaningless because there is no terminal parsing them.
|
|
case '--force-non-tty':
|
|
args.forceNonTty = true
|
|
break
|
|
default:
|
|
throw new Error(`Unknown argument: ${argv[i]}`)
|
|
}
|
|
}
|
|
if (!Number.isFinite(args.sizeMb) || args.sizeMb <= 0) {
|
|
throw new Error('--size-mb must be > 0')
|
|
}
|
|
if (!Number.isInteger(args.iterations) || args.iterations <= 0) {
|
|
throw new Error('--iterations must be a positive integer')
|
|
}
|
|
return args
|
|
}
|
|
|
|
// Deterministic PRNG so every terminal parses byte-identical fixtures.
|
|
function mulberry32(seed) {
|
|
let a = seed >>> 0
|
|
return () => {
|
|
a = (a + 0x6d2b79f5) | 0
|
|
let t = Math.imul(a ^ (a >>> 15), 1 | a)
|
|
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
|
|
return ((t ^ (t >>> 14)) >>> 0) / 4294967296
|
|
}
|
|
}
|
|
|
|
function pick(rng, list) {
|
|
return list[Math.floor(rng() * list.length)]
|
|
}
|
|
|
|
const LOG_WORDS =
|
|
'build compile transform resolve bundle emit chunk module dependency vite electron terminal daemon renderer checkpoint snapshot worktree agent session'.split(
|
|
' '
|
|
)
|
|
|
|
function asciiLogFixture(targetBytes) {
|
|
const rng = mulberry32(1)
|
|
const colors = ['32', '33', '36', '90', '35']
|
|
const parts = []
|
|
let bytes = 0
|
|
let line = 0
|
|
while (bytes < targetBytes) {
|
|
line++
|
|
const words = Array.from({ length: 6 + Math.floor(rng() * 8) }, () => pick(rng, LOG_WORDS))
|
|
const s = `${ESC}[${pick(rng, colors)}m[build ${String(line).padStart(6, '0')}]${ESC}[0m ${words.join(' ')} (${Math.floor(rng() * 5000)}ms)\r\n`
|
|
parts.push(s)
|
|
bytes += s.length // fixture is ASCII + escapes: length === bytes
|
|
}
|
|
return parts.join('')
|
|
}
|
|
|
|
function cjkEmojiFixture(targetBytes) {
|
|
const rng = mulberry32(2)
|
|
const cjk = '終端の性能を測定する漢字混じりの長い行を生成します電光石火'
|
|
const emoji = ['🚀', '🐋', '✅', '⚠️', '👨👩👧👦', '🇯🇵']
|
|
const parts = []
|
|
let bytes = 0
|
|
let line = 0
|
|
while (bytes < targetBytes) {
|
|
line++
|
|
const body = Array.from(
|
|
{ length: 8 + Math.floor(rng() * 10) },
|
|
() => cjk[Math.floor(rng() * cjk.length)]
|
|
).join('')
|
|
const s = `${String(line).padStart(6, '0')} ${body} ${pick(rng, emoji)}\r\n`
|
|
parts.push(s)
|
|
bytes += Buffer.byteLength(s, 'utf8')
|
|
}
|
|
return parts.join('')
|
|
}
|
|
|
|
// Inline Claude-shaped transcript plus DEC 2026 bottom-status redraw, not an alt-screen TUI.
|
|
function agentTuiFixture(targetBytes, cols, rows) {
|
|
const rng = mulberry32(3)
|
|
const statusRows = Math.min(10, Math.max(4, rows - 4))
|
|
const spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴']
|
|
const parts = []
|
|
let bytes = 0
|
|
let frame = 0
|
|
let painted = false
|
|
const push = (s) => {
|
|
parts.push(s)
|
|
bytes += Buffer.byteLength(s, 'utf8')
|
|
}
|
|
while (bytes < targetBytes) {
|
|
frame++
|
|
push(`${ESC}[?2026h`)
|
|
if (painted) {
|
|
// Erase status only, retaining transcript above as agent CLIs do.
|
|
push(`${ESC}[${statusRows}A${ESC}[0J`)
|
|
}
|
|
for (let t = 0; t < 2; t++) {
|
|
const words = Array.from({ length: 8 }, () => pick(rng, LOG_WORDS))
|
|
push(`${ESC}[2m●${ESC}[0m ${words.join(' ')}\r\n`)
|
|
}
|
|
for (let r = 0; r < statusRows; r++) {
|
|
const fill = Math.floor(rng() * Math.max(10, cols - 20))
|
|
push(
|
|
`${ESC}[38;5;${33 + (r % 6)}m${pick(rng, spinner)} task ${frame % 100}·${r}${ESC}[0m ${'▇'.repeat(fill % 40)}\r\n`
|
|
)
|
|
}
|
|
painted = true
|
|
push(`${ESC}[?2026l`)
|
|
}
|
|
return parts.join('')
|
|
}
|
|
|
|
// Label per-cell SGR as a pathological ceiling probe, not representative workload.
|
|
function stylesStressFixture(targetBytes, cols) {
|
|
const rng = mulberry32(4)
|
|
const parts = []
|
|
let bytes = 0
|
|
const push = (s) => {
|
|
parts.push(s)
|
|
bytes += s.length
|
|
}
|
|
while (bytes < targetBytes) {
|
|
for (let c = 0; c < cols && bytes < targetBytes; c++) {
|
|
push(`${ESC}[38;5;${Math.floor(rng() * 255)}mX`)
|
|
}
|
|
push(`${ESC}[0m\r\n`)
|
|
}
|
|
return parts.join('')
|
|
}
|
|
|
|
export function buildFixture(name, targetBytes, cols, rows) {
|
|
switch (name) {
|
|
case 'ascii-log':
|
|
return asciiLogFixture(targetBytes)
|
|
case 'cjk-emoji':
|
|
return cjkEmojiFixture(targetBytes)
|
|
case 'agent-tui':
|
|
return agentTuiFixture(targetBytes, cols, rows)
|
|
case 'styles-stress':
|
|
return stylesStressFixture(targetBytes, cols)
|
|
default:
|
|
throw new Error(`Unknown fixture: ${name}`)
|
|
}
|
|
}
|
|
|
|
const dsrWaiters = []
|
|
let stdinBuffer = ''
|
|
|
|
function onStdinData(chunk) {
|
|
stdinBuffer += chunk.toString('utf8')
|
|
// Ctrl+C must still work while raw mode is on.
|
|
if (stdinBuffer.includes('\x03')) {
|
|
cleanupAndExit(130)
|
|
}
|
|
let match
|
|
while (dsrWaiters.length > 0 && (match = DSR_REPLY_RE.exec(stdinBuffer))) {
|
|
stdinBuffer = stdinBuffer.slice(match.index + match[0].length)
|
|
dsrWaiters.shift().resolve(performance.now())
|
|
}
|
|
// Bound stray non-reply input (accidental keystrokes).
|
|
if (stdinBuffer.length > 4096) {
|
|
stdinBuffer = stdinBuffer.slice(-256)
|
|
}
|
|
}
|
|
|
|
function dsrRoundTrip() {
|
|
return new Promise((resolvePromise, reject) => {
|
|
const timer = setTimeout(() => {
|
|
const idx = dsrWaiters.indexOf(waiter)
|
|
if (idx !== -1) {
|
|
dsrWaiters.splice(idx, 1)
|
|
}
|
|
reject(new Error('DSR reply timeout'))
|
|
}, dsrTimeoutMs)
|
|
const waiter = {
|
|
resolve: (endTs) => {
|
|
clearTimeout(timer)
|
|
resolvePromise(endTs - startTs)
|
|
}
|
|
}
|
|
dsrWaiters.push(waiter)
|
|
const startTs = performance.now()
|
|
process.stdout.write(DSR_QUERY)
|
|
})
|
|
}
|
|
|
|
function writeChunk(data) {
|
|
return new Promise((resolvePromise) => {
|
|
if (process.stdout.write(data)) {
|
|
resolvePromise()
|
|
} else {
|
|
process.stdout.once('drain', resolvePromise)
|
|
}
|
|
})
|
|
}
|
|
|
|
async function streamFixture(fixture) {
|
|
for (let i = 0; i < fixture.length; i += CHUNK_BYTES) {
|
|
await writeChunk(fixture.slice(i, i + CHUNK_BYTES))
|
|
}
|
|
}
|
|
|
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
|
|
|
|
function stats(samples) {
|
|
if (samples.length === 0) {
|
|
return null
|
|
}
|
|
const sorted = [...samples].sort((a, b) => a - b)
|
|
const at = (q) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]
|
|
return {
|
|
count: sorted.length,
|
|
p50: at(0.5),
|
|
p90: at(0.9),
|
|
p99: at(0.99),
|
|
max: sorted.at(-1)
|
|
}
|
|
}
|
|
|
|
async function measureDsrIdle(count) {
|
|
const samples = []
|
|
let timeouts = 0
|
|
for (let i = 0; i < count; i++) {
|
|
try {
|
|
samples.push(await dsrRoundTrip())
|
|
} catch {
|
|
timeouts++
|
|
}
|
|
await sleep(5)
|
|
}
|
|
return { ...stats(samples), timeouts }
|
|
}
|
|
|
|
async function measureThroughput(name, fixture, iterations) {
|
|
const fixtureBytes = Buffer.byteLength(fixture, 'utf8')
|
|
// Warmup pass primes glyph atlases / JIT so iteration 1 isn't an outlier.
|
|
await streamFixture(fixture.slice(0, Math.min(fixture.length, 512 * 1024)))
|
|
await dsrRoundTrip().catch(() => {})
|
|
const runs = []
|
|
for (let i = 0; i < iterations; i++) {
|
|
const start = performance.now()
|
|
await streamFixture(fixture)
|
|
// Fence: the terminal can only answer after parsing everything above.
|
|
await dsrRoundTrip()
|
|
runs.push(performance.now() - start)
|
|
}
|
|
const mbPerSec = runs.map((ms) => fixtureBytes / 1024 / 1024 / (ms / 1000))
|
|
return { fixture: name, fixtureBytes, wallMs: stats(runs), mbPerSec: stats(mbPerSec) }
|
|
}
|
|
|
|
async function measureDsrUnderLoad(fixture, paceBytesPerSec, sampleCount) {
|
|
const samples = []
|
|
let timeouts = 0
|
|
let stop = false
|
|
const paceChunk = Math.min(CHUNK_BYTES, Math.floor(paceBytesPerSec / 20))
|
|
const producer = (async () => {
|
|
let offset = 0
|
|
while (!stop) {
|
|
await writeChunk(fixture.slice(offset, offset + paceChunk))
|
|
offset = (offset + paceChunk) % fixture.length
|
|
await sleep(50) // paceChunk every 50ms ≈ paceBytesPerSec
|
|
}
|
|
})()
|
|
for (let i = 0; i < sampleCount; i++) {
|
|
try {
|
|
samples.push(await dsrRoundTrip())
|
|
} catch {
|
|
timeouts++
|
|
}
|
|
await sleep(100)
|
|
}
|
|
stop = true
|
|
await producer
|
|
return { paceBytesPerSec, ...stats(samples), timeouts }
|
|
}
|
|
|
|
function restoreTerminal() {
|
|
// Always reset SGR, show cursor and leave synchronized mode, including aborted runs.
|
|
process.stdout.write(`${ESC}[0m${ESC}[?25h${ESC}[?2026l\r\n`)
|
|
if (process.stdin.isTTY) {
|
|
process.stdin.setRawMode(false)
|
|
}
|
|
process.stdin.pause()
|
|
}
|
|
|
|
function cleanupAndExit(code) {
|
|
restoreTerminal()
|
|
process.exit(code)
|
|
}
|
|
|
|
function fmt(n, digits = 1) {
|
|
return n === null || n === undefined ? '—' : n.toFixed(digits)
|
|
}
|
|
|
|
function runReport(labelFilter) {
|
|
let files = []
|
|
try {
|
|
files = readdirSync(resultsDir).filter((f) => f.startsWith('terminal-pipeline-'))
|
|
} catch {
|
|
/* no results dir yet */
|
|
}
|
|
const rows = []
|
|
for (const f of files) {
|
|
try {
|
|
const r = JSON.parse(readFileSync(join(resultsDir, f), 'utf8'))
|
|
if (labelFilter && labelFilter !== 'run' && !r.label.includes(labelFilter)) {
|
|
continue
|
|
}
|
|
rows.push(r)
|
|
} catch {
|
|
/* skip unreadable */
|
|
}
|
|
}
|
|
if (rows.length === 0) {
|
|
console.log('No results found. Run the benchmark in each terminal first.')
|
|
return
|
|
}
|
|
rows.sort((a, b) => (a.terminal.program || '').localeCompare(b.terminal.program || ''))
|
|
console.log(
|
|
'\nterminal | dsr idle p50/p90 (ms) | dsr load p50/p90 (ms) | ascii MB/s | cjk MB/s | tui MB/s | stress MB/s'
|
|
)
|
|
console.log('-'.repeat(120))
|
|
for (const r of rows) {
|
|
const t = `${r.terminal.program ?? r.terminal.term ?? '?'} ${r.terminal.version ?? ''}`.trim()
|
|
const tp = (name) => {
|
|
const e = r.throughput?.find((x) => x.fixture === name)
|
|
// Smoke-mode (--force-non-tty) entries have no mbPerSec — show a dash.
|
|
return e?.mbPerSec ? fmt(e.mbPerSec.p50) : '—'
|
|
}
|
|
console.log(
|
|
`${t.padEnd(21)} | ${fmt(r.dsrIdle?.p50, 2)}/${fmt(r.dsrIdle?.p90, 2).padEnd(12)} | ${fmt(r.dsrUnderLoad?.p50, 2)}/${fmt(r.dsrUnderLoad?.p90, 2).padEnd(12)} | ${tp('ascii-log').padStart(10)} | ${tp('cjk-emoji').padStart(8)} | ${tp('agent-tui').padStart(8)} | ${tp('styles-stress').padStart(11)}`
|
|
)
|
|
}
|
|
console.log(`\n${rows.length} result file(s) from ${resultsDir}`)
|
|
}
|
|
|
|
async function main() {
|
|
const args = parseArgs(process.argv)
|
|
if (args.mode === 'report') {
|
|
runReport(args.label)
|
|
return
|
|
}
|
|
|
|
const isTty = process.stdout.isTTY && process.stdin.isTTY
|
|
if (!isTty && !args.forceNonTty) {
|
|
console.error(
|
|
'stdout/stdin is not a TTY — run this inside the terminal under test.\n' +
|
|
'(--force-non-tty runs anyway for script smoke tests; numbers are meaningless.)'
|
|
)
|
|
process.exit(1)
|
|
}
|
|
|
|
const cols = process.stdout.columns ?? 80
|
|
const rows = process.stdout.rows ?? 24
|
|
const targetBytes = Math.floor(args.sizeMb * 1024 * 1024)
|
|
|
|
if (process.stdin.isTTY) {
|
|
process.stdin.setRawMode(true)
|
|
}
|
|
process.stdin.resume()
|
|
process.stdin.on('data', onStdinData)
|
|
process.on('SIGINT', () => cleanupAndExit(130))
|
|
process.on('SIGTERM', () => cleanupAndExit(143))
|
|
|
|
const meta = {
|
|
benchmark: 'terminal-pipeline',
|
|
label: args.label,
|
|
timestamp: new Date().toISOString(),
|
|
terminal: {
|
|
program: process.env.TERM_PROGRAM ?? null,
|
|
version: process.env.TERM_PROGRAM_VERSION ?? null,
|
|
term: process.env.TERM ?? null,
|
|
cols,
|
|
rows,
|
|
tty: isTty
|
|
},
|
|
machine: {
|
|
platform: process.platform,
|
|
arch: process.arch,
|
|
cpu: os.cpus()[0]?.model ?? 'unknown',
|
|
release: os.release(),
|
|
node: process.version
|
|
},
|
|
config: { sizeMb: args.sizeMb, iterations: args.iterations, dsrCount: args.dsrCount }
|
|
}
|
|
|
|
console.log(
|
|
`terminal-pipeline-bench: ${meta.terminal.program ?? meta.terminal.term} ${meta.terminal.version ?? ''} (${cols}x${rows})`
|
|
)
|
|
console.log('Hands off the keyboard until the summary prints.\n')
|
|
|
|
const result = { ...meta }
|
|
|
|
if (isTty) {
|
|
process.stdout.write('Phase 1/3: DSR idle latency...\r\n')
|
|
result.dsrIdle = await measureDsrIdle(args.dsrCount)
|
|
} else {
|
|
result.dsrIdle = null
|
|
}
|
|
|
|
process.stdout.write('Phase 2/3: throughput fixtures...\r\n')
|
|
result.throughput = []
|
|
for (const name of args.fixtures) {
|
|
const fixture = buildFixture(name, targetBytes, cols, rows)
|
|
if (isTty) {
|
|
result.throughput.push(await measureThroughput(name, fixture, args.iterations))
|
|
} else {
|
|
// Smoke mode: exercise generation + streaming only.
|
|
await streamFixture(fixture.slice(0, CHUNK_BYTES))
|
|
result.throughput.push({ fixture: name, fixtureBytes: Buffer.byteLength(fixture, 'utf8') })
|
|
}
|
|
process.stdout.write(`${ESC}[0m\r\n[done: ${name}]\r\n`)
|
|
}
|
|
|
|
if (isTty && !args.skipLoad) {
|
|
process.stdout.write('Phase 3/3: DSR latency under paced agent-TUI load...\r\n')
|
|
const loadFixture = buildFixture('agent-tui', targetBytes, cols, rows)
|
|
result.dsrUnderLoad = await measureDsrUnderLoad(loadFixture, 1024 * 1024, 50)
|
|
} else {
|
|
result.dsrUnderLoad = null
|
|
}
|
|
|
|
restoreTerminal()
|
|
|
|
mkdirSync(resultsDir, { recursive: true })
|
|
const stamp = meta.timestamp.replace(/[:.]/g, '-')
|
|
const outPath = join(resultsDir, `terminal-pipeline-${args.label}-${stamp}.json`)
|
|
writeFileSync(outPath, JSON.stringify(result, null, 2))
|
|
|
|
console.log('\n=== Summary ===')
|
|
if (result.dsrIdle) {
|
|
console.log(
|
|
`DSR idle: p50 ${fmt(result.dsrIdle.p50, 2)}ms p90 ${fmt(result.dsrIdle.p90, 2)}ms p99 ${fmt(result.dsrIdle.p99, 2)}ms (timeouts: ${result.dsrIdle.timeouts})`
|
|
)
|
|
}
|
|
if (result.dsrUnderLoad) {
|
|
console.log(
|
|
`DSR under load: p50 ${fmt(result.dsrUnderLoad.p50, 2)}ms p90 ${fmt(result.dsrUnderLoad.p90, 2)}ms p99 ${fmt(result.dsrUnderLoad.p99, 2)}ms (timeouts: ${result.dsrUnderLoad.timeouts})`
|
|
)
|
|
}
|
|
for (const t of result.throughput) {
|
|
if (t.mbPerSec) {
|
|
console.log(
|
|
`${t.fixture.padEnd(14)} p50 ${fmt(t.mbPerSec.p50)} MB/s (wall p50 ${fmt(t.wallMs.p50, 0)}ms over ${(t.fixtureBytes / 1024 / 1024).toFixed(1)}MB)`
|
|
)
|
|
}
|
|
}
|
|
console.log(`\nSaved: ${outPath}`)
|
|
console.log('Compare terminals: node tests/tools/benchmarks/terminal-pipeline-bench.mjs report')
|
|
process.exit(0)
|
|
}
|
|
|
|
// Sibling analysis scripts import buildFixture; import must not start the benchmark.
|
|
const invokedDirectly =
|
|
process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href
|
|
if (invokedDirectly) {
|
|
main().catch((err) => {
|
|
restoreTerminal()
|
|
console.error(err)
|
|
process.exit(1)
|
|
})
|
|
}
|