diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 595ee5a6b97..8c030818cbc 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -2250,6 +2250,102 @@ ], "demotionRule": "Cannot promote while Windows E2E is flaky, silently skipped, or screenshot-only." }, + { + "id": "terminal-performance.cold-restore-replay-budget", + "title": "Daemon cold restore keeps replay work and retained payloads bounded", + "maturity": "experimental", + "protection": "partial", + "owner": "terminal-runtime", + "layer": "main-daemon-unit", + "surfaces": [ + "startup restore", + "daemon history replay", + "sleep and hibernation restore", + "main-process memory" + ], + "platforms": ["macos", "linux", "windows"], + "providers": ["daemon", "wsl"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["daemon"], + "coverageNotes": "Deterministic main-process tests cover byte-bounded cache eviction and ACK release, one-at-a-time replay admission, a fixed per-turn replay budget within one large output record, UTF-16 boundary preservation, and checkpoint-only restore bypass while another replay is paused. The same HistoryReader path carries WSL context, but live WSL and cross-platform startup-scale runs remain gaps.", + "motivatingLinks": [ + "https://github.com/stablyai/orca/issues/9971", + "https://github.com/stablyai/orca/pull/9990", + "https://github.com/stablyai/orca/issues/9441" + ], + "invariant": "Cold restore must reproduce persisted terminal output while admitting at most one scratch-emulator replay, yielding after at most 64 Ki UTF-16 code units or 1,024 replay operations, keeping sticky restore payloads within 16 MiB, and allowing header-only checkpoint restores to bypass the replay queue.", + "oracle": "Pause setImmediate during two single-batch restores larger than one replay slice and require exactly one admitted yield at a time, preserved text across a surrogate-pair slice boundary, and a concurrent header-only checkpoint restore to finish without consuming a replay slot. Cache tests require least-recently-used eviction, rejection of one oversized payload, and zero retained cache bytes after renderer ACK.", + "commands": [ + "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/cold-restore-payload-cache.test.ts src/main/daemon/history-reader.test.ts src/main/daemon/terminal-history-incremental-restore.test.ts src/main/daemon/hibernation-cold-restore-repro.test.ts src/main/daemon/daemon-pty-adapter.test.ts" + ], + "testFiles": [ + "src/main/daemon/cold-restore-payload-cache.test.ts", + "src/main/daemon/history-reader.test.ts", + "src/main/daemon/terminal-history-incremental-restore.test.ts", + "src/main/daemon/hibernation-cold-restore-repro.test.ts", + "src/main/daemon/daemon-pty-adapter.test.ts" + ], + "assertionRefs": [ + { + "file": "src/main/daemon/terminal-history-incremental-restore.test.ts", + "assertions": [ + "large single-batch replays yield within the record, preserve a surrogate pair at the slice boundary, and admit only one scratch replay at a time", + "a header-only checkpoint restore completes while an unrelated incremental replay is paused" + ] + }, + { + "file": "src/main/daemon/cold-restore-payload-cache.test.ts", + "assertions": [ + "least-recently-used payloads are evicted to the aggregate byte budget", + "one payload larger than the entire budget is not retained" + ] + }, + { + "file": "src/main/daemon/daemon-pty-adapter.test.ts", + "assertions": [ + "StrictMode remount receives sticky cold-restore data until renderer ACK clears its retained bytes" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-07-22", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/cold-restore-payload-cache.test.ts src/main/daemon/history-reader.test.ts src/main/daemon/terminal-history-incremental-restore.test.ts src/main/daemon/hibernation-cold-restore-repro.test.ts src/main/daemon/daemon-pty-adapter.test.ts", + "result": "passed", + "durationSeconds": 5.68, + "summary": "Five focused files passed 150 tests, including deterministic single-record replay slicing, one-at-a-time admission, UTF-16 boundary preservation, header-only queue bypass, byte-bounded LRU eviction, and ACK cleanup." + } + ], + "runtimeBudget": { + "p95Seconds": 15, + "scope": "focused main-process cold-restore unit contract" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "The focused deterministic slice is new and has no CI or soak history yet." + }, + "redGreenEvidence": { + "status": "partial", + "evidence": "The prior implementation had no yield inside one large batch and queued header-only restores behind the shared semaphore by inspection; an intentional pre-fix test run was not recorded." + }, + "performanceBudget": { + "required": true, + "evidence": "Production admits one emulator replay globally, yields after a deterministic 64 Ki character or 1,024-operation budget even within one record, bypasses the semaphore for the common header-only final-checkpoint path, and caps sticky payloads at 16 MiB. No polling, subprocess, session inventory, or renderer wake loop is added." + }, + "promotionCriteria": [ + "Record an intentional-break red run for both the within-record yield and header-only bypass assertions.", + "Collect startup event-loop-delay evidence with dozens of near-cap histories on representative macOS, Windows, and Linux hardware.", + "Add live WSL restore evidence before claiming WSL coverage." + ], + "knownGaps": [ + "The log decoder and final headless snapshot serialization remain synchronous inside the one-at-a-time replay slot; the gate bounds replay writes, not every CPU phase.", + "No live Electron startup-scale run currently proves first-pane paint order or end-to-end restore latency with dozens of histories.", + "SSH, remote-runtime, relay, and mobile do not use this local daemon history reader and are unaffected." + ], + "demotionRule": "Keep experimental or demote to protection none if output differs across replay slices, header-only restores consume a replay slot, retained payload bytes exceed the cap, or the focused gate flakes." + }, { "id": "terminal-performance.no-hot-list-sessions", "title": "Hot terminal interactions do not call global PTY session listing", diff --git a/src/main/daemon/cold-restore-replay-writer.ts b/src/main/daemon/cold-restore-replay-writer.ts new file mode 100644 index 00000000000..c2c8c95d03b --- /dev/null +++ b/src/main/daemon/cold-restore-replay-writer.ts @@ -0,0 +1,73 @@ +import type { HeadlessEmulator } from './headless-emulator' + +const REPLAY_CHARS_PER_TURN = 64 * 1024 +const REPLAY_OPERATIONS_PER_TURN = 1024 + +export class ColdRestoreReplayWriter { + private chars = 0 + private operations = 0 + + constructor(private readonly emulator: HeadlessEmulator) {} + + async write(data: string): Promise { + let offset = 0 + while (offset < data.length) { + const pendingYield = this.takeBudgetYield() + if (pendingYield) { + await pendingYield + } + const remainingBudget = REPLAY_CHARS_PER_TURN - this.chars + let end = Math.min(data.length, offset + remainingBudget) + // Why: xterm must receive UTF-16 surrogate pairs together when a replay slice lands between them. + const leftCodeUnit = data.charCodeAt(end - 1) + const rightCodeUnit = data.charCodeAt(end) + const splitsSurrogatePair = + end < data.length && + leftCodeUnit >= 0xd800 && + leftCodeUnit <= 0xdbff && + rightCodeUnit >= 0xdc00 && + rightCodeUnit <= 0xdfff + if (splitsSurrogatePair) { + end += end === offset + 1 ? 1 : -1 + } + if (!this.emulator.writeSync(data.slice(offset, end))) { + return false + } + this.chars += end - offset + this.operations += 1 + offset = end + } + return true + } + + async resize(cols: number, rows: number): Promise { + const pendingYield = this.takeBudgetYield() + if (pendingYield) { + await pendingYield + } + this.emulator.resize(cols, rows) + this.operations += 1 + } + + async clearScrollback(): Promise { + const pendingYield = this.takeBudgetYield() + if (pendingYield) { + await pendingYield + } + this.emulator.clearScrollback() + this.operations += 1 + } + + private takeBudgetYield(): Promise | null { + if (this.chars < REPLAY_CHARS_PER_TURN && this.operations < REPLAY_OPERATIONS_PER_TURN) { + return null + } + return new Promise((resolve) => { + setImmediate(() => { + this.chars = 0 + this.operations = 0 + resolve() + }) + }) + } +} diff --git a/src/main/daemon/history-reader.ts b/src/main/daemon/history-reader.ts index c47f432af08..e68eca75ecd 100644 --- a/src/main/daemon/history-reader.ts +++ b/src/main/daemon/history-reader.ts @@ -1,13 +1,14 @@ import { join } from 'node:path' import { readFileSync, existsSync, readdirSync } from 'node:fs' -import { readFile } from 'node:fs/promises' +import { readFile, stat } from 'node:fs/promises' import type { SessionMeta } from './history-manager' import type { TerminalCheckpointFile, TerminalModes } from './types' import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges' import { getHistorySessionDirName } from './history-paths' -import { decodeTerminalHistoryLog } from './terminal-history-log' +import { decodeTerminalHistoryLog, LOG_HEADER_BYTES } from './terminal-history-log' import { HeadlessEmulator } from './headless-emulator' import { PrioritySemaphore } from './priority-semaphore' +import { ColdRestoreReplayWriter } from './cold-restore-replay-writer' export type ColdRestoreInfo = { snapshotAnsi: string @@ -25,10 +26,6 @@ const ALT_SCREEN_OFF = '\x1b[?1049l' // Why: parallel pane mounts should interleave with main-process work without multiplying replay slices per turn. const coldRestoreReplaySemaphore = new PrioritySemaphore(1) -function yieldToEventLoop(): Promise { - return new Promise((resolve) => setImmediate(resolve)) -} - export class HistoryReader { private basePath: string @@ -138,11 +135,20 @@ export class HistoryReader { checkpoint: TerminalCheckpointFile | null, wslDistro?: string ): Promise { + const logPath = join(sessionDir, 'output.log') + try { + // Why: final checkpoints leave a header-only log; they need no scarce replay slot and must not queue sleep teardown behind startup restores. + if ((await stat(logPath)).size <= LOG_HEADER_BYTES) { + return null + } + } catch { + return null + } const release = await coldRestoreReplaySemaphore.acquire(0) try { let logBuffer: Buffer try { - logBuffer = await readFile(join(sessionDir, 'output.log')) + logBuffer = await readFile(logPath) } catch { return null } @@ -167,34 +173,30 @@ export class HistoryReader { rows: checkpoint?.rows ?? meta.rows, wslDistro }) + const replay = new ColdRestoreReplayWriter(emulator) try { if (checkpoint) { if ( - !emulator.writeSync( - (checkpoint.scrollbackAnsi ?? '') + - checkpoint.rehydrateSequences + - checkpoint.snapshotAnsi - ) + !(await replay.write(checkpoint.scrollbackAnsi ?? '')) || + !(await replay.write(checkpoint.rehydrateSequences)) || + !(await replay.write(checkpoint.snapshotAnsi)) ) { return null } emulator.setRestoredOscLinks(checkpoint.oscLinks) } - for (const [batchIndex, batch] of log.batches.entries()) { + for (const batch of log.batches) { for (const record of batch.records) { if (record.kind === 'output') { - if (!emulator.writeSync(record.data)) { + if (!(await replay.write(record.data))) { return null } } else if (record.kind === 'resize') { - emulator.resize(record.cols, record.rows) + await replay.resize(record.cols, record.rows) } else { - emulator.clearScrollback() + await replay.clearScrollback() } } - if (batchIndex < log.batches.length - 1) { - await yieldToEventLoop() - } } const snapshot = emulator.getSnapshot() return this.coldRestoreInfoFromSnapshot( diff --git a/src/main/daemon/terminal-history-incremental-restore.test.ts b/src/main/daemon/terminal-history-incremental-restore.test.ts index 9411342976e..8c43e0e82de 100644 --- a/src/main/daemon/terminal-history-incremental-restore.test.ts +++ b/src/main/daemon/terminal-history-incremental-restore.test.ts @@ -252,12 +252,13 @@ describe('incremental terminal history restore', () => { expect(restore!.scrollbackAnsi).toContain('after relaunch') }) - it('yields between batches and admits only one replay at a time', async () => { + it('bounds large single-batch replay slices and admits only one replay at a time', async () => { const secondSessionId = `${SESSION_ID}-second` await manager.openSession(secondSessionId, { cwd: '/home/user', cols: 80, rows: 24 }) for (const sessionId of [SESSION_ID, secondSessionId]) { - await manager.appendIncrements(sessionId, 1, [{ kind: 'output', data: 'first\r\n' }]) - await manager.appendIncrements(sessionId, 2, [{ kind: 'output', data: 'second\r\n' }]) + await manager.appendIncrements(sessionId, 1, [ + { kind: 'output', data: `${'x'.repeat(64 * 1024 - 1)}😀second\r\n` } + ]) } const pendingYields: (() => void)[] = [] @@ -269,20 +270,55 @@ describe('incremental terminal history restore', () => { return {} as NodeJS.Immediate }) as typeof setImmediate) + const firstReplay = reader.detectColdRestore(SESSION_ID) + const secondReplay = reader.detectColdRestore(secondSessionId) try { - const firstReplay = reader.detectColdRestore(SESSION_ID) - const secondReplay = reader.detectColdRestore(secondSessionId) await vi.waitFor(() => expect(pendingYields).toHaveLength(1)) pendingYields.shift()!() - expect((await firstReplay)?.scrollbackAnsi).toContain('second') + expect((await firstReplay)?.scrollbackAnsi).toContain('😀second') await vi.waitFor(() => expect(pendingYields).toHaveLength(1)) pendingYields.shift()!() - expect((await secondReplay)?.scrollbackAnsi).toContain('second') + expect((await secondReplay)?.scrollbackAnsi).toContain('😀second') expect(pendingYields).toHaveLength(0) } finally { + for (const resume of pendingYields.splice(0)) { + resume() + } immediateSpy.mockRestore() + await Promise.allSettled([firstReplay, secondReplay]) + } + }) + + it('does not queue header-only checkpoint restores behind replay work', async () => { + const checkpointOnlySessionId = `${SESSION_ID}-checkpoint-only` + await manager.openSession(checkpointOnlySessionId, { cwd: '/home/user', cols: 80, rows: 24 }) + await manager.checkpoint(checkpointOnlySessionId, snapshotOf(['checkpoint only\r\n'])) + await manager.appendIncrements(SESSION_ID, 1, [ + { kind: 'output', data: `${'x'.repeat(64 * 1024)}slow replay\r\n` } + ]) + + const pendingYields: (() => void)[] = [] + const immediateSpy = vi.spyOn(globalThis, 'setImmediate').mockImplementation((( + callback: (...args: unknown[]) => void, + ...args: unknown[] + ) => { + pendingYields.push(() => callback(...args)) + return {} as NodeJS.Immediate + }) as typeof setImmediate) + + const replay = reader.detectColdRestore(SESSION_ID) + try { + await vi.waitFor(() => expect(pendingYields).toHaveLength(1)) + const checkpointOnlyRestore = await reader.detectColdRestore(checkpointOnlySessionId) + + expect(checkpointOnlyRestore?.scrollbackAnsi).toContain('checkpoint only') + expect(pendingYields).toHaveLength(1) + } finally { + pendingYields.shift()?.() + immediateSpy.mockRestore() + await replay } }) })