fix(terminal): bound cold restore replay slices

This commit is contained in:
Brennan Benson
2026-07-22 15:42:55 -07:00
parent 047c954219
commit 846ad263cd
4 changed files with 233 additions and 26 deletions
+96
View File
@@ -2250,6 +2250,102 @@
],
"demotionRule": "Cannot promote while Windows E2E is flaky, silently skipped, or screenshot-only."
},
{
"id": "terminal-performance.cold-restore-replay-budget",
"title": "Daemon cold restore keeps replay work and retained payloads bounded",
"maturity": "experimental",
"protection": "partial",
"owner": "terminal-runtime",
"layer": "main-daemon-unit",
"surfaces": [
"startup restore",
"daemon history replay",
"sleep and hibernation restore",
"main-process memory"
],
"platforms": ["macos", "linux", "windows"],
"providers": ["daemon", "wsl"],
"coveredPlatforms": ["macos"],
"coveredProviders": ["daemon"],
"coverageNotes": "Deterministic main-process tests cover byte-bounded cache eviction and ACK release, one-at-a-time replay admission, a fixed per-turn replay budget within one large output record, UTF-16 boundary preservation, and checkpoint-only restore bypass while another replay is paused. The same HistoryReader path carries WSL context, but live WSL and cross-platform startup-scale runs remain gaps.",
"motivatingLinks": [
"https://github.com/stablyai/orca/issues/9971",
"https://github.com/stablyai/orca/pull/9990",
"https://github.com/stablyai/orca/issues/9441"
],
"invariant": "Cold restore must reproduce persisted terminal output while admitting at most one scratch-emulator replay, yielding after at most 64 Ki UTF-16 code units or 1,024 replay operations, keeping sticky restore payloads within 16 MiB, and allowing header-only checkpoint restores to bypass the replay queue.",
"oracle": "Pause setImmediate during two single-batch restores larger than one replay slice and require exactly one admitted yield at a time, preserved text across a surrogate-pair slice boundary, and a concurrent header-only checkpoint restore to finish without consuming a replay slot. Cache tests require least-recently-used eviction, rejection of one oversized payload, and zero retained cache bytes after renderer ACK.",
"commands": [
"pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/cold-restore-payload-cache.test.ts src/main/daemon/history-reader.test.ts src/main/daemon/terminal-history-incremental-restore.test.ts src/main/daemon/hibernation-cold-restore-repro.test.ts src/main/daemon/daemon-pty-adapter.test.ts"
],
"testFiles": [
"src/main/daemon/cold-restore-payload-cache.test.ts",
"src/main/daemon/history-reader.test.ts",
"src/main/daemon/terminal-history-incremental-restore.test.ts",
"src/main/daemon/hibernation-cold-restore-repro.test.ts",
"src/main/daemon/daemon-pty-adapter.test.ts"
],
"assertionRefs": [
{
"file": "src/main/daemon/terminal-history-incremental-restore.test.ts",
"assertions": [
"large single-batch replays yield within the record, preserve a surrogate pair at the slice boundary, and admit only one scratch replay at a time",
"a header-only checkpoint restore completes while an unrelated incremental replay is paused"
]
},
{
"file": "src/main/daemon/cold-restore-payload-cache.test.ts",
"assertions": [
"least-recently-used payloads are evicted to the aggregate byte budget",
"one payload larger than the entire budget is not retained"
]
},
{
"file": "src/main/daemon/daemon-pty-adapter.test.ts",
"assertions": [
"StrictMode remount receives sticky cold-restore data until renderer ACK clears its retained bytes"
]
}
],
"evidenceRuns": [
{
"date": "2026-07-22",
"runner": "local",
"platform": "macos",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/cold-restore-payload-cache.test.ts src/main/daemon/history-reader.test.ts src/main/daemon/terminal-history-incremental-restore.test.ts src/main/daemon/hibernation-cold-restore-repro.test.ts src/main/daemon/daemon-pty-adapter.test.ts",
"result": "passed",
"durationSeconds": 5.68,
"summary": "Five focused files passed 150 tests, including deterministic single-record replay slicing, one-at-a-time admission, UTF-16 boundary preservation, header-only queue bypass, byte-bounded LRU eviction, and ACK cleanup."
}
],
"runtimeBudget": {
"p95Seconds": 15,
"scope": "focused main-process cold-restore unit contract"
},
"flakeHistory": {
"status": "unknown",
"evidence": "The focused deterministic slice is new and has no CI or soak history yet."
},
"redGreenEvidence": {
"status": "partial",
"evidence": "The prior implementation had no yield inside one large batch and queued header-only restores behind the shared semaphore by inspection; an intentional pre-fix test run was not recorded."
},
"performanceBudget": {
"required": true,
"evidence": "Production admits one emulator replay globally, yields after a deterministic 64 Ki character or 1,024-operation budget even within one record, bypasses the semaphore for the common header-only final-checkpoint path, and caps sticky payloads at 16 MiB. No polling, subprocess, session inventory, or renderer wake loop is added."
},
"promotionCriteria": [
"Record an intentional-break red run for both the within-record yield and header-only bypass assertions.",
"Collect startup event-loop-delay evidence with dozens of near-cap histories on representative macOS, Windows, and Linux hardware.",
"Add live WSL restore evidence before claiming WSL coverage."
],
"knownGaps": [
"The log decoder and final headless snapshot serialization remain synchronous inside the one-at-a-time replay slot; the gate bounds replay writes, not every CPU phase.",
"No live Electron startup-scale run currently proves first-pane paint order or end-to-end restore latency with dozens of histories.",
"SSH, remote-runtime, relay, and mobile do not use this local daemon history reader and are unaffected."
],
"demotionRule": "Keep experimental or demote to protection none if output differs across replay slices, header-only restores consume a replay slot, retained payload bytes exceed the cap, or the focused gate flakes."
},
{
"id": "terminal-performance.no-hot-list-sessions",
"title": "Hot terminal interactions do not call global PTY session listing",
@@ -0,0 +1,73 @@
import type { HeadlessEmulator } from './headless-emulator'
const REPLAY_CHARS_PER_TURN = 64 * 1024
const REPLAY_OPERATIONS_PER_TURN = 1024
export class ColdRestoreReplayWriter {
private chars = 0
private operations = 0
constructor(private readonly emulator: HeadlessEmulator) {}
async write(data: string): Promise<boolean> {
let offset = 0
while (offset < data.length) {
const pendingYield = this.takeBudgetYield()
if (pendingYield) {
await pendingYield
}
const remainingBudget = REPLAY_CHARS_PER_TURN - this.chars
let end = Math.min(data.length, offset + remainingBudget)
// Why: xterm must receive UTF-16 surrogate pairs together when a replay slice lands between them.
const leftCodeUnit = data.charCodeAt(end - 1)
const rightCodeUnit = data.charCodeAt(end)
const splitsSurrogatePair =
end < data.length &&
leftCodeUnit >= 0xd800 &&
leftCodeUnit <= 0xdbff &&
rightCodeUnit >= 0xdc00 &&
rightCodeUnit <= 0xdfff
if (splitsSurrogatePair) {
end += end === offset + 1 ? 1 : -1
}
if (!this.emulator.writeSync(data.slice(offset, end))) {
return false
}
this.chars += end - offset
this.operations += 1
offset = end
}
return true
}
async resize(cols: number, rows: number): Promise<void> {
const pendingYield = this.takeBudgetYield()
if (pendingYield) {
await pendingYield
}
this.emulator.resize(cols, rows)
this.operations += 1
}
async clearScrollback(): Promise<void> {
const pendingYield = this.takeBudgetYield()
if (pendingYield) {
await pendingYield
}
this.emulator.clearScrollback()
this.operations += 1
}
private takeBudgetYield(): Promise<void> | null {
if (this.chars < REPLAY_CHARS_PER_TURN && this.operations < REPLAY_OPERATIONS_PER_TURN) {
return null
}
return new Promise<void>((resolve) => {
setImmediate(() => {
this.chars = 0
this.operations = 0
resolve()
})
})
}
}
+21 -19
View File
@@ -1,13 +1,14 @@
import { join } from 'node:path'
import { readFileSync, existsSync, readdirSync } from 'node:fs'
import { readFile } from 'node:fs/promises'
import { readFile, stat } from 'node:fs/promises'
import type { SessionMeta } from './history-manager'
import type { TerminalCheckpointFile, TerminalModes } from './types'
import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges'
import { getHistorySessionDirName } from './history-paths'
import { decodeTerminalHistoryLog } from './terminal-history-log'
import { decodeTerminalHistoryLog, LOG_HEADER_BYTES } from './terminal-history-log'
import { HeadlessEmulator } from './headless-emulator'
import { PrioritySemaphore } from './priority-semaphore'
import { ColdRestoreReplayWriter } from './cold-restore-replay-writer'
export type ColdRestoreInfo = {
snapshotAnsi: string
@@ -25,10 +26,6 @@ const ALT_SCREEN_OFF = '\x1b[?1049l'
// Why: parallel pane mounts should interleave with main-process work without multiplying replay slices per turn.
const coldRestoreReplaySemaphore = new PrioritySemaphore(1)
function yieldToEventLoop(): Promise<void> {
return new Promise((resolve) => setImmediate(resolve))
}
export class HistoryReader {
private basePath: string
@@ -138,11 +135,20 @@ export class HistoryReader {
checkpoint: TerminalCheckpointFile | null,
wslDistro?: string
): Promise<ColdRestoreInfo | null> {
const logPath = join(sessionDir, 'output.log')
try {
// Why: final checkpoints leave a header-only log; they need no scarce replay slot and must not queue sleep teardown behind startup restores.
if ((await stat(logPath)).size <= LOG_HEADER_BYTES) {
return null
}
} catch {
return null
}
const release = await coldRestoreReplaySemaphore.acquire(0)
try {
let logBuffer: Buffer
try {
logBuffer = await readFile(join(sessionDir, 'output.log'))
logBuffer = await readFile(logPath)
} catch {
return null
}
@@ -167,34 +173,30 @@ export class HistoryReader {
rows: checkpoint?.rows ?? meta.rows,
wslDistro
})
const replay = new ColdRestoreReplayWriter(emulator)
try {
if (checkpoint) {
if (
!emulator.writeSync(
(checkpoint.scrollbackAnsi ?? '') +
checkpoint.rehydrateSequences +
checkpoint.snapshotAnsi
)
!(await replay.write(checkpoint.scrollbackAnsi ?? '')) ||
!(await replay.write(checkpoint.rehydrateSequences)) ||
!(await replay.write(checkpoint.snapshotAnsi))
) {
return null
}
emulator.setRestoredOscLinks(checkpoint.oscLinks)
}
for (const [batchIndex, batch] of log.batches.entries()) {
for (const batch of log.batches) {
for (const record of batch.records) {
if (record.kind === 'output') {
if (!emulator.writeSync(record.data)) {
if (!(await replay.write(record.data))) {
return null
}
} else if (record.kind === 'resize') {
emulator.resize(record.cols, record.rows)
await replay.resize(record.cols, record.rows)
} else {
emulator.clearScrollback()
await replay.clearScrollback()
}
}
if (batchIndex < log.batches.length - 1) {
await yieldToEventLoop()
}
}
const snapshot = emulator.getSnapshot()
return this.coldRestoreInfoFromSnapshot(
@@ -252,12 +252,13 @@ describe('incremental terminal history restore', () => {
expect(restore!.scrollbackAnsi).toContain('after relaunch')
})
it('yields between batches and admits only one replay at a time', async () => {
it('bounds large single-batch replay slices and admits only one replay at a time', async () => {
const secondSessionId = `${SESSION_ID}-second`
await manager.openSession(secondSessionId, { cwd: '/home/user', cols: 80, rows: 24 })
for (const sessionId of [SESSION_ID, secondSessionId]) {
await manager.appendIncrements(sessionId, 1, [{ kind: 'output', data: 'first\r\n' }])
await manager.appendIncrements(sessionId, 2, [{ kind: 'output', data: 'second\r\n' }])
await manager.appendIncrements(sessionId, 1, [
{ kind: 'output', data: `${'x'.repeat(64 * 1024 - 1)}😀second\r\n` }
])
}
const pendingYields: (() => void)[] = []
@@ -269,20 +270,55 @@ describe('incremental terminal history restore', () => {
return {} as NodeJS.Immediate
}) as typeof setImmediate)
const firstReplay = reader.detectColdRestore(SESSION_ID)
const secondReplay = reader.detectColdRestore(secondSessionId)
try {
const firstReplay = reader.detectColdRestore(SESSION_ID)
const secondReplay = reader.detectColdRestore(secondSessionId)
await vi.waitFor(() => expect(pendingYields).toHaveLength(1))
pendingYields.shift()!()
expect((await firstReplay)?.scrollbackAnsi).toContain('second')
expect((await firstReplay)?.scrollbackAnsi).toContain('😀second')
await vi.waitFor(() => expect(pendingYields).toHaveLength(1))
pendingYields.shift()!()
expect((await secondReplay)?.scrollbackAnsi).toContain('second')
expect((await secondReplay)?.scrollbackAnsi).toContain('😀second')
expect(pendingYields).toHaveLength(0)
} finally {
for (const resume of pendingYields.splice(0)) {
resume()
}
immediateSpy.mockRestore()
await Promise.allSettled([firstReplay, secondReplay])
}
})
it('does not queue header-only checkpoint restores behind replay work', async () => {
const checkpointOnlySessionId = `${SESSION_ID}-checkpoint-only`
await manager.openSession(checkpointOnlySessionId, { cwd: '/home/user', cols: 80, rows: 24 })
await manager.checkpoint(checkpointOnlySessionId, snapshotOf(['checkpoint only\r\n']))
await manager.appendIncrements(SESSION_ID, 1, [
{ kind: 'output', data: `${'x'.repeat(64 * 1024)}slow replay\r\n` }
])
const pendingYields: (() => void)[] = []
const immediateSpy = vi.spyOn(globalThis, 'setImmediate').mockImplementation(((
callback: (...args: unknown[]) => void,
...args: unknown[]
) => {
pendingYields.push(() => callback(...args))
return {} as NodeJS.Immediate
}) as typeof setImmediate)
const replay = reader.detectColdRestore(SESSION_ID)
try {
await vi.waitFor(() => expect(pendingYields).toHaveLength(1))
const checkpointOnlyRestore = await reader.detectColdRestore(checkpointOnlySessionId)
expect(checkpointOnlyRestore?.scrollbackAnsi).toContain('checkpoint only')
expect(pendingYields).toHaveLength(1)
} finally {
pendingYields.shift()?.()
immediateSpy.mockRestore()
await replay
}
})
})