mirror of
https://github.com/stablyai/orca.git
synced 2026-10-18 16:03:17 +00:00
fix(terminal): bound cold restore replay slices
This commit is contained in:
@@ -2250,6 +2250,102 @@
|
||||
],
|
||||
"demotionRule": "Cannot promote while Windows E2E is flaky, silently skipped, or screenshot-only."
|
||||
},
|
||||
{
|
||||
"id": "terminal-performance.cold-restore-replay-budget",
|
||||
"title": "Daemon cold restore keeps replay work and retained payloads bounded",
|
||||
"maturity": "experimental",
|
||||
"protection": "partial",
|
||||
"owner": "terminal-runtime",
|
||||
"layer": "main-daemon-unit",
|
||||
"surfaces": [
|
||||
"startup restore",
|
||||
"daemon history replay",
|
||||
"sleep and hibernation restore",
|
||||
"main-process memory"
|
||||
],
|
||||
"platforms": ["macos", "linux", "windows"],
|
||||
"providers": ["daemon", "wsl"],
|
||||
"coveredPlatforms": ["macos"],
|
||||
"coveredProviders": ["daemon"],
|
||||
"coverageNotes": "Deterministic main-process tests cover byte-bounded cache eviction and ACK release, one-at-a-time replay admission, a fixed per-turn replay budget within one large output record, UTF-16 boundary preservation, and checkpoint-only restore bypass while another replay is paused. The same HistoryReader path carries WSL context, but live WSL and cross-platform startup-scale runs remain gaps.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/issues/9971",
|
||||
"https://github.com/stablyai/orca/pull/9990",
|
||||
"https://github.com/stablyai/orca/issues/9441"
|
||||
],
|
||||
"invariant": "Cold restore must reproduce persisted terminal output while admitting at most one scratch-emulator replay, yielding after at most 64 Ki UTF-16 code units or 1,024 replay operations, keeping sticky restore payloads within 16 MiB, and allowing header-only checkpoint restores to bypass the replay queue.",
|
||||
"oracle": "Pause setImmediate during two single-batch restores larger than one replay slice and require exactly one admitted yield at a time, preserved text across a surrogate-pair slice boundary, and a concurrent header-only checkpoint restore to finish without consuming a replay slot. Cache tests require least-recently-used eviction, rejection of one oversized payload, and zero retained cache bytes after renderer ACK.",
|
||||
"commands": [
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/cold-restore-payload-cache.test.ts src/main/daemon/history-reader.test.ts src/main/daemon/terminal-history-incremental-restore.test.ts src/main/daemon/hibernation-cold-restore-repro.test.ts src/main/daemon/daemon-pty-adapter.test.ts"
|
||||
],
|
||||
"testFiles": [
|
||||
"src/main/daemon/cold-restore-payload-cache.test.ts",
|
||||
"src/main/daemon/history-reader.test.ts",
|
||||
"src/main/daemon/terminal-history-incremental-restore.test.ts",
|
||||
"src/main/daemon/hibernation-cold-restore-repro.test.ts",
|
||||
"src/main/daemon/daemon-pty-adapter.test.ts"
|
||||
],
|
||||
"assertionRefs": [
|
||||
{
|
||||
"file": "src/main/daemon/terminal-history-incremental-restore.test.ts",
|
||||
"assertions": [
|
||||
"large single-batch replays yield within the record, preserve a surrogate pair at the slice boundary, and admit only one scratch replay at a time",
|
||||
"a header-only checkpoint restore completes while an unrelated incremental replay is paused"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/daemon/cold-restore-payload-cache.test.ts",
|
||||
"assertions": [
|
||||
"least-recently-used payloads are evicted to the aggregate byte budget",
|
||||
"one payload larger than the entire budget is not retained"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/daemon/daemon-pty-adapter.test.ts",
|
||||
"assertions": [
|
||||
"StrictMode remount receives sticky cold-restore data until renderer ACK clears its retained bytes"
|
||||
]
|
||||
}
|
||||
],
|
||||
"evidenceRuns": [
|
||||
{
|
||||
"date": "2026-07-22",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/daemon/cold-restore-payload-cache.test.ts src/main/daemon/history-reader.test.ts src/main/daemon/terminal-history-incremental-restore.test.ts src/main/daemon/hibernation-cold-restore-repro.test.ts src/main/daemon/daemon-pty-adapter.test.ts",
|
||||
"result": "passed",
|
||||
"durationSeconds": 5.68,
|
||||
"summary": "Five focused files passed 150 tests, including deterministic single-record replay slicing, one-at-a-time admission, UTF-16 boundary preservation, header-only queue bypass, byte-bounded LRU eviction, and ACK cleanup."
|
||||
}
|
||||
],
|
||||
"runtimeBudget": {
|
||||
"p95Seconds": 15,
|
||||
"scope": "focused main-process cold-restore unit contract"
|
||||
},
|
||||
"flakeHistory": {
|
||||
"status": "unknown",
|
||||
"evidence": "The focused deterministic slice is new and has no CI or soak history yet."
|
||||
},
|
||||
"redGreenEvidence": {
|
||||
"status": "partial",
|
||||
"evidence": "The prior implementation had no yield inside one large batch and queued header-only restores behind the shared semaphore by inspection; an intentional pre-fix test run was not recorded."
|
||||
},
|
||||
"performanceBudget": {
|
||||
"required": true,
|
||||
"evidence": "Production admits one emulator replay globally, yields after a deterministic 64 Ki character or 1,024-operation budget even within one record, bypasses the semaphore for the common header-only final-checkpoint path, and caps sticky payloads at 16 MiB. No polling, subprocess, session inventory, or renderer wake loop is added."
|
||||
},
|
||||
"promotionCriteria": [
|
||||
"Record an intentional-break red run for both the within-record yield and header-only bypass assertions.",
|
||||
"Collect startup event-loop-delay evidence with dozens of near-cap histories on representative macOS, Windows, and Linux hardware.",
|
||||
"Add live WSL restore evidence before claiming WSL coverage."
|
||||
],
|
||||
"knownGaps": [
|
||||
"The log decoder and final headless snapshot serialization remain synchronous inside the one-at-a-time replay slot; the gate bounds replay writes, not every CPU phase.",
|
||||
"No live Electron startup-scale run currently proves first-pane paint order or end-to-end restore latency with dozens of histories.",
|
||||
"SSH, remote-runtime, relay, and mobile do not use this local daemon history reader and are unaffected."
|
||||
],
|
||||
"demotionRule": "Keep experimental or demote to protection none if output differs across replay slices, header-only restores consume a replay slot, retained payload bytes exceed the cap, or the focused gate flakes."
|
||||
},
|
||||
{
|
||||
"id": "terminal-performance.no-hot-list-sessions",
|
||||
"title": "Hot terminal interactions do not call global PTY session listing",
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
import type { HeadlessEmulator } from './headless-emulator'
|
||||
|
||||
const REPLAY_CHARS_PER_TURN = 64 * 1024
|
||||
const REPLAY_OPERATIONS_PER_TURN = 1024
|
||||
|
||||
export class ColdRestoreReplayWriter {
|
||||
private chars = 0
|
||||
private operations = 0
|
||||
|
||||
constructor(private readonly emulator: HeadlessEmulator) {}
|
||||
|
||||
async write(data: string): Promise<boolean> {
|
||||
let offset = 0
|
||||
while (offset < data.length) {
|
||||
const pendingYield = this.takeBudgetYield()
|
||||
if (pendingYield) {
|
||||
await pendingYield
|
||||
}
|
||||
const remainingBudget = REPLAY_CHARS_PER_TURN - this.chars
|
||||
let end = Math.min(data.length, offset + remainingBudget)
|
||||
// Why: xterm must receive UTF-16 surrogate pairs together when a replay slice lands between them.
|
||||
const leftCodeUnit = data.charCodeAt(end - 1)
|
||||
const rightCodeUnit = data.charCodeAt(end)
|
||||
const splitsSurrogatePair =
|
||||
end < data.length &&
|
||||
leftCodeUnit >= 0xd800 &&
|
||||
leftCodeUnit <= 0xdbff &&
|
||||
rightCodeUnit >= 0xdc00 &&
|
||||
rightCodeUnit <= 0xdfff
|
||||
if (splitsSurrogatePair) {
|
||||
end += end === offset + 1 ? 1 : -1
|
||||
}
|
||||
if (!this.emulator.writeSync(data.slice(offset, end))) {
|
||||
return false
|
||||
}
|
||||
this.chars += end - offset
|
||||
this.operations += 1
|
||||
offset = end
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
async resize(cols: number, rows: number): Promise<void> {
|
||||
const pendingYield = this.takeBudgetYield()
|
||||
if (pendingYield) {
|
||||
await pendingYield
|
||||
}
|
||||
this.emulator.resize(cols, rows)
|
||||
this.operations += 1
|
||||
}
|
||||
|
||||
async clearScrollback(): Promise<void> {
|
||||
const pendingYield = this.takeBudgetYield()
|
||||
if (pendingYield) {
|
||||
await pendingYield
|
||||
}
|
||||
this.emulator.clearScrollback()
|
||||
this.operations += 1
|
||||
}
|
||||
|
||||
private takeBudgetYield(): Promise<void> | null {
|
||||
if (this.chars < REPLAY_CHARS_PER_TURN && this.operations < REPLAY_OPERATIONS_PER_TURN) {
|
||||
return null
|
||||
}
|
||||
return new Promise<void>((resolve) => {
|
||||
setImmediate(() => {
|
||||
this.chars = 0
|
||||
this.operations = 0
|
||||
resolve()
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,14 @@
|
||||
import { join } from 'node:path'
|
||||
import { readFileSync, existsSync, readdirSync } from 'node:fs'
|
||||
import { readFile } from 'node:fs/promises'
|
||||
import { readFile, stat } from 'node:fs/promises'
|
||||
import type { SessionMeta } from './history-manager'
|
||||
import type { TerminalCheckpointFile, TerminalModes } from './types'
|
||||
import type { TerminalOscLinkRange } from '../../shared/terminal-osc-link-ranges'
|
||||
import { getHistorySessionDirName } from './history-paths'
|
||||
import { decodeTerminalHistoryLog } from './terminal-history-log'
|
||||
import { decodeTerminalHistoryLog, LOG_HEADER_BYTES } from './terminal-history-log'
|
||||
import { HeadlessEmulator } from './headless-emulator'
|
||||
import { PrioritySemaphore } from './priority-semaphore'
|
||||
import { ColdRestoreReplayWriter } from './cold-restore-replay-writer'
|
||||
|
||||
export type ColdRestoreInfo = {
|
||||
snapshotAnsi: string
|
||||
@@ -25,10 +26,6 @@ const ALT_SCREEN_OFF = '\x1b[?1049l'
|
||||
// Why: parallel pane mounts should interleave with main-process work without multiplying replay slices per turn.
|
||||
const coldRestoreReplaySemaphore = new PrioritySemaphore(1)
|
||||
|
||||
function yieldToEventLoop(): Promise<void> {
|
||||
return new Promise((resolve) => setImmediate(resolve))
|
||||
}
|
||||
|
||||
export class HistoryReader {
|
||||
private basePath: string
|
||||
|
||||
@@ -138,11 +135,20 @@ export class HistoryReader {
|
||||
checkpoint: TerminalCheckpointFile | null,
|
||||
wslDistro?: string
|
||||
): Promise<ColdRestoreInfo | null> {
|
||||
const logPath = join(sessionDir, 'output.log')
|
||||
try {
|
||||
// Why: final checkpoints leave a header-only log; they need no scarce replay slot and must not queue sleep teardown behind startup restores.
|
||||
if ((await stat(logPath)).size <= LOG_HEADER_BYTES) {
|
||||
return null
|
||||
}
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
const release = await coldRestoreReplaySemaphore.acquire(0)
|
||||
try {
|
||||
let logBuffer: Buffer
|
||||
try {
|
||||
logBuffer = await readFile(join(sessionDir, 'output.log'))
|
||||
logBuffer = await readFile(logPath)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
@@ -167,34 +173,30 @@ export class HistoryReader {
|
||||
rows: checkpoint?.rows ?? meta.rows,
|
||||
wslDistro
|
||||
})
|
||||
const replay = new ColdRestoreReplayWriter(emulator)
|
||||
try {
|
||||
if (checkpoint) {
|
||||
if (
|
||||
!emulator.writeSync(
|
||||
(checkpoint.scrollbackAnsi ?? '') +
|
||||
checkpoint.rehydrateSequences +
|
||||
checkpoint.snapshotAnsi
|
||||
)
|
||||
!(await replay.write(checkpoint.scrollbackAnsi ?? '')) ||
|
||||
!(await replay.write(checkpoint.rehydrateSequences)) ||
|
||||
!(await replay.write(checkpoint.snapshotAnsi))
|
||||
) {
|
||||
return null
|
||||
}
|
||||
emulator.setRestoredOscLinks(checkpoint.oscLinks)
|
||||
}
|
||||
for (const [batchIndex, batch] of log.batches.entries()) {
|
||||
for (const batch of log.batches) {
|
||||
for (const record of batch.records) {
|
||||
if (record.kind === 'output') {
|
||||
if (!emulator.writeSync(record.data)) {
|
||||
if (!(await replay.write(record.data))) {
|
||||
return null
|
||||
}
|
||||
} else if (record.kind === 'resize') {
|
||||
emulator.resize(record.cols, record.rows)
|
||||
await replay.resize(record.cols, record.rows)
|
||||
} else {
|
||||
emulator.clearScrollback()
|
||||
await replay.clearScrollback()
|
||||
}
|
||||
}
|
||||
if (batchIndex < log.batches.length - 1) {
|
||||
await yieldToEventLoop()
|
||||
}
|
||||
}
|
||||
const snapshot = emulator.getSnapshot()
|
||||
return this.coldRestoreInfoFromSnapshot(
|
||||
|
||||
@@ -252,12 +252,13 @@ describe('incremental terminal history restore', () => {
|
||||
expect(restore!.scrollbackAnsi).toContain('after relaunch')
|
||||
})
|
||||
|
||||
it('yields between batches and admits only one replay at a time', async () => {
|
||||
it('bounds large single-batch replay slices and admits only one replay at a time', async () => {
|
||||
const secondSessionId = `${SESSION_ID}-second`
|
||||
await manager.openSession(secondSessionId, { cwd: '/home/user', cols: 80, rows: 24 })
|
||||
for (const sessionId of [SESSION_ID, secondSessionId]) {
|
||||
await manager.appendIncrements(sessionId, 1, [{ kind: 'output', data: 'first\r\n' }])
|
||||
await manager.appendIncrements(sessionId, 2, [{ kind: 'output', data: 'second\r\n' }])
|
||||
await manager.appendIncrements(sessionId, 1, [
|
||||
{ kind: 'output', data: `${'x'.repeat(64 * 1024 - 1)}😀second\r\n` }
|
||||
])
|
||||
}
|
||||
|
||||
const pendingYields: (() => void)[] = []
|
||||
@@ -269,20 +270,55 @@ describe('incremental terminal history restore', () => {
|
||||
return {} as NodeJS.Immediate
|
||||
}) as typeof setImmediate)
|
||||
|
||||
const firstReplay = reader.detectColdRestore(SESSION_ID)
|
||||
const secondReplay = reader.detectColdRestore(secondSessionId)
|
||||
try {
|
||||
const firstReplay = reader.detectColdRestore(SESSION_ID)
|
||||
const secondReplay = reader.detectColdRestore(secondSessionId)
|
||||
await vi.waitFor(() => expect(pendingYields).toHaveLength(1))
|
||||
|
||||
pendingYields.shift()!()
|
||||
expect((await firstReplay)?.scrollbackAnsi).toContain('second')
|
||||
expect((await firstReplay)?.scrollbackAnsi).toContain('😀second')
|
||||
await vi.waitFor(() => expect(pendingYields).toHaveLength(1))
|
||||
|
||||
pendingYields.shift()!()
|
||||
expect((await secondReplay)?.scrollbackAnsi).toContain('second')
|
||||
expect((await secondReplay)?.scrollbackAnsi).toContain('😀second')
|
||||
expect(pendingYields).toHaveLength(0)
|
||||
} finally {
|
||||
for (const resume of pendingYields.splice(0)) {
|
||||
resume()
|
||||
}
|
||||
immediateSpy.mockRestore()
|
||||
await Promise.allSettled([firstReplay, secondReplay])
|
||||
}
|
||||
})
|
||||
|
||||
it('does not queue header-only checkpoint restores behind replay work', async () => {
|
||||
const checkpointOnlySessionId = `${SESSION_ID}-checkpoint-only`
|
||||
await manager.openSession(checkpointOnlySessionId, { cwd: '/home/user', cols: 80, rows: 24 })
|
||||
await manager.checkpoint(checkpointOnlySessionId, snapshotOf(['checkpoint only\r\n']))
|
||||
await manager.appendIncrements(SESSION_ID, 1, [
|
||||
{ kind: 'output', data: `${'x'.repeat(64 * 1024)}slow replay\r\n` }
|
||||
])
|
||||
|
||||
const pendingYields: (() => void)[] = []
|
||||
const immediateSpy = vi.spyOn(globalThis, 'setImmediate').mockImplementation(((
|
||||
callback: (...args: unknown[]) => void,
|
||||
...args: unknown[]
|
||||
) => {
|
||||
pendingYields.push(() => callback(...args))
|
||||
return {} as NodeJS.Immediate
|
||||
}) as typeof setImmediate)
|
||||
|
||||
const replay = reader.detectColdRestore(SESSION_ID)
|
||||
try {
|
||||
await vi.waitFor(() => expect(pendingYields).toHaveLength(1))
|
||||
const checkpointOnlyRestore = await reader.detectColdRestore(checkpointOnlySessionId)
|
||||
|
||||
expect(checkpointOnlyRestore?.scrollbackAnsi).toContain('checkpoint only')
|
||||
expect(pendingYields).toHaveLength(1)
|
||||
} finally {
|
||||
pendingYields.shift()?.()
|
||||
immediateSpy.mockRestore()
|
||||
await replay
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user