diff --git a/src/main/daemon/daemon-scrollback-budget.test.ts b/src/main/daemon/daemon-scrollback-budget.test.ts new file mode 100644 index 00000000000..864bde04d5a --- /dev/null +++ b/src/main/daemon/daemon-scrollback-budget.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest' +import { + computeSessionScrollbackRows, + DAEMON_SCROLLBACK_BUDGET_ROWS, + DAEMON_SCROLLBACK_FULL_ROWS, + DAEMON_SCROLLBACK_MIN_ROWS +} from './daemon-scrollback-budget' + +describe('computeSessionScrollbackRows', () => { + it('keeps full depth while the budget is not binding', () => { + expect(computeSessionScrollbackRows(1, { budgetRows: 100_000 })).toBe( + DAEMON_SCROLLBACK_FULL_ROWS + ) + expect(computeSessionScrollbackRows(20, { budgetRows: 100_000 })).toBe( + DAEMON_SCROLLBACK_FULL_ROWS + ) + }) + + it('splits the budget evenly once it binds', () => { + expect(computeSessionScrollbackRows(40, { budgetRows: 100_000 })).toBe(2500) + expect(computeSessionScrollbackRows(50, { budgetRows: 100_000 })).toBe(2000) + }) + + it('never drops below the floor', () => { + expect(computeSessionScrollbackRows(10_000, { budgetRows: 100_000 })).toBe( + DAEMON_SCROLLBACK_MIN_ROWS + ) + }) + + it('holds total retained rows within the budget until the floor binds', () => { + for (const sessions of [30, 60, 120, 200]) { + const total = computeSessionScrollbackRows(sessions, { budgetRows: 100_000 }) * sessions + expect(total).toBeLessThanOrEqual(100_000) + } + }) + + it('lets the floor override the budget, trading bounded growth for a usable restore', () => { + // Past ~200 sessions the even split would fall below MIN_ROWS, which reads as data loss on + // reattach. Growth past that point is deliberate but shallow: 500 rows, not 5000. + const perSession = computeSessionScrollbackRows(1000, { budgetRows: 100_000 }) + expect(perSession).toBe(DAEMON_SCROLLBACK_MIN_ROWS) + expect(perSession * 1000).toBeGreaterThan(100_000) + expect(perSession * 1000).toBeLessThan(DAEMON_SCROLLBACK_FULL_ROWS * 1000) + }) + + it('caps the session count that broke the Windows host well below its old retention', () => { + // 100+ live terminals previously retained 100 x 5000 = 500k rows (~1 GB of grid). + const perSession = computeSessionScrollbackRows(100) + expect(perSession).toBeLessThan(DAEMON_SCROLLBACK_FULL_ROWS) + expect(perSession * 100).toBeLessThanOrEqual(DAEMON_SCROLLBACK_BUDGET_ROWS) + }) + + it('treats a degenerate session count as a single session', () => { + expect(computeSessionScrollbackRows(0)).toBe(DAEMON_SCROLLBACK_FULL_ROWS) + expect(computeSessionScrollbackRows(Number.NaN)).toBe(DAEMON_SCROLLBACK_FULL_ROWS) + }) + + it('accepts an env override and ignores an unusable one', () => { + const env = { ORCA_DAEMON_SCROLLBACK_BUDGET_ROWS: '20000' } as NodeJS.ProcessEnv + expect(computeSessionScrollbackRows(10, { env })).toBe(2000) + // Why: an unparseable or non-positive override must not disable the budget — unbounded is the bug. + for (const raw of ['nonsense', '0', '-5']) { + const bad = { ORCA_DAEMON_SCROLLBACK_BUDGET_ROWS: raw } as NodeJS.ProcessEnv + expect(computeSessionScrollbackRows(100, { env: bad })).toBe( + computeSessionScrollbackRows(100, { budgetRows: DAEMON_SCROLLBACK_BUDGET_ROWS }) + ) + } + }) +}) diff --git a/src/main/daemon/daemon-scrollback-budget.ts b/src/main/daemon/daemon-scrollback-budget.ts new file mode 100644 index 00000000000..81025d9623c --- /dev/null +++ b/src/main/daemon/daemon-scrollback-budget.ts @@ -0,0 +1,43 @@ +const DAEMON_SCROLLBACK_ENV_VAR = 'ORCA_DAEMON_SCROLLBACK_BUDGET_ROWS' + +/** Depth a session retains while the daemon is under budget; matches HeadlessEmulator's default. */ +export const DAEMON_SCROLLBACK_FULL_ROWS = 5000 +// Why: below this a reattach loses the command that produced the visible screen, which reads as data loss +// rather than degradation. Bounded growth past the floor is preferable to an unusable restore. +export const DAEMON_SCROLLBACK_MIN_ROWS = 500 +// Why: retained rows are the daemon's dominant heap term (an xterm row is ~2 KB at typical widths), and +// session count is unbounded — a host owning 100+ terminals retained ~1 GB of grid and was OOM-killed, +// taking its sessions with it. ~128k rows caps the grid near 256 MB, which is ~25 sessions at full depth. +export const DAEMON_SCROLLBACK_BUDGET_ROWS = 128_000 + +function parseBudgetOverrideRows(value: string | undefined): number | undefined { + if (value === undefined) { + return undefined + } + const parsed = Number(value.trim()) + // Why: ignore an unparseable or non-positive override rather than disabling the budget entirely — + // an unbounded daemon is the failure this module exists to prevent. + if (!Number.isFinite(parsed) || parsed <= 0) { + return undefined + } + return Math.floor(parsed) +} + +/** + * Rows each live session may retain so the daemon's total stays within the budget. Full depth until the + * budget binds, then an even split down to the floor. Pure so the tiers are unit-testable without a daemon. + */ +export function computeSessionScrollbackRows( + liveSessionCount: number, + options: { budgetRows?: number; env?: NodeJS.ProcessEnv } = {} +): number { + const budgetRows = + options.budgetRows ?? + parseBudgetOverrideRows((options.env ?? process.env)[DAEMON_SCROLLBACK_ENV_VAR]) ?? + DAEMON_SCROLLBACK_BUDGET_ROWS + if (!Number.isFinite(liveSessionCount) || liveSessionCount <= 1) { + return DAEMON_SCROLLBACK_FULL_ROWS + } + const evenShare = Math.floor(budgetRows / liveSessionCount) + return Math.min(DAEMON_SCROLLBACK_FULL_ROWS, Math.max(DAEMON_SCROLLBACK_MIN_ROWS, evenShare)) +} diff --git a/src/main/daemon/headless-emulator-modes.ts b/src/main/daemon/headless-emulator-modes.ts new file mode 100644 index 00000000000..bf65d527a3c --- /dev/null +++ b/src/main/daemon/headless-emulator-modes.ts @@ -0,0 +1,32 @@ +import type { Terminal } from '@xterm/headless' +import type { TerminalMouseModeMirror } from './terminal-mouse-mode-mirror' +import type { TerminalModes } from './types' + +type TerminalWithKittyKeyboard = Terminal & { + // Why: kitty keyboard flags aren't on the public IModes; read the core service the CSI u handlers mutate. + _core?: { coreService?: { kittyKeyboard?: { flags?: number } } } +} + +export function readKittyKeyboardFlags(terminal: Terminal): number { + const flags = (terminal as TerminalWithKittyKeyboard)._core?.coreService?.kittyKeyboard?.flags + return typeof flags === 'number' ? flags : 0 +} + +/** Mode state a snapshot must carry so a restored pane behaves like the live one. */ +export function readTerminalModes( + terminal: Terminal, + mouseModes: TerminalMouseModeMirror +): TerminalModes { + const buffer = terminal.buffer.active + const mouseTrackingMode = mouseModes.mouseTrackingMode + return { + bracketedPaste: terminal.modes.bracketedPasteMode, + mouseTracking: mouseTrackingMode !== 'none', + mouseTrackingMode, + sgrMouseMode: mouseModes.sgrMouseMode, + sgrMousePixelsMode: mouseModes.sgrMousePixelsMode, + applicationCursor: buffer.type === 'normal' ? terminal.modes.applicationCursorKeysMode : false, + alternateScreen: buffer.type === 'alternate', + kittyKeyboardFlags: readKittyKeyboardFlags(terminal) + } +} diff --git a/src/main/daemon/headless-emulator.ts b/src/main/daemon/headless-emulator.ts index ca8bb6c331f..4137ffa3e57 100644 --- a/src/main/daemon/headless-emulator.ts +++ b/src/main/daemon/headless-emulator.ts @@ -10,6 +10,7 @@ import { import { advancePartialEscapeTail } from '../../shared/terminal-partial-escape-tail' import type { TerminalViewAttributes } from '../../shared/terminal-view-attributes' import { collectHeadlessOscLinkRanges } from './headless-osc-link-ranges' +import { readTerminalModes } from './headless-emulator-modes' import { buildRehydrateSequences } from './terminal-mode-rehydrate-sequences' import { TerminalMouseModeMirror } from './terminal-mouse-mode-mirror' import { TerminalOscCwdTitleScanner } from './terminal-osc-cwd-title-scanner' @@ -330,30 +331,24 @@ export class HeadlessEmulator { this.terminal.clear() } + // Why: xterm trims the normal buffer on assignment, so lowering this frees retained rows immediately. + setRetainedScrollbackRows(rows: number): void { + if (this.disposed || !Number.isFinite(rows) || rows < 0) { + return + } + const next = Math.floor(rows) + if (this.terminal.options.scrollback === next) { + return + } + this.terminal.options.scrollback = next + } + dispose(): void { this.disposed = true this.terminal.dispose() } private getModes(): TerminalModes { - const buffer = this.terminal.buffer.active - const mouseTrackingMode = this.mouseModes.mouseTrackingMode - return { - bracketedPaste: this.terminal.modes.bracketedPasteMode, - mouseTracking: mouseTrackingMode !== 'none', - mouseTrackingMode, - sgrMouseMode: this.mouseModes.sgrMouseMode, - sgrMousePixelsMode: this.mouseModes.sgrMousePixelsMode, - applicationCursor: - buffer.type === 'normal' ? this.terminal.modes.applicationCursorKeysMode : false, - alternateScreen: buffer.type === 'alternate', - kittyKeyboardFlags: this.getKittyKeyboardFlags() - } - } - - private getKittyKeyboardFlags(): number { - const flags = (this.terminal as TerminalWithSynchronousWrite)._core?.coreService?.kittyKeyboard - ?.flags - return typeof flags === 'number' ? flags : 0 + return readTerminalModes(this.terminal, this.mouseModes) } } diff --git a/src/main/daemon/session.ts b/src/main/daemon/session.ts index d9c5553d3e8..28c68887e49 100644 --- a/src/main/daemon/session.ts +++ b/src/main/daemon/session.ts @@ -469,6 +469,13 @@ export class Session { return this.subprocess.confirmForegroundProcess?.() ?? this.subprocess.getForegroundProcess() } + setRetainedScrollbackRows(rows: number): void { + if (this._disposed) { + return + } + this.emulator.setRetainedScrollbackRows(rows) + } + clearScrollback(): void { if (this._disposed) { return diff --git a/src/main/daemon/terminal-host-scrollback-budget.test.ts b/src/main/daemon/terminal-host-scrollback-budget.test.ts new file mode 100644 index 00000000000..232f638f612 --- /dev/null +++ b/src/main/daemon/terminal-host-scrollback-budget.test.ts @@ -0,0 +1,119 @@ +/** + * OOM regression: a daemon owning many terminals retained ~5000 rows per session with no aggregate + * bound, grew to ~1.9 GB, and was killed under system memory pressure — taking every session it owned + * with it. TerminalHost must re-split its row budget as the live session count changes. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { TerminalHost } from './terminal-host' +import type { SubprocessHandle } from './session' +import { HeadlessEmulator } from './headless-emulator' +import { + computeSessionScrollbackRows, + DAEMON_SCROLLBACK_FULL_ROWS +} from './daemon-scrollback-budget' + +function createMockSubprocess(): SubprocessHandle & { _onExitCb: ((code: number) => void) | null } { + let onExitCb: ((code: number) => void) | null = null + return { + pid: 4242, + getForegroundProcess: vi.fn(() => null), + write: vi.fn(), + resize: vi.fn(), + kill: vi.fn(() => { + setTimeout(() => onExitCb?.(0), 1) + }), + forceKill: vi.fn(() => onExitCb?.(137)), + signal: vi.fn(), + onData() {}, + onExit(cb) { + onExitCb = cb + }, + dispose: vi.fn(), + get _onExitCb() { + return onExitCb + } + } as SubprocessHandle & { _onExitCb: ((code: number) => void) | null } +} + +describe('TerminalHost scrollback budget', () => { + let host: TerminalHost + let subprocesses: ReturnType[] + let applyRows: ReturnType + + beforeEach(() => { + subprocesses = [] + applyRows = vi.spyOn(HeadlessEmulator.prototype, 'setRetainedScrollbackRows') + host = new TerminalHost({ + spawnSubprocess: () => { + const sub = createMockSubprocess() + subprocesses.push(sub) + return sub + } + }) + }) + + afterEach(async () => { + await host.dispose() + applyRows.mockRestore() + }) + + async function create(sessionId: string): Promise { + await host.createOrAttach({ + sessionId, + cols: 80, + rows: 24, + streamClient: { onData: vi.fn(), onExit: vi.fn() } + }) + } + + function lastAppliedRows(): number { + const call = applyRows.mock.calls.at(-1) + return call?.[0] as number + } + + it('leaves a lightly loaded daemon at full depth', async () => { + await create('a') + await create('b') + expect(lastAppliedRows()).toBe(DAEMON_SCROLLBACK_FULL_ROWS) + }) + + it('applies the budget to every live session, not just the new one', async () => { + for (let i = 0; i < 4; i++) { + await create(`session-${i}`) + } + // The final create re-applies to all four sessions. + const lastFour = applyRows.mock.calls.slice(-4) + expect(lastFour).toHaveLength(4) + for (const call of lastFour) { + expect(call[0]).toBe(computeSessionScrollbackRows(4)) + } + }) + + it('reduces per-session depth as the session count grows past the budget', async () => { + const SESSIONS = 60 + for (let i = 0; i < SESSIONS; i++) { + await create(`session-${i}`) + } + const applied = lastAppliedRows() + expect(applied).toBe(computeSessionScrollbackRows(SESSIONS)) + expect(applied).toBeLessThan(DAEMON_SCROLLBACK_FULL_ROWS) + }) + + it('restores depth to survivors when sessions exit', async () => { + const SESSIONS = 60 + for (let i = 0; i < SESSIONS; i++) { + await create(`session-${i}`) + } + const underLoad = lastAppliedRows() + + // Drain back down to a handful of terminals. + for (let i = 0; i < SESSIONS - 2; i++) { + subprocesses[i]._onExitCb?.(0) + } + + expect(host.listSessions()).toHaveLength(2) + const afterDrain = lastAppliedRows() + expect(afterDrain).toBeGreaterThan(underLoad) + expect(afterDrain).toBe(DAEMON_SCROLLBACK_FULL_ROWS) + }) +}) diff --git a/src/main/daemon/terminal-host.ts b/src/main/daemon/terminal-host.ts index ae348a3854e..c4463d94e15 100644 --- a/src/main/daemon/terminal-host.ts +++ b/src/main/daemon/terminal-host.ts @@ -16,6 +16,7 @@ import { resolveTerminalHostSessionCwd } from './terminal-host-session-cwd' import { TerminalHostTombstones } from './terminal-host-tombstones' import { listLiveTerminalHostSessions } from './terminal-host-session-listing' import { createOrAttachTerminalSession } from './terminal-host-session-create' +import { computeSessionScrollbackRows } from './daemon-scrollback-budget' import { isShellProcess } from '../../shared/agent-detection' export type { CreateOrAttachOptions, CreateOrAttachResult } from './terminal-host-create-contract' @@ -57,7 +58,7 @@ export class TerminalHost { if (options.agentSessionGeneration && this.sessions.get(options.sessionId)?.isAlive) { throw new Error('agent_session_claim_unavailable') } - return await createOrAttachTerminalSession(options, { + const result = await createOrAttachTerminalSession(options, { sessions: this.sessions, sessionTeardown: this.sessionTeardown, killedTombstones: this.killedTombstones, @@ -72,10 +73,21 @@ export class TerminalHost { this.reapSession(sessionId) } }) + this.applyScrollbackBudget() + return result } }) } + // Why: retained rows are O(sessions × depth) and session count is unbounded, so re-split the + // daemon's row budget whenever the count changes rather than letting every session hold full depth. + private applyScrollbackBudget(): void { + const rows = computeSessionScrollbackRows(this.sessions.size) + for (const session of this.sessions.values()) { + session.setRetainedScrollbackRows(rows) + } + } + write(sessionId: string, data: string): void { this.getAliveSession(sessionId).write(data) } @@ -122,6 +134,7 @@ export class TerminalHost { } session.dispose() this.sessions.delete(sessionId) + this.applyScrollbackBudget() this.onSessionReaped?.(sessionId) }