From a348e01a38c2e2082a97e66b5138d15c1e526cd9 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sat, 1 Aug 2026 06:35:21 -0700 Subject: [PATCH] fix(crash): measure the three heaps that dominate a renderer OOM The highwater breadcrumb named store collections only, so the biggest retainers were invisible and every OOM report looked like "the store grew". All three are counted, never copied; each read is O(1). - terminal output backlog: a module-global Map in the output scheduler, capped per terminal (2MB default) with nothing bounding the aggregate. 170 terminals x that cap is ~340MB the store profile structurally cannot see. - live xterm buffers: reported as a pane count, but each retained row is a BufferLine holding 3 words per cell. 412 panes x 5,000 rows is ~412MB, and a pane count is equally consistent with megabytes. - Monaco models: a global registry that outlives the React panel that opened it, so editorContent.chars reads ~0 for exactly the leak that matters. Each instrument registers from a leaf module imported by the store, so a zeroed key means "nothing held" and a missing key means the instrument never ran. Conflating those two is what made the existing profile unreadable. Also counts editor census readers dropped at the cap: an undercount with no signal is worse than no number in the one artifact meant to settle where the heap went. --- .../editor-content-memory-census.test.ts | 22 +++++- .../editor/editor-content-memory-census.ts | 15 +++- .../lib/monaco-model-memory-census.test.ts | 28 ++++++++ .../src/lib/monaco-model-memory-census.ts | 33 +++++++++ src/renderer/src/lib/monaco-setup.ts | 20 ++++++ .../pane-manager-registry.test.ts | 72 +++++++++++++++++++ .../lib/pane-manager/pane-manager-registry.ts | 46 ++++++++++++ .../pane-terminal-output-scheduler.ts | 13 ++++ .../terminal-output-backlog-census.test.ts | 43 +++++++++++ .../src/lib/terminal-output-backlog-census.ts | 40 +++++++++++ src/renderer/src/store/index.ts | 7 ++ .../terminal-scrollback-memory-census.ts | 5 ++ 12 files changed, 341 insertions(+), 3 deletions(-) create mode 100644 src/renderer/src/lib/monaco-model-memory-census.test.ts create mode 100644 src/renderer/src/lib/monaco-model-memory-census.ts create mode 100644 src/renderer/src/lib/terminal-output-backlog-census.test.ts create mode 100644 src/renderer/src/lib/terminal-output-backlog-census.ts diff --git a/src/renderer/src/components/editor/editor-content-memory-census.test.ts b/src/renderer/src/components/editor/editor-content-memory-census.test.ts index 7cc45849d78..858e59957c8 100644 --- a/src/renderer/src/components/editor/editor-content-memory-census.test.ts +++ b/src/renderer/src/components/editor/editor-content-memory-census.test.ts @@ -1,8 +1,9 @@ -import { describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it } from 'vitest' import { collectRendererMemoryProfileCounts } from '../../lib/renderer-memory-profile' import { measureEditorFileContents, - registerEditorContentCensusReader + registerEditorContentCensusReader, + resetEditorContentCensusForTesting } from './editor-content-memory-census' describe('measureEditorFileContents', () => { @@ -57,4 +58,21 @@ describe('editor content memory profile contributor', () => { releaseA() releaseB() }) + + it('reports how many panels the reader cap dropped instead of undercounting silently', () => { + // Why: past the cap the census stops seeing new panels. A stuck `panels: 64` + // is indistinguishable from "exactly 64 panels" unless the drops are named. + for (let i = 0; i < 66; i += 1) { + registerEditorContentCensusReader(() => ({ files: 1, chars: 10 })) + } + + expect(collectRendererMemoryProfileCounts()).toMatchObject({ + 'editorContent.panels': 64, + 'editorContent.droppedPanels': 2 + }) + }) + + afterEach(() => { + resetEditorContentCensusForTesting() + }) }) diff --git a/src/renderer/src/components/editor/editor-content-memory-census.ts b/src/renderer/src/components/editor/editor-content-memory-census.ts index d63b401453e..0e331bf3346 100644 --- a/src/renderer/src/components/editor/editor-content-memory-census.ts +++ b/src/renderer/src/components/editor/editor-content-memory-census.ts @@ -16,9 +16,16 @@ const readers = new Set<() => EditorContentCensus>() // leaked its unmount would make this diagnostic a retainer of what it measures. const MAX_CENSUS_READERS = 64 +// Why counted: hitting the cap makes the census undercount, and an undercount with +// no signal is worse than no number at all in the one artifact meant to settle +// where the heap went. Cumulative, never decremented — it reports "we dropped at +// least N panels", not a live size. +let droppedReaders = 0 + /** Registers one live panel's contents; call the returned function on unmount. */ export function registerEditorContentCensusReader(read: () => EditorContentCensus): () => void { if (readers.size >= MAX_CENSUS_READERS) { + droppedReaders += 1 return () => {} } readers.add(read) @@ -52,5 +59,11 @@ registerRendererMemoryProfileContributor('editorContent', () => { files += census.files chars += census.chars } - return { panels, files, chars } + return { panels, files, chars, droppedPanels: droppedReaders } }) + +/** Test-only: the cap is process-wide, so suites must be able to clear it. */ +export function resetEditorContentCensusForTesting(): void { + readers.clear() + droppedReaders = 0 +} diff --git a/src/renderer/src/lib/monaco-model-memory-census.test.ts b/src/renderer/src/lib/monaco-model-memory-census.test.ts new file mode 100644 index 00000000000..d19a583d5c3 --- /dev/null +++ b/src/renderer/src/lib/monaco-model-memory-census.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest' +import { collectRendererMemoryProfileCounts } from './renderer-memory-profile' +import { readMonacoModelCensus, setMonacoModelCensusReader } from './monaco-model-memory-census' + +describe('monaco model memory census', () => { + it('reports zeros before monaco loads, so a missing key means the instrument never ran', () => { + expect(readMonacoModelCensus()).toEqual({ models: 0, chars: 0, lines: 0 }) + expect(collectRendererMemoryProfileCounts()).toMatchObject({ + 'monacoModels.models': 0, + 'monacoModels.chars': 0, + 'monacoModels.lines': 0 + }) + }) + + it('surfaces models retained past their panel, which editorContent.chars reads as zero', () => { + // Why: models live in monaco's global registry and are disposed only on tab + // close, so an unmounted panel leaves its text fully retained and unmeasured. + setMonacoModelCensusReader(() => ({ models: 214, chars: 91_324_887, lines: 2_140_000 })) + + expect(collectRendererMemoryProfileCounts()).toMatchObject({ + 'monacoModels.models': 214, + 'monacoModels.chars': 91_324_887, + 'monacoModels.lines': 2_140_000 + }) + + setMonacoModelCensusReader(() => ({ models: 0, chars: 0, lines: 0 })) + }) +}) diff --git a/src/renderer/src/lib/monaco-model-memory-census.ts b/src/renderer/src/lib/monaco-model-memory-census.ts new file mode 100644 index 00000000000..fdea1096985 --- /dev/null +++ b/src/renderer/src/lib/monaco-model-memory-census.ts @@ -0,0 +1,33 @@ +/** + * Size of the Monaco text models the renderer keeps alive, in models and characters. + * + * Why this is separate from the editor-content census: that one measures the raw + * strings a *mounted* React panel holds, while Monaco keeps its own global model + * registry. A model outlives the panel that opened it (only useClosedEditorTabCleanup + * disposes one, on tab close) and stores a piece tree plus per-line tokenization and + * decoration state, so it retains several times its text length. Without this, + * `editorContent.chars` reads ~0 for exactly the case that matters — panels + * unmounted, models retained. + * + * Monaco installs the reader from monaco-setup because importing monaco-editor here + * would pull the whole editor bundle into the store chunk. Registering from this leaf + * keeps the key present (zeroed) when the editor never loaded, so a missing key means + * the instrument never ran rather than "no file was opened". + */ +import { registerRendererMemoryProfileContributor } from './renderer-memory-profile' + +export type MonacoModelCensus = { models: number; chars: number; lines: number } + +const EMPTY_CENSUS: MonacoModelCensus = { models: 0, chars: 0, lines: 0 } + +let readModels: (() => MonacoModelCensus) | null = null + +export function setMonacoModelCensusReader(read: () => MonacoModelCensus): void { + readModels = read +} + +export function readMonacoModelCensus(): MonacoModelCensus { + return readModels?.() ?? EMPTY_CENSUS +} + +registerRendererMemoryProfileContributor('monacoModels', readMonacoModelCensus) diff --git a/src/renderer/src/lib/monaco-setup.ts b/src/renderer/src/lib/monaco-setup.ts index 523ecf8e77b..214762bbeda 100644 --- a/src/renderer/src/lib/monaco-setup.ts +++ b/src/renderer/src/lib/monaco-setup.ts @@ -16,6 +16,7 @@ import { installMonacoDelayerCancellationGuard } from './monaco-delayer-cancella import { installMonacoDiffEditorDisposalGuard } from './monaco-diff-editor-disposal' import { installMonacoPeekReferencesPreviewOptions } from './monaco-peek-preview-options' import { installMonacoContextMenuPaste } from '@/components/editor/install-monaco-context-menu-paste' +import { setMonacoModelCensusReader } from './monaco-model-memory-census' globalThis.MonacoEnvironment = { getWorker(_workerId, label) { @@ -87,6 +88,25 @@ installMonacoPeekReferencesPreviewOptions() // so right-click Paste works like Cmd+V (which already works via native events). installMonacoContextMenuPaste(monaco) +// Why: models outlive the React panels that opened them, so an OOM report cannot +// otherwise tell "editor holds 200MB of open files" from "Monaco is retaining models +// for files whose panel unmounted". getValueLength/getLineCount are both O(1) reads +// off the piece tree, so this stays safe to run inside a near-OOM breadcrumb. +setMonacoModelCensusReader(() => { + let models = 0 + let chars = 0 + let lines = 0 + for (const model of monaco.editor.getModels()) { + models += 1 + if (model.isDisposed()) { + continue + } + chars += model.getValueLength() + lines += model.getLineCount() + } + return { models, chars, lines } +}) + // Configure Monaco to use the locally bundled editor instead of CDN loader.config({ monaco }) diff --git a/src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts b/src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts index c423004e05c..af06366d733 100644 --- a/src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts +++ b/src/renderer/src/lib/pane-manager/pane-manager-registry.test.ts @@ -1,7 +1,9 @@ import { afterEach, describe, expect, it, vi, type Mock } from 'vitest' +import { collectRendererMemoryProfileCounts } from '../renderer-memory-profile' import { forEachLivePaneForDesyncSentinel, getLivePaneCensus, + getLiveTerminalBufferCensus, refitAndRefreshAllTerminalPanes, registerLivePaneManager, resetAndRefreshAllTerminalWebglAtlases, @@ -237,3 +239,73 @@ describe('pane manager registry', () => { expect(getLivePaneCensus()).toEqual({ managers: 2, panes: 1 }) }) }) + +describe('live terminal buffer census', () => { + const registeredManagers: { resetWebglTextureAtlases(): void }[] = [] + + function registerPanes(...panes: { id: number; terminal: unknown }[]): void { + const manager = { + resetWebglTextureAtlases: vi.fn<() => void>(), + getPanes: () => panes + } + registerLivePaneManager(manager) + registeredManagers.push(manager) + } + + function terminal(cols: number, lines: number): unknown { + return { cols, buffer: { active: { length: lines } } } + } + + afterEach(() => { + for (const manager of registeredManagers.splice(0)) { + unregisterLivePaneManager(manager) + } + }) + + it('separates retained scrollback size from the live terminal count', () => { + // Why: `terminalElements` says 2 either way; only lines/cells tell an idle + // pair apart from a pair sitting at the 50k-row scrollback preset. + registerPanes( + { id: 1, terminal: terminal(120, 24) }, + { id: 2, terminal: terminal(120, 50_000) } + ) + + expect(getLiveTerminalBufferCensus()).toEqual({ + panes: 2, + lines: 50_024, + cells: 6_002_880 + }) + }) + + it('reports into the renderer memory profile', () => { + registerPanes({ id: 1, terminal: terminal(80, 5_000) }) + + expect(collectRendererMemoryProfileCounts()).toMatchObject({ + 'liveTerminalBuffers.panes': 1, + 'liveTerminalBuffers.lines': 5_000, + 'liveTerminalBuffers.cells': 400_000 + }) + }) + + it('keeps counting siblings when a disposed terminal throws on buffer access', () => { + registerPanes( + { + id: 1, + terminal: { + get buffer(): never { + throw new Error('disposed') + } + } + }, + { id: 2, terminal: terminal(100, 10) } + ) + + expect(getLiveTerminalBufferCensus()).toEqual({ panes: 1, lines: 10, cells: 1_000 }) + }) + + it('tolerates a terminal with no buffer rather than emitting NaN', () => { + registerPanes({ id: 1, terminal: {} }) + + expect(getLiveTerminalBufferCensus()).toEqual({ panes: 1, lines: 0, cells: 0 }) + }) +}) diff --git a/src/renderer/src/lib/pane-manager/pane-manager-registry.ts b/src/renderer/src/lib/pane-manager/pane-manager-registry.ts index 6ebe2b4924f..bb244450216 100644 --- a/src/renderer/src/lib/pane-manager/pane-manager-registry.ts +++ b/src/renderer/src/lib/pane-manager/pane-manager-registry.ts @@ -1,4 +1,5 @@ import { recordTerminalWebglDiagnostic } from '../../../../shared/terminal-webgl-diagnostics' +import { registerRendererMemoryProfileContributor } from '../renderer-memory-profile' import type { PaneRenderingDiagnostics } from './pane-manager-types' type RegisteredPaneManager = { @@ -110,6 +111,51 @@ export function getLivePaneCensus(): { managers: number; panes: number } { return { managers: liveManagers.size, panes } } +type XtermBufferShape = { cols?: number; buffer?: { active?: { length?: number } } } + +/** + * Retained size of the live xterm buffers, in scrollback lines and cells. + * + * Why lines/cells and not `terminalElements`: every retained row is a BufferLine + * object holding a Uint32Array of 3 words per cell, so one terminal at the 50k-row + * scrollback preset retains 50k of them while an idle one retains ~24. A count of + * live terminals is therefore consistent with anything from megabytes to gigabytes + * — the same ambiguity the scrollback census resolves for stored buffers, for the + * larger live copy. `buffer.active.length` and `cols` are both O(1) reads, so this + * stays safe to run inside a near-OOM highwater breadcrumb. + */ +export function getLiveTerminalBufferCensus(): { + panes: number + lines: number + cells: number +} { + let panes = 0 + let lines = 0 + let cells = 0 + for (const manager of liveManagers) { + let managerPanes: { id: number; terminal: unknown }[] = [] + try { + managerPanes = manager.getPanes?.() ?? [] + } catch { + continue + } + for (const pane of managerPanes) { + try { + const terminal = pane.terminal as XtermBufferShape | undefined + const length = terminal?.buffer?.active?.length ?? 0 + panes += 1 + lines += length + cells += length * (terminal?.cols ?? 0) + } catch { + // Why: a disposed terminal throws on buffer access; siblings still count. + } + } + } + return { panes, lines, cells } +} + +registerRendererMemoryProfileContributor('liveTerminalBuffers', getLiveTerminalBufferCensus) + /** * Iterates every live pane for the render-desync sentinel. Weakly-held manager * ids stay stable when an earlier manager unregisters without retaining it. diff --git a/src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler.ts b/src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler.ts index 4272d4c0913..8819225e7af 100644 --- a/src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler.ts +++ b/src/renderer/src/lib/pane-manager/pane-terminal-output-scheduler.ts @@ -7,6 +7,7 @@ import { } from './pane-terminal-foreground-render-settle' import { runGuardedWriteCompletionStep } from './xterm-write-callback-guard' import { recordRendererCrashBreadcrumb } from '@/lib/crash-breadcrumb-recorder' +import { setTerminalOutputBacklogCensusReader } from '@/lib/terminal-output-backlog-census' import { discardInFlightTerminalOutputAckCredits, registerTerminalOutputAckCredits @@ -240,6 +241,18 @@ function readQueueDebugSnapshot(): { } } +// Why unconditionally (not behind debugEnabled): the snapshot above is the only +// place the renderer knows how many chars of un-parsed PTY output it is holding, +// and an OOM report cannot otherwise tell a backlog blowup from a data leak. +setTerminalOutputBacklogCensusReader(() => { + const snapshot = readQueueDebugSnapshot() + return { + terminals: snapshot.queuedTerminalCount, + chars: snapshot.queuedChars, + maxTerminalChars: snapshot.queuedCharsByTerminal + } +}) + function recordQueueDebugPressure(): void { if (!debugEnabled) { return diff --git a/src/renderer/src/lib/terminal-output-backlog-census.test.ts b/src/renderer/src/lib/terminal-output-backlog-census.test.ts new file mode 100644 index 00000000000..38f88a0346b --- /dev/null +++ b/src/renderer/src/lib/terminal-output-backlog-census.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { collectRendererMemoryProfileCounts } from './renderer-memory-profile' +import { + readTerminalOutputBacklogCensus, + setTerminalOutputBacklogCensusReader +} from './terminal-output-backlog-census' + +describe('terminal output backlog census', () => { + it('reports zeros before the scheduler installs a reader, so a missing key means the instrument never ran', () => { + expect(readTerminalOutputBacklogCensus()).toEqual({ + terminals: 0, + chars: 0, + maxTerminalChars: 0 + }) + expect(collectRendererMemoryProfileCounts()).toMatchObject({ + 'terminalOutputBacklog.terminals': 0, + 'terminalOutputBacklog.chars': 0, + 'terminalOutputBacklog.maxTerminalChars': 0 + }) + }) + + it('surfaces the queued characters the store profile structurally cannot see', () => { + // Why: the queue is a module-global Map in the output scheduler, capped per + // terminal (2MB at the default scrollback) but unbounded across terminals. + setTerminalOutputBacklogCensusReader(() => ({ + terminals: 170, + chars: 356_515_840, + maxTerminalChars: 2_097_152 + })) + + expect(collectRendererMemoryProfileCounts()).toMatchObject({ + 'terminalOutputBacklog.terminals': 170, + 'terminalOutputBacklog.chars': 356_515_840, + 'terminalOutputBacklog.maxTerminalChars': 2_097_152 + }) + + setTerminalOutputBacklogCensusReader(() => ({ + terminals: 0, + chars: 0, + maxTerminalChars: 0 + })) + }) +}) diff --git a/src/renderer/src/lib/terminal-output-backlog-census.ts b/src/renderer/src/lib/terminal-output-backlog-census.ts new file mode 100644 index 00000000000..a975b2a1026 --- /dev/null +++ b/src/renderer/src/lib/terminal-output-backlog-census.ts @@ -0,0 +1,40 @@ +/** + * Size of the PTY output the renderer holds queued before xterm parses it. + * + * Why this is separate from the scrollback census: the backlog is a module-global + * Map inside the output scheduler, not store state, so neither + * summarizeStateCollectionSizes nor terminalLayoutsByTabId can see it. The cap is + * per terminal (terminalOutputBacklogCapChars: 2MB at the default scrollback, 6MB + * at 50k rows) and nothing bounds the aggregate, so N background terminals whose + * PTYs keep writing while Chromium throttles their drain retain N x that cap in + * plain JS strings — the exact growth shape an OOM highwater breadcrumb must name. + * + * The scheduler installs the reader because importing it here would pull the whole + * pane-manager into the store chunk; registering from this leaf keeps the key + * present (zeroed) even when no terminal ever loaded, so a missing key means the + * instrument never ran rather than "no terminal output". + */ +import { registerRendererMemoryProfileContributor } from './renderer-memory-profile' + +export type TerminalOutputBacklogCensus = { + terminals: number + chars: number + maxTerminalChars: number +} + +const EMPTY_CENSUS: TerminalOutputBacklogCensus = { terminals: 0, chars: 0, maxTerminalChars: 0 } + +let readBacklog: (() => TerminalOutputBacklogCensus) | null = null + +/** Installed by the output scheduler, which owns the queue. */ +export function setTerminalOutputBacklogCensusReader( + read: () => TerminalOutputBacklogCensus +): void { + readBacklog = read +} + +export function readTerminalOutputBacklogCensus(): TerminalOutputBacklogCensus { + return readBacklog?.() ?? EMPTY_CENSUS +} + +registerRendererMemoryProfileContributor('terminalOutputBacklog', readTerminalOutputBacklogCensus) diff --git a/src/renderer/src/store/index.ts b/src/renderer/src/store/index.ts index 8da246fe404..ede342ecdd5 100644 --- a/src/renderer/src/store/index.ts +++ b/src/renderer/src/store/index.ts @@ -48,6 +48,13 @@ import { installStoreListenerCensus } from './store-listener-census' // loads only if an editor panel mounts — making a missing editorContent.* key // ambiguous between "no editor was opened" and "the instrument never ran". import '@/components/editor/editor-content-memory-census' +// Same reasoning for the live terminal and Monaco instruments: each registers from a +// leaf whose owning subsystem (output scheduler, pane manager, monaco-setup) only +// loads on first use, so a zeroed key means "nothing held" and a missing key means +// the instrument never ran. +import '@/lib/terminal-output-backlog-census' +import '@/lib/pane-manager/pane-manager-registry' +import '@/lib/monaco-model-memory-census' import { measureTerminalScrollbackBuffers } from './terminal-scrollback-memory-census' import { registerRendererMemoryProfileContributor, diff --git a/src/renderer/src/store/terminal-scrollback-memory-census.ts b/src/renderer/src/store/terminal-scrollback-memory-census.ts index e17788c9cf1..354de9f8a25 100644 --- a/src/renderer/src/store/terminal-scrollback-memory-census.ts +++ b/src/renderer/src/store/terminal-scrollback-memory-census.ts @@ -7,6 +7,11 @@ * TERMINAL_SCROLLBACK_SESSION_BUFFER_BYTE_LIMIT (512KB). A count of 170 is * consistent with anywhere from kilobytes to ~1GB, which is the exact ambiguity * an OOM highwater breadcrumb needs resolved. + * + * Read coldRestores/coldRestoreChars as a tripwire, not evidence: nothing currently + * writes a key into pendingColdRestoreByPtyId (the live cold-restore payload goes + * straight to xterm via writeReplayData), so today they are 0 by construction. They + * stay so a future writer is measured on arrival rather than silently unmeasured. */ import type { TerminalLayoutSnapshot } from '../../../shared/types'