fix(crash): measure the three heaps that dominate a renderer OOM

The highwater breadcrumb named store collections only, so the biggest
retainers were invisible and every OOM report looked like "the store grew".
All three are counted, never copied; each read is O(1).

- terminal output backlog: a module-global Map in the output scheduler,
  capped per terminal (2MB default) with nothing bounding the aggregate.
  170 terminals x that cap is ~340MB the store profile structurally cannot see.
- live xterm buffers: reported as a pane count, but each retained row is a
  BufferLine holding 3 words per cell. 412 panes x 5,000 rows is ~412MB, and a
  pane count is equally consistent with megabytes.
- Monaco models: a global registry that outlives the React panel that opened
  it, so editorContent.chars reads ~0 for exactly the leak that matters.

Each instrument registers from a leaf module imported by the store, so a
zeroed key means "nothing held" and a missing key means the instrument never
ran. Conflating those two is what made the existing profile unreadable.

Also counts editor census readers dropped at the cap: an undercount with no
signal is worse than no number in the one artifact meant to settle where the
heap went.
This commit is contained in:
Neil
2026-08-01 06:35:21 -07:00
parent c3b3ba69cc
commit a348e01a38
12 changed files with 341 additions and 3 deletions
@@ -1,8 +1,9 @@
import { describe, expect, it } from 'vitest'
import { afterEach, describe, expect, it } from 'vitest'
import { collectRendererMemoryProfileCounts } from '../../lib/renderer-memory-profile'
import {
measureEditorFileContents,
registerEditorContentCensusReader
registerEditorContentCensusReader,
resetEditorContentCensusForTesting
} from './editor-content-memory-census'
describe('measureEditorFileContents', () => {
@@ -57,4 +58,21 @@ describe('editor content memory profile contributor', () => {
releaseA()
releaseB()
})
it('reports how many panels the reader cap dropped instead of undercounting silently', () => {
// Why: past the cap the census stops seeing new panels. A stuck `panels: 64`
// is indistinguishable from "exactly 64 panels" unless the drops are named.
for (let i = 0; i < 66; i += 1) {
registerEditorContentCensusReader(() => ({ files: 1, chars: 10 }))
}
expect(collectRendererMemoryProfileCounts()).toMatchObject({
'editorContent.panels': 64,
'editorContent.droppedPanels': 2
})
})
afterEach(() => {
resetEditorContentCensusForTesting()
})
})
@@ -16,9 +16,16 @@ const readers = new Set<() => EditorContentCensus>()
// leaked its unmount would make this diagnostic a retainer of what it measures.
const MAX_CENSUS_READERS = 64
// Why counted: hitting the cap makes the census undercount, and an undercount with
// no signal is worse than no number at all in the one artifact meant to settle
// where the heap went. Cumulative, never decremented — it reports "we dropped at
// least N panels", not a live size.
let droppedReaders = 0
/** Registers one live panel's contents; call the returned function on unmount. */
export function registerEditorContentCensusReader(read: () => EditorContentCensus): () => void {
if (readers.size >= MAX_CENSUS_READERS) {
droppedReaders += 1
return () => {}
}
readers.add(read)
@@ -52,5 +59,11 @@ registerRendererMemoryProfileContributor('editorContent', () => {
files += census.files
chars += census.chars
}
return { panels, files, chars }
return { panels, files, chars, droppedPanels: droppedReaders }
})
/** Test-only: the cap is process-wide, so suites must be able to clear it. */
export function resetEditorContentCensusForTesting(): void {
readers.clear()
droppedReaders = 0
}
@@ -0,0 +1,28 @@
import { describe, expect, it } from 'vitest'
import { collectRendererMemoryProfileCounts } from './renderer-memory-profile'
import { readMonacoModelCensus, setMonacoModelCensusReader } from './monaco-model-memory-census'
describe('monaco model memory census', () => {
it('reports zeros before monaco loads, so a missing key means the instrument never ran', () => {
expect(readMonacoModelCensus()).toEqual({ models: 0, chars: 0, lines: 0 })
expect(collectRendererMemoryProfileCounts()).toMatchObject({
'monacoModels.models': 0,
'monacoModels.chars': 0,
'monacoModels.lines': 0
})
})
it('surfaces models retained past their panel, which editorContent.chars reads as zero', () => {
// Why: models live in monaco's global registry and are disposed only on tab
// close, so an unmounted panel leaves its text fully retained and unmeasured.
setMonacoModelCensusReader(() => ({ models: 214, chars: 91_324_887, lines: 2_140_000 }))
expect(collectRendererMemoryProfileCounts()).toMatchObject({
'monacoModels.models': 214,
'monacoModels.chars': 91_324_887,
'monacoModels.lines': 2_140_000
})
setMonacoModelCensusReader(() => ({ models: 0, chars: 0, lines: 0 }))
})
})
@@ -0,0 +1,33 @@
/**
* Size of the Monaco text models the renderer keeps alive, in models and characters.
*
* Why this is separate from the editor-content census: that one measures the raw
* strings a *mounted* React panel holds, while Monaco keeps its own global model
* registry. A model outlives the panel that opened it (only useClosedEditorTabCleanup
* disposes one, on tab close) and stores a piece tree plus per-line tokenization and
* decoration state, so it retains several times its text length. Without this,
* `editorContent.chars` reads ~0 for exactly the case that matters — panels
* unmounted, models retained.
*
* Monaco installs the reader from monaco-setup because importing monaco-editor here
* would pull the whole editor bundle into the store chunk. Registering from this leaf
* keeps the key present (zeroed) when the editor never loaded, so a missing key means
* the instrument never ran rather than "no file was opened".
*/
import { registerRendererMemoryProfileContributor } from './renderer-memory-profile'
export type MonacoModelCensus = { models: number; chars: number; lines: number }
const EMPTY_CENSUS: MonacoModelCensus = { models: 0, chars: 0, lines: 0 }
let readModels: (() => MonacoModelCensus) | null = null
export function setMonacoModelCensusReader(read: () => MonacoModelCensus): void {
readModels = read
}
export function readMonacoModelCensus(): MonacoModelCensus {
return readModels?.() ?? EMPTY_CENSUS
}
registerRendererMemoryProfileContributor('monacoModels', readMonacoModelCensus)
+20
View File
@@ -16,6 +16,7 @@ import { installMonacoDelayerCancellationGuard } from './monaco-delayer-cancella
import { installMonacoDiffEditorDisposalGuard } from './monaco-diff-editor-disposal'
import { installMonacoPeekReferencesPreviewOptions } from './monaco-peek-preview-options'
import { installMonacoContextMenuPaste } from '@/components/editor/install-monaco-context-menu-paste'
import { setMonacoModelCensusReader } from './monaco-model-memory-census'
globalThis.MonacoEnvironment = {
getWorker(_workerId, label) {
@@ -87,6 +88,25 @@ installMonacoPeekReferencesPreviewOptions()
// so right-click Paste works like Cmd+V (which already works via native events).
installMonacoContextMenuPaste(monaco)
// Why: models outlive the React panels that opened them, so an OOM report cannot
// otherwise tell "editor holds 200MB of open files" from "Monaco is retaining models
// for files whose panel unmounted". getValueLength/getLineCount are both O(1) reads
// off the piece tree, so this stays safe to run inside a near-OOM breadcrumb.
setMonacoModelCensusReader(() => {
let models = 0
let chars = 0
let lines = 0
for (const model of monaco.editor.getModels()) {
models += 1
if (model.isDisposed()) {
continue
}
chars += model.getValueLength()
lines += model.getLineCount()
}
return { models, chars, lines }
})
// Configure Monaco to use the locally bundled editor instead of CDN
loader.config({ monaco })
@@ -1,7 +1,9 @@
import { afterEach, describe, expect, it, vi, type Mock } from 'vitest'
import { collectRendererMemoryProfileCounts } from '../renderer-memory-profile'
import {
forEachLivePaneForDesyncSentinel,
getLivePaneCensus,
getLiveTerminalBufferCensus,
refitAndRefreshAllTerminalPanes,
registerLivePaneManager,
resetAndRefreshAllTerminalWebglAtlases,
@@ -237,3 +239,73 @@ describe('pane manager registry', () => {
expect(getLivePaneCensus()).toEqual({ managers: 2, panes: 1 })
})
})
describe('live terminal buffer census', () => {
const registeredManagers: { resetWebglTextureAtlases(): void }[] = []
function registerPanes(...panes: { id: number; terminal: unknown }[]): void {
const manager = {
resetWebglTextureAtlases: vi.fn<() => void>(),
getPanes: () => panes
}
registerLivePaneManager(manager)
registeredManagers.push(manager)
}
function terminal(cols: number, lines: number): unknown {
return { cols, buffer: { active: { length: lines } } }
}
afterEach(() => {
for (const manager of registeredManagers.splice(0)) {
unregisterLivePaneManager(manager)
}
})
it('separates retained scrollback size from the live terminal count', () => {
// Why: `terminalElements` says 2 either way; only lines/cells tell an idle
// pair apart from a pair sitting at the 50k-row scrollback preset.
registerPanes(
{ id: 1, terminal: terminal(120, 24) },
{ id: 2, terminal: terminal(120, 50_000) }
)
expect(getLiveTerminalBufferCensus()).toEqual({
panes: 2,
lines: 50_024,
cells: 6_002_880
})
})
it('reports into the renderer memory profile', () => {
registerPanes({ id: 1, terminal: terminal(80, 5_000) })
expect(collectRendererMemoryProfileCounts()).toMatchObject({
'liveTerminalBuffers.panes': 1,
'liveTerminalBuffers.lines': 5_000,
'liveTerminalBuffers.cells': 400_000
})
})
it('keeps counting siblings when a disposed terminal throws on buffer access', () => {
registerPanes(
{
id: 1,
terminal: {
get buffer(): never {
throw new Error('disposed')
}
}
},
{ id: 2, terminal: terminal(100, 10) }
)
expect(getLiveTerminalBufferCensus()).toEqual({ panes: 1, lines: 10, cells: 1_000 })
})
it('tolerates a terminal with no buffer rather than emitting NaN', () => {
registerPanes({ id: 1, terminal: {} })
expect(getLiveTerminalBufferCensus()).toEqual({ panes: 1, lines: 0, cells: 0 })
})
})
@@ -1,4 +1,5 @@
import { recordTerminalWebglDiagnostic } from '../../../../shared/terminal-webgl-diagnostics'
import { registerRendererMemoryProfileContributor } from '../renderer-memory-profile'
import type { PaneRenderingDiagnostics } from './pane-manager-types'
type RegisteredPaneManager = {
@@ -110,6 +111,51 @@ export function getLivePaneCensus(): { managers: number; panes: number } {
return { managers: liveManagers.size, panes }
}
type XtermBufferShape = { cols?: number; buffer?: { active?: { length?: number } } }
/**
* Retained size of the live xterm buffers, in scrollback lines and cells.
*
* Why lines/cells and not `terminalElements`: every retained row is a BufferLine
* object holding a Uint32Array of 3 words per cell, so one terminal at the 50k-row
* scrollback preset retains 50k of them while an idle one retains ~24. A count of
* live terminals is therefore consistent with anything from megabytes to gigabytes
* — the same ambiguity the scrollback census resolves for stored buffers, for the
* larger live copy. `buffer.active.length` and `cols` are both O(1) reads, so this
* stays safe to run inside a near-OOM highwater breadcrumb.
*/
export function getLiveTerminalBufferCensus(): {
panes: number
lines: number
cells: number
} {
let panes = 0
let lines = 0
let cells = 0
for (const manager of liveManagers) {
let managerPanes: { id: number; terminal: unknown }[] = []
try {
managerPanes = manager.getPanes?.() ?? []
} catch {
continue
}
for (const pane of managerPanes) {
try {
const terminal = pane.terminal as XtermBufferShape | undefined
const length = terminal?.buffer?.active?.length ?? 0
panes += 1
lines += length
cells += length * (terminal?.cols ?? 0)
} catch {
// Why: a disposed terminal throws on buffer access; siblings still count.
}
}
}
return { panes, lines, cells }
}
registerRendererMemoryProfileContributor('liveTerminalBuffers', getLiveTerminalBufferCensus)
/**
* Iterates every live pane for the render-desync sentinel. Weakly-held manager
* ids stay stable when an earlier manager unregisters without retaining it.
@@ -7,6 +7,7 @@ import {
} from './pane-terminal-foreground-render-settle'
import { runGuardedWriteCompletionStep } from './xterm-write-callback-guard'
import { recordRendererCrashBreadcrumb } from '@/lib/crash-breadcrumb-recorder'
import { setTerminalOutputBacklogCensusReader } from '@/lib/terminal-output-backlog-census'
import {
discardInFlightTerminalOutputAckCredits,
registerTerminalOutputAckCredits
@@ -240,6 +241,18 @@ function readQueueDebugSnapshot(): {
}
}
// Why unconditionally (not behind debugEnabled): the snapshot above is the only
// place the renderer knows how many chars of un-parsed PTY output it is holding,
// and an OOM report cannot otherwise tell a backlog blowup from a data leak.
setTerminalOutputBacklogCensusReader(() => {
const snapshot = readQueueDebugSnapshot()
return {
terminals: snapshot.queuedTerminalCount,
chars: snapshot.queuedChars,
maxTerminalChars: snapshot.queuedCharsByTerminal
}
})
function recordQueueDebugPressure(): void {
if (!debugEnabled) {
return
@@ -0,0 +1,43 @@
import { describe, expect, it } from 'vitest'
import { collectRendererMemoryProfileCounts } from './renderer-memory-profile'
import {
readTerminalOutputBacklogCensus,
setTerminalOutputBacklogCensusReader
} from './terminal-output-backlog-census'
describe('terminal output backlog census', () => {
it('reports zeros before the scheduler installs a reader, so a missing key means the instrument never ran', () => {
expect(readTerminalOutputBacklogCensus()).toEqual({
terminals: 0,
chars: 0,
maxTerminalChars: 0
})
expect(collectRendererMemoryProfileCounts()).toMatchObject({
'terminalOutputBacklog.terminals': 0,
'terminalOutputBacklog.chars': 0,
'terminalOutputBacklog.maxTerminalChars': 0
})
})
it('surfaces the queued characters the store profile structurally cannot see', () => {
// Why: the queue is a module-global Map in the output scheduler, capped per
// terminal (2MB at the default scrollback) but unbounded across terminals.
setTerminalOutputBacklogCensusReader(() => ({
terminals: 170,
chars: 356_515_840,
maxTerminalChars: 2_097_152
}))
expect(collectRendererMemoryProfileCounts()).toMatchObject({
'terminalOutputBacklog.terminals': 170,
'terminalOutputBacklog.chars': 356_515_840,
'terminalOutputBacklog.maxTerminalChars': 2_097_152
})
setTerminalOutputBacklogCensusReader(() => ({
terminals: 0,
chars: 0,
maxTerminalChars: 0
}))
})
})
@@ -0,0 +1,40 @@
/**
* Size of the PTY output the renderer holds queued before xterm parses it.
*
* Why this is separate from the scrollback census: the backlog is a module-global
* Map inside the output scheduler, not store state, so neither
* summarizeStateCollectionSizes nor terminalLayoutsByTabId can see it. The cap is
* per terminal (terminalOutputBacklogCapChars: 2MB at the default scrollback, 6MB
* at 50k rows) and nothing bounds the aggregate, so N background terminals whose
* PTYs keep writing while Chromium throttles their drain retain N x that cap in
* plain JS strings — the exact growth shape an OOM highwater breadcrumb must name.
*
* The scheduler installs the reader because importing it here would pull the whole
* pane-manager into the store chunk; registering from this leaf keeps the key
* present (zeroed) even when no terminal ever loaded, so a missing key means the
* instrument never ran rather than "no terminal output".
*/
import { registerRendererMemoryProfileContributor } from './renderer-memory-profile'
export type TerminalOutputBacklogCensus = {
terminals: number
chars: number
maxTerminalChars: number
}
const EMPTY_CENSUS: TerminalOutputBacklogCensus = { terminals: 0, chars: 0, maxTerminalChars: 0 }
let readBacklog: (() => TerminalOutputBacklogCensus) | null = null
/** Installed by the output scheduler, which owns the queue. */
export function setTerminalOutputBacklogCensusReader(
read: () => TerminalOutputBacklogCensus
): void {
readBacklog = read
}
export function readTerminalOutputBacklogCensus(): TerminalOutputBacklogCensus {
return readBacklog?.() ?? EMPTY_CENSUS
}
registerRendererMemoryProfileContributor('terminalOutputBacklog', readTerminalOutputBacklogCensus)
+7
View File
@@ -48,6 +48,13 @@ import { installStoreListenerCensus } from './store-listener-census'
// loads only if an editor panel mounts — making a missing editorContent.* key
// ambiguous between "no editor was opened" and "the instrument never ran".
import '@/components/editor/editor-content-memory-census'
// Same reasoning for the live terminal and Monaco instruments: each registers from a
// leaf whose owning subsystem (output scheduler, pane manager, monaco-setup) only
// loads on first use, so a zeroed key means "nothing held" and a missing key means
// the instrument never ran.
import '@/lib/terminal-output-backlog-census'
import '@/lib/pane-manager/pane-manager-registry'
import '@/lib/monaco-model-memory-census'
import { measureTerminalScrollbackBuffers } from './terminal-scrollback-memory-census'
import {
registerRendererMemoryProfileContributor,
@@ -7,6 +7,11 @@
* TERMINAL_SCROLLBACK_SESSION_BUFFER_BYTE_LIMIT (512KB). A count of 170 is
* consistent with anywhere from kilobytes to ~1GB, which is the exact ambiguity
* an OOM highwater breadcrumb needs resolved.
*
* Read coldRestores/coldRestoreChars as a tripwire, not evidence: nothing currently
* writes a key into pendingColdRestoreByPtyId (the live cold-restore payload goes
* straight to xterm via writeReplayData), so today they are 0 by construction. They
* stay so a future writer is measured on arrival rather than silently unmeasured.
*/
import type { TerminalLayoutSnapshot } from '../../../shared/types'