mirror of
https://github.com/stablyai/orca.git
synced 2026-10-27 16:02:42 +00:00
fix(crash): measure the three heaps that dominate a renderer OOM
The highwater breadcrumb named store collections only, so the biggest retainers were invisible and every OOM report looked like "the store grew". All three are counted, never copied; each read is O(1). - terminal output backlog: a module-global Map in the output scheduler, capped per terminal (2MB default) with nothing bounding the aggregate. 170 terminals x that cap is ~340MB the store profile structurally cannot see. - live xterm buffers: reported as a pane count, but each retained row is a BufferLine holding 3 words per cell. 412 panes x 5,000 rows is ~412MB, and a pane count is equally consistent with megabytes. - Monaco models: a global registry that outlives the React panel that opened it, so editorContent.chars reads ~0 for exactly the leak that matters. Each instrument registers from a leaf module imported by the store, so a zeroed key means "nothing held" and a missing key means the instrument never ran. Conflating those two is what made the existing profile unreadable. Also counts editor census readers dropped at the cap: an undercount with no signal is worse than no number in the one artifact meant to settle where the heap went.
This commit is contained in:
@@ -1,8 +1,9 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { collectRendererMemoryProfileCounts } from '../../lib/renderer-memory-profile'
|
||||
import {
|
||||
measureEditorFileContents,
|
||||
registerEditorContentCensusReader
|
||||
registerEditorContentCensusReader,
|
||||
resetEditorContentCensusForTesting
|
||||
} from './editor-content-memory-census'
|
||||
|
||||
describe('measureEditorFileContents', () => {
|
||||
@@ -57,4 +58,21 @@ describe('editor content memory profile contributor', () => {
|
||||
releaseA()
|
||||
releaseB()
|
||||
})
|
||||
|
||||
it('reports how many panels the reader cap dropped instead of undercounting silently', () => {
|
||||
// Why: past the cap the census stops seeing new panels. A stuck `panels: 64`
|
||||
// is indistinguishable from "exactly 64 panels" unless the drops are named.
|
||||
for (let i = 0; i < 66; i += 1) {
|
||||
registerEditorContentCensusReader(() => ({ files: 1, chars: 10 }))
|
||||
}
|
||||
|
||||
expect(collectRendererMemoryProfileCounts()).toMatchObject({
|
||||
'editorContent.panels': 64,
|
||||
'editorContent.droppedPanels': 2
|
||||
})
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
resetEditorContentCensusForTesting()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -16,9 +16,16 @@ const readers = new Set<() => EditorContentCensus>()
|
||||
// leaked its unmount would make this diagnostic a retainer of what it measures.
|
||||
const MAX_CENSUS_READERS = 64
|
||||
|
||||
// Why counted: hitting the cap makes the census undercount, and an undercount with
|
||||
// no signal is worse than no number at all in the one artifact meant to settle
|
||||
// where the heap went. Cumulative, never decremented — it reports "we dropped at
|
||||
// least N panels", not a live size.
|
||||
let droppedReaders = 0
|
||||
|
||||
/** Registers one live panel's contents; call the returned function on unmount. */
|
||||
export function registerEditorContentCensusReader(read: () => EditorContentCensus): () => void {
|
||||
if (readers.size >= MAX_CENSUS_READERS) {
|
||||
droppedReaders += 1
|
||||
return () => {}
|
||||
}
|
||||
readers.add(read)
|
||||
@@ -52,5 +59,11 @@ registerRendererMemoryProfileContributor('editorContent', () => {
|
||||
files += census.files
|
||||
chars += census.chars
|
||||
}
|
||||
return { panels, files, chars }
|
||||
return { panels, files, chars, droppedPanels: droppedReaders }
|
||||
})
|
||||
|
||||
/** Test-only: the cap is process-wide, so suites must be able to clear it. */
|
||||
export function resetEditorContentCensusForTesting(): void {
|
||||
readers.clear()
|
||||
droppedReaders = 0
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { collectRendererMemoryProfileCounts } from './renderer-memory-profile'
|
||||
import { readMonacoModelCensus, setMonacoModelCensusReader } from './monaco-model-memory-census'
|
||||
|
||||
describe('monaco model memory census', () => {
|
||||
it('reports zeros before monaco loads, so a missing key means the instrument never ran', () => {
|
||||
expect(readMonacoModelCensus()).toEqual({ models: 0, chars: 0, lines: 0 })
|
||||
expect(collectRendererMemoryProfileCounts()).toMatchObject({
|
||||
'monacoModels.models': 0,
|
||||
'monacoModels.chars': 0,
|
||||
'monacoModels.lines': 0
|
||||
})
|
||||
})
|
||||
|
||||
it('surfaces models retained past their panel, which editorContent.chars reads as zero', () => {
|
||||
// Why: models live in monaco's global registry and are disposed only on tab
|
||||
// close, so an unmounted panel leaves its text fully retained and unmeasured.
|
||||
setMonacoModelCensusReader(() => ({ models: 214, chars: 91_324_887, lines: 2_140_000 }))
|
||||
|
||||
expect(collectRendererMemoryProfileCounts()).toMatchObject({
|
||||
'monacoModels.models': 214,
|
||||
'monacoModels.chars': 91_324_887,
|
||||
'monacoModels.lines': 2_140_000
|
||||
})
|
||||
|
||||
setMonacoModelCensusReader(() => ({ models: 0, chars: 0, lines: 0 }))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,33 @@
|
||||
/**
|
||||
* Size of the Monaco text models the renderer keeps alive, in models and characters.
|
||||
*
|
||||
* Why this is separate from the editor-content census: that one measures the raw
|
||||
* strings a *mounted* React panel holds, while Monaco keeps its own global model
|
||||
* registry. A model outlives the panel that opened it (only useClosedEditorTabCleanup
|
||||
* disposes one, on tab close) and stores a piece tree plus per-line tokenization and
|
||||
* decoration state, so it retains several times its text length. Without this,
|
||||
* `editorContent.chars` reads ~0 for exactly the case that matters — panels
|
||||
* unmounted, models retained.
|
||||
*
|
||||
* Monaco installs the reader from monaco-setup because importing monaco-editor here
|
||||
* would pull the whole editor bundle into the store chunk. Registering from this leaf
|
||||
* keeps the key present (zeroed) when the editor never loaded, so a missing key means
|
||||
* the instrument never ran rather than "no file was opened".
|
||||
*/
|
||||
import { registerRendererMemoryProfileContributor } from './renderer-memory-profile'
|
||||
|
||||
export type MonacoModelCensus = { models: number; chars: number; lines: number }
|
||||
|
||||
const EMPTY_CENSUS: MonacoModelCensus = { models: 0, chars: 0, lines: 0 }
|
||||
|
||||
let readModels: (() => MonacoModelCensus) | null = null
|
||||
|
||||
export function setMonacoModelCensusReader(read: () => MonacoModelCensus): void {
|
||||
readModels = read
|
||||
}
|
||||
|
||||
export function readMonacoModelCensus(): MonacoModelCensus {
|
||||
return readModels?.() ?? EMPTY_CENSUS
|
||||
}
|
||||
|
||||
registerRendererMemoryProfileContributor('monacoModels', readMonacoModelCensus)
|
||||
@@ -16,6 +16,7 @@ import { installMonacoDelayerCancellationGuard } from './monaco-delayer-cancella
|
||||
import { installMonacoDiffEditorDisposalGuard } from './monaco-diff-editor-disposal'
|
||||
import { installMonacoPeekReferencesPreviewOptions } from './monaco-peek-preview-options'
|
||||
import { installMonacoContextMenuPaste } from '@/components/editor/install-monaco-context-menu-paste'
|
||||
import { setMonacoModelCensusReader } from './monaco-model-memory-census'
|
||||
|
||||
globalThis.MonacoEnvironment = {
|
||||
getWorker(_workerId, label) {
|
||||
@@ -87,6 +88,25 @@ installMonacoPeekReferencesPreviewOptions()
|
||||
// so right-click Paste works like Cmd+V (which already works via native events).
|
||||
installMonacoContextMenuPaste(monaco)
|
||||
|
||||
// Why: models outlive the React panels that opened them, so an OOM report cannot
|
||||
// otherwise tell "editor holds 200MB of open files" from "Monaco is retaining models
|
||||
// for files whose panel unmounted". getValueLength/getLineCount are both O(1) reads
|
||||
// off the piece tree, so this stays safe to run inside a near-OOM breadcrumb.
|
||||
setMonacoModelCensusReader(() => {
|
||||
let models = 0
|
||||
let chars = 0
|
||||
let lines = 0
|
||||
for (const model of monaco.editor.getModels()) {
|
||||
models += 1
|
||||
if (model.isDisposed()) {
|
||||
continue
|
||||
}
|
||||
chars += model.getValueLength()
|
||||
lines += model.getLineCount()
|
||||
}
|
||||
return { models, chars, lines }
|
||||
})
|
||||
|
||||
// Configure Monaco to use the locally bundled editor instead of CDN
|
||||
loader.config({ monaco })
|
||||
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import { afterEach, describe, expect, it, vi, type Mock } from 'vitest'
|
||||
import { collectRendererMemoryProfileCounts } from '../renderer-memory-profile'
|
||||
import {
|
||||
forEachLivePaneForDesyncSentinel,
|
||||
getLivePaneCensus,
|
||||
getLiveTerminalBufferCensus,
|
||||
refitAndRefreshAllTerminalPanes,
|
||||
registerLivePaneManager,
|
||||
resetAndRefreshAllTerminalWebglAtlases,
|
||||
@@ -237,3 +239,73 @@ describe('pane manager registry', () => {
|
||||
expect(getLivePaneCensus()).toEqual({ managers: 2, panes: 1 })
|
||||
})
|
||||
})
|
||||
|
||||
describe('live terminal buffer census', () => {
|
||||
const registeredManagers: { resetWebglTextureAtlases(): void }[] = []
|
||||
|
||||
function registerPanes(...panes: { id: number; terminal: unknown }[]): void {
|
||||
const manager = {
|
||||
resetWebglTextureAtlases: vi.fn<() => void>(),
|
||||
getPanes: () => panes
|
||||
}
|
||||
registerLivePaneManager(manager)
|
||||
registeredManagers.push(manager)
|
||||
}
|
||||
|
||||
function terminal(cols: number, lines: number): unknown {
|
||||
return { cols, buffer: { active: { length: lines } } }
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const manager of registeredManagers.splice(0)) {
|
||||
unregisterLivePaneManager(manager)
|
||||
}
|
||||
})
|
||||
|
||||
it('separates retained scrollback size from the live terminal count', () => {
|
||||
// Why: `terminalElements` says 2 either way; only lines/cells tell an idle
|
||||
// pair apart from a pair sitting at the 50k-row scrollback preset.
|
||||
registerPanes(
|
||||
{ id: 1, terminal: terminal(120, 24) },
|
||||
{ id: 2, terminal: terminal(120, 50_000) }
|
||||
)
|
||||
|
||||
expect(getLiveTerminalBufferCensus()).toEqual({
|
||||
panes: 2,
|
||||
lines: 50_024,
|
||||
cells: 6_002_880
|
||||
})
|
||||
})
|
||||
|
||||
it('reports into the renderer memory profile', () => {
|
||||
registerPanes({ id: 1, terminal: terminal(80, 5_000) })
|
||||
|
||||
expect(collectRendererMemoryProfileCounts()).toMatchObject({
|
||||
'liveTerminalBuffers.panes': 1,
|
||||
'liveTerminalBuffers.lines': 5_000,
|
||||
'liveTerminalBuffers.cells': 400_000
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps counting siblings when a disposed terminal throws on buffer access', () => {
|
||||
registerPanes(
|
||||
{
|
||||
id: 1,
|
||||
terminal: {
|
||||
get buffer(): never {
|
||||
throw new Error('disposed')
|
||||
}
|
||||
}
|
||||
},
|
||||
{ id: 2, terminal: terminal(100, 10) }
|
||||
)
|
||||
|
||||
expect(getLiveTerminalBufferCensus()).toEqual({ panes: 1, lines: 10, cells: 1_000 })
|
||||
})
|
||||
|
||||
it('tolerates a terminal with no buffer rather than emitting NaN', () => {
|
||||
registerPanes({ id: 1, terminal: {} })
|
||||
|
||||
expect(getLiveTerminalBufferCensus()).toEqual({ panes: 1, lines: 0, cells: 0 })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { recordTerminalWebglDiagnostic } from '../../../../shared/terminal-webgl-diagnostics'
|
||||
import { registerRendererMemoryProfileContributor } from '../renderer-memory-profile'
|
||||
import type { PaneRenderingDiagnostics } from './pane-manager-types'
|
||||
|
||||
type RegisteredPaneManager = {
|
||||
@@ -110,6 +111,51 @@ export function getLivePaneCensus(): { managers: number; panes: number } {
|
||||
return { managers: liveManagers.size, panes }
|
||||
}
|
||||
|
||||
type XtermBufferShape = { cols?: number; buffer?: { active?: { length?: number } } }
|
||||
|
||||
/**
|
||||
* Retained size of the live xterm buffers, in scrollback lines and cells.
|
||||
*
|
||||
* Why lines/cells and not `terminalElements`: every retained row is a BufferLine
|
||||
* object holding a Uint32Array of 3 words per cell, so one terminal at the 50k-row
|
||||
* scrollback preset retains 50k of them while an idle one retains ~24. A count of
|
||||
* live terminals is therefore consistent with anything from megabytes to gigabytes
|
||||
* — the same ambiguity the scrollback census resolves for stored buffers, for the
|
||||
* larger live copy. `buffer.active.length` and `cols` are both O(1) reads, so this
|
||||
* stays safe to run inside a near-OOM highwater breadcrumb.
|
||||
*/
|
||||
export function getLiveTerminalBufferCensus(): {
|
||||
panes: number
|
||||
lines: number
|
||||
cells: number
|
||||
} {
|
||||
let panes = 0
|
||||
let lines = 0
|
||||
let cells = 0
|
||||
for (const manager of liveManagers) {
|
||||
let managerPanes: { id: number; terminal: unknown }[] = []
|
||||
try {
|
||||
managerPanes = manager.getPanes?.() ?? []
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
for (const pane of managerPanes) {
|
||||
try {
|
||||
const terminal = pane.terminal as XtermBufferShape | undefined
|
||||
const length = terminal?.buffer?.active?.length ?? 0
|
||||
panes += 1
|
||||
lines += length
|
||||
cells += length * (terminal?.cols ?? 0)
|
||||
} catch {
|
||||
// Why: a disposed terminal throws on buffer access; siblings still count.
|
||||
}
|
||||
}
|
||||
}
|
||||
return { panes, lines, cells }
|
||||
}
|
||||
|
||||
registerRendererMemoryProfileContributor('liveTerminalBuffers', getLiveTerminalBufferCensus)
|
||||
|
||||
/**
|
||||
* Iterates every live pane for the render-desync sentinel. Weakly-held manager
|
||||
* ids stay stable when an earlier manager unregisters without retaining it.
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
} from './pane-terminal-foreground-render-settle'
|
||||
import { runGuardedWriteCompletionStep } from './xterm-write-callback-guard'
|
||||
import { recordRendererCrashBreadcrumb } from '@/lib/crash-breadcrumb-recorder'
|
||||
import { setTerminalOutputBacklogCensusReader } from '@/lib/terminal-output-backlog-census'
|
||||
import {
|
||||
discardInFlightTerminalOutputAckCredits,
|
||||
registerTerminalOutputAckCredits
|
||||
@@ -240,6 +241,18 @@ function readQueueDebugSnapshot(): {
|
||||
}
|
||||
}
|
||||
|
||||
// Why unconditionally (not behind debugEnabled): the snapshot above is the only
|
||||
// place the renderer knows how many chars of un-parsed PTY output it is holding,
|
||||
// and an OOM report cannot otherwise tell a backlog blowup from a data leak.
|
||||
setTerminalOutputBacklogCensusReader(() => {
|
||||
const snapshot = readQueueDebugSnapshot()
|
||||
return {
|
||||
terminals: snapshot.queuedTerminalCount,
|
||||
chars: snapshot.queuedChars,
|
||||
maxTerminalChars: snapshot.queuedCharsByTerminal
|
||||
}
|
||||
})
|
||||
|
||||
function recordQueueDebugPressure(): void {
|
||||
if (!debugEnabled) {
|
||||
return
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { collectRendererMemoryProfileCounts } from './renderer-memory-profile'
|
||||
import {
|
||||
readTerminalOutputBacklogCensus,
|
||||
setTerminalOutputBacklogCensusReader
|
||||
} from './terminal-output-backlog-census'
|
||||
|
||||
describe('terminal output backlog census', () => {
|
||||
it('reports zeros before the scheduler installs a reader, so a missing key means the instrument never ran', () => {
|
||||
expect(readTerminalOutputBacklogCensus()).toEqual({
|
||||
terminals: 0,
|
||||
chars: 0,
|
||||
maxTerminalChars: 0
|
||||
})
|
||||
expect(collectRendererMemoryProfileCounts()).toMatchObject({
|
||||
'terminalOutputBacklog.terminals': 0,
|
||||
'terminalOutputBacklog.chars': 0,
|
||||
'terminalOutputBacklog.maxTerminalChars': 0
|
||||
})
|
||||
})
|
||||
|
||||
it('surfaces the queued characters the store profile structurally cannot see', () => {
|
||||
// Why: the queue is a module-global Map in the output scheduler, capped per
|
||||
// terminal (2MB at the default scrollback) but unbounded across terminals.
|
||||
setTerminalOutputBacklogCensusReader(() => ({
|
||||
terminals: 170,
|
||||
chars: 356_515_840,
|
||||
maxTerminalChars: 2_097_152
|
||||
}))
|
||||
|
||||
expect(collectRendererMemoryProfileCounts()).toMatchObject({
|
||||
'terminalOutputBacklog.terminals': 170,
|
||||
'terminalOutputBacklog.chars': 356_515_840,
|
||||
'terminalOutputBacklog.maxTerminalChars': 2_097_152
|
||||
})
|
||||
|
||||
setTerminalOutputBacklogCensusReader(() => ({
|
||||
terminals: 0,
|
||||
chars: 0,
|
||||
maxTerminalChars: 0
|
||||
}))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* Size of the PTY output the renderer holds queued before xterm parses it.
|
||||
*
|
||||
* Why this is separate from the scrollback census: the backlog is a module-global
|
||||
* Map inside the output scheduler, not store state, so neither
|
||||
* summarizeStateCollectionSizes nor terminalLayoutsByTabId can see it. The cap is
|
||||
* per terminal (terminalOutputBacklogCapChars: 2MB at the default scrollback, 6MB
|
||||
* at 50k rows) and nothing bounds the aggregate, so N background terminals whose
|
||||
* PTYs keep writing while Chromium throttles their drain retain N x that cap in
|
||||
* plain JS strings — the exact growth shape an OOM highwater breadcrumb must name.
|
||||
*
|
||||
* The scheduler installs the reader because importing it here would pull the whole
|
||||
* pane-manager into the store chunk; registering from this leaf keeps the key
|
||||
* present (zeroed) even when no terminal ever loaded, so a missing key means the
|
||||
* instrument never ran rather than "no terminal output".
|
||||
*/
|
||||
import { registerRendererMemoryProfileContributor } from './renderer-memory-profile'
|
||||
|
||||
export type TerminalOutputBacklogCensus = {
|
||||
terminals: number
|
||||
chars: number
|
||||
maxTerminalChars: number
|
||||
}
|
||||
|
||||
const EMPTY_CENSUS: TerminalOutputBacklogCensus = { terminals: 0, chars: 0, maxTerminalChars: 0 }
|
||||
|
||||
let readBacklog: (() => TerminalOutputBacklogCensus) | null = null
|
||||
|
||||
/** Installed by the output scheduler, which owns the queue. */
|
||||
export function setTerminalOutputBacklogCensusReader(
|
||||
read: () => TerminalOutputBacklogCensus
|
||||
): void {
|
||||
readBacklog = read
|
||||
}
|
||||
|
||||
export function readTerminalOutputBacklogCensus(): TerminalOutputBacklogCensus {
|
||||
return readBacklog?.() ?? EMPTY_CENSUS
|
||||
}
|
||||
|
||||
registerRendererMemoryProfileContributor('terminalOutputBacklog', readTerminalOutputBacklogCensus)
|
||||
@@ -48,6 +48,13 @@ import { installStoreListenerCensus } from './store-listener-census'
|
||||
// loads only if an editor panel mounts — making a missing editorContent.* key
|
||||
// ambiguous between "no editor was opened" and "the instrument never ran".
|
||||
import '@/components/editor/editor-content-memory-census'
|
||||
// Same reasoning for the live terminal and Monaco instruments: each registers from a
|
||||
// leaf whose owning subsystem (output scheduler, pane manager, monaco-setup) only
|
||||
// loads on first use, so a zeroed key means "nothing held" and a missing key means
|
||||
// the instrument never ran.
|
||||
import '@/lib/terminal-output-backlog-census'
|
||||
import '@/lib/pane-manager/pane-manager-registry'
|
||||
import '@/lib/monaco-model-memory-census'
|
||||
import { measureTerminalScrollbackBuffers } from './terminal-scrollback-memory-census'
|
||||
import {
|
||||
registerRendererMemoryProfileContributor,
|
||||
|
||||
@@ -7,6 +7,11 @@
|
||||
* TERMINAL_SCROLLBACK_SESSION_BUFFER_BYTE_LIMIT (512KB). A count of 170 is
|
||||
* consistent with anywhere from kilobytes to ~1GB, which is the exact ambiguity
|
||||
* an OOM highwater breadcrumb needs resolved.
|
||||
*
|
||||
* Read coldRestores/coldRestoreChars as a tripwire, not evidence: nothing currently
|
||||
* writes a key into pendingColdRestoreByPtyId (the live cold-restore payload goes
|
||||
* straight to xterm via writeReplayData), so today they are 0 by construction. They
|
||||
* stay so a future writer is measured on arrival rather than silently unmeasured.
|
||||
*/
|
||||
import type { TerminalLayoutSnapshot } from '../../../shared/types'
|
||||
|
||||
|
||||
Reference in New Issue
Block a user