fix(diagnostics): name what the renderer heap holds before it OOMs

Crash cluster D: 54 renderer OOM aborts (exit 5), median 94% of a 4,192MB
limit. Across 51 diagnostics bundles the 60% and 80% highwater profiles are
byte-identical — 883MB retained with every recorded counter unchanged — and
mallocedMB is 1 in all 44,832 samples. The instrument, not the heap, is the
blind spot.

- Record every field process.getHeapStatistics() exposes and we discarded:
  physical, available, executable (code space) and peak malloced, on the
  one-shot highwater breadcrumb only. A per-space breakdown is not reachable:
  v8.getHeapSpaceStatistics() needs the `v8` module, which a sandboxed preload
  cannot require (verified against Electron 43).
- Report the zustand subscriber count, which was already computed and dropped.
- Report editor file bodies held in React state, which the store-only profile
  is structurally unable to walk.
- Cap restored editor tabs at 100 per workspace. openFilesByWorktree is
  append-only and never pruned; bundles show 3,711 tabs restoring into one
  renderer. Unsaved drafts and the focused tab always survive the cap.
This commit is contained in:
Neil
2026-08-01 04:40:19 -07:00
parent 16c5526dfd
commit 1b2122e194
13 changed files with 417 additions and 5 deletions
@@ -29,7 +29,11 @@ describe('readRendererHeapStatistics', () => {
usedHeapKB: 1536,
totalHeapKB: 2048,
heapLimitKB: 4_292_608,
physicalKB: 2048,
availableKB: 4_000_000,
executableKB: 0,
mallocedKB: 64,
peakMallocedKB: 96,
blinkAllocatedKB: 1227
})
})
@@ -50,7 +54,11 @@ describe('readRendererHeapStatistics', () => {
usedHeapKB: 1536,
totalHeapKB: 2048,
heapLimitKB: 4_292_608,
physicalKB: 2048,
availableKB: 4_000_000,
executableKB: 0,
mallocedKB: 64,
peakMallocedKB: 96,
blinkAllocatedKB: undefined
})
})
@@ -31,7 +31,11 @@ export function readRendererHeapStatistics(
usedHeapKB: heap.usedHeapSize,
totalHeapKB: heap.totalHeapSize,
heapLimitKB: heap.heapSizeLimit,
physicalKB: heap.totalPhysicalSize,
availableKB: heap.totalAvailableSize,
executableKB: heap.totalHeapSizeExecutable,
mallocedKB: heap.mallocedMemory,
peakMallocedKB: heap.peakMallocedMemory,
blinkAllocatedKB
}
}
+9 -5
View File
@@ -137,6 +137,7 @@ import {
collectFolderWorkspaceKeysFromSession,
collectWorktreeHydrationRepoIdsFromSession
} from './lib/workspace-session-hydration-keys'
import { capRestoredOpenFilesByWorktree } from './lib/workspace-session-open-file-cap'
import {
getStartupErrorFallbackUI,
hydratePersistedUIAfterStartupRead
@@ -963,17 +964,20 @@ function App(): React.JSX.Element {
actions.awaitLocalRepoCatalogSettlement()
)
if (!cancelled) {
// Why: cap before any slice reads it — editor, tab and tab-group hydration
// all derive from openFilesByWorktree and must agree on the same list.
const restoredSession = capRestoredOpenFilesByWorktree(sessionRead.session)
const sessionHydrationOptions = {
additionalValidWorkspaceKeys: collectFolderWorkspaceKeysFromSession(sessionRead.session)
additionalValidWorkspaceKeys: collectFolderWorkspaceKeysFromSession(restoredSession)
}
timeRendererStartupSyncStep('hydrate-session-stores', () => {
actions.hydrateWorkspaceSession(sessionRead.session, {
actions.hydrateWorkspaceSession(restoredSession, {
...sessionHydrationOptions,
runtimeHostIdByWorkspaceSessionKey: sessionRead.runtimeHostIdByWorkspaceSessionKey
})
actions.hydrateTabsSession(sessionRead.session, sessionHydrationOptions)
actions.hydrateEditorSession(sessionRead.session, sessionHydrationOptions)
actions.hydrateBrowserSession(sessionRead.session, sessionHydrationOptions)
actions.hydrateTabsSession(restoredSession, sessionHydrationOptions)
actions.hydrateEditorSession(restoredSession, sessionHydrationOptions)
actions.hydrateBrowserSession(restoredSession, sessionHydrationOptions)
})
// Why: prune visit timestamps AFTER hydration (earlier, worktreesByRepo may be empty and prune would drop entries for worktrees about to appear); seed the active worktree if missing.
// See docs/cmd-j-empty-query-ordering.md.
@@ -0,0 +1,60 @@
import { describe, expect, it } from 'vitest'
import { collectRendererMemoryProfileCounts } from '../../lib/renderer-memory-profile'
import {
measureEditorFileContents,
registerEditorContentCensusReader
} from './editor-content-memory-census'
describe('measureEditorFileContents', () => {
it('counts entries and total characters of the loaded file bodies', () => {
expect(
measureEditorFileContents({
a: { content: 'abc', isBinary: false },
b: { content: 'de', isBinary: false }
})
).toEqual({ files: 2, chars: 5 })
})
it('tolerates an entry with no content string', () => {
expect(
measureEditorFileContents({
a: { content: undefined as unknown as string, isBinary: true }
})
).toEqual({ files: 1, chars: 0 })
})
})
describe('editor content memory profile contributor', () => {
it('reports React-held file bodies, which the store-only profile cannot see', () => {
// Why: crash bundles show gigabytes retained with every store collection
// count unchanged. File contents live in useState, outside that profile.
const release = registerEditorContentCensusReader(() => ({ files: 3, chars: 4_000_000 }))
expect(collectRendererMemoryProfileCounts()).toMatchObject({
'editorContent.panels': 1,
'editorContent.files': 3,
'editorContent.chars': 4_000_000
})
release()
expect(collectRendererMemoryProfileCounts()).toMatchObject({
'editorContent.panels': 0,
'editorContent.files': 0,
'editorContent.chars': 0
})
})
it('sums every mounted panel so a split editor is not undercounted', () => {
const releaseA = registerEditorContentCensusReader(() => ({ files: 2, chars: 100 }))
const releaseB = registerEditorContentCensusReader(() => ({ files: 5, chars: 900 }))
expect(collectRendererMemoryProfileCounts()).toMatchObject({
'editorContent.panels': 2,
'editorContent.files': 7,
'editorContent.chars': 1000
})
releaseA()
releaseB()
})
})
@@ -0,0 +1,49 @@
/**
* Sizes of the file bodies each mounted editor panel holds in React state.
*
* Why this is separate from the store profile: renderer-memory-profile only walks
* zustand collections, so the largest strings in the renderer — loaded file
* contents — are structurally invisible when an OOM highwater breadcrumb fires.
*/
import { registerRendererMemoryProfileContributor } from '../../lib/renderer-memory-profile'
import type { FileContent } from './editor-panel-content-types'
export type EditorContentCensus = { files: number; chars: number }
const readers = new Set<() => EditorContentCensus>()
/** Registers one live panel's contents; call the returned function on unmount. */
export function registerEditorContentCensusReader(read: () => EditorContentCensus): () => void {
readers.add(read)
return () => {
readers.delete(read)
}
}
export function measureEditorFileContents(
fileContents: Record<string, FileContent>
): EditorContentCensus {
let files = 0
let chars = 0
for (const key in fileContents) {
if (!Object.hasOwn(fileContents, key)) {
continue
}
files += 1
chars += fileContents[key]?.content?.length ?? 0
}
return { files, chars }
}
registerRendererMemoryProfileContributor('editorContent', () => {
let panels = 0
let files = 0
let chars = 0
for (const read of readers) {
const census = read()
panels += 1
files += census.files
chars += census.chars
}
return { panels, files, chars }
})
@@ -20,6 +20,10 @@ import {
type DiffContent,
type FileContent
} from './editor-panel-content-types'
import {
measureEditorFileContents,
registerEditorContentCensusReader
} from './editor-content-memory-census'
import { canUseChangesModeForFile } from './editor-panel-file-mode'
import {
isReloadableSingleFileDiffTab,
@@ -102,6 +106,15 @@ export function useEditorPanelContentState({
const [diffContents, setDiffContents] = useState<Record<string, DiffContent>>({})
const diffContentsRef = useRef(diffContents)
diffContentsRef.current = diffContents
const fileContentsRef = useRef(fileContents)
fileContentsRef.current = fileContents
// Why: file bodies live in React state, which the store-only memory profile
// cannot walk — without this an OOM report never names them.
useEffect(
() =>
registerEditorContentCensusReader(() => measureEditorFileContents(fileContentsRef.current)),
[]
)
const fileLoadRetryAttemptsRef = useRef<Record<string, number>>({})
// Why: per-tab read generations let a forced/external reload supersede an
// older in-flight read so a slower stale promise cannot overwrite fresh state.
@@ -316,6 +316,43 @@ describe('renderer crash diagnostics', () => {
})
})
it('profiles the V8 space breakdown on highwater but keeps the interval sample lean', () => {
// Why: crash bundles pinned 883MB of growth with every recorded counter
// byte-identical. These four are the only extra shapes Electron exposes.
readHeapStatistics.mockReturnValue({
usedHeapKB: 400 * KB,
totalHeapKB: 800 * KB,
heapLimitKB: 512 * KB,
physicalKB: 780 * KB,
availableKB: 112 * KB,
executableKB: 9 * KB,
mallocedKB: 3 * KB,
peakMallocedKB: 64 * KB,
blinkAllocatedKB: 7 * KB
})
vi.stubGlobal('document', {
getElementsByTagName: () => ({ length: 1 }),
querySelectorAll: () => ({ length: 0 })
})
diagnostics.installRendererCrashDiagnostics()
const named = (name: string): { data: Record<string, unknown> } =>
recordBreadcrumbMock.mock.calls.find(
([entry]) => (entry as { name: string }).name === name
)?.[0] as { data: Record<string, unknown> }
expect(named('renderer_memory_highwater').data).toMatchObject({
physicalMB: 780,
availableMB: 112,
executableMB: 9,
peakMallocedMB: 64
})
// Why: this one fires every 60s for the whole session; four more numbers
// per sample buys nothing the one-shot profile does not already answer.
expect(named('renderer_memory').data).not.toHaveProperty('physicalMB')
expect(named('renderer_memory').data).not.toHaveProperty('executableMB')
})
it('omits the Blink field instead of emitting a junk value for it', () => {
// Why: `undefined * 1024` is NaN. The breadcrumb must carry no
// blinkAllocatedMB at all rather than a meaningless number.
+15
View File
@@ -27,6 +27,11 @@ type BrowserPerformanceMemory = {
type HeapMetrics = BrowserPerformanceMemory & {
mallocedBytes?: number
blinkAllocatedBytes?: number
/** Highwater-only shapes; absent on the quantized `performance.memory` path. */
physicalBytes?: number
availableBytes?: number
executableBytes?: number
peakMallocedBytes?: number
exact: boolean
}
@@ -170,6 +175,12 @@ function recordRendererMemoryHighwater(
heapSource: memory.exact ? 'v8' : 'quantized',
mallocedMB: toMegabytes(memory.mallocedBytes),
blinkAllocatedMB: toMegabytes(memory.blinkAllocatedBytes),
// Why highwater-only: these four separate fragmentation, headroom and compiled
// code from real object retention, and cost nothing at 60s sampling rates.
physicalMB: toMegabytes(memory.physicalBytes),
availableMB: toMegabytes(memory.availableBytes),
executableMB: toMegabytes(memory.executableBytes),
peakMallocedMB: toMegabytes(memory.peakMallocedBytes),
domNodes: document.getElementsByTagName('*').length,
terminalElements: document.querySelectorAll('.xterm').length,
browserWebviews: browserWebviews.browserWebviewCount,
@@ -217,6 +228,10 @@ function readHeapMetrics(): HeapMetrics | undefined {
totalJSHeapSize: exact.totalHeapKB * BYTES_PER_KILOBYTE,
jsHeapSizeLimit: exact.heapLimitKB * BYTES_PER_KILOBYTE,
mallocedBytes: exact.mallocedKB * BYTES_PER_KILOBYTE,
physicalBytes: exact.physicalKB * BYTES_PER_KILOBYTE,
availableBytes: exact.availableKB * BYTES_PER_KILOBYTE,
executableBytes: exact.executableKB * BYTES_PER_KILOBYTE,
peakMallocedBytes: exact.peakMallocedKB * BYTES_PER_KILOBYTE,
// Why guarded: undefined * 1024 is NaN, which would emit a junk field.
blinkAllocatedBytes:
exact.blinkAllocatedKB === undefined
@@ -0,0 +1,115 @@
import { describe, expect, it } from 'vitest'
import type { PersistedOpenFile, WorkspaceSessionState } from '../../../shared/types'
import {
capRestoredOpenFilesByWorktree,
MAX_RESTORED_OPEN_FILES_PER_WORKSPACE
} from './workspace-session-open-file-cap'
function persistedFile(index: number, overrides: Partial<PersistedOpenFile> = {}) {
return {
filePath: `/repo/src/file-${index}.ts`,
relativePath: `src/file-${index}.ts`,
worktreeId: 'wt-1',
language: 'typescript',
...overrides
} satisfies PersistedOpenFile
}
function sessionWith(
openFilesByWorktree: Record<string, PersistedOpenFile[]>,
activeFileIdByWorktree?: Record<string, string>
): WorkspaceSessionState {
return { openFilesByWorktree, activeFileIdByWorktree } as unknown as WorkspaceSessionState
}
describe('capRestoredOpenFilesByWorktree', () => {
it('returns the same session object when every workspace is under the cap', () => {
const session = sessionWith({
'wt-1': Array.from({ length: MAX_RESTORED_OPEN_FILES_PER_WORKSPACE }, (_, i) =>
persistedFile(i)
)
})
expect(capRestoredOpenFilesByWorktree(session)).toBe(session)
})
it('keeps the newest entries when a workspace exceeds the cap', () => {
const total = MAX_RESTORED_OPEN_FILES_PER_WORKSPACE + 50
const session = sessionWith({
'wt-1': Array.from({ length: total }, (_, i) => persistedFile(i))
})
const kept = capRestoredOpenFilesByWorktree(session).openFilesByWorktree?.['wt-1'] ?? []
expect(kept).toHaveLength(MAX_RESTORED_OPEN_FILES_PER_WORKSPACE)
expect(kept[0]?.filePath).toBe(persistedFile(50).filePath)
expect(kept.at(-1)?.filePath).toBe(persistedFile(total - 1).filePath)
})
it('never drops a file with an unsaved draft, even from the oldest entries', () => {
const total = MAX_RESTORED_OPEN_FILES_PER_WORKSPACE + 50
const files = Array.from({ length: total }, (_, i) =>
persistedFile(i, i === 0 ? { dirtyDraftContent: 'unsaved work' } : {})
)
const kept = capRestoredOpenFilesByWorktree(sessionWith({ 'wt-1': files }))
.openFilesByWorktree?.['wt-1']
expect(kept?.[0]?.dirtyDraftContent).toBe('unsaved work')
expect(kept).toHaveLength(MAX_RESTORED_OPEN_FILES_PER_WORKSPACE)
})
it('keeps every dirty file even when they alone exceed the cap', () => {
const total = MAX_RESTORED_OPEN_FILES_PER_WORKSPACE + 20
const files = Array.from({ length: total }, (_, i) =>
persistedFile(i, { dirtyDraftContent: `draft ${i}` })
)
const kept = capRestoredOpenFilesByWorktree(sessionWith({ 'wt-1': files }))
.openFilesByWorktree?.['wt-1']
expect(kept).toHaveLength(total)
})
it('keeps the focused tab addressed by its raw path id', () => {
const total = MAX_RESTORED_OPEN_FILES_PER_WORKSPACE + 50
const files = Array.from({ length: total }, (_, i) => persistedFile(i))
const session = sessionWith({ 'wt-1': files }, { 'wt-1': files[0].filePath })
const kept = capRestoredOpenFilesByWorktree(session).openFilesByWorktree?.['wt-1'] ?? []
expect(kept[0]?.filePath).toBe(files[0].filePath)
})
it('keeps the focused tab addressed by its owner-qualified id', () => {
const total = MAX_RESTORED_OPEN_FILES_PER_WORKSPACE + 50
const files = Array.from({ length: total }, (_, i) => persistedFile(i))
const ownedId = `editor:wt-1:local:${encodeURIComponent(files[0].filePath)}`
const session = sessionWith({ 'wt-1': files }, { 'wt-1': ownedId })
const kept = capRestoredOpenFilesByWorktree(session).openFilesByWorktree?.['wt-1'] ?? []
expect(kept[0]?.filePath).toBe(files[0].filePath)
})
it('caps each workspace independently and leaves untouched ones referentially stable', () => {
const small = [persistedFile(1), persistedFile(2)]
const large = Array.from({ length: MAX_RESTORED_OPEN_FILES_PER_WORKSPACE + 5 }, (_, i) =>
persistedFile(i)
)
const session = sessionWith({ 'wt-small': small, 'wt-large': large })
const capped = capRestoredOpenFilesByWorktree(session)
expect(capped.openFilesByWorktree?.['wt-small']).toBe(small)
expect(capped.openFilesByWorktree?.['wt-large']).toHaveLength(
MAX_RESTORED_OPEN_FILES_PER_WORKSPACE
)
})
it('tolerates a session with no persisted editor files', () => {
const session = { activeFileIdByWorktree: {} } as unknown as WorkspaceSessionState
expect(capRestoredOpenFilesByWorktree(session)).toBe(session)
})
})
@@ -0,0 +1,70 @@
import type { PersistedOpenFile, WorkspaceSessionState } from '../../../shared/types'
/**
* Restore-time bound on persisted editor tabs, per workspace.
*
* Why: `openFilesByWorktree` is append-only and never pruned, so it accumulates
* across every restart. Crash bundles show sessions restoring 3,711 editor tabs
* into one renderer. 100 is far above any real working set.
*/
export const MAX_RESTORED_OPEN_FILES_PER_WORKSPACE = 100
/**
* Applied to the session once, before any slice hydrates it: the editor, tab and
* tab-group slices all derive their state from `openFilesByWorktree`, so capping
* it here keeps them consistent instead of leaving orphaned tabs behind.
*/
export function capRestoredOpenFilesByWorktree(
session: WorkspaceSessionState
): WorkspaceSessionState {
const openFilesByWorktree = session.openFilesByWorktree
if (!openFilesByWorktree) {
return session
}
let capped: Record<string, PersistedOpenFile[]> | null = null
for (const [workspaceKey, files] of Object.entries(openFilesByWorktree)) {
if (files.length <= MAX_RESTORED_OPEN_FILES_PER_WORKSPACE) {
continue
}
capped ??= { ...openFilesByWorktree }
capped[workspaceKey] = selectRestorableOpenFiles(
files,
session.activeFileIdByWorktree?.[workspaceKey]
)
}
return capped ? { ...session, openFilesByWorktree: capped } : session
}
function selectRestorableOpenFiles(
files: readonly PersistedOpenFile[],
activeFileId: string | null | undefined
): PersistedOpenFile[] {
const keep = new Set<number>()
files.forEach((file, index) => {
// Why: an unsaved draft is unrecoverable once dropped, and losing the tab the
// user left focused is the one omission they would notice immediately.
if (file.dirtyDraftContent !== undefined || addressesFile(activeFileId, file.filePath)) {
keep.add(index)
}
})
// Why: newly opened files are appended, so the tail is the most recent working set.
for (
let index = files.length - 1;
index >= 0 && keep.size < MAX_RESTORED_OPEN_FILES_PER_WORKSPACE;
index -= 1
) {
keep.add(index)
}
return files.filter((_, index) => keep.has(index))
}
// Why: hydration keys a file either by its raw path or by an owner-qualified id
// whose final segment is the encoded path; both forms reach activeFileIdByWorktree.
function addressesFile(activeFileId: string | null | undefined, filePath: string): boolean {
if (!activeFileId) {
return false
}
return activeFileId === filePath || activeFileId.endsWith(`:${encodeURIComponent(filePath)}`)
}
@@ -7,6 +7,7 @@
import { describe, expect, it } from 'vitest'
import { createStore } from 'zustand/vanilla'
import { useStore } from 'zustand'
import { collectRendererMemoryProfileCounts } from '../lib/renderer-memory-profile'
import { installStoreListenerCensus, readStoreListenerCount } from './store-listener-census'
type CensusState = { n: number }
@@ -76,6 +77,18 @@ describe('store listener census', () => {
expect(readStoreListenerCount()).toBe(0)
})
it('reports the live count into the renderer memory profile', () => {
// Why: the count was already computed and then discarded. OOM crash reports
// need it to tell a subscriber leak apart from a data leak.
const { api } = createCensusStore()
const unsubscribe = api.subscribe(() => undefined)
expect(collectRendererMemoryProfileCounts()).toMatchObject({ 'storeSubscribers.live': 1 })
unsubscribe()
expect(collectRendererMemoryProfileCounts()).toMatchObject({ 'storeSubscribers.live': 0 })
})
it('still delivers state updates to a counted listener', () => {
const { api } = createCensusStore()
let seen = 0
@@ -1,3 +1,8 @@
import {
registerRendererMemoryProfileContributor,
type RendererMemoryProfileCounts
} from '../lib/renderer-memory-profile'
/**
* Live count of zustand store subscribers, for the typing-latency census.
*
@@ -18,6 +23,13 @@ type StoreListenerCensusApi<T> = {
let liveListenerCount: number | null = null
// Why: an OOM report cannot otherwise separate a subscriber leak (components that
// never unmount) from a data leak, and the count is already being computed.
registerRendererMemoryProfileContributor('storeSubscribers', (): RendererMemoryProfileCounts => {
const live = readStoreListenerCount()
return live === null ? {} : { live }
})
export function readStoreListenerCount(): number | null {
return liveListenerCount
}
+12
View File
@@ -9,13 +9,25 @@
* works in a sandboxed, context-isolated preload.
*
* Electron reports these in kilobytes; we keep that unit unconverted here.
*
* No per-space breakdown: `v8.getHeapSpaceStatistics()` needs the `v8` module,
* which a sandboxed preload cannot require. These are every field Electron's own
* `process.getHeapStatistics()` exposes.
*/
export type RendererHeapStatistics = {
usedHeapKB: number
totalHeapKB: number
heapLimitKB: number
/** Committed vs reserved: separates fragmentation from real retention. */
physicalKB: number
/** Headroom left before the OOM abort, which `heapLimitKB` alone cannot track. */
availableKB: number
/** V8 code space — the one growth shape (compiled code) no JS counter reveals. */
executableKB: number
/** Off-heap V8 allocations, which `usedJSHeapSize` never included. */
mallocedKB: number
/** High-water mark for `mallocedKB`, which only ever reports the instant value. */
peakMallocedKB: number
/**
* Blink's own allocator (DOM, layout), invisible to the V8 heap counters.
* Optional: supplementary, so its absence must never discard the V8 numbers.