feat(crash-diagnostics): sample the near-death renderer heap

The 42-crash OOM cluster (macOS exit 5, Windows 0xE0000008/-36861, one Linux
SIGKILL) dies at 90-97% of heapLimitMB, but highwater profiles stopped at 80%
— roughly 60 minutes stale at the observed ~25MB/min ramp — and the routine
60s crumb carried no byte attribution at all, so reports held two isolated
snapshots and named no retainer.

- Add 0.9 and 0.95 highwater levels; the 0.95 profile now lands ~2min before
  death.
- Carry truncated byte attribution on every routine renderer_memory breadcrumb
  so reports show a trend, not two points: storeKB (__totalKB + top 4 slices)
  and terminals (estBufferKB + estPanes). Both are needed — F0BMB7LAAMU grew
  857MB with every store field frozen, i.e. outside the store, where xterm
  scrollback lives; and F0BM6V4KEBV died at 78% of its limit, so no highwater
  level fired at all and terminals would have been sampled exactly once.
- Tag the routine crumb with rendererSurface, matching the highwater crumb: main
  and the dashboard popout share one 30-slot ring, so an untagged storeKB series
  interleaves two unrelated heaps.
- Derive the main-process retained-breadcrumb budget from thresholds x renderer
  surfaces in shared/renderer-memory-highwater.ts. The retained key is
  `<surface>:<thresholdPct>`, so 4 levels x 2 surfaces demanded 8 slots against a
  hand-written 4, and oldest-first eviction dropped every main-window profile —
  including the 95% one this change exists to produce.
- Share one contributor census per sample between the trend counts and the
  highwater profile, so crossing 90/95% no longer walks the whole store twice
  with no heap headroom left.
- Assert both budgets against the contributors the app really registers rather
  than stand-ins: 41 counts of MAX_PROFILE_COUNTS=64 at their widest, and 7 of
  MAX_TREND_COUNTS=8 on the routine crumb.

Diagnostics only — no leak fix and no heap-ceiling change.

Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
Neil
2026-08-04 01:22:26 -07:00
co-authored by Orca
parent dbbeca0916
commit 41abe354a5
10 changed files with 602 additions and 25 deletions
@@ -5,6 +5,11 @@ import {
recordCoalescedCrashBreadcrumb,
recordCrashBreadcrumb
} from './crash-breadcrumb-store'
import {
MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS,
RENDERER_MEMORY_HIGHWATER_RATIOS,
RENDERER_MEMORY_SURFACES
} from '../../shared/renderer-memory-highwater'
afterEach(() => {
vi.useRealTimers()
@@ -50,7 +55,8 @@ describe('crash breadcrumb store', () => {
})
it('caps retained high-water profiles', () => {
for (let index = 0; index < 5; index += 1) {
const overflow = MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS + 1
for (let index = 0; index < overflow; index += 1) {
recordCrashBreadcrumb('renderer_memory_highwater', {
rendererSurface: `surface-${index}`,
thresholdPct: 80
@@ -59,7 +65,66 @@ describe('crash breadcrumb store', () => {
expect(
getCrashBreadcrumbSnapshot().map((breadcrumb) => breadcrumb.data?.rendererSurface)
).toEqual(['surface-1', 'surface-2', 'surface-3', 'surface-4'])
).toEqual(
Array.from(
{ length: MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS },
(_, index) => `surface-${index + 1}`
)
)
})
// The retained key is `<surface>:<thresholdPct>`, so the budget must track
// thresholds x surfaces. It once lagged at 4 against 8 emitted profiles, and
// oldest-first eviction dropped every main-window one — including the 95%
// near-death profile the OOM reports exist to carry.
it('retains every threshold profile that both renderer surfaces can emit', () => {
for (const surface of RENDERER_MEMORY_SURFACES) {
for (const ratio of RENDERER_MEMORY_HIGHWATER_RATIOS) {
recordCrashBreadcrumb('renderer_memory_highwater', {
rendererSurface: surface,
thresholdPct: Math.round(ratio * 100)
})
}
}
const profiles = getCrashBreadcrumbSnapshot().map(
(breadcrumb) => `${breadcrumb.data?.rendererSurface}:${breadcrumb.data?.thresholdPct}`
)
expect(profiles).toHaveLength(
RENDERER_MEMORY_SURFACES.length * RENDERER_MEMORY_HIGHWATER_RATIOS.length
)
for (const surface of RENDERER_MEMORY_SURFACES) {
for (const ratio of RENDERER_MEMORY_HIGHWATER_RATIOS) {
expect(profiles).toContain(`${surface}:${Math.round(ratio * 100)}`)
}
}
})
// Retained profiles spend the ring budget rather than extending it, so the
// newest events lose slots one-for-one instead of the snapshot growing.
it('never exceeds the ring size once high-water profiles are retained', () => {
for (const surface of RENDERER_MEMORY_SURFACES) {
for (const ratio of RENDERER_MEMORY_HIGHWATER_RATIOS) {
recordCrashBreadcrumb('renderer_memory_highwater', {
rendererSurface: surface,
thresholdPct: Math.round(ratio * 100)
})
}
}
for (let index = 0; index < 40; index += 1) {
recordCrashBreadcrumb(`event_${index}`, { index })
}
const snapshot = getCrashBreadcrumbSnapshot()
expect(snapshot).toHaveLength(30)
expect(
snapshot.filter((breadcrumb) => breadcrumb.name === 'renderer_memory_highwater')
).toHaveLength(MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS)
expect(snapshot.at(-1)?.name).toBe('event_39')
expect(snapshot.map((breadcrumb) => breadcrumb.name)).not.toContain(
`event_${39 - (30 - MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS)}`
)
})
it('redacts sensitive breadcrumb fields before they can be snapshotted', () => {
@@ -4,10 +4,12 @@ import {
type CrashReportBreadcrumbData,
type CrashReportBreadcrumb
} from '../../shared/crash-reporting'
import { MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS } from '../../shared/renderer-memory-highwater'
const MAX_BREADCRUMBS = 30
// Why: retain two thresholds for each renderer surface without growing the ring.
const MAX_RETAINED_BREADCRUMBS = 4
// Why derived: eviction is oldest-first, so a budget smaller than the emitted
// (surface x threshold) profiles drops the near-death one.
const MAX_RETAINED_BREADCRUMBS = MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS
// Why: coalesceKey embeds an open-string agentType (length-trimmed only, never
// enum-checked), so the key space is unbounded over a long multi-agent/SSH session.
// Bound the coalesce map the same way ProcessGoneDedupe bounds its key map.
@@ -163,7 +165,11 @@ export function getCrashBreadcrumbSnapshot(): CrashReportBreadcrumb[] {
resolveAllPendingCoalescedBreadcrumbs()
// Why: long sessions must retain threshold profiles without growing the 30-entry budget.
const retained = [...retainedBreadcrumbs.values()]
const recent = breadcrumbs.slice(-(MAX_BREADCRUMBS - retained.length))
// Why a start index rather than slice(-budget): a retained budget that ever
// reaches MAX_BREADCRUMBS makes the budget 0, and slice(-0) returns the whole
// ring. Counting forward degrades to [] with no branch to leave untested.
const recentBudget = MAX_BREADCRUMBS - retained.length
const recent = breadcrumbs.slice(Math.max(0, breadcrumbs.length - recentBudget))
return [...retained, ...recent]
.sort((left, right) => left.createdAt.localeCompare(right.createdAt))
.map((breadcrumb) => ({
@@ -78,6 +78,7 @@ describe('renderer crash diagnostics', () => {
name: 'renderer_memory',
data: {
reason: 'startup',
rendererSurface: 'main',
usedHeapMB: 32,
totalHeapMB: 64,
heapLimitMB: 512,
@@ -222,6 +223,145 @@ describe('renderer crash diagnostics', () => {
unregister()
})
it('emits highwater samples inside the last 20% of the heap limit', async () => {
// Why: the 42-crash OOM cluster dies at 90-97% of the limit. With samples
// only at 60/80% the nearest profile is ~60min stale and names nothing.
const memory = (window.performance as unknown as { memory: Record<string, number> }).memory
vi.stubGlobal('document', {
getElementsByTagName: () => ({ length: 1 }),
querySelectorAll: () => ({ length: 0 })
})
diagnostics.installRendererCrashDiagnostics()
const tick = setIntervalMock.mock.calls[0][0] as () => void
memory.usedJSHeapSize = 0.92 * memory.jsHeapSizeLimit
tick()
memory.usedJSHeapSize = 0.96 * memory.jsHeapSizeLimit
tick()
const thresholds = recordBreadcrumbMock.mock.calls
.filter((call) => (call[0] as { name: string }).name === 'renderer_memory_highwater')
.map((call) => (call[0] as { data: { thresholdPct: number } }).data.thresholdPct)
expect(thresholds).toEqual([60, 80, 90, 95])
})
it('carries both shipped byte contributors on the routine renderer_memory breadcrumb', async () => {
// Why the real registrations rather than stand-ins: the sub-A leak grows
// outside the zustand store, so a trend carrying only storeKB still cannot
// name it. F0BM6V4KEBV died at 78% of the limit, where no highwater level
// fires and this crumb is the only attribution there is — which is also why
// no document is stubbed here: 32MB of 512MB crosses nothing.
const { useAppStore } = await import('../store')
const { registerLivePaneManager, unregisterLivePaneManager } =
await import('./pane-manager/pane-manager-registry')
useAppStore.setState({ worktrees: [{ id: 'w1' }, { id: 'w2' }] } as never)
const paneManager = {
resetWebglTextureAtlases: () => undefined,
getPaneCount: () => 2,
getPanes: () =>
[1, 2].map((id) => ({
id,
terminal: { cols: 200, buffer: { active: { length: 5000 } } }
}))
}
registerLivePaneManager(paneManager)
diagnostics.installRendererCrashDiagnostics()
const routine = recordBreadcrumbMock.mock.calls.find(
(call) => (call[0] as { name: string }).name === 'renderer_memory'
)?.[0] as { data: Record<string, number> }
expect(routine.data['storeKB.__totalKB']).toBeGreaterThan(0)
// 2 panes x 5000 rows x 200 cols x 16B ~= 31MB of scrollback.
expect(routine.data['terminals.estBufferKB']).toBeGreaterThan(30_000)
expect(routine.data['terminals.estPanes']).toBe(2)
unregisterLivePaneManager(paneManager)
})
it('carries truncated byte attribution on the routine renderer_memory breadcrumb', async () => {
// Why: two isolated highwater snapshots cannot show a ramp; the 60s crumb
// must carry a byte TREND so a report localizes the retainer on its own.
const profile = await import('./renderer-memory-profile')
const unregister = profile.registerRendererMemoryProfileContributor(
'storeKB',
() => ({ worktrees: 120, __totalKB: 4200, tabs: 900, agents: 5, prs: 60, drafts: 2 }),
{ trendLimit: 5 }
)
diagnostics.installRendererCrashDiagnostics()
const routine = recordBreadcrumbMock.mock.calls.find(
(call) => (call[0] as { name: string }).name === 'renderer_memory'
)?.[0] as { data: Record<string, unknown> }
expect(routine.data).toMatchObject({
'storeKB.__totalKB': 4200,
'storeKB.tabs': 900,
'storeKB.worktrees': 120,
'storeKB.prs': 60,
'storeKB.agents': 5
})
expect(routine.data).not.toHaveProperty('storeKB.drafts')
unregister()
})
it('censuses the store once on a sample that also profiles the highwater', async () => {
// Why: the trend and the highwater profile share one contributor. Walking the
// store twice is worst exactly at 90/95%, where there is no headroom for the
// transient allocation the census makes.
const memory = (window.performance as unknown as { memory: Record<string, number> }).memory
memory.usedJSHeapSize = 0.96 * memory.jsHeapSizeLimit
vi.stubGlobal('document', {
getElementsByTagName: () => ({ length: 1 }),
querySelectorAll: () => ({ length: 0 })
})
let censuses = 0
const profile = await import('./renderer-memory-profile')
const unregister = profile.registerRendererMemoryProfileContributor(
'storeKB',
() => {
censuses += 1
return { __totalKB: 4200 }
},
{ trendLimit: 5 }
)
diagnostics.installRendererCrashDiagnostics()
expect(
recordBreadcrumbMock.mock.calls.filter(
(call) => (call[0] as { name: string }).name === 'renderer_memory_highwater'
)
).toHaveLength(4)
expect(censuses).toBe(1)
unregister()
})
it('tags every routine renderer_memory sample with its renderer surface', async () => {
// Why: main and the dashboard popout push into the same 30-slot main-process
// ring, each with its own storeKB census. Untagged, the byte trend this crumb
// exists to expose interleaves two unrelated heaps into one unreadable series.
const profile = await import('./renderer-memory-profile')
const unregister = profile.registerRendererMemoryProfileContributor(
'storeKB',
() => ({ __totalKB: 12 }),
{ trendLimit: 5 }
)
diagnostics.installRendererCrashDiagnostics('dashboard-popout')
const tick = setIntervalMock.mock.calls[0][0] as () => void
tick()
const surfaces = recordBreadcrumbMock.mock.calls
.filter((call) => (call[0] as { name: string }).name === 'renderer_memory')
.map((call) => (call[0] as { data: { rendererSurface?: string } }).data.rendererSurface)
expect(surfaces).toEqual(['dashboard-popout', 'dashboard-popout'])
unregister()
})
it('skips highwater emission when heap readings are not finite', () => {
const memory = (window.performance as unknown as { memory: Record<string, number> }).memory
memory.usedJSHeapSize = Number.NaN
+26 -9
View File
@@ -2,20 +2,27 @@ import type {
CrashReportBreadcrumbData,
CrashReportDetailValue
} from '../../../shared/crash-reporting'
// Why the levels live in shared/: one detailed breadcrumb per (surface,
// threshold) is retained in the main process, whose budget is derived from them.
import {
RENDERER_MEMORY_HIGHWATER_RATIOS,
type RendererSurface
} from '../../../shared/renderer-memory-highwater'
import {
getBrowserWebviewMemoryProfile,
type BrowserWebviewMemoryProfile
} from '../components/browser-pane/webview-registry'
import { recordRendererCrashBreadcrumb } from './crash-breadcrumb-recorder'
import { collectRendererMemoryProfileCounts } from './renderer-memory-profile'
import {
collectRendererMemoryProfileCounts,
collectRendererMemoryTrendCounts,
createRendererMemoryCensus,
type RendererMemoryCensus
} from './renderer-memory-profile'
const RENDERER_MEMORY_SAMPLE_INTERVAL_MS = 60_000
const BYTES_PER_MEGABYTE = 1024 * 1024
const BYTES_PER_KILOBYTE = 1024
// Why: one detailed breadcrumb per threshold names what grew before an OOM.
const RENDERER_MEMORY_HIGHWATER_RATIOS = [0.6, 0.8] as const
type RendererSurface = 'main' | 'dashboard-popout'
type BrowserPerformanceMemory = {
usedJSHeapSize?: number
@@ -117,11 +124,17 @@ function recordRendererMemory(reason: string): void {
return
}
const browserWebviews = getBrowserWebviewMemoryProfile()
// Why one census per sample: the highwater profile below repeats this pass's
// contributors, and near-OOM is the worst moment to walk the store twice.
const census = createRendererMemoryCensus()
recordRendererCrashBreadcrumb(
'renderer_memory',
compactBreadcrumbData({
reason,
// Why: main and the dashboard popout each sample into the same ring, so an
// untagged series reads as a sawtooth between two unrelated heaps.
rendererSurface,
usedHeapMB: toMegabytes(memory.usedJSHeapSize),
totalHeapMB: toMegabytes(memory.totalJSHeapSize),
heapLimitMB: toMegabytes(memory.jsHeapSizeLimit),
@@ -129,15 +142,19 @@ function recordRendererMemory(reason: string): void {
mallocedMB: toMegabytes(memory.mallocedBytes),
blinkAllocatedMB: toMegabytes(memory.blinkAllocatedBytes),
browserWebviews: browserWebviews.browserWebviewCount,
registeredBrowserGuests: browserWebviews.registeredBrowserGuestCount
registeredBrowserGuests: browserWebviews.registeredBrowserGuestCount,
// Why on every sample: a byte trend across the session localizes a
// retainer that two near-death snapshots cannot.
...collectRendererMemoryTrendCounts(census)
})
)
recordRendererMemoryHighwater(memory, browserWebviews)
recordRendererMemoryHighwater(memory, browserWebviews, census)
}
function recordRendererMemoryHighwater(
memory: HeapMetrics,
browserWebviews: BrowserWebviewMemoryProfile
browserWebviews: BrowserWebviewMemoryProfile,
census: RendererMemoryCensus
): void {
const used = memory.usedJSHeapSize
const limit = memory.jsHeapSizeLimit
@@ -170,7 +187,7 @@ function recordRendererMemoryHighwater(
terminalElements: document.querySelectorAll('.xterm').length,
browserWebviews: browserWebviews.browserWebviewCount,
registeredBrowserGuests: browserWebviews.registeredBrowserGuestCount,
...collectRendererMemoryProfileCounts()
...collectRendererMemoryProfileCounts(census)
})
for (const threshold of RENDERER_MEMORY_HIGHWATER_RATIOS) {
if (ratio < threshold || emittedHighwaterRatios.has(threshold)) {
@@ -237,4 +237,11 @@ export function getLivePaneMemoryProfileCounts(): Record<string, number> {
// Why here: contributors push in (crash-diagnostics stays a leaf); same-name
// re-registration on dev HMR overwrites, so no disposal hook is needed.
registerRendererMemoryProfileContributor('terminals', getLivePaneMemoryProfileCounts)
// Why trendLimit: scrollback bytes live outside the zustand store, so storeKB
// cannot see them and the one-shot highwater profiles sample them once — the
// 78%-of-limit death in F0BM6V4KEBV never reached even the 90% level. The
// census above is sample-capped arithmetic, far cheaper than the storeKB walk
// it rides along with. 2 keys carries estBufferKB plus the pane count it scales.
registerRendererMemoryProfileContributor('terminals', getLivePaneMemoryProfileCounts, {
trendLimit: 2
})
@@ -1,6 +1,8 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
collectRendererMemoryProfileCounts,
collectRendererMemoryTrendCounts,
createRendererMemoryCensus,
registerRendererMemoryProfileContributor,
summarizeStateCollectionSizes
} from './renderer-memory-profile'
@@ -135,6 +137,115 @@ describe('collectRendererMemoryProfileCounts', () => {
})
})
describe('collectRendererMemoryTrendCounts', () => {
it('keeps only the heaviest keys of trend-tagged contributors', () => {
unregisters.push(
registerRendererMemoryProfileContributor(
'storeKB',
() => ({ tabs: 900, __totalKB: 4200, worktrees: 120, prs: 60, agents: 5, drafts: 2 }),
{ trendLimit: 5 }
)
)
register('store', () => ({ worktrees: 40 }))
expect(collectRendererMemoryTrendCounts()).toEqual({
'storeKB.__totalKB': 4200,
'storeKB.tabs': 900,
'storeKB.worktrees': 120,
'storeKB.prs': 60,
'storeKB.agents': 5
})
})
it('bounds the aggregate trend budget and contains throwing contributors', () => {
unregisters.push(
registerRendererMemoryProfileContributor(
'broken',
() => {
throw new Error('boom')
},
{ trendLimit: 4 }
)
)
unregisters.push(
registerRendererMemoryProfileContributor(
'runaway',
() => Object.fromEntries(Array.from({ length: 500 }, (_, index) => [`key${index}`, index])),
{ trendLimit: 500 }
)
)
const counts = collectRendererMemoryTrendCounts()
expect(counts['broken.error']).toBe(1)
expect(Object.keys(counts)).toHaveLength(8)
})
})
// A sample that crosses a threshold runs both censuses, and storeKB walks every
// top-level store slice with a transient allocation the size of what it measures
// — the last thing to do twice with no heap headroom left.
describe('a census shared across one sample pass', () => {
it('reads each contributor once for both the trend and the profile', () => {
let reads = 0
unregisters.push(
registerRendererMemoryProfileContributor(
'storeKB',
() => {
reads += 1
return { __totalKB: 4200, tabs: 900 }
},
{ trendLimit: 5 }
)
)
const census = createRendererMemoryCensus()
const trend = collectRendererMemoryTrendCounts(census)
const profile = collectRendererMemoryProfileCounts(census)
expect(reads).toBe(1)
expect(trend).toEqual({ 'storeKB.__totalKB': 4200, 'storeKB.tabs': 900 })
expect(profile).toEqual({ 'storeKB.__totalKB': 4200, 'storeKB.tabs': 900 })
})
it('keeps a fresh census per sample so the trend still moves', () => {
let size = 10
unregisters.push(
registerRendererMemoryProfileContributor('storeKB', () => ({ __totalKB: (size += 10) }), {
trendLimit: 1
})
)
expect(collectRendererMemoryTrendCounts(createRendererMemoryCensus())).toEqual({
'storeKB.__totalKB': 20
})
expect(collectRendererMemoryTrendCounts(createRendererMemoryCensus())).toEqual({
'storeKB.__totalKB': 30
})
})
it('reports a throwing contributor in both censuses without re-running it', () => {
let reads = 0
unregisters.push(
registerRendererMemoryProfileContributor(
'broken',
() => {
reads += 1
throw new Error('boom')
},
{ trendLimit: 4 }
)
)
const census = createRendererMemoryCensus()
expect(collectRendererMemoryTrendCounts(census)).toEqual({ 'broken.error': 1 })
expect(collectRendererMemoryProfileCounts(census)).toEqual({ 'broken.error': 1 })
expect(reads).toBe(1)
})
})
// The shipped contributors are measured against MAX_PROFILE_COUNTS in
// renderer-memory-shipped-contributors.test.ts, which registers them for real.
describe('summarizeStateCollectionSizes', () => {
it('reports the largest collections first, capped at the limit', () => {
const state = {
+115 -8
View File
@@ -10,21 +10,48 @@ export type RendererMemoryProfileCounts = Record<string, number>
type RendererMemoryProfileContributor = () => RendererMemoryProfileCounts
const contributors = new Map<string, RendererMemoryProfileContributor>()
type RendererMemoryProfileOptions = {
/** Heaviest keys to also carry on the routine renderer_memory breadcrumb. */
trendLimit?: number
}
type RegisteredContributor = {
collect: RendererMemoryProfileContributor
trendLimit: number
}
/**
* Per-sample memo shared by the trend and profile censuses.
*
* Why: crossing 90%/95% runs both in one pass, and the storeKB contributor walks
* every top-level store slice with a transient allocation the size of what it
* measures — exactly when the heap has no headroom to walk it twice.
*/
export type RendererMemoryCensus = Map<string, RendererMemoryProfileCounts | null>
export function createRendererMemoryCensus(): RendererMemoryCensus {
return new Map()
}
const contributors = new Map<string, RegisteredContributor>()
// Why: breadcrumbs are retained per session; a misbehaving contributor must not
// bloat every crash report. 32 counts is plenty to name a leaking subsystem.
const MAX_COUNTS_PER_CONTRIBUTOR = 32
// Why: individually bounded contributors can still create unbounded near-OOM work in aggregate.
const MAX_PROFILE_COUNTS = 64
export const MAX_PROFILE_COUNTS = 64
// Why: empty contributors do not consume the count budget but must not make collection unbounded.
const MAX_PROFILE_CONTRIBUTORS = 64
const MAX_CONTRIBUTOR_NAME_LENGTH = 64
const MAX_COUNT_KEY_LENGTH = 80
// Why far below MAX_PROFILE_COUNTS: this rides every 60s sample, not just the
// near-OOM ones, and every retained breadcrumb pays for it.
export const MAX_TREND_COUNTS = 8
export function registerRendererMemoryProfileContributor(
name: string,
contributor: RendererMemoryProfileContributor
contributor: RendererMemoryProfileContributor,
options?: RendererMemoryProfileOptions
): () => void {
if (
name.length === 0 ||
@@ -33,26 +60,106 @@ export function registerRendererMemoryProfileContributor(
) {
return () => undefined
}
contributors.set(name, contributor)
contributors.set(name, { collect: contributor, trendLimit: options?.trendLimit ?? 0 })
return () => {
if (contributors.get(name) === contributor) {
if (contributors.get(name)?.collect === contributor) {
contributors.delete(name)
}
}
}
export function collectRendererMemoryProfileCounts(): RendererMemoryProfileCounts {
/**
* Heaviest keys of trend-tagged contributors, for the routine renderer_memory
* breadcrumb.
*
* Why: highwater profiles are two isolated snapshots. A crash report can only
* name a retainer if the byte attribution appears as a ramp across the whole
* session, so a truncated slice of it rides every sample.
*/
export function collectRendererMemoryTrendCounts(
census?: RendererMemoryCensus
): RendererMemoryProfileCounts {
const counts: RendererMemoryProfileCounts = {}
let collected = 0
for (const [name, entry] of contributors) {
if (entry.trendLimit <= 0 || collected >= MAX_TREND_COUNTS) {
continue
}
try {
const ranked = rankContributorCounts(readContribution(name, entry, census))
const keep = Math.min(entry.trendLimit, MAX_TREND_COUNTS - collected)
for (const [key, value] of ranked.slice(0, keep)) {
counts[`${name}.${key}`] = value
collected += 1
}
} catch {
counts[`${name}.error`] = 1
collected += 1
}
}
return counts
}
/** Throws on failure so both censuses keep funnelling it into `${name}.error`. */
function readContribution(
name: string,
entry: RegisteredContributor,
census?: RendererMemoryCensus
): RendererMemoryProfileCounts {
const memoized = census?.get(name)
if (memoized) {
return memoized
}
if (memoized === null) {
throw new Error(`renderer memory contributor failed: ${name}`)
}
try {
const contribution = entry.collect()
census?.set(name, contribution)
return contribution
} catch (error) {
census?.set(name, null)
throw error
}
}
function rankContributorCounts(contribution: RendererMemoryProfileCounts): [string, number][] {
const ranked: [string, number][] = []
let inspected = 0
for (const key in contribution) {
if (inspected >= MAX_COUNTS_PER_CONTRIBUTOR) {
break
}
inspected += 1
if (
!Object.hasOwn(contribution, key) ||
key.length === 0 ||
key.length > MAX_COUNT_KEY_LENGTH
) {
continue
}
const value = contribution[key]
if (typeof value === 'number' && Number.isFinite(value)) {
ranked.push([key, value])
}
}
return ranked.sort((a, b) => b[1] - a[1])
}
export function collectRendererMemoryProfileCounts(
census?: RendererMemoryCensus
): RendererMemoryProfileCounts {
const counts: RendererMemoryProfileCounts = {}
let collected = 0
let visited = 0
for (const [name, contributor] of contributors) {
for (const [name, entry] of contributors) {
if (collected >= MAX_PROFILE_COUNTS || visited >= MAX_PROFILE_CONTRIBUTORS) {
break
}
visited += 1
// Why: a broken contributor must never take down memory reporting itself.
try {
const contribution = contributor()
const contribution = readContribution(name, entry, census)
let inspected = 0
for (const key in contribution) {
if (inspected >= MAX_COUNTS_PER_CONTRIBUTOR || collected >= MAX_PROFILE_COUNTS) {
@@ -0,0 +1,97 @@
import { afterEach, describe, expect, it } from 'vitest'
import {
registerLivePaneManager,
unregisterLivePaneManager
} from './pane-manager/pane-manager-registry'
import {
MAX_PROFILE_COUNTS,
MAX_TREND_COUNTS,
collectRendererMemoryProfileCounts,
collectRendererMemoryTrendCounts,
createRendererMemoryCensus
} from './renderer-memory-profile'
import { useAppStore } from '../store'
import type { AppState } from '../store/types'
/**
* Drives the contributors the app actually registers — importing ../store and
* pane-manager-registry for their registration side effects — instead of
* hand-written stand-ins that can only restate the test's own setup.
*/
const paneManager = {
resetWebglTextureAtlases: () => undefined,
getPaneCount: () => 3,
getPanes: () =>
[1, 2, 3].map((id) => ({
id,
terminal: { cols: 200, buffer: { active: { length: 5000 } } }
}))
}
/** Saturates every slice limit so each contributor emits its widest census. */
function fillStoreWithProbeSlices(): string[] {
const keys = Array.from({ length: 40 }, (_, index) => `__profileProbe${index}`)
useAppStore.setState(
Object.fromEntries(
keys.map((key, index) => [key, Array.from({ length: index + 1 }, () => 'x'.repeat(64))])
) as unknown as Partial<AppState>
)
return keys
}
afterEach(() => {
unregisterLivePaneManager(paneManager)
})
describe('shipped renderer memory contributors', () => {
it('fits the widest census of every registered contributor inside MAX_PROFILE_COUNTS', () => {
// Why the real registry: a fake census cannot notice the store raising its
// slice limit, a fourth contributor arriving, or terminals adding keys —
// any of which silently truncates the near-death profile mid-contributor.
const probeKeys = fillStoreWithProbeSlices()
registerLivePaneManager(paneManager)
const counts = collectRendererMemoryProfileCounts()
const prefixes = new Set(Object.keys(counts).map((key) => key.split('.')[0]))
expect(Object.keys(counts).length).toBeLessThanOrEqual(MAX_PROFILE_COUNTS)
expect([...prefixes].sort()).toEqual(['store', 'storeKB', 'terminals'])
// Truncation stops mid-contributor, so the last registered one loses keys first.
expect(counts['terminals.estBufferKB']).toBeGreaterThan(0)
expect(counts['storeKB.__totalKB']).toBeGreaterThan(0)
expect(Object.keys(counts).filter((key) => key.startsWith('store.'))).toHaveLength(20)
expect(probeKeys.some((key) => counts[`store.${key}`] !== undefined)).toBe(true)
})
it('carries both byte contributors on the routine trend within MAX_TREND_COUNTS', () => {
// Why terminals too: the sub-A leak grows outside the zustand store, and
// xterm scrollback is the byte source storeKB structurally cannot see.
fillStoreWithProbeSlices()
registerLivePaneManager(paneManager)
const counts = collectRendererMemoryTrendCounts(createRendererMemoryCensus())
expect(Object.keys(counts).length).toBeLessThanOrEqual(MAX_TREND_COUNTS)
expect(counts['storeKB.__totalKB']).toBeGreaterThan(0)
// 3 panes x 5000 rows x 200 cols x 16B ~= 46MB of scrollback.
expect(counts['terminals.estBufferKB']).toBeGreaterThan(40_000)
expect(Object.keys(counts).filter((key) => key.startsWith('terminals.'))).toHaveLength(2)
})
it('reads each contributor once when a sample both trends and profiles', () => {
// Why: crossing 90/95% runs both censuses in one pass, with no heap headroom
// left to walk the store twice.
fillStoreWithProbeSlices()
registerLivePaneManager(paneManager)
const census = createRendererMemoryCensus()
collectRendererMemoryTrendCounts(census)
const before = census.get('storeKB')
collectRendererMemoryProfileCounts(census)
expect(before).toBeDefined()
expect(census.get('storeKB')).toBe(before)
expect(census.get('terminals')).toBeDefined()
})
})
+6 -2
View File
@@ -109,8 +109,12 @@ registerRendererMemoryProfileContributor('store', () =>
// Why bytes too: counts miss value-weight growth (97b9e86d leaked ~700MB while
// its biggest slice grew by 4 entries); sampled KB names what got FAT.
registerRendererMemoryProfileContributor('storeKB', () =>
estimateStateCollectionKB(useAppStore.getState(), 16)
// Why trendLimit 5: __totalKB always outranks any single slice, so 5 carries
// the total plus the top 4 slices onto every routine renderer_memory crumb.
registerRendererMemoryProfileContributor(
'storeKB',
() => estimateStateCollectionKB(useAppStore.getState(), 16),
{ trendLimit: 5 }
)
export type { AppState } from './types'
+23
View File
@@ -0,0 +1,23 @@
/**
* Shape of the renderer_memory_highwater breadcrumb series, shared so the
* main-process retention budget cannot drift from what the renderer emits.
*/
// Why these levels: the OOM cluster dies at 90-97% of the heap limit; at the
// observed ~25MB/min ramp an 0.8-only profile is ~60min stale by then.
export const RENDERER_MEMORY_HIGHWATER_RATIOS = [0.6, 0.8, 0.9, 0.95] as const
// Each surface samples its own heap into the same breadcrumb ring.
export const RENDERER_MEMORY_SURFACES = ['main', 'dashboard-popout'] as const
export type RendererSurface = (typeof RENDERER_MEMORY_SURFACES)[number]
/**
* One retained breadcrumb per (surface, threshold) profile.
*
* Why derived: a hand-written cap silently evicts oldest-first, so adding a
* threshold would drop the near-death main-window profile these reports exist
* to carry.
*/
export const MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS =
RENDERER_MEMORY_HIGHWATER_RATIOS.length * RENDERER_MEMORY_SURFACES.length