mirror of
https://github.com/stablyai/orca.git
synced 2026-10-21 00:03:46 +00:00
feat(crash-diagnostics): sample the near-death renderer heap
The 42-crash OOM cluster (macOS exit 5, Windows 0xE0000008/-36861, one Linux SIGKILL) dies at 90-97% of heapLimitMB, but highwater profiles stopped at 80% — roughly 60 minutes stale at the observed ~25MB/min ramp — and the routine 60s crumb carried no byte attribution at all, so reports held two isolated snapshots and named no retainer. - Add 0.9 and 0.95 highwater levels; the 0.95 profile now lands ~2min before death. - Carry truncated byte attribution on every routine renderer_memory breadcrumb so reports show a trend, not two points: storeKB (__totalKB + top 4 slices) and terminals (estBufferKB + estPanes). Both are needed — F0BMB7LAAMU grew 857MB with every store field frozen, i.e. outside the store, where xterm scrollback lives; and F0BM6V4KEBV died at 78% of its limit, so no highwater level fired at all and terminals would have been sampled exactly once. - Tag the routine crumb with rendererSurface, matching the highwater crumb: main and the dashboard popout share one 30-slot ring, so an untagged storeKB series interleaves two unrelated heaps. - Derive the main-process retained-breadcrumb budget from thresholds x renderer surfaces in shared/renderer-memory-highwater.ts. The retained key is `<surface>:<thresholdPct>`, so 4 levels x 2 surfaces demanded 8 slots against a hand-written 4, and oldest-first eviction dropped every main-window profile — including the 95% one this change exists to produce. - Share one contributor census per sample between the trend counts and the highwater profile, so crossing 90/95% no longer walks the whole store twice with no heap headroom left. - Assert both budgets against the contributors the app really registers rather than stand-ins: 41 counts of MAX_PROFILE_COUNTS=64 at their widest, and 7 of MAX_TREND_COUNTS=8 on the routine crumb. Diagnostics only — no leak fix and no heap-ceiling change. Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
@@ -5,6 +5,11 @@ import {
|
||||
recordCoalescedCrashBreadcrumb,
|
||||
recordCrashBreadcrumb
|
||||
} from './crash-breadcrumb-store'
|
||||
import {
|
||||
MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS,
|
||||
RENDERER_MEMORY_HIGHWATER_RATIOS,
|
||||
RENDERER_MEMORY_SURFACES
|
||||
} from '../../shared/renderer-memory-highwater'
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
@@ -50,7 +55,8 @@ describe('crash breadcrumb store', () => {
|
||||
})
|
||||
|
||||
it('caps retained high-water profiles', () => {
|
||||
for (let index = 0; index < 5; index += 1) {
|
||||
const overflow = MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS + 1
|
||||
for (let index = 0; index < overflow; index += 1) {
|
||||
recordCrashBreadcrumb('renderer_memory_highwater', {
|
||||
rendererSurface: `surface-${index}`,
|
||||
thresholdPct: 80
|
||||
@@ -59,7 +65,66 @@ describe('crash breadcrumb store', () => {
|
||||
|
||||
expect(
|
||||
getCrashBreadcrumbSnapshot().map((breadcrumb) => breadcrumb.data?.rendererSurface)
|
||||
).toEqual(['surface-1', 'surface-2', 'surface-3', 'surface-4'])
|
||||
).toEqual(
|
||||
Array.from(
|
||||
{ length: MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS },
|
||||
(_, index) => `surface-${index + 1}`
|
||||
)
|
||||
)
|
||||
})
|
||||
|
||||
// The retained key is `<surface>:<thresholdPct>`, so the budget must track
|
||||
// thresholds x surfaces. It once lagged at 4 against 8 emitted profiles, and
|
||||
// oldest-first eviction dropped every main-window one — including the 95%
|
||||
// near-death profile the OOM reports exist to carry.
|
||||
it('retains every threshold profile that both renderer surfaces can emit', () => {
|
||||
for (const surface of RENDERER_MEMORY_SURFACES) {
|
||||
for (const ratio of RENDERER_MEMORY_HIGHWATER_RATIOS) {
|
||||
recordCrashBreadcrumb('renderer_memory_highwater', {
|
||||
rendererSurface: surface,
|
||||
thresholdPct: Math.round(ratio * 100)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const profiles = getCrashBreadcrumbSnapshot().map(
|
||||
(breadcrumb) => `${breadcrumb.data?.rendererSurface}:${breadcrumb.data?.thresholdPct}`
|
||||
)
|
||||
|
||||
expect(profiles).toHaveLength(
|
||||
RENDERER_MEMORY_SURFACES.length * RENDERER_MEMORY_HIGHWATER_RATIOS.length
|
||||
)
|
||||
for (const surface of RENDERER_MEMORY_SURFACES) {
|
||||
for (const ratio of RENDERER_MEMORY_HIGHWATER_RATIOS) {
|
||||
expect(profiles).toContain(`${surface}:${Math.round(ratio * 100)}`)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Retained profiles spend the ring budget rather than extending it, so the
|
||||
// newest events lose slots one-for-one instead of the snapshot growing.
|
||||
it('never exceeds the ring size once high-water profiles are retained', () => {
|
||||
for (const surface of RENDERER_MEMORY_SURFACES) {
|
||||
for (const ratio of RENDERER_MEMORY_HIGHWATER_RATIOS) {
|
||||
recordCrashBreadcrumb('renderer_memory_highwater', {
|
||||
rendererSurface: surface,
|
||||
thresholdPct: Math.round(ratio * 100)
|
||||
})
|
||||
}
|
||||
}
|
||||
for (let index = 0; index < 40; index += 1) {
|
||||
recordCrashBreadcrumb(`event_${index}`, { index })
|
||||
}
|
||||
|
||||
const snapshot = getCrashBreadcrumbSnapshot()
|
||||
expect(snapshot).toHaveLength(30)
|
||||
expect(
|
||||
snapshot.filter((breadcrumb) => breadcrumb.name === 'renderer_memory_highwater')
|
||||
).toHaveLength(MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS)
|
||||
expect(snapshot.at(-1)?.name).toBe('event_39')
|
||||
expect(snapshot.map((breadcrumb) => breadcrumb.name)).not.toContain(
|
||||
`event_${39 - (30 - MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS)}`
|
||||
)
|
||||
})
|
||||
|
||||
it('redacts sensitive breadcrumb fields before they can be snapshotted', () => {
|
||||
|
||||
@@ -4,10 +4,12 @@ import {
|
||||
type CrashReportBreadcrumbData,
|
||||
type CrashReportBreadcrumb
|
||||
} from '../../shared/crash-reporting'
|
||||
import { MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS } from '../../shared/renderer-memory-highwater'
|
||||
|
||||
const MAX_BREADCRUMBS = 30
|
||||
// Why: retain two thresholds for each renderer surface without growing the ring.
|
||||
const MAX_RETAINED_BREADCRUMBS = 4
|
||||
// Why derived: eviction is oldest-first, so a budget smaller than the emitted
|
||||
// (surface x threshold) profiles drops the near-death one.
|
||||
const MAX_RETAINED_BREADCRUMBS = MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS
|
||||
// Why: coalesceKey embeds an open-string agentType (length-trimmed only, never
|
||||
// enum-checked), so the key space is unbounded over a long multi-agent/SSH session.
|
||||
// Bound the coalesce map the same way ProcessGoneDedupe bounds its key map.
|
||||
@@ -163,7 +165,11 @@ export function getCrashBreadcrumbSnapshot(): CrashReportBreadcrumb[] {
|
||||
resolveAllPendingCoalescedBreadcrumbs()
|
||||
// Why: long sessions must retain threshold profiles without growing the 30-entry budget.
|
||||
const retained = [...retainedBreadcrumbs.values()]
|
||||
const recent = breadcrumbs.slice(-(MAX_BREADCRUMBS - retained.length))
|
||||
// Why a start index rather than slice(-budget): a retained budget that ever
|
||||
// reaches MAX_BREADCRUMBS makes the budget 0, and slice(-0) returns the whole
|
||||
// ring. Counting forward degrades to [] with no branch to leave untested.
|
||||
const recentBudget = MAX_BREADCRUMBS - retained.length
|
||||
const recent = breadcrumbs.slice(Math.max(0, breadcrumbs.length - recentBudget))
|
||||
return [...retained, ...recent]
|
||||
.sort((left, right) => left.createdAt.localeCompare(right.createdAt))
|
||||
.map((breadcrumb) => ({
|
||||
|
||||
@@ -78,6 +78,7 @@ describe('renderer crash diagnostics', () => {
|
||||
name: 'renderer_memory',
|
||||
data: {
|
||||
reason: 'startup',
|
||||
rendererSurface: 'main',
|
||||
usedHeapMB: 32,
|
||||
totalHeapMB: 64,
|
||||
heapLimitMB: 512,
|
||||
@@ -222,6 +223,145 @@ describe('renderer crash diagnostics', () => {
|
||||
unregister()
|
||||
})
|
||||
|
||||
it('emits highwater samples inside the last 20% of the heap limit', async () => {
|
||||
// Why: the 42-crash OOM cluster dies at 90-97% of the limit. With samples
|
||||
// only at 60/80% the nearest profile is ~60min stale and names nothing.
|
||||
const memory = (window.performance as unknown as { memory: Record<string, number> }).memory
|
||||
vi.stubGlobal('document', {
|
||||
getElementsByTagName: () => ({ length: 1 }),
|
||||
querySelectorAll: () => ({ length: 0 })
|
||||
})
|
||||
|
||||
diagnostics.installRendererCrashDiagnostics()
|
||||
const tick = setIntervalMock.mock.calls[0][0] as () => void
|
||||
memory.usedJSHeapSize = 0.92 * memory.jsHeapSizeLimit
|
||||
tick()
|
||||
memory.usedJSHeapSize = 0.96 * memory.jsHeapSizeLimit
|
||||
tick()
|
||||
|
||||
const thresholds = recordBreadcrumbMock.mock.calls
|
||||
.filter((call) => (call[0] as { name: string }).name === 'renderer_memory_highwater')
|
||||
.map((call) => (call[0] as { data: { thresholdPct: number } }).data.thresholdPct)
|
||||
expect(thresholds).toEqual([60, 80, 90, 95])
|
||||
})
|
||||
|
||||
it('carries both shipped byte contributors on the routine renderer_memory breadcrumb', async () => {
|
||||
// Why the real registrations rather than stand-ins: the sub-A leak grows
|
||||
// outside the zustand store, so a trend carrying only storeKB still cannot
|
||||
// name it. F0BM6V4KEBV died at 78% of the limit, where no highwater level
|
||||
// fires and this crumb is the only attribution there is — which is also why
|
||||
// no document is stubbed here: 32MB of 512MB crosses nothing.
|
||||
const { useAppStore } = await import('../store')
|
||||
const { registerLivePaneManager, unregisterLivePaneManager } =
|
||||
await import('./pane-manager/pane-manager-registry')
|
||||
useAppStore.setState({ worktrees: [{ id: 'w1' }, { id: 'w2' }] } as never)
|
||||
const paneManager = {
|
||||
resetWebglTextureAtlases: () => undefined,
|
||||
getPaneCount: () => 2,
|
||||
getPanes: () =>
|
||||
[1, 2].map((id) => ({
|
||||
id,
|
||||
terminal: { cols: 200, buffer: { active: { length: 5000 } } }
|
||||
}))
|
||||
}
|
||||
registerLivePaneManager(paneManager)
|
||||
|
||||
diagnostics.installRendererCrashDiagnostics()
|
||||
|
||||
const routine = recordBreadcrumbMock.mock.calls.find(
|
||||
(call) => (call[0] as { name: string }).name === 'renderer_memory'
|
||||
)?.[0] as { data: Record<string, number> }
|
||||
expect(routine.data['storeKB.__totalKB']).toBeGreaterThan(0)
|
||||
// 2 panes x 5000 rows x 200 cols x 16B ~= 31MB of scrollback.
|
||||
expect(routine.data['terminals.estBufferKB']).toBeGreaterThan(30_000)
|
||||
expect(routine.data['terminals.estPanes']).toBe(2)
|
||||
|
||||
unregisterLivePaneManager(paneManager)
|
||||
})
|
||||
|
||||
it('carries truncated byte attribution on the routine renderer_memory breadcrumb', async () => {
|
||||
// Why: two isolated highwater snapshots cannot show a ramp; the 60s crumb
|
||||
// must carry a byte TREND so a report localizes the retainer on its own.
|
||||
const profile = await import('./renderer-memory-profile')
|
||||
const unregister = profile.registerRendererMemoryProfileContributor(
|
||||
'storeKB',
|
||||
() => ({ worktrees: 120, __totalKB: 4200, tabs: 900, agents: 5, prs: 60, drafts: 2 }),
|
||||
{ trendLimit: 5 }
|
||||
)
|
||||
|
||||
diagnostics.installRendererCrashDiagnostics()
|
||||
|
||||
const routine = recordBreadcrumbMock.mock.calls.find(
|
||||
(call) => (call[0] as { name: string }).name === 'renderer_memory'
|
||||
)?.[0] as { data: Record<string, unknown> }
|
||||
expect(routine.data).toMatchObject({
|
||||
'storeKB.__totalKB': 4200,
|
||||
'storeKB.tabs': 900,
|
||||
'storeKB.worktrees': 120,
|
||||
'storeKB.prs': 60,
|
||||
'storeKB.agents': 5
|
||||
})
|
||||
expect(routine.data).not.toHaveProperty('storeKB.drafts')
|
||||
|
||||
unregister()
|
||||
})
|
||||
|
||||
it('censuses the store once on a sample that also profiles the highwater', async () => {
|
||||
// Why: the trend and the highwater profile share one contributor. Walking the
|
||||
// store twice is worst exactly at 90/95%, where there is no headroom for the
|
||||
// transient allocation the census makes.
|
||||
const memory = (window.performance as unknown as { memory: Record<string, number> }).memory
|
||||
memory.usedJSHeapSize = 0.96 * memory.jsHeapSizeLimit
|
||||
vi.stubGlobal('document', {
|
||||
getElementsByTagName: () => ({ length: 1 }),
|
||||
querySelectorAll: () => ({ length: 0 })
|
||||
})
|
||||
let censuses = 0
|
||||
const profile = await import('./renderer-memory-profile')
|
||||
const unregister = profile.registerRendererMemoryProfileContributor(
|
||||
'storeKB',
|
||||
() => {
|
||||
censuses += 1
|
||||
return { __totalKB: 4200 }
|
||||
},
|
||||
{ trendLimit: 5 }
|
||||
)
|
||||
|
||||
diagnostics.installRendererCrashDiagnostics()
|
||||
|
||||
expect(
|
||||
recordBreadcrumbMock.mock.calls.filter(
|
||||
(call) => (call[0] as { name: string }).name === 'renderer_memory_highwater'
|
||||
)
|
||||
).toHaveLength(4)
|
||||
expect(censuses).toBe(1)
|
||||
|
||||
unregister()
|
||||
})
|
||||
|
||||
it('tags every routine renderer_memory sample with its renderer surface', async () => {
|
||||
// Why: main and the dashboard popout push into the same 30-slot main-process
|
||||
// ring, each with its own storeKB census. Untagged, the byte trend this crumb
|
||||
// exists to expose interleaves two unrelated heaps into one unreadable series.
|
||||
const profile = await import('./renderer-memory-profile')
|
||||
const unregister = profile.registerRendererMemoryProfileContributor(
|
||||
'storeKB',
|
||||
() => ({ __totalKB: 12 }),
|
||||
{ trendLimit: 5 }
|
||||
)
|
||||
|
||||
diagnostics.installRendererCrashDiagnostics('dashboard-popout')
|
||||
const tick = setIntervalMock.mock.calls[0][0] as () => void
|
||||
tick()
|
||||
|
||||
const surfaces = recordBreadcrumbMock.mock.calls
|
||||
.filter((call) => (call[0] as { name: string }).name === 'renderer_memory')
|
||||
.map((call) => (call[0] as { data: { rendererSurface?: string } }).data.rendererSurface)
|
||||
expect(surfaces).toEqual(['dashboard-popout', 'dashboard-popout'])
|
||||
|
||||
unregister()
|
||||
})
|
||||
|
||||
it('skips highwater emission when heap readings are not finite', () => {
|
||||
const memory = (window.performance as unknown as { memory: Record<string, number> }).memory
|
||||
memory.usedJSHeapSize = Number.NaN
|
||||
|
||||
@@ -2,20 +2,27 @@ import type {
|
||||
CrashReportBreadcrumbData,
|
||||
CrashReportDetailValue
|
||||
} from '../../../shared/crash-reporting'
|
||||
// Why the levels live in shared/: one detailed breadcrumb per (surface,
|
||||
// threshold) is retained in the main process, whose budget is derived from them.
|
||||
import {
|
||||
RENDERER_MEMORY_HIGHWATER_RATIOS,
|
||||
type RendererSurface
|
||||
} from '../../../shared/renderer-memory-highwater'
|
||||
import {
|
||||
getBrowserWebviewMemoryProfile,
|
||||
type BrowserWebviewMemoryProfile
|
||||
} from '../components/browser-pane/webview-registry'
|
||||
import { recordRendererCrashBreadcrumb } from './crash-breadcrumb-recorder'
|
||||
import { collectRendererMemoryProfileCounts } from './renderer-memory-profile'
|
||||
import {
|
||||
collectRendererMemoryProfileCounts,
|
||||
collectRendererMemoryTrendCounts,
|
||||
createRendererMemoryCensus,
|
||||
type RendererMemoryCensus
|
||||
} from './renderer-memory-profile'
|
||||
|
||||
const RENDERER_MEMORY_SAMPLE_INTERVAL_MS = 60_000
|
||||
const BYTES_PER_MEGABYTE = 1024 * 1024
|
||||
const BYTES_PER_KILOBYTE = 1024
|
||||
// Why: one detailed breadcrumb per threshold names what grew before an OOM.
|
||||
const RENDERER_MEMORY_HIGHWATER_RATIOS = [0.6, 0.8] as const
|
||||
|
||||
type RendererSurface = 'main' | 'dashboard-popout'
|
||||
|
||||
type BrowserPerformanceMemory = {
|
||||
usedJSHeapSize?: number
|
||||
@@ -117,11 +124,17 @@ function recordRendererMemory(reason: string): void {
|
||||
return
|
||||
}
|
||||
const browserWebviews = getBrowserWebviewMemoryProfile()
|
||||
// Why one census per sample: the highwater profile below repeats this pass's
|
||||
// contributors, and near-OOM is the worst moment to walk the store twice.
|
||||
const census = createRendererMemoryCensus()
|
||||
|
||||
recordRendererCrashBreadcrumb(
|
||||
'renderer_memory',
|
||||
compactBreadcrumbData({
|
||||
reason,
|
||||
// Why: main and the dashboard popout each sample into the same ring, so an
|
||||
// untagged series reads as a sawtooth between two unrelated heaps.
|
||||
rendererSurface,
|
||||
usedHeapMB: toMegabytes(memory.usedJSHeapSize),
|
||||
totalHeapMB: toMegabytes(memory.totalJSHeapSize),
|
||||
heapLimitMB: toMegabytes(memory.jsHeapSizeLimit),
|
||||
@@ -129,15 +142,19 @@ function recordRendererMemory(reason: string): void {
|
||||
mallocedMB: toMegabytes(memory.mallocedBytes),
|
||||
blinkAllocatedMB: toMegabytes(memory.blinkAllocatedBytes),
|
||||
browserWebviews: browserWebviews.browserWebviewCount,
|
||||
registeredBrowserGuests: browserWebviews.registeredBrowserGuestCount
|
||||
registeredBrowserGuests: browserWebviews.registeredBrowserGuestCount,
|
||||
// Why on every sample: a byte trend across the session localizes a
|
||||
// retainer that two near-death snapshots cannot.
|
||||
...collectRendererMemoryTrendCounts(census)
|
||||
})
|
||||
)
|
||||
recordRendererMemoryHighwater(memory, browserWebviews)
|
||||
recordRendererMemoryHighwater(memory, browserWebviews, census)
|
||||
}
|
||||
|
||||
function recordRendererMemoryHighwater(
|
||||
memory: HeapMetrics,
|
||||
browserWebviews: BrowserWebviewMemoryProfile
|
||||
browserWebviews: BrowserWebviewMemoryProfile,
|
||||
census: RendererMemoryCensus
|
||||
): void {
|
||||
const used = memory.usedJSHeapSize
|
||||
const limit = memory.jsHeapSizeLimit
|
||||
@@ -170,7 +187,7 @@ function recordRendererMemoryHighwater(
|
||||
terminalElements: document.querySelectorAll('.xterm').length,
|
||||
browserWebviews: browserWebviews.browserWebviewCount,
|
||||
registeredBrowserGuests: browserWebviews.registeredBrowserGuestCount,
|
||||
...collectRendererMemoryProfileCounts()
|
||||
...collectRendererMemoryProfileCounts(census)
|
||||
})
|
||||
for (const threshold of RENDERER_MEMORY_HIGHWATER_RATIOS) {
|
||||
if (ratio < threshold || emittedHighwaterRatios.has(threshold)) {
|
||||
|
||||
@@ -237,4 +237,11 @@ export function getLivePaneMemoryProfileCounts(): Record<string, number> {
|
||||
|
||||
// Why here: contributors push in (crash-diagnostics stays a leaf); same-name
|
||||
// re-registration on dev HMR overwrites, so no disposal hook is needed.
|
||||
registerRendererMemoryProfileContributor('terminals', getLivePaneMemoryProfileCounts)
|
||||
// Why trendLimit: scrollback bytes live outside the zustand store, so storeKB
|
||||
// cannot see them and the one-shot highwater profiles sample them once — the
|
||||
// 78%-of-limit death in F0BM6V4KEBV never reached even the 90% level. The
|
||||
// census above is sample-capped arithmetic, far cheaper than the storeKB walk
|
||||
// it rides along with. 2 keys carries estBufferKB plus the pane count it scales.
|
||||
registerRendererMemoryProfileContributor('terminals', getLivePaneMemoryProfileCounts, {
|
||||
trendLimit: 2
|
||||
})
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
collectRendererMemoryProfileCounts,
|
||||
collectRendererMemoryTrendCounts,
|
||||
createRendererMemoryCensus,
|
||||
registerRendererMemoryProfileContributor,
|
||||
summarizeStateCollectionSizes
|
||||
} from './renderer-memory-profile'
|
||||
@@ -135,6 +137,115 @@ describe('collectRendererMemoryProfileCounts', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('collectRendererMemoryTrendCounts', () => {
|
||||
it('keeps only the heaviest keys of trend-tagged contributors', () => {
|
||||
unregisters.push(
|
||||
registerRendererMemoryProfileContributor(
|
||||
'storeKB',
|
||||
() => ({ tabs: 900, __totalKB: 4200, worktrees: 120, prs: 60, agents: 5, drafts: 2 }),
|
||||
{ trendLimit: 5 }
|
||||
)
|
||||
)
|
||||
register('store', () => ({ worktrees: 40 }))
|
||||
|
||||
expect(collectRendererMemoryTrendCounts()).toEqual({
|
||||
'storeKB.__totalKB': 4200,
|
||||
'storeKB.tabs': 900,
|
||||
'storeKB.worktrees': 120,
|
||||
'storeKB.prs': 60,
|
||||
'storeKB.agents': 5
|
||||
})
|
||||
})
|
||||
|
||||
it('bounds the aggregate trend budget and contains throwing contributors', () => {
|
||||
unregisters.push(
|
||||
registerRendererMemoryProfileContributor(
|
||||
'broken',
|
||||
() => {
|
||||
throw new Error('boom')
|
||||
},
|
||||
{ trendLimit: 4 }
|
||||
)
|
||||
)
|
||||
unregisters.push(
|
||||
registerRendererMemoryProfileContributor(
|
||||
'runaway',
|
||||
() => Object.fromEntries(Array.from({ length: 500 }, (_, index) => [`key${index}`, index])),
|
||||
{ trendLimit: 500 }
|
||||
)
|
||||
)
|
||||
|
||||
const counts = collectRendererMemoryTrendCounts()
|
||||
expect(counts['broken.error']).toBe(1)
|
||||
expect(Object.keys(counts)).toHaveLength(8)
|
||||
})
|
||||
})
|
||||
|
||||
// A sample that crosses a threshold runs both censuses, and storeKB walks every
|
||||
// top-level store slice with a transient allocation the size of what it measures
|
||||
// — the last thing to do twice with no heap headroom left.
|
||||
describe('a census shared across one sample pass', () => {
|
||||
it('reads each contributor once for both the trend and the profile', () => {
|
||||
let reads = 0
|
||||
unregisters.push(
|
||||
registerRendererMemoryProfileContributor(
|
||||
'storeKB',
|
||||
() => {
|
||||
reads += 1
|
||||
return { __totalKB: 4200, tabs: 900 }
|
||||
},
|
||||
{ trendLimit: 5 }
|
||||
)
|
||||
)
|
||||
const census = createRendererMemoryCensus()
|
||||
|
||||
const trend = collectRendererMemoryTrendCounts(census)
|
||||
const profile = collectRendererMemoryProfileCounts(census)
|
||||
|
||||
expect(reads).toBe(1)
|
||||
expect(trend).toEqual({ 'storeKB.__totalKB': 4200, 'storeKB.tabs': 900 })
|
||||
expect(profile).toEqual({ 'storeKB.__totalKB': 4200, 'storeKB.tabs': 900 })
|
||||
})
|
||||
|
||||
it('keeps a fresh census per sample so the trend still moves', () => {
|
||||
let size = 10
|
||||
unregisters.push(
|
||||
registerRendererMemoryProfileContributor('storeKB', () => ({ __totalKB: (size += 10) }), {
|
||||
trendLimit: 1
|
||||
})
|
||||
)
|
||||
|
||||
expect(collectRendererMemoryTrendCounts(createRendererMemoryCensus())).toEqual({
|
||||
'storeKB.__totalKB': 20
|
||||
})
|
||||
expect(collectRendererMemoryTrendCounts(createRendererMemoryCensus())).toEqual({
|
||||
'storeKB.__totalKB': 30
|
||||
})
|
||||
})
|
||||
|
||||
it('reports a throwing contributor in both censuses without re-running it', () => {
|
||||
let reads = 0
|
||||
unregisters.push(
|
||||
registerRendererMemoryProfileContributor(
|
||||
'broken',
|
||||
() => {
|
||||
reads += 1
|
||||
throw new Error('boom')
|
||||
},
|
||||
{ trendLimit: 4 }
|
||||
)
|
||||
)
|
||||
const census = createRendererMemoryCensus()
|
||||
|
||||
expect(collectRendererMemoryTrendCounts(census)).toEqual({ 'broken.error': 1 })
|
||||
expect(collectRendererMemoryProfileCounts(census)).toEqual({ 'broken.error': 1 })
|
||||
expect(reads).toBe(1)
|
||||
})
|
||||
})
|
||||
|
||||
// The shipped contributors are measured against MAX_PROFILE_COUNTS in
|
||||
// renderer-memory-shipped-contributors.test.ts, which registers them for real.
|
||||
|
||||
describe('summarizeStateCollectionSizes', () => {
|
||||
it('reports the largest collections first, capped at the limit', () => {
|
||||
const state = {
|
||||
|
||||
@@ -10,21 +10,48 @@ export type RendererMemoryProfileCounts = Record<string, number>
|
||||
|
||||
type RendererMemoryProfileContributor = () => RendererMemoryProfileCounts
|
||||
|
||||
const contributors = new Map<string, RendererMemoryProfileContributor>()
|
||||
type RendererMemoryProfileOptions = {
|
||||
/** Heaviest keys to also carry on the routine renderer_memory breadcrumb. */
|
||||
trendLimit?: number
|
||||
}
|
||||
|
||||
type RegisteredContributor = {
|
||||
collect: RendererMemoryProfileContributor
|
||||
trendLimit: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-sample memo shared by the trend and profile censuses.
|
||||
*
|
||||
* Why: crossing 90%/95% runs both in one pass, and the storeKB contributor walks
|
||||
* every top-level store slice with a transient allocation the size of what it
|
||||
* measures — exactly when the heap has no headroom to walk it twice.
|
||||
*/
|
||||
export type RendererMemoryCensus = Map<string, RendererMemoryProfileCounts | null>
|
||||
|
||||
export function createRendererMemoryCensus(): RendererMemoryCensus {
|
||||
return new Map()
|
||||
}
|
||||
|
||||
const contributors = new Map<string, RegisteredContributor>()
|
||||
|
||||
// Why: breadcrumbs are retained per session; a misbehaving contributor must not
|
||||
// bloat every crash report. 32 counts is plenty to name a leaking subsystem.
|
||||
const MAX_COUNTS_PER_CONTRIBUTOR = 32
|
||||
// Why: individually bounded contributors can still create unbounded near-OOM work in aggregate.
|
||||
const MAX_PROFILE_COUNTS = 64
|
||||
export const MAX_PROFILE_COUNTS = 64
|
||||
// Why: empty contributors do not consume the count budget but must not make collection unbounded.
|
||||
const MAX_PROFILE_CONTRIBUTORS = 64
|
||||
const MAX_CONTRIBUTOR_NAME_LENGTH = 64
|
||||
const MAX_COUNT_KEY_LENGTH = 80
|
||||
// Why far below MAX_PROFILE_COUNTS: this rides every 60s sample, not just the
|
||||
// near-OOM ones, and every retained breadcrumb pays for it.
|
||||
export const MAX_TREND_COUNTS = 8
|
||||
|
||||
export function registerRendererMemoryProfileContributor(
|
||||
name: string,
|
||||
contributor: RendererMemoryProfileContributor
|
||||
contributor: RendererMemoryProfileContributor,
|
||||
options?: RendererMemoryProfileOptions
|
||||
): () => void {
|
||||
if (
|
||||
name.length === 0 ||
|
||||
@@ -33,26 +60,106 @@ export function registerRendererMemoryProfileContributor(
|
||||
) {
|
||||
return () => undefined
|
||||
}
|
||||
contributors.set(name, contributor)
|
||||
contributors.set(name, { collect: contributor, trendLimit: options?.trendLimit ?? 0 })
|
||||
return () => {
|
||||
if (contributors.get(name) === contributor) {
|
||||
if (contributors.get(name)?.collect === contributor) {
|
||||
contributors.delete(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function collectRendererMemoryProfileCounts(): RendererMemoryProfileCounts {
|
||||
/**
|
||||
* Heaviest keys of trend-tagged contributors, for the routine renderer_memory
|
||||
* breadcrumb.
|
||||
*
|
||||
* Why: highwater profiles are two isolated snapshots. A crash report can only
|
||||
* name a retainer if the byte attribution appears as a ramp across the whole
|
||||
* session, so a truncated slice of it rides every sample.
|
||||
*/
|
||||
export function collectRendererMemoryTrendCounts(
|
||||
census?: RendererMemoryCensus
|
||||
): RendererMemoryProfileCounts {
|
||||
const counts: RendererMemoryProfileCounts = {}
|
||||
let collected = 0
|
||||
for (const [name, entry] of contributors) {
|
||||
if (entry.trendLimit <= 0 || collected >= MAX_TREND_COUNTS) {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
const ranked = rankContributorCounts(readContribution(name, entry, census))
|
||||
const keep = Math.min(entry.trendLimit, MAX_TREND_COUNTS - collected)
|
||||
for (const [key, value] of ranked.slice(0, keep)) {
|
||||
counts[`${name}.${key}`] = value
|
||||
collected += 1
|
||||
}
|
||||
} catch {
|
||||
counts[`${name}.error`] = 1
|
||||
collected += 1
|
||||
}
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
/** Throws on failure so both censuses keep funnelling it into `${name}.error`. */
|
||||
function readContribution(
|
||||
name: string,
|
||||
entry: RegisteredContributor,
|
||||
census?: RendererMemoryCensus
|
||||
): RendererMemoryProfileCounts {
|
||||
const memoized = census?.get(name)
|
||||
if (memoized) {
|
||||
return memoized
|
||||
}
|
||||
if (memoized === null) {
|
||||
throw new Error(`renderer memory contributor failed: ${name}`)
|
||||
}
|
||||
try {
|
||||
const contribution = entry.collect()
|
||||
census?.set(name, contribution)
|
||||
return contribution
|
||||
} catch (error) {
|
||||
census?.set(name, null)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function rankContributorCounts(contribution: RendererMemoryProfileCounts): [string, number][] {
|
||||
const ranked: [string, number][] = []
|
||||
let inspected = 0
|
||||
for (const key in contribution) {
|
||||
if (inspected >= MAX_COUNTS_PER_CONTRIBUTOR) {
|
||||
break
|
||||
}
|
||||
inspected += 1
|
||||
if (
|
||||
!Object.hasOwn(contribution, key) ||
|
||||
key.length === 0 ||
|
||||
key.length > MAX_COUNT_KEY_LENGTH
|
||||
) {
|
||||
continue
|
||||
}
|
||||
const value = contribution[key]
|
||||
if (typeof value === 'number' && Number.isFinite(value)) {
|
||||
ranked.push([key, value])
|
||||
}
|
||||
}
|
||||
return ranked.sort((a, b) => b[1] - a[1])
|
||||
}
|
||||
|
||||
export function collectRendererMemoryProfileCounts(
|
||||
census?: RendererMemoryCensus
|
||||
): RendererMemoryProfileCounts {
|
||||
const counts: RendererMemoryProfileCounts = {}
|
||||
let collected = 0
|
||||
let visited = 0
|
||||
for (const [name, contributor] of contributors) {
|
||||
for (const [name, entry] of contributors) {
|
||||
if (collected >= MAX_PROFILE_COUNTS || visited >= MAX_PROFILE_CONTRIBUTORS) {
|
||||
break
|
||||
}
|
||||
visited += 1
|
||||
// Why: a broken contributor must never take down memory reporting itself.
|
||||
try {
|
||||
const contribution = contributor()
|
||||
const contribution = readContribution(name, entry, census)
|
||||
let inspected = 0
|
||||
for (const key in contribution) {
|
||||
if (inspected >= MAX_COUNTS_PER_CONTRIBUTOR || collected >= MAX_PROFILE_COUNTS) {
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
registerLivePaneManager,
|
||||
unregisterLivePaneManager
|
||||
} from './pane-manager/pane-manager-registry'
|
||||
import {
|
||||
MAX_PROFILE_COUNTS,
|
||||
MAX_TREND_COUNTS,
|
||||
collectRendererMemoryProfileCounts,
|
||||
collectRendererMemoryTrendCounts,
|
||||
createRendererMemoryCensus
|
||||
} from './renderer-memory-profile'
|
||||
import { useAppStore } from '../store'
|
||||
import type { AppState } from '../store/types'
|
||||
|
||||
/**
|
||||
* Drives the contributors the app actually registers — importing ../store and
|
||||
* pane-manager-registry for their registration side effects — instead of
|
||||
* hand-written stand-ins that can only restate the test's own setup.
|
||||
*/
|
||||
|
||||
const paneManager = {
|
||||
resetWebglTextureAtlases: () => undefined,
|
||||
getPaneCount: () => 3,
|
||||
getPanes: () =>
|
||||
[1, 2, 3].map((id) => ({
|
||||
id,
|
||||
terminal: { cols: 200, buffer: { active: { length: 5000 } } }
|
||||
}))
|
||||
}
|
||||
|
||||
/** Saturates every slice limit so each contributor emits its widest census. */
|
||||
function fillStoreWithProbeSlices(): string[] {
|
||||
const keys = Array.from({ length: 40 }, (_, index) => `__profileProbe${index}`)
|
||||
useAppStore.setState(
|
||||
Object.fromEntries(
|
||||
keys.map((key, index) => [key, Array.from({ length: index + 1 }, () => 'x'.repeat(64))])
|
||||
) as unknown as Partial<AppState>
|
||||
)
|
||||
return keys
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
unregisterLivePaneManager(paneManager)
|
||||
})
|
||||
|
||||
describe('shipped renderer memory contributors', () => {
|
||||
it('fits the widest census of every registered contributor inside MAX_PROFILE_COUNTS', () => {
|
||||
// Why the real registry: a fake census cannot notice the store raising its
|
||||
// slice limit, a fourth contributor arriving, or terminals adding keys —
|
||||
// any of which silently truncates the near-death profile mid-contributor.
|
||||
const probeKeys = fillStoreWithProbeSlices()
|
||||
registerLivePaneManager(paneManager)
|
||||
|
||||
const counts = collectRendererMemoryProfileCounts()
|
||||
const prefixes = new Set(Object.keys(counts).map((key) => key.split('.')[0]))
|
||||
|
||||
expect(Object.keys(counts).length).toBeLessThanOrEqual(MAX_PROFILE_COUNTS)
|
||||
expect([...prefixes].sort()).toEqual(['store', 'storeKB', 'terminals'])
|
||||
// Truncation stops mid-contributor, so the last registered one loses keys first.
|
||||
expect(counts['terminals.estBufferKB']).toBeGreaterThan(0)
|
||||
expect(counts['storeKB.__totalKB']).toBeGreaterThan(0)
|
||||
expect(Object.keys(counts).filter((key) => key.startsWith('store.'))).toHaveLength(20)
|
||||
expect(probeKeys.some((key) => counts[`store.${key}`] !== undefined)).toBe(true)
|
||||
})
|
||||
|
||||
it('carries both byte contributors on the routine trend within MAX_TREND_COUNTS', () => {
|
||||
// Why terminals too: the sub-A leak grows outside the zustand store, and
|
||||
// xterm scrollback is the byte source storeKB structurally cannot see.
|
||||
fillStoreWithProbeSlices()
|
||||
registerLivePaneManager(paneManager)
|
||||
|
||||
const counts = collectRendererMemoryTrendCounts(createRendererMemoryCensus())
|
||||
|
||||
expect(Object.keys(counts).length).toBeLessThanOrEqual(MAX_TREND_COUNTS)
|
||||
expect(counts['storeKB.__totalKB']).toBeGreaterThan(0)
|
||||
// 3 panes x 5000 rows x 200 cols x 16B ~= 46MB of scrollback.
|
||||
expect(counts['terminals.estBufferKB']).toBeGreaterThan(40_000)
|
||||
expect(Object.keys(counts).filter((key) => key.startsWith('terminals.'))).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('reads each contributor once when a sample both trends and profiles', () => {
|
||||
// Why: crossing 90/95% runs both censuses in one pass, with no heap headroom
|
||||
// left to walk the store twice.
|
||||
fillStoreWithProbeSlices()
|
||||
registerLivePaneManager(paneManager)
|
||||
const census = createRendererMemoryCensus()
|
||||
|
||||
collectRendererMemoryTrendCounts(census)
|
||||
const before = census.get('storeKB')
|
||||
collectRendererMemoryProfileCounts(census)
|
||||
|
||||
expect(before).toBeDefined()
|
||||
expect(census.get('storeKB')).toBe(before)
|
||||
expect(census.get('terminals')).toBeDefined()
|
||||
})
|
||||
})
|
||||
@@ -109,8 +109,12 @@ registerRendererMemoryProfileContributor('store', () =>
|
||||
|
||||
// Why bytes too: counts miss value-weight growth (97b9e86d leaked ~700MB while
|
||||
// its biggest slice grew by 4 entries); sampled KB names what got FAT.
|
||||
registerRendererMemoryProfileContributor('storeKB', () =>
|
||||
estimateStateCollectionKB(useAppStore.getState(), 16)
|
||||
// Why trendLimit 5: __totalKB always outranks any single slice, so 5 carries
|
||||
// the total plus the top 4 slices onto every routine renderer_memory crumb.
|
||||
registerRendererMemoryProfileContributor(
|
||||
'storeKB',
|
||||
() => estimateStateCollectionKB(useAppStore.getState(), 16),
|
||||
{ trendLimit: 5 }
|
||||
)
|
||||
|
||||
export type { AppState } from './types'
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
/**
|
||||
* Shape of the renderer_memory_highwater breadcrumb series, shared so the
|
||||
* main-process retention budget cannot drift from what the renderer emits.
|
||||
*/
|
||||
|
||||
// Why these levels: the OOM cluster dies at 90-97% of the heap limit; at the
|
||||
// observed ~25MB/min ramp an 0.8-only profile is ~60min stale by then.
|
||||
export const RENDERER_MEMORY_HIGHWATER_RATIOS = [0.6, 0.8, 0.9, 0.95] as const
|
||||
|
||||
// Each surface samples its own heap into the same breadcrumb ring.
|
||||
export const RENDERER_MEMORY_SURFACES = ['main', 'dashboard-popout'] as const
|
||||
|
||||
export type RendererSurface = (typeof RENDERER_MEMORY_SURFACES)[number]
|
||||
|
||||
/**
|
||||
* One retained breadcrumb per (surface, threshold) profile.
|
||||
*
|
||||
* Why derived: a hand-written cap silently evicts oldest-first, so adding a
|
||||
* threshold would drop the near-death main-window profile these reports exist
|
||||
* to carry.
|
||||
*/
|
||||
export const MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS =
|
||||
RENDERER_MEMORY_HIGHWATER_RATIOS.length * RENDERER_MEMORY_SURFACES.length
|
||||
Reference in New Issue
Block a user