From 41abe354a56a80099bbf16550b78b39cd2e18e91 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Mon, 3 Aug 2026 23:13:08 -0700 Subject: [PATCH] feat(crash-diagnostics): sample the near-death renderer heap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 42-crash OOM cluster (macOS exit 5, Windows 0xE0000008/-36861, one Linux SIGKILL) dies at 90-97% of heapLimitMB, but highwater profiles stopped at 80% — roughly 60 minutes stale at the observed ~25MB/min ramp — and the routine 60s crumb carried no byte attribution at all, so reports held two isolated snapshots and named no retainer. - Add 0.9 and 0.95 highwater levels; the 0.95 profile now lands ~2min before death. - Carry truncated byte attribution on every routine renderer_memory breadcrumb so reports show a trend, not two points: storeKB (__totalKB + top 4 slices) and terminals (estBufferKB + estPanes). Both are needed — F0BMB7LAAMU grew 857MB with every store field frozen, i.e. outside the store, where xterm scrollback lives; and F0BM6V4KEBV died at 78% of its limit, so no highwater level fired at all and terminals would have been sampled exactly once. - Tag the routine crumb with rendererSurface, matching the highwater crumb: main and the dashboard popout share one 30-slot ring, so an untagged storeKB series interleaves two unrelated heaps. - Derive the main-process retained-breadcrumb budget from thresholds x renderer surfaces in shared/renderer-memory-highwater.ts. The retained key is `:`, so 4 levels x 2 surfaces demanded 8 slots against a hand-written 4, and oldest-first eviction dropped every main-window profile — including the 95% one this change exists to produce. - Share one contributor census per sample between the trend counts and the highwater profile, so crossing 90/95% no longer walks the whole store twice with no heap headroom left. - Assert both budgets against the contributors the app really registers rather than stand-ins: 41 counts of MAX_PROFILE_COUNTS=64 at their widest, and 7 of MAX_TREND_COUNTS=8 on the routine crumb. Diagnostics only — no leak fix and no heap-ceiling change. Co-authored-by: Orca --- .../crash-breadcrumb-store.test.ts | 69 ++++++++- .../crash-reporting/crash-breadcrumb-store.ts | 12 +- .../src/lib/crash-diagnostics.test.ts | 140 ++++++++++++++++++ src/renderer/src/lib/crash-diagnostics.ts | 35 +++-- .../lib/pane-manager/pane-manager-registry.ts | 9 +- .../src/lib/renderer-memory-profile.test.ts | 111 ++++++++++++++ .../src/lib/renderer-memory-profile.ts | 123 ++++++++++++++- ...nderer-memory-shipped-contributors.test.ts | 97 ++++++++++++ src/renderer/src/store/index.ts | 8 +- src/shared/renderer-memory-highwater.ts | 23 +++ 10 files changed, 602 insertions(+), 25 deletions(-) create mode 100644 src/renderer/src/lib/renderer-memory-shipped-contributors.test.ts create mode 100644 src/shared/renderer-memory-highwater.ts diff --git a/src/main/crash-reporting/crash-breadcrumb-store.test.ts b/src/main/crash-reporting/crash-breadcrumb-store.test.ts index 5a456e8dd32..07e6681dc1e 100644 --- a/src/main/crash-reporting/crash-breadcrumb-store.test.ts +++ b/src/main/crash-reporting/crash-breadcrumb-store.test.ts @@ -5,6 +5,11 @@ import { recordCoalescedCrashBreadcrumb, recordCrashBreadcrumb } from './crash-breadcrumb-store' +import { + MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS, + RENDERER_MEMORY_HIGHWATER_RATIOS, + RENDERER_MEMORY_SURFACES +} from '../../shared/renderer-memory-highwater' afterEach(() => { vi.useRealTimers() @@ -50,7 +55,8 @@ describe('crash breadcrumb store', () => { }) it('caps retained high-water profiles', () => { - for (let index = 0; index < 5; index += 1) { + const overflow = MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS + 1 + for (let index = 0; index < overflow; index += 1) { recordCrashBreadcrumb('renderer_memory_highwater', { rendererSurface: `surface-${index}`, thresholdPct: 80 @@ -59,7 +65,66 @@ describe('crash breadcrumb store', () => { expect( getCrashBreadcrumbSnapshot().map((breadcrumb) => breadcrumb.data?.rendererSurface) - ).toEqual(['surface-1', 'surface-2', 'surface-3', 'surface-4']) + ).toEqual( + Array.from( + { length: MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS }, + (_, index) => `surface-${index + 1}` + ) + ) + }) + + // The retained key is `:`, so the budget must track + // thresholds x surfaces. It once lagged at 4 against 8 emitted profiles, and + // oldest-first eviction dropped every main-window one — including the 95% + // near-death profile the OOM reports exist to carry. + it('retains every threshold profile that both renderer surfaces can emit', () => { + for (const surface of RENDERER_MEMORY_SURFACES) { + for (const ratio of RENDERER_MEMORY_HIGHWATER_RATIOS) { + recordCrashBreadcrumb('renderer_memory_highwater', { + rendererSurface: surface, + thresholdPct: Math.round(ratio * 100) + }) + } + } + + const profiles = getCrashBreadcrumbSnapshot().map( + (breadcrumb) => `${breadcrumb.data?.rendererSurface}:${breadcrumb.data?.thresholdPct}` + ) + + expect(profiles).toHaveLength( + RENDERER_MEMORY_SURFACES.length * RENDERER_MEMORY_HIGHWATER_RATIOS.length + ) + for (const surface of RENDERER_MEMORY_SURFACES) { + for (const ratio of RENDERER_MEMORY_HIGHWATER_RATIOS) { + expect(profiles).toContain(`${surface}:${Math.round(ratio * 100)}`) + } + } + }) + + // Retained profiles spend the ring budget rather than extending it, so the + // newest events lose slots one-for-one instead of the snapshot growing. + it('never exceeds the ring size once high-water profiles are retained', () => { + for (const surface of RENDERER_MEMORY_SURFACES) { + for (const ratio of RENDERER_MEMORY_HIGHWATER_RATIOS) { + recordCrashBreadcrumb('renderer_memory_highwater', { + rendererSurface: surface, + thresholdPct: Math.round(ratio * 100) + }) + } + } + for (let index = 0; index < 40; index += 1) { + recordCrashBreadcrumb(`event_${index}`, { index }) + } + + const snapshot = getCrashBreadcrumbSnapshot() + expect(snapshot).toHaveLength(30) + expect( + snapshot.filter((breadcrumb) => breadcrumb.name === 'renderer_memory_highwater') + ).toHaveLength(MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS) + expect(snapshot.at(-1)?.name).toBe('event_39') + expect(snapshot.map((breadcrumb) => breadcrumb.name)).not.toContain( + `event_${39 - (30 - MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS)}` + ) }) it('redacts sensitive breadcrumb fields before they can be snapshotted', () => { diff --git a/src/main/crash-reporting/crash-breadcrumb-store.ts b/src/main/crash-reporting/crash-breadcrumb-store.ts index f33856f5e77..149aa911ed6 100644 --- a/src/main/crash-reporting/crash-breadcrumb-store.ts +++ b/src/main/crash-reporting/crash-breadcrumb-store.ts @@ -4,10 +4,12 @@ import { type CrashReportBreadcrumbData, type CrashReportBreadcrumb } from '../../shared/crash-reporting' +import { MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS } from '../../shared/renderer-memory-highwater' const MAX_BREADCRUMBS = 30 -// Why: retain two thresholds for each renderer surface without growing the ring. -const MAX_RETAINED_BREADCRUMBS = 4 +// Why derived: eviction is oldest-first, so a budget smaller than the emitted +// (surface x threshold) profiles drops the near-death one. +const MAX_RETAINED_BREADCRUMBS = MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS // Why: coalesceKey embeds an open-string agentType (length-trimmed only, never // enum-checked), so the key space is unbounded over a long multi-agent/SSH session. // Bound the coalesce map the same way ProcessGoneDedupe bounds its key map. @@ -163,7 +165,11 @@ export function getCrashBreadcrumbSnapshot(): CrashReportBreadcrumb[] { resolveAllPendingCoalescedBreadcrumbs() // Why: long sessions must retain threshold profiles without growing the 30-entry budget. const retained = [...retainedBreadcrumbs.values()] - const recent = breadcrumbs.slice(-(MAX_BREADCRUMBS - retained.length)) + // Why a start index rather than slice(-budget): a retained budget that ever + // reaches MAX_BREADCRUMBS makes the budget 0, and slice(-0) returns the whole + // ring. Counting forward degrades to [] with no branch to leave untested. + const recentBudget = MAX_BREADCRUMBS - retained.length + const recent = breadcrumbs.slice(Math.max(0, breadcrumbs.length - recentBudget)) return [...retained, ...recent] .sort((left, right) => left.createdAt.localeCompare(right.createdAt)) .map((breadcrumb) => ({ diff --git a/src/renderer/src/lib/crash-diagnostics.test.ts b/src/renderer/src/lib/crash-diagnostics.test.ts index 6e96efecf0c..736da555767 100644 --- a/src/renderer/src/lib/crash-diagnostics.test.ts +++ b/src/renderer/src/lib/crash-diagnostics.test.ts @@ -78,6 +78,7 @@ describe('renderer crash diagnostics', () => { name: 'renderer_memory', data: { reason: 'startup', + rendererSurface: 'main', usedHeapMB: 32, totalHeapMB: 64, heapLimitMB: 512, @@ -222,6 +223,145 @@ describe('renderer crash diagnostics', () => { unregister() }) + it('emits highwater samples inside the last 20% of the heap limit', async () => { + // Why: the 42-crash OOM cluster dies at 90-97% of the limit. With samples + // only at 60/80% the nearest profile is ~60min stale and names nothing. + const memory = (window.performance as unknown as { memory: Record }).memory + vi.stubGlobal('document', { + getElementsByTagName: () => ({ length: 1 }), + querySelectorAll: () => ({ length: 0 }) + }) + + diagnostics.installRendererCrashDiagnostics() + const tick = setIntervalMock.mock.calls[0][0] as () => void + memory.usedJSHeapSize = 0.92 * memory.jsHeapSizeLimit + tick() + memory.usedJSHeapSize = 0.96 * memory.jsHeapSizeLimit + tick() + + const thresholds = recordBreadcrumbMock.mock.calls + .filter((call) => (call[0] as { name: string }).name === 'renderer_memory_highwater') + .map((call) => (call[0] as { data: { thresholdPct: number } }).data.thresholdPct) + expect(thresholds).toEqual([60, 80, 90, 95]) + }) + + it('carries both shipped byte contributors on the routine renderer_memory breadcrumb', async () => { + // Why the real registrations rather than stand-ins: the sub-A leak grows + // outside the zustand store, so a trend carrying only storeKB still cannot + // name it. F0BM6V4KEBV died at 78% of the limit, where no highwater level + // fires and this crumb is the only attribution there is — which is also why + // no document is stubbed here: 32MB of 512MB crosses nothing. + const { useAppStore } = await import('../store') + const { registerLivePaneManager, unregisterLivePaneManager } = + await import('./pane-manager/pane-manager-registry') + useAppStore.setState({ worktrees: [{ id: 'w1' }, { id: 'w2' }] } as never) + const paneManager = { + resetWebglTextureAtlases: () => undefined, + getPaneCount: () => 2, + getPanes: () => + [1, 2].map((id) => ({ + id, + terminal: { cols: 200, buffer: { active: { length: 5000 } } } + })) + } + registerLivePaneManager(paneManager) + + diagnostics.installRendererCrashDiagnostics() + + const routine = recordBreadcrumbMock.mock.calls.find( + (call) => (call[0] as { name: string }).name === 'renderer_memory' + )?.[0] as { data: Record } + expect(routine.data['storeKB.__totalKB']).toBeGreaterThan(0) + // 2 panes x 5000 rows x 200 cols x 16B ~= 31MB of scrollback. + expect(routine.data['terminals.estBufferKB']).toBeGreaterThan(30_000) + expect(routine.data['terminals.estPanes']).toBe(2) + + unregisterLivePaneManager(paneManager) + }) + + it('carries truncated byte attribution on the routine renderer_memory breadcrumb', async () => { + // Why: two isolated highwater snapshots cannot show a ramp; the 60s crumb + // must carry a byte TREND so a report localizes the retainer on its own. + const profile = await import('./renderer-memory-profile') + const unregister = profile.registerRendererMemoryProfileContributor( + 'storeKB', + () => ({ worktrees: 120, __totalKB: 4200, tabs: 900, agents: 5, prs: 60, drafts: 2 }), + { trendLimit: 5 } + ) + + diagnostics.installRendererCrashDiagnostics() + + const routine = recordBreadcrumbMock.mock.calls.find( + (call) => (call[0] as { name: string }).name === 'renderer_memory' + )?.[0] as { data: Record } + expect(routine.data).toMatchObject({ + 'storeKB.__totalKB': 4200, + 'storeKB.tabs': 900, + 'storeKB.worktrees': 120, + 'storeKB.prs': 60, + 'storeKB.agents': 5 + }) + expect(routine.data).not.toHaveProperty('storeKB.drafts') + + unregister() + }) + + it('censuses the store once on a sample that also profiles the highwater', async () => { + // Why: the trend and the highwater profile share one contributor. Walking the + // store twice is worst exactly at 90/95%, where there is no headroom for the + // transient allocation the census makes. + const memory = (window.performance as unknown as { memory: Record }).memory + memory.usedJSHeapSize = 0.96 * memory.jsHeapSizeLimit + vi.stubGlobal('document', { + getElementsByTagName: () => ({ length: 1 }), + querySelectorAll: () => ({ length: 0 }) + }) + let censuses = 0 + const profile = await import('./renderer-memory-profile') + const unregister = profile.registerRendererMemoryProfileContributor( + 'storeKB', + () => { + censuses += 1 + return { __totalKB: 4200 } + }, + { trendLimit: 5 } + ) + + diagnostics.installRendererCrashDiagnostics() + + expect( + recordBreadcrumbMock.mock.calls.filter( + (call) => (call[0] as { name: string }).name === 'renderer_memory_highwater' + ) + ).toHaveLength(4) + expect(censuses).toBe(1) + + unregister() + }) + + it('tags every routine renderer_memory sample with its renderer surface', async () => { + // Why: main and the dashboard popout push into the same 30-slot main-process + // ring, each with its own storeKB census. Untagged, the byte trend this crumb + // exists to expose interleaves two unrelated heaps into one unreadable series. + const profile = await import('./renderer-memory-profile') + const unregister = profile.registerRendererMemoryProfileContributor( + 'storeKB', + () => ({ __totalKB: 12 }), + { trendLimit: 5 } + ) + + diagnostics.installRendererCrashDiagnostics('dashboard-popout') + const tick = setIntervalMock.mock.calls[0][0] as () => void + tick() + + const surfaces = recordBreadcrumbMock.mock.calls + .filter((call) => (call[0] as { name: string }).name === 'renderer_memory') + .map((call) => (call[0] as { data: { rendererSurface?: string } }).data.rendererSurface) + expect(surfaces).toEqual(['dashboard-popout', 'dashboard-popout']) + + unregister() + }) + it('skips highwater emission when heap readings are not finite', () => { const memory = (window.performance as unknown as { memory: Record }).memory memory.usedJSHeapSize = Number.NaN diff --git a/src/renderer/src/lib/crash-diagnostics.ts b/src/renderer/src/lib/crash-diagnostics.ts index 95c9bfe213c..6df7308a8db 100644 --- a/src/renderer/src/lib/crash-diagnostics.ts +++ b/src/renderer/src/lib/crash-diagnostics.ts @@ -2,20 +2,27 @@ import type { CrashReportBreadcrumbData, CrashReportDetailValue } from '../../../shared/crash-reporting' +// Why the levels live in shared/: one detailed breadcrumb per (surface, +// threshold) is retained in the main process, whose budget is derived from them. +import { + RENDERER_MEMORY_HIGHWATER_RATIOS, + type RendererSurface +} from '../../../shared/renderer-memory-highwater' import { getBrowserWebviewMemoryProfile, type BrowserWebviewMemoryProfile } from '../components/browser-pane/webview-registry' import { recordRendererCrashBreadcrumb } from './crash-breadcrumb-recorder' -import { collectRendererMemoryProfileCounts } from './renderer-memory-profile' +import { + collectRendererMemoryProfileCounts, + collectRendererMemoryTrendCounts, + createRendererMemoryCensus, + type RendererMemoryCensus +} from './renderer-memory-profile' const RENDERER_MEMORY_SAMPLE_INTERVAL_MS = 60_000 const BYTES_PER_MEGABYTE = 1024 * 1024 const BYTES_PER_KILOBYTE = 1024 -// Why: one detailed breadcrumb per threshold names what grew before an OOM. -const RENDERER_MEMORY_HIGHWATER_RATIOS = [0.6, 0.8] as const - -type RendererSurface = 'main' | 'dashboard-popout' type BrowserPerformanceMemory = { usedJSHeapSize?: number @@ -117,11 +124,17 @@ function recordRendererMemory(reason: string): void { return } const browserWebviews = getBrowserWebviewMemoryProfile() + // Why one census per sample: the highwater profile below repeats this pass's + // contributors, and near-OOM is the worst moment to walk the store twice. + const census = createRendererMemoryCensus() recordRendererCrashBreadcrumb( 'renderer_memory', compactBreadcrumbData({ reason, + // Why: main and the dashboard popout each sample into the same ring, so an + // untagged series reads as a sawtooth between two unrelated heaps. + rendererSurface, usedHeapMB: toMegabytes(memory.usedJSHeapSize), totalHeapMB: toMegabytes(memory.totalJSHeapSize), heapLimitMB: toMegabytes(memory.jsHeapSizeLimit), @@ -129,15 +142,19 @@ function recordRendererMemory(reason: string): void { mallocedMB: toMegabytes(memory.mallocedBytes), blinkAllocatedMB: toMegabytes(memory.blinkAllocatedBytes), browserWebviews: browserWebviews.browserWebviewCount, - registeredBrowserGuests: browserWebviews.registeredBrowserGuestCount + registeredBrowserGuests: browserWebviews.registeredBrowserGuestCount, + // Why on every sample: a byte trend across the session localizes a + // retainer that two near-death snapshots cannot. + ...collectRendererMemoryTrendCounts(census) }) ) - recordRendererMemoryHighwater(memory, browserWebviews) + recordRendererMemoryHighwater(memory, browserWebviews, census) } function recordRendererMemoryHighwater( memory: HeapMetrics, - browserWebviews: BrowserWebviewMemoryProfile + browserWebviews: BrowserWebviewMemoryProfile, + census: RendererMemoryCensus ): void { const used = memory.usedJSHeapSize const limit = memory.jsHeapSizeLimit @@ -170,7 +187,7 @@ function recordRendererMemoryHighwater( terminalElements: document.querySelectorAll('.xterm').length, browserWebviews: browserWebviews.browserWebviewCount, registeredBrowserGuests: browserWebviews.registeredBrowserGuestCount, - ...collectRendererMemoryProfileCounts() + ...collectRendererMemoryProfileCounts(census) }) for (const threshold of RENDERER_MEMORY_HIGHWATER_RATIOS) { if (ratio < threshold || emittedHighwaterRatios.has(threshold)) { diff --git a/src/renderer/src/lib/pane-manager/pane-manager-registry.ts b/src/renderer/src/lib/pane-manager/pane-manager-registry.ts index d864ce0de37..482d1586f73 100644 --- a/src/renderer/src/lib/pane-manager/pane-manager-registry.ts +++ b/src/renderer/src/lib/pane-manager/pane-manager-registry.ts @@ -237,4 +237,11 @@ export function getLivePaneMemoryProfileCounts(): Record { // Why here: contributors push in (crash-diagnostics stays a leaf); same-name // re-registration on dev HMR overwrites, so no disposal hook is needed. -registerRendererMemoryProfileContributor('terminals', getLivePaneMemoryProfileCounts) +// Why trendLimit: scrollback bytes live outside the zustand store, so storeKB +// cannot see them and the one-shot highwater profiles sample them once — the +// 78%-of-limit death in F0BM6V4KEBV never reached even the 90% level. The +// census above is sample-capped arithmetic, far cheaper than the storeKB walk +// it rides along with. 2 keys carries estBufferKB plus the pane count it scales. +registerRendererMemoryProfileContributor('terminals', getLivePaneMemoryProfileCounts, { + trendLimit: 2 +}) diff --git a/src/renderer/src/lib/renderer-memory-profile.test.ts b/src/renderer/src/lib/renderer-memory-profile.test.ts index 9daf083c7f4..460da3599b6 100644 --- a/src/renderer/src/lib/renderer-memory-profile.test.ts +++ b/src/renderer/src/lib/renderer-memory-profile.test.ts @@ -1,6 +1,8 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { collectRendererMemoryProfileCounts, + collectRendererMemoryTrendCounts, + createRendererMemoryCensus, registerRendererMemoryProfileContributor, summarizeStateCollectionSizes } from './renderer-memory-profile' @@ -135,6 +137,115 @@ describe('collectRendererMemoryProfileCounts', () => { }) }) +describe('collectRendererMemoryTrendCounts', () => { + it('keeps only the heaviest keys of trend-tagged contributors', () => { + unregisters.push( + registerRendererMemoryProfileContributor( + 'storeKB', + () => ({ tabs: 900, __totalKB: 4200, worktrees: 120, prs: 60, agents: 5, drafts: 2 }), + { trendLimit: 5 } + ) + ) + register('store', () => ({ worktrees: 40 })) + + expect(collectRendererMemoryTrendCounts()).toEqual({ + 'storeKB.__totalKB': 4200, + 'storeKB.tabs': 900, + 'storeKB.worktrees': 120, + 'storeKB.prs': 60, + 'storeKB.agents': 5 + }) + }) + + it('bounds the aggregate trend budget and contains throwing contributors', () => { + unregisters.push( + registerRendererMemoryProfileContributor( + 'broken', + () => { + throw new Error('boom') + }, + { trendLimit: 4 } + ) + ) + unregisters.push( + registerRendererMemoryProfileContributor( + 'runaway', + () => Object.fromEntries(Array.from({ length: 500 }, (_, index) => [`key${index}`, index])), + { trendLimit: 500 } + ) + ) + + const counts = collectRendererMemoryTrendCounts() + expect(counts['broken.error']).toBe(1) + expect(Object.keys(counts)).toHaveLength(8) + }) +}) + +// A sample that crosses a threshold runs both censuses, and storeKB walks every +// top-level store slice with a transient allocation the size of what it measures +// — the last thing to do twice with no heap headroom left. +describe('a census shared across one sample pass', () => { + it('reads each contributor once for both the trend and the profile', () => { + let reads = 0 + unregisters.push( + registerRendererMemoryProfileContributor( + 'storeKB', + () => { + reads += 1 + return { __totalKB: 4200, tabs: 900 } + }, + { trendLimit: 5 } + ) + ) + const census = createRendererMemoryCensus() + + const trend = collectRendererMemoryTrendCounts(census) + const profile = collectRendererMemoryProfileCounts(census) + + expect(reads).toBe(1) + expect(trend).toEqual({ 'storeKB.__totalKB': 4200, 'storeKB.tabs': 900 }) + expect(profile).toEqual({ 'storeKB.__totalKB': 4200, 'storeKB.tabs': 900 }) + }) + + it('keeps a fresh census per sample so the trend still moves', () => { + let size = 10 + unregisters.push( + registerRendererMemoryProfileContributor('storeKB', () => ({ __totalKB: (size += 10) }), { + trendLimit: 1 + }) + ) + + expect(collectRendererMemoryTrendCounts(createRendererMemoryCensus())).toEqual({ + 'storeKB.__totalKB': 20 + }) + expect(collectRendererMemoryTrendCounts(createRendererMemoryCensus())).toEqual({ + 'storeKB.__totalKB': 30 + }) + }) + + it('reports a throwing contributor in both censuses without re-running it', () => { + let reads = 0 + unregisters.push( + registerRendererMemoryProfileContributor( + 'broken', + () => { + reads += 1 + throw new Error('boom') + }, + { trendLimit: 4 } + ) + ) + const census = createRendererMemoryCensus() + + expect(collectRendererMemoryTrendCounts(census)).toEqual({ 'broken.error': 1 }) + expect(collectRendererMemoryProfileCounts(census)).toEqual({ 'broken.error': 1 }) + expect(reads).toBe(1) + }) +}) + +// The shipped contributors are measured against MAX_PROFILE_COUNTS in +// renderer-memory-shipped-contributors.test.ts, which registers them for real. + describe('summarizeStateCollectionSizes', () => { it('reports the largest collections first, capped at the limit', () => { const state = { diff --git a/src/renderer/src/lib/renderer-memory-profile.ts b/src/renderer/src/lib/renderer-memory-profile.ts index f9043af1e68..7dc31ca207a 100644 --- a/src/renderer/src/lib/renderer-memory-profile.ts +++ b/src/renderer/src/lib/renderer-memory-profile.ts @@ -10,21 +10,48 @@ export type RendererMemoryProfileCounts = Record type RendererMemoryProfileContributor = () => RendererMemoryProfileCounts -const contributors = new Map() +type RendererMemoryProfileOptions = { + /** Heaviest keys to also carry on the routine renderer_memory breadcrumb. */ + trendLimit?: number +} + +type RegisteredContributor = { + collect: RendererMemoryProfileContributor + trendLimit: number +} + +/** + * Per-sample memo shared by the trend and profile censuses. + * + * Why: crossing 90%/95% runs both in one pass, and the storeKB contributor walks + * every top-level store slice with a transient allocation the size of what it + * measures — exactly when the heap has no headroom to walk it twice. + */ +export type RendererMemoryCensus = Map + +export function createRendererMemoryCensus(): RendererMemoryCensus { + return new Map() +} + +const contributors = new Map() // Why: breadcrumbs are retained per session; a misbehaving contributor must not // bloat every crash report. 32 counts is plenty to name a leaking subsystem. const MAX_COUNTS_PER_CONTRIBUTOR = 32 // Why: individually bounded contributors can still create unbounded near-OOM work in aggregate. -const MAX_PROFILE_COUNTS = 64 +export const MAX_PROFILE_COUNTS = 64 // Why: empty contributors do not consume the count budget but must not make collection unbounded. const MAX_PROFILE_CONTRIBUTORS = 64 const MAX_CONTRIBUTOR_NAME_LENGTH = 64 const MAX_COUNT_KEY_LENGTH = 80 +// Why far below MAX_PROFILE_COUNTS: this rides every 60s sample, not just the +// near-OOM ones, and every retained breadcrumb pays for it. +export const MAX_TREND_COUNTS = 8 export function registerRendererMemoryProfileContributor( name: string, - contributor: RendererMemoryProfileContributor + contributor: RendererMemoryProfileContributor, + options?: RendererMemoryProfileOptions ): () => void { if ( name.length === 0 || @@ -33,26 +60,106 @@ export function registerRendererMemoryProfileContributor( ) { return () => undefined } - contributors.set(name, contributor) + contributors.set(name, { collect: contributor, trendLimit: options?.trendLimit ?? 0 }) return () => { - if (contributors.get(name) === contributor) { + if (contributors.get(name)?.collect === contributor) { contributors.delete(name) } } } -export function collectRendererMemoryProfileCounts(): RendererMemoryProfileCounts { +/** + * Heaviest keys of trend-tagged contributors, for the routine renderer_memory + * breadcrumb. + * + * Why: highwater profiles are two isolated snapshots. A crash report can only + * name a retainer if the byte attribution appears as a ramp across the whole + * session, so a truncated slice of it rides every sample. + */ +export function collectRendererMemoryTrendCounts( + census?: RendererMemoryCensus +): RendererMemoryProfileCounts { + const counts: RendererMemoryProfileCounts = {} + let collected = 0 + for (const [name, entry] of contributors) { + if (entry.trendLimit <= 0 || collected >= MAX_TREND_COUNTS) { + continue + } + try { + const ranked = rankContributorCounts(readContribution(name, entry, census)) + const keep = Math.min(entry.trendLimit, MAX_TREND_COUNTS - collected) + for (const [key, value] of ranked.slice(0, keep)) { + counts[`${name}.${key}`] = value + collected += 1 + } + } catch { + counts[`${name}.error`] = 1 + collected += 1 + } + } + return counts +} + +/** Throws on failure so both censuses keep funnelling it into `${name}.error`. */ +function readContribution( + name: string, + entry: RegisteredContributor, + census?: RendererMemoryCensus +): RendererMemoryProfileCounts { + const memoized = census?.get(name) + if (memoized) { + return memoized + } + if (memoized === null) { + throw new Error(`renderer memory contributor failed: ${name}`) + } + try { + const contribution = entry.collect() + census?.set(name, contribution) + return contribution + } catch (error) { + census?.set(name, null) + throw error + } +} + +function rankContributorCounts(contribution: RendererMemoryProfileCounts): [string, number][] { + const ranked: [string, number][] = [] + let inspected = 0 + for (const key in contribution) { + if (inspected >= MAX_COUNTS_PER_CONTRIBUTOR) { + break + } + inspected += 1 + if ( + !Object.hasOwn(contribution, key) || + key.length === 0 || + key.length > MAX_COUNT_KEY_LENGTH + ) { + continue + } + const value = contribution[key] + if (typeof value === 'number' && Number.isFinite(value)) { + ranked.push([key, value]) + } + } + return ranked.sort((a, b) => b[1] - a[1]) +} + +export function collectRendererMemoryProfileCounts( + census?: RendererMemoryCensus +): RendererMemoryProfileCounts { const counts: RendererMemoryProfileCounts = {} let collected = 0 let visited = 0 - for (const [name, contributor] of contributors) { + for (const [name, entry] of contributors) { if (collected >= MAX_PROFILE_COUNTS || visited >= MAX_PROFILE_CONTRIBUTORS) { break } visited += 1 // Why: a broken contributor must never take down memory reporting itself. try { - const contribution = contributor() + const contribution = readContribution(name, entry, census) let inspected = 0 for (const key in contribution) { if (inspected >= MAX_COUNTS_PER_CONTRIBUTOR || collected >= MAX_PROFILE_COUNTS) { diff --git a/src/renderer/src/lib/renderer-memory-shipped-contributors.test.ts b/src/renderer/src/lib/renderer-memory-shipped-contributors.test.ts new file mode 100644 index 00000000000..6cab1008fa9 --- /dev/null +++ b/src/renderer/src/lib/renderer-memory-shipped-contributors.test.ts @@ -0,0 +1,97 @@ +import { afterEach, describe, expect, it } from 'vitest' +import { + registerLivePaneManager, + unregisterLivePaneManager +} from './pane-manager/pane-manager-registry' +import { + MAX_PROFILE_COUNTS, + MAX_TREND_COUNTS, + collectRendererMemoryProfileCounts, + collectRendererMemoryTrendCounts, + createRendererMemoryCensus +} from './renderer-memory-profile' +import { useAppStore } from '../store' +import type { AppState } from '../store/types' + +/** + * Drives the contributors the app actually registers — importing ../store and + * pane-manager-registry for their registration side effects — instead of + * hand-written stand-ins that can only restate the test's own setup. + */ + +const paneManager = { + resetWebglTextureAtlases: () => undefined, + getPaneCount: () => 3, + getPanes: () => + [1, 2, 3].map((id) => ({ + id, + terminal: { cols: 200, buffer: { active: { length: 5000 } } } + })) +} + +/** Saturates every slice limit so each contributor emits its widest census. */ +function fillStoreWithProbeSlices(): string[] { + const keys = Array.from({ length: 40 }, (_, index) => `__profileProbe${index}`) + useAppStore.setState( + Object.fromEntries( + keys.map((key, index) => [key, Array.from({ length: index + 1 }, () => 'x'.repeat(64))]) + ) as unknown as Partial + ) + return keys +} + +afterEach(() => { + unregisterLivePaneManager(paneManager) +}) + +describe('shipped renderer memory contributors', () => { + it('fits the widest census of every registered contributor inside MAX_PROFILE_COUNTS', () => { + // Why the real registry: a fake census cannot notice the store raising its + // slice limit, a fourth contributor arriving, or terminals adding keys — + // any of which silently truncates the near-death profile mid-contributor. + const probeKeys = fillStoreWithProbeSlices() + registerLivePaneManager(paneManager) + + const counts = collectRendererMemoryProfileCounts() + const prefixes = new Set(Object.keys(counts).map((key) => key.split('.')[0])) + + expect(Object.keys(counts).length).toBeLessThanOrEqual(MAX_PROFILE_COUNTS) + expect([...prefixes].sort()).toEqual(['store', 'storeKB', 'terminals']) + // Truncation stops mid-contributor, so the last registered one loses keys first. + expect(counts['terminals.estBufferKB']).toBeGreaterThan(0) + expect(counts['storeKB.__totalKB']).toBeGreaterThan(0) + expect(Object.keys(counts).filter((key) => key.startsWith('store.'))).toHaveLength(20) + expect(probeKeys.some((key) => counts[`store.${key}`] !== undefined)).toBe(true) + }) + + it('carries both byte contributors on the routine trend within MAX_TREND_COUNTS', () => { + // Why terminals too: the sub-A leak grows outside the zustand store, and + // xterm scrollback is the byte source storeKB structurally cannot see. + fillStoreWithProbeSlices() + registerLivePaneManager(paneManager) + + const counts = collectRendererMemoryTrendCounts(createRendererMemoryCensus()) + + expect(Object.keys(counts).length).toBeLessThanOrEqual(MAX_TREND_COUNTS) + expect(counts['storeKB.__totalKB']).toBeGreaterThan(0) + // 3 panes x 5000 rows x 200 cols x 16B ~= 46MB of scrollback. + expect(counts['terminals.estBufferKB']).toBeGreaterThan(40_000) + expect(Object.keys(counts).filter((key) => key.startsWith('terminals.'))).toHaveLength(2) + }) + + it('reads each contributor once when a sample both trends and profiles', () => { + // Why: crossing 90/95% runs both censuses in one pass, with no heap headroom + // left to walk the store twice. + fillStoreWithProbeSlices() + registerLivePaneManager(paneManager) + const census = createRendererMemoryCensus() + + collectRendererMemoryTrendCounts(census) + const before = census.get('storeKB') + collectRendererMemoryProfileCounts(census) + + expect(before).toBeDefined() + expect(census.get('storeKB')).toBe(before) + expect(census.get('terminals')).toBeDefined() + }) +}) diff --git a/src/renderer/src/store/index.ts b/src/renderer/src/store/index.ts index 1c6ebf327ac..453fe82d662 100644 --- a/src/renderer/src/store/index.ts +++ b/src/renderer/src/store/index.ts @@ -109,8 +109,12 @@ registerRendererMemoryProfileContributor('store', () => // Why bytes too: counts miss value-weight growth (97b9e86d leaked ~700MB while // its biggest slice grew by 4 entries); sampled KB names what got FAT. -registerRendererMemoryProfileContributor('storeKB', () => - estimateStateCollectionKB(useAppStore.getState(), 16) +// Why trendLimit 5: __totalKB always outranks any single slice, so 5 carries +// the total plus the top 4 slices onto every routine renderer_memory crumb. +registerRendererMemoryProfileContributor( + 'storeKB', + () => estimateStateCollectionKB(useAppStore.getState(), 16), + { trendLimit: 5 } ) export type { AppState } from './types' diff --git a/src/shared/renderer-memory-highwater.ts b/src/shared/renderer-memory-highwater.ts new file mode 100644 index 00000000000..448fd9db2b6 --- /dev/null +++ b/src/shared/renderer-memory-highwater.ts @@ -0,0 +1,23 @@ +/** + * Shape of the renderer_memory_highwater breadcrumb series, shared so the + * main-process retention budget cannot drift from what the renderer emits. + */ + +// Why these levels: the OOM cluster dies at 90-97% of the heap limit; at the +// observed ~25MB/min ramp an 0.8-only profile is ~60min stale by then. +export const RENDERER_MEMORY_HIGHWATER_RATIOS = [0.6, 0.8, 0.9, 0.95] as const + +// Each surface samples its own heap into the same breadcrumb ring. +export const RENDERER_MEMORY_SURFACES = ['main', 'dashboard-popout'] as const + +export type RendererSurface = (typeof RENDERER_MEMORY_SURFACES)[number] + +/** + * One retained breadcrumb per (surface, threshold) profile. + * + * Why derived: a hand-written cap silently evicts oldest-first, so adding a + * threshold would drop the near-death main-window profile these reports exist + * to carry. + */ +export const MAX_RETAINED_RENDERER_MEMORY_BREADCRUMBS = + RENDERER_MEMORY_HIGHWATER_RATIOS.length * RENDERER_MEMORY_SURFACES.length