diff --git a/src/main/crash-reporting/process-gone-heartbeat-silence.test.ts b/src/main/crash-reporting/process-gone-heartbeat-silence.test.ts new file mode 100644 index 00000000000..9a5df968fc0 --- /dev/null +++ b/src/main/crash-reporting/process-gone-heartbeat-silence.test.ts @@ -0,0 +1,288 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +vi.mock('electron', () => ({ + app: { + getVersion: () => '1.2.3-test', + getAppMetrics: (): unknown[] => [] + } +})) + +import { rendererCrashBreadcrumbOrigin } from '../../shared/crash-breadcrumb-origin' +import { clearCrashBreadcrumbsForTest, recordCrashBreadcrumb } from './crash-breadcrumb-store' +import { ProcessGoneDedupe } from './process-gone-dedupe' +import { recordProcessGoneCrash, type ProcessGoneCrashEvent } from './process-gone-recorder' +import { _resetTracerForTests, setActiveSink, type TracerSink } from '../observability/tracer' + +const CRASHED_WEB_CONTENTS_ID = 11 +const SIBLING_WEB_CONTENTS_ID = 22 +const noMinidump = async () => null + +function killedEvent(overrides: Partial = {}): ProcessGoneCrashEvent { + return { + source: 'renderer', + processType: 'renderer', + reason: 'killed', + exitCode: 1, + expectedTeardown: 'none', + details: { processType: 'renderer' }, + webContentsId: CRASHED_WEB_CONTENTS_ID, + ...overrides + } +} + +function recordedDetails(record: ReturnType): Record { + const input = record.mock.calls[0]?.[0] as { details?: Record } | undefined + return input?.details ?? {} +} + +type SpanRecord = { name?: string; attributes?: Record } + +let spans: SpanRecord[] + +function spanNamed(name: string): SpanRecord | undefined { + return spans.find((span) => span.name === name) +} + +beforeEach(() => { + spans = [] + const sink: TracerSink = { + push: (record) => spans.push(record as SpanRecord), + flush: vi.fn(), + close: vi.fn() + } + setActiveSink(sink) + clearCrashBreadcrumbsForTest() +}) + +afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + _resetTracerForTests() + clearCrashBreadcrumbsForTest() +}) + +describe('renderer silence on process-gone reports', () => { + it('stamps how long the crashing renderer had been silent before a killed/1 death', () => { + vi.useFakeTimers() + vi.setSystemTime(Date.parse('2026-08-20T11:01:20.000Z')) + recordCrashBreadcrumb( + 'renderer_memory', + { reason: 'interval', usedHeapMB: 120 }, + rendererCrashBreadcrumbOrigin(CRASHED_WEB_CONTENTS_ID) + ) + vi.setSystemTime(Date.parse('2026-08-20T11:14:50.000Z')) + const record = vi.fn().mockResolvedValue({ id: 'report-1' }) + + recordProcessGoneCrash( + { record, attachDetails: async () => null } as never, + killedEvent(), + new ProcessGoneDedupe(), + noMinidump + ) + + expect(recordedDetails(record)).toMatchObject({ + rendererHeartbeatStatus: 'observed', + rendererHeartbeatAttribution: 'crashed-renderer', + rendererHeartbeatSilenceMs: 810_000, + rendererHeartbeatMissedIntervals: 13 + }) + }) + + it('does not credit the crashing renderer with a sibling renderer heartbeat', () => { + vi.useFakeTimers() + vi.setSystemTime(Date.parse('2026-08-20T11:01:20.000Z')) + recordCrashBreadcrumb( + 'renderer_memory', + { reason: 'interval' }, + rendererCrashBreadcrumbOrigin(CRASHED_WEB_CONTENTS_ID) + ) + vi.setSystemTime(Date.parse('2026-08-20T11:14:20.000Z')) + recordCrashBreadcrumb( + 'renderer_memory', + { reason: 'interval' }, + rendererCrashBreadcrumbOrigin(SIBLING_WEB_CONTENTS_ID) + ) + vi.setSystemTime(Date.parse('2026-08-20T11:14:50.000Z')) + const record = vi.fn().mockResolvedValue({ id: 'report-1' }) + + recordProcessGoneCrash( + { record, attachDetails: async () => null } as never, + killedEvent(), + new ProcessGoneDedupe(), + noMinidump + ) + + expect(recordedDetails(record)).toMatchObject({ + rendererHeartbeatSilenceMs: 810_000, + rendererHeartbeatAttribution: 'crashed-renderer' + }) + }) + + it('does not read an OS sleep as a wedged renderer', () => { + vi.useFakeTimers() + vi.setSystemTime(Date.parse('2026-08-20T03:00:20.000Z')) + recordCrashBreadcrumb( + 'renderer_memory', + { reason: 'interval', usedHeapMB: 120 }, + rendererCrashBreadcrumbOrigin(CRASHED_WEB_CONTENTS_ID) + ) + vi.setSystemTime(Date.parse('2026-08-20T11:14:20.000Z')) + recordCrashBreadcrumb('system_slept', { suspendedForMs: 29_580_000 }) + vi.setSystemTime(Date.parse('2026-08-20T11:14:50.000Z')) + const record = vi.fn().mockResolvedValue({ id: 'report-1' }) + + recordProcessGoneCrash( + { record, attachDetails: async () => null } as never, + killedEvent(), + new ProcessGoneDedupe(), + noMinidump + ) + + expect(recordedDetails(record)).toMatchObject({ + rendererHeartbeatSilenceMs: 29_670_000, + rendererHeartbeatSuspendedMs: 29_580_000, + rendererHeartbeatAwakeSilenceMs: 90_000, + rendererHeartbeatMissedIntervals: 1 + }) + }) + + it('records that no heartbeat was seen rather than omitting the field', () => { + const record = vi.fn().mockResolvedValue({ id: 'report-1' }) + + recordProcessGoneCrash( + { record, attachDetails: async () => null } as never, + killedEvent(), + new ProcessGoneDedupe(), + noMinidump + ) + + expect(recordedDetails(record)).toMatchObject({ + rendererHeartbeatStatus: 'none' + }) + }) + + it('does not stamp a renderer heartbeat verdict on a child-process death', () => { + vi.useFakeTimers() + vi.setSystemTime(Date.parse('2026-08-20T11:00:20.000Z')) + recordCrashBreadcrumb( + 'renderer_memory', + { reason: 'interval', usedHeapMB: 120 }, + rendererCrashBreadcrumbOrigin(CRASHED_WEB_CONTENTS_ID) + ) + vi.setSystemTime(Date.parse('2026-08-20T11:14:50.000Z')) + const record = vi.fn().mockResolvedValue({ id: 'report-1' }) + + recordProcessGoneCrash( + { record, attachDetails: async () => null } as never, + killedEvent({ + source: 'child', + processType: 'utility', + reason: 'crashed', + exitCode: 133, + details: { processType: 'utility', serviceName: 'storage.mojom.StorageService' }, + webContentsId: undefined + }), + new ProcessGoneDedupe(), + noMinidump + ) + + const details = recordedDetails(record) + expect(details).not.toHaveProperty('rendererHeartbeatStatus') + expect(details).not.toHaveProperty('rendererHeartbeatSilenceMs') + expect(details).not.toHaveProperty('rendererHeartbeatMissedIntervals') + }) +}) + +describe('minidump status on process-gone reports', () => { + // Why: the dump wait runs up to 8s after the record is written, so a main + // process that exits with the renderer (the killed/1 shape) never attaches. + it('stamps a pending minidump status at record time', () => { + const record = vi.fn().mockResolvedValue({ id: 'report-1' }) + + recordProcessGoneCrash( + { record, attachDetails: async () => null } as never, + killedEvent(), + new ProcessGoneDedupe(), + () => new Promise(() => {}) + ) + + expect(recordedDetails(record)).toMatchObject({ + minidumpStatus: 'pending' + }) + }) + + it('replaces the pending status once the dump wait resolves', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report-1' }) + const attach = vi.fn().mockResolvedValue(null) + + recordProcessGoneCrash( + { record, attachDetails: attach } as never, + killedEvent(), + new ProcessGoneDedupe(), + noMinidump + ) + + await vi.waitFor(() => + expect(attach).toHaveBeenCalledWith('report-1', { + minidumpStatus: 'absent' + }) + ) + }) + + it('replaces the pending status when the terminal attach write fails', async () => { + vi.spyOn(console, 'error').mockImplementation(() => {}) + const record = vi.fn().mockResolvedValue({ id: 'report-1' }) + const attach = vi.fn().mockRejectedValueOnce(new Error('EPERM')).mockResolvedValue(null) + + recordProcessGoneCrash( + { record, attachDetails: attach } as never, + killedEvent(), + new ProcessGoneDedupe(), + noMinidump + ) + + await vi.waitFor(() => + expect(attach).toHaveBeenCalledWith('report-1', { + minidumpStatus: 'attach-failed' + }) + ) + }) +}) + +describe('minidump status on the process-gone span', () => { + it('does not stamp a dump status the span is serialized too early to know', () => { + const record = vi.fn().mockResolvedValue({ id: 'report-1' }) + + recordProcessGoneCrash( + { record, attachDetails: async () => null } as never, + killedEvent(), + new ProcessGoneDedupe(), + noMinidump + ) + + const details = spanNamed('electron.process_gone')?.attributes?.details as Record< + string, + unknown + > + expect(details).toBeDefined() + expect(details).not.toHaveProperty('minidumpStatus') + }) + + it('emits the terminal dump status on its own span once the wait resolves', async () => { + const record = vi.fn().mockResolvedValue({ id: 'report-1' }) + + recordProcessGoneCrash( + { record, attachDetails: async () => null } as never, + killedEvent(), + new ProcessGoneDedupe(), + noMinidump + ) + + await vi.waitFor(() => + expect(spanNamed('electron.minidump_signature')?.attributes).toMatchObject({ + 'crash.report_id': 'report-1', + 'crash.minidump_status': 'absent' + }) + ) + }) +}) diff --git a/src/main/crash-reporting/process-gone-minidump-attachment.ts b/src/main/crash-reporting/process-gone-minidump-attachment.ts new file mode 100644 index 00000000000..87304b23fe2 --- /dev/null +++ b/src/main/crash-reporting/process-gone-minidump-attachment.ts @@ -0,0 +1,98 @@ +import { sanitizeCrashReportDetails } from '../../shared/crash-reporting' +import type { CrashReportStore } from './crash-report-store' +import { captureMinidumpSignature, type CapturedMinidump } from './crashpad-capture' +import { minidumpSignatureDetails } from './minidump-crash-signature' +import type { ProcessGoneSource } from './process-gone-classification' +import { flushActiveSink, startSpan } from '../observability/tracer' + +type MinidumpDetailStore = Pick + +/** Injectable so tests can drive the pairing without a Crashpad handler. */ +export type MinidumpCapture = ( + crashedAtMs: number, + expectedProcessType: string +) => Promise + +const CHILD_CRASHPAD_PROCESS_TYPES: Readonly> = { + gpu: 'gpu-process', + utility: 'utility', + zygote: 'zygote' +} + +export function expectedCrashpadProcessType( + source: ProcessGoneSource, + processType: string +): string | null { + return source === 'renderer' + ? 'renderer' + : (CHILD_CRASHPAD_PROCESS_TYPES[processType.trim().toLowerCase()] ?? null) +} + +export const captureProcessMinidump: MinidumpCapture = (crashedAtMs, expectedProcessType) => + captureMinidumpSignature(crashedAtMs, { expectedProcessType }) + +// Why: the crash-report record is capped at 5 entries and is user-facing; the span +// is what makes the outcome countable in the diagnostics bundle. Emitted for the +// absent case too, so a trace can tell "no dump was written" from a process-gone +// span with no status at all -- the main process exited first (the killed/1 shape). +function recordMinidumpStatusSpan( + reportId: string, + status: 'captured' | 'absent', + attributes: Record = {} +): void { + const span = startSpan('electron.minidump_signature', { + attributes: { 'crash.report_id': reportId, 'crash.minidump_status': status, ...attributes } + }) + span.end() + flushActiveSink() +} + +/** + * Best effort, so a store that is failing writes cannot leave `pending` on disk — + * which reads as "the dump wait never ran" rather than "the attach write failed". + */ +export async function markMinidumpAttachFailed( + store: MinidumpDetailStore, + reportId: string +): Promise { + try { + await store.attachDetails(reportId, { minidumpStatus: 'attach-failed' }) + } catch { + // The caller already logged and breadcrumbed the original failure. + } +} + +/** + * Folds the Crashpad signature into a report that is already on disk. + * + * Why separate from the record write: an exit code of 0x80000003 only says "a + * CHECK fired"; the name, file and line live in the dump, which Crashpad is + * still writing when process-gone fires. Waiting inline would stall recovery. + * The record carries `minidumpStatus: 'pending'` until this lands, so a wait that + * never completes leaves that state visible instead of no field. + */ +export async function attachMinidumpSignature( + store: MinidumpDetailStore, + reportId: string, + crashedAtMs: number, + expectedProcessType: string | null, + capture: MinidumpCapture +): Promise { + const captured = expectedProcessType ? await capture(crashedAtMs, expectedProcessType) : null + if (!captured) { + await store.attachDetails(reportId, { minidumpStatus: 'absent' }) + recordMinidumpStatusSpan(reportId, 'absent') + return + } + const signatureDetails = sanitizeCrashReportDetails(minidumpSignatureDetails(captured.signature)) + await store.attachDetails(reportId, { + ...signatureDetails, + minidumpStatus: 'captured', + minidumpPath: captured.filePath, + minidumpBytes: captured.sizeBytes + }) + recordMinidumpStatusSpan(reportId, 'captured', { + 'crash.minidump_bytes': captured.sizeBytes, + ...signatureDetails + }) +} diff --git a/src/main/crash-reporting/process-gone-recorder.ts b/src/main/crash-reporting/process-gone-recorder.ts index 47ac561c4a7..cce75b5f978 100644 --- a/src/main/crash-reporting/process-gone-recorder.ts +++ b/src/main/crash-reporting/process-gone-recorder.ts @@ -2,7 +2,6 @@ import os from 'node:os' import { app } from 'electron' import { isCrashReportReason, - sanitizeCrashReportDetails, sanitizeCrashReportString, type CrashReportBreadcrumbData } from '../../shared/crash-reporting' @@ -24,6 +23,7 @@ import { type ProcessGoneSource } from './process-gone-classification' import { buildProcessGoneCrashDetails } from './process-gone-diagnostics' +import { rendererHeartbeatSilenceDetails } from './renderer-heartbeat-silence' import { buildSuppressedProcessGoneBreadcrumbData } from './suppressed-process-gone-breadcrumb' import { getProcessGoneDedupeKey, @@ -35,12 +35,14 @@ import { siblingProcessDeathDetails } from './process-gone-sibling-correlation' import { getMainProcessLifecycleIdentity } from './main-process-lifecycle-identity' +import { scheduleCrashpadDumpPrune } from './crashpad-capture' import { - captureMinidumpSignature, - scheduleCrashpadDumpPrune, - type CapturedMinidump -} from './crashpad-capture' -import { minidumpSignatureDetails } from './minidump-crash-signature' + attachMinidumpSignature, + captureProcessMinidump, + expectedCrashpadProcessType, + markMinidumpAttachFailed, + type MinidumpCapture +} from './process-gone-minidump-attachment' import { flushActiveSink, startSpan } from '../observability/tracer' export type ProcessGoneCrashEvent = { @@ -55,27 +57,6 @@ export type ProcessGoneCrashEvent = { type CrashReportRecorderStore = Pick -/** Injectable so tests can drive the pairing without a Crashpad handler. */ -export type MinidumpCapture = ( - crashedAtMs: number, - expectedProcessType: string -) => Promise - -const CHILD_CRASHPAD_PROCESS_TYPES: Readonly> = { - gpu: 'gpu-process', - utility: 'utility', - zygote: 'zygote' -} - -function expectedCrashpadProcessType(event: ProcessGoneCrashEvent): string | null { - return event.source === 'renderer' - ? 'renderer' - : (CHILD_CRASHPAD_PROCESS_TYPES[event.processType.trim().toLowerCase()] ?? null) -} - -const captureProcessMinidump: MinidumpCapture = (crashedAtMs, expectedProcessType) => - captureMinidumpSignature(crashedAtMs, { expectedProcessType }) - // Why: the coalesce map prunes every key against the calling window, so a shorter // one here would weaken the other 30s coalescers. Stay uniform with them. const SUPPRESSED_PROCESS_GONE_COALESCE_MS = 30_000 @@ -129,45 +110,6 @@ function persistFailureData(event: ProcessGoneCrashEvent, error: unknown) { } } -/** - * Folds the Crashpad signature into a report that is already on disk. - * - * Why separate from the record write: an exit code of 0x80000003 only says "a - * CHECK fired"; the name, file and line live in the dump, which Crashpad is - * still writing when process-gone fires. Waiting inline would stall recovery. - */ -async function attachMinidumpSignature( - store: CrashReportRecorderStore, - reportId: string, - crashedAtMs: number, - expectedProcessType: string | null, - capture: MinidumpCapture -): Promise { - const captured = expectedProcessType ? await capture(crashedAtMs, expectedProcessType) : null - if (!captured) { - await store.attachDetails(reportId, { minidumpStatus: 'absent' }) - return - } - const signatureDetails = sanitizeCrashReportDetails(minidumpSignatureDetails(captured.signature)) - await store.attachDetails(reportId, { - ...signatureDetails, - minidumpStatus: 'captured', - minidumpPath: captured.filePath, - minidumpBytes: captured.sizeBytes - }) - // Why: the crash-report record is capped at 5 entries and is user-facing; - // the span is what makes the signature countable in the diagnostics bundle. - const span = startSpan('electron.minidump_signature', { - attributes: { - 'crash.report_id': reportId, - 'crash.minidump_bytes': captured.sizeBytes, - ...signatureDetails - } - }) - span.end() - flushActiveSink() -} - export function recordProcessGoneCrash( store: CrashReportRecorderStore | null, event: ProcessGoneCrashEvent, @@ -246,16 +188,29 @@ export function recordProcessGoneCrash( : [] const siblingDetails = siblingDeaths.length > 0 ? siblingProcessDeathDetails(siblingDeaths, goneAt) : {} + const reporterOrigin = processGoneRendererOrigin(event) + const breadcrumbs = getCrashBreadcrumbSnapshot(reporterOrigin) + const reportBreadcrumbs = breadcrumbs?.map(({ origin: _origin, ...breadcrumb }) => breadcrumb) const crashDetails = buildProcessGoneCrashDetails( { ...event.details, ...mainProcessLifecycle, - ...siblingDetails + ...siblingDetails, + // Why: only a renderer emits the heartbeat, and a child death has no origin to + // scope the ring by, so it would be stamped with an unrelated renderer's crumb. + ...(event.source === 'renderer' + ? rendererHeartbeatSilenceDetails(breadcrumbs, reporterOrigin, goneAt) + : {}), + // Overwritten by attachMinidumpSignature; surviving as `pending` means the dump + // wait never completed -- most often the main process exited first (killed/1). + minidumpStatus: 'pending' }, event.processType ) - const breadcrumbs = getCrashBreadcrumbSnapshot(processGoneRendererOrigin(event)) - const reportBreadcrumbs = breadcrumbs?.map(({ origin: _origin, ...breadcrumb }) => breadcrumb) + // Why: this span is serialized at fail(), before the dump wait even starts, so a + // `pending` on it would read as terminal on every crash. The terminal value goes + // out on the minidump span; only the on-disk record carries the pending state. + const { minidumpStatus: _pendingMinidumpStatus, ...spanDetails } = crashDetails const span = startSpan('electron.process_gone', { attributes: { 'crash.source': event.source, @@ -272,7 +227,7 @@ export function recordProcessGoneCrash( 'app.main_process.pid': mainProcessLifecycle.mainProcessPid, 'app.main_process.launch_id': mainProcessLifecycle.mainProcessLaunchId, 'app.main_process.started_at': mainProcessLifecycle.mainProcessStartedAt, - details: crashDetails, + details: spanDetails, breadcrumbs: reportBreadcrumbs } }) @@ -284,7 +239,7 @@ export function recordProcessGoneCrash( flushActiveSink() const crashedAtMs = Date.now() - const expectedProcessType = expectedCrashpadProcessType(event) + const expectedProcessType = expectedCrashpadProcessType(event.source, event.processType) const recorded = store.record({ source: event.source, processType: event.processType, @@ -306,7 +261,7 @@ export function recordProcessGoneCrash( (reportId, details) => store.attachDetails(reportId, details), recorded, processGoneBreadcrumbData(event), - processGoneRendererOrigin(event) + reporterOrigin ) void recorded .then((report) => { @@ -320,11 +275,14 @@ export function recordProcessGoneCrash( capture ).catch((error) => { console.error('[crash-reporting] Failed to attach minidump signature:', error) + // Why: without a terminal status the report keeps `pending`, which reads as + // evidence the main process died first even when the dump was captured. + void markMinidumpAttachFailed(store, report.id) recordDurableCrashBreadcrumb( 'minidump_signature_attach_failed', processGoneBreadcrumbData(event), error instanceof Error ? error.message : String(error), - processGoneRendererOrigin(event) + reporterOrigin ) }) }) @@ -336,7 +294,7 @@ export function recordProcessGoneCrash( 'crash_report_persist_failed', data, `${String(data.errorName)}: ${String(data.errorMessage)}`, - processGoneRendererOrigin(event) + reporterOrigin ) }) } diff --git a/src/main/crash-reporting/renderer-heartbeat-silence.test.ts b/src/main/crash-reporting/renderer-heartbeat-silence.test.ts new file mode 100644 index 00000000000..15be28c2d7c --- /dev/null +++ b/src/main/crash-reporting/renderer-heartbeat-silence.test.ts @@ -0,0 +1,203 @@ +import { describe, expect, it } from 'vitest' +import type { CrashReportBreadcrumb } from '../../shared/crash-reporting' +import { rendererHeartbeatSilenceDetails } from './renderer-heartbeat-silence' + +const CRASHED_ORIGIN = 'renderer:11' +const GONE_AT = Date.parse('2026-08-20T11:14:50.000Z') + +function heartbeat(createdAt: string, origin?: string): CrashReportBreadcrumb { + return { + createdAt, + name: 'renderer_memory', + data: { reason: 'interval', usedHeapMB: 120 }, + ...(origin ? { origin } : {}) + } +} + +/** Stamped at resume, so `createdAt` is the wake instant and the span runs backwards from it. */ +function slept(resumedAt: string, suspendedForMs: unknown): CrashReportBreadcrumb { + return { + createdAt: resumedAt, + name: 'system_slept', + data: { suspendedForMs } as CrashReportBreadcrumb['data'] + } +} + +describe('rendererHeartbeatSilenceDetails', () => { + it('measures how long the crashing renderer had been silent before it died', () => { + const breadcrumbs = [ + heartbeat('2026-08-20T11:00:20.000Z', CRASHED_ORIGIN), + heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN) + ] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toEqual({ + rendererHeartbeatStatus: 'observed', + rendererHeartbeatAttribution: 'crashed-renderer', + rendererHeartbeatLastAt: '2026-08-20T11:01:20.000Z', + rendererHeartbeatSilenceMs: 810_000, + rendererHeartbeatIntervalMs: 60_000, + rendererHeartbeatMissedIntervals: 13 + }) + }) + + it('reports no missed intervals for a renderer that was beating up to the crash', () => { + const breadcrumbs = [heartbeat('2026-08-20T11:14:20.000Z', CRASHED_ORIGIN)] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({ + rendererHeartbeatSilenceMs: 30_000, + rendererHeartbeatMissedIntervals: 0 + }) + }) + + it('marks the value unattributed when the heartbeat names no emitting renderer', () => { + const breadcrumbs = [heartbeat('2026-08-20T11:01:20.000Z')] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({ + rendererHeartbeatAttribution: 'unattributed', + rendererHeartbeatSilenceMs: 810_000 + }) + }) + + it('has no attributable heartbeat for a child-process death', () => { + const breadcrumbs = [heartbeat('2026-08-20T11:14:20.000Z', CRASHED_ORIGIN)] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, undefined, GONE_AT)).toMatchObject({ + rendererHeartbeatAttribution: 'unattributed' + }) + }) + + it('prefers the crashed renderer own heartbeat over a newer unattributed one', () => { + const breadcrumbs = [ + heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN), + heartbeat('2026-08-20T11:14:20.000Z') + ] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({ + rendererHeartbeatAttribution: 'crashed-renderer', + rendererHeartbeatLastAt: '2026-08-20T11:01:20.000Z', + rendererHeartbeatSilenceMs: 810_000 + }) + }) + + it('records the absence explicitly instead of omitting the field', () => { + expect( + rendererHeartbeatSilenceDetails( + [{ createdAt: '2026-08-20T11:01:20.000Z', name: 'app_started' }], + CRASHED_ORIGIN, + GONE_AT + ) + ).toEqual({ rendererHeartbeatStatus: 'none' }) + }) + + it('ignores the one-shot highwater profile, which is not on the sampling cadence', () => { + const breadcrumbs: CrashReportBreadcrumb[] = [ + heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN), + { + createdAt: '2026-08-20T11:14:40.000Z', + name: 'renderer_memory_highwater', + data: { thresholdPct: 80 }, + origin: CRASHED_ORIGIN + } + ] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({ + rendererHeartbeatLastAt: '2026-08-20T11:01:20.000Z' + }) + }) + + it('clamps a backwards clock correction instead of reporting negative silence', () => { + const breadcrumbs = [heartbeat('2026-08-20T11:20:00.000Z', CRASHED_ORIGIN)] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({ + rendererHeartbeatSilenceMs: 0, + rendererHeartbeatMissedIntervals: 0 + }) + }) + + it('skips a breadcrumb whose timestamp cannot be parsed', () => { + const breadcrumbs = [ + heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN), + heartbeat('not-a-timestamp', CRASHED_ORIGIN) + ] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({ + rendererHeartbeatLastAt: '2026-08-20T11:01:20.000Z' + }) + }) + + it('discounts an OS sleep span instead of calling it a renderer wedge', () => { + const breadcrumbs = [ + heartbeat('2026-08-20T03:00:20.000Z', CRASHED_ORIGIN), + slept('2026-08-20T11:14:20.000Z', 29_580_000) + ] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toEqual({ + rendererHeartbeatStatus: 'observed', + rendererHeartbeatAttribution: 'crashed-renderer', + rendererHeartbeatLastAt: '2026-08-20T03:00:20.000Z', + rendererHeartbeatSilenceMs: 29_670_000, + rendererHeartbeatSuspendedMs: 29_580_000, + rendererHeartbeatAwakeSilenceMs: 90_000, + rendererHeartbeatIntervalMs: 60_000, + rendererHeartbeatMissedIntervals: 1 + }) + }) + + it('counts only the part of a sleep that overlaps the silence window', () => { + const breadcrumbs = [ + heartbeat('2026-08-20T11:04:50.000Z', CRASHED_ORIGIN), + // Slept 11:00:50 -> 11:10:50, so only the 6m after the heartbeat is explained. + slept('2026-08-20T11:10:50.000Z', 600_000) + ] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({ + rendererHeartbeatSilenceMs: 600_000, + rendererHeartbeatSuspendedMs: 360_000, + rendererHeartbeatAwakeSilenceMs: 240_000, + rendererHeartbeatMissedIntervals: 4 + }) + }) + + it('leaves silence undiscounted when the sleep ended before the last heartbeat', () => { + const breadcrumbs = [ + slept('2026-08-20T11:00:20.000Z', 600_000), + heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN) + ] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toEqual({ + rendererHeartbeatStatus: 'observed', + rendererHeartbeatAttribution: 'crashed-renderer', + rendererHeartbeatLastAt: '2026-08-20T11:01:20.000Z', + rendererHeartbeatSilenceMs: 810_000, + rendererHeartbeatIntervalMs: 60_000, + rendererHeartbeatMissedIntervals: 13 + }) + }) + + it('sums repeated sleeps inside one silence window', () => { + const breadcrumbs = [ + heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN), + slept('2026-08-20T11:05:20.000Z', 120_000), + slept('2026-08-20T11:12:20.000Z', 180_000) + ] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({ + rendererHeartbeatSilenceMs: 810_000, + rendererHeartbeatSuspendedMs: 300_000, + rendererHeartbeatAwakeSilenceMs: 510_000, + rendererHeartbeatMissedIntervals: 8 + }) + }) + + it('ignores a sleep crumb whose span is not a usable number', () => { + const breadcrumbs = [ + heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN), + slept('2026-08-20T11:12:20.000Z', 'a while') + ] + + expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({ + rendererHeartbeatSilenceMs: 810_000, + rendererHeartbeatMissedIntervals: 13 + }) + }) +}) diff --git a/src/main/crash-reporting/renderer-heartbeat-silence.ts b/src/main/crash-reporting/renderer-heartbeat-silence.ts new file mode 100644 index 00000000000..7b37d2bfdfd --- /dev/null +++ b/src/main/crash-reporting/renderer-heartbeat-silence.ts @@ -0,0 +1,135 @@ +import type { CrashReportBreadcrumb, CrashReportDetailValue } from '../../shared/crash-reporting' +import { + RENDERER_MEMORY_HEARTBEAT_BREADCRUMB, + RENDERER_MEMORY_HEARTBEAT_INTERVAL_MS +} from '../../shared/renderer-memory-heartbeat' +import { SYSTEM_SLEPT_BREADCRUMB } from '../../shared/system-sleep-breadcrumb' + +// ─── Renderer silence before a process-gone death ─────────────────── +// Why: the renderer emits a memory breadcrumb on a fixed interval, so the wall +// time between the last one and the report separates "a healthy app was force +// killed" from "the renderer had been wedged for 13 minutes" — the distinction +// the killed/1 cluster otherwise cannot make. +// +// Honesty limits, deliberately left to the reader instead of a verdict field: +// - Attribution is stamped, never assumed. `crashed-renderer` means the crumb +// carried the dead renderer's origin; `unattributed` means it could be any +// renderer (child-process deaths, or an event with no webContentsId). +// - Silence is not proof of a wedge. Chromium throttles background timers, so a +// hidden or occluded window stretches the cadence on its own. +// - OS sleep stops renderer timers outright, so recorded suspend spans are +// subtracted and reported separately; a sleep too short to record, a dark wake +// that never fires resume, or a resume crumb evicted from the ring still lands +// in the awake figure. +// - It is read off the report's own breadcrumb snapshot, so ring eviction shows +// up as `none` rather than as a silently wrong duration. + +type CrashReportDetails = Record + +type HeartbeatMatch = { + createdAt: string + atMs: number + attributed: boolean +} + +function heartbeatMatch( + breadcrumb: CrashReportBreadcrumb, + reporterOrigin: string | undefined +): HeartbeatMatch | null { + if (breadcrumb.name !== RENDERER_MEMORY_HEARTBEAT_BREADCRUMB) { + return null + } + const atMs = Date.parse(breadcrumb.createdAt) + if (!Number.isFinite(atMs)) { + return null + } + return { + createdAt: breadcrumb.createdAt, + atMs, + attributed: reporterOrigin !== undefined && breadcrumb.origin === reporterOrigin + } +} + +/** Newest heartbeat, preferring a provably-attributed one over a newer anonymous one. */ +function lastHeartbeat( + breadcrumbs: readonly CrashReportBreadcrumb[], + reporterOrigin: string | undefined +): HeartbeatMatch | null { + let best: HeartbeatMatch | null = null + for (const breadcrumb of breadcrumbs) { + const match = heartbeatMatch(breadcrumb, reporterOrigin) + if (!match) { + continue + } + if ( + !best || + (match.attributed === best.attributed ? match.atMs > best.atMs : match.attributed) + ) { + best = match + } + } + return best +} + +/** Sleep is stamped at resume, so the crumb's own time is the wake edge of the span. */ +function suspendedMsWithin( + breadcrumbs: readonly CrashReportBreadcrumb[], + fromMs: number, + toMs: number +): number { + let total = 0 + for (const breadcrumb of breadcrumbs) { + if (breadcrumb.name !== SYSTEM_SLEPT_BREADCRUMB) { + continue + } + const resumedAtMs = Date.parse(breadcrumb.createdAt) + const suspendedForMs = breadcrumb.data?.suspendedForMs + if ( + !Number.isFinite(resumedAtMs) || + typeof suspendedForMs !== 'number' || + !Number.isFinite(suspendedForMs) || + suspendedForMs <= 0 + ) { + continue + } + total += Math.max( + 0, + Math.min(toMs, resumedAtMs) - Math.max(fromMs, resumedAtMs - suspendedForMs) + ) + } + return total +} + +export function rendererHeartbeatSilenceDetails( + breadcrumbs: readonly CrashReportBreadcrumb[], + reporterOrigin: string | undefined, + nowMs: number +): CrashReportDetails { + const last = lastHeartbeat(breadcrumbs, reporterOrigin) + if (!last) { + // Explicit, so a reader can tell "no heartbeat in evidence" from a build + // that never stamped the field. + return { rendererHeartbeatStatus: 'none' } + } + const silenceMs = Math.max(0, nowMs - last.atMs) + // Clamped: overlapping suspend crumbs must never explain more than the gap itself. + const suspendedMs = Math.min(silenceMs, suspendedMsWithin(breadcrumbs, last.atMs, nowMs)) + const awakeSilenceMs = silenceMs - suspendedMs + return { + rendererHeartbeatStatus: 'observed', + rendererHeartbeatAttribution: last.attributed ? 'crashed-renderer' : 'unattributed', + rendererHeartbeatLastAt: last.createdAt, + rendererHeartbeatSilenceMs: silenceMs, + ...(suspendedMs > 0 + ? { + rendererHeartbeatSuspendedMs: suspendedMs, + rendererHeartbeatAwakeSilenceMs: awakeSilenceMs + } + : {}), + rendererHeartbeatIntervalMs: RENDERER_MEMORY_HEARTBEAT_INTERVAL_MS, + // Why: counted off the awake span, so an overnight sleep is not a 479-interval wedge. + rendererHeartbeatMissedIntervals: Math.floor( + awakeSilenceMs / RENDERER_MEMORY_HEARTBEAT_INTERVAL_MS + ) + } +} diff --git a/src/main/system-resume-broadcast.ts b/src/main/system-resume-broadcast.ts index 5ae10cdc884..ff7189cc158 100644 --- a/src/main/system-resume-broadcast.ts +++ b/src/main/system-resume-broadcast.ts @@ -1,5 +1,6 @@ import { BrowserWindow, powerMonitor } from 'electron' import { recordCrashBreadcrumb } from './crash-reporting/crash-breadcrumb-store' +import { SYSTEM_SLEPT_BREADCRUMB } from '../shared/system-sleep-breadcrumb' import { publishSystemResume, publishSystemSuspend } from './system-power-lifecycle' export const SYSTEM_RESUMED_CHANNEL = 'system:resumed' @@ -53,7 +54,7 @@ export function registerSystemResumeBroadcast( const suspendedForMs = suspendedAt === null ? null : Math.max(0, now() - suspendedAt) suspendedAt = null if (suspendedForMs !== null && suspendedForMs >= MIN_REPORTABLE_SUSPEND_MS) { - recordCrashBreadcrumb('system_slept', { suspendedForMs }) + recordCrashBreadcrumb(SYSTEM_SLEPT_BREADCRUMB, { suspendedForMs }) } publishSystemResume() for (const window of getWindows()) { diff --git a/src/renderer/src/lib/crash-diagnostics.ts b/src/renderer/src/lib/crash-diagnostics.ts index 25a6b75caae..dba8ff361b5 100644 --- a/src/renderer/src/lib/crash-diagnostics.ts +++ b/src/renderer/src/lib/crash-diagnostics.ts @@ -2,6 +2,10 @@ import type { CrashReportBreadcrumbData, CrashReportDetailValue } from '../../../shared/crash-reporting' +import { + RENDERER_MEMORY_HEARTBEAT_BREADCRUMB, + RENDERER_MEMORY_HEARTBEAT_INTERVAL_MS +} from '../../../shared/renderer-memory-heartbeat' import { getBrowserWebviewMemoryProfile, type BrowserWebviewMemoryProfile @@ -9,7 +13,6 @@ import { import { recordRendererCrashBreadcrumb } from './crash-breadcrumb-recorder' import { collectRendererMemoryProfileCounts } from './renderer-memory-profile' -const RENDERER_MEMORY_SAMPLE_INTERVAL_MS = 60_000 const BYTES_PER_MEGABYTE = 1024 * 1024 const BYTES_PER_KILOBYTE = 1024 // Why: one detailed breadcrumb per threshold names what grew before an OOM. @@ -54,7 +57,7 @@ export function installRendererCrashDiagnostics(surface: RendererSurface = 'main recordRendererMemory('startup') rendererMemoryInterval = window.setInterval( () => recordRendererMemory('interval'), - RENDERER_MEMORY_SAMPLE_INTERVAL_MS + RENDERER_MEMORY_HEARTBEAT_INTERVAL_MS ) } } @@ -119,7 +122,7 @@ function recordRendererMemory(reason: string): void { const browserWebviews = getBrowserWebviewMemoryProfile() recordRendererCrashBreadcrumb( - 'renderer_memory', + RENDERER_MEMORY_HEARTBEAT_BREADCRUMB, compactBreadcrumbData({ reason, usedHeapMB: toMegabytes(memory.usedJSHeapSize), diff --git a/src/shared/renderer-memory-heartbeat.ts b/src/shared/renderer-memory-heartbeat.ts new file mode 100644 index 00000000000..f7b8105064a --- /dev/null +++ b/src/shared/renderer-memory-heartbeat.ts @@ -0,0 +1,8 @@ +/** + * The renderer's periodic memory sample doubles as a liveness heartbeat: the + * gap between the last one and a crash report measures how long the renderer + * had been silent before it died. Shared so main reads the cadence it is + * measuring against instead of hardcoding a copy of the renderer's interval. + */ +export const RENDERER_MEMORY_HEARTBEAT_BREADCRUMB = 'renderer_memory' +export const RENDERER_MEMORY_HEARTBEAT_INTERVAL_MS = 60_000 diff --git a/src/shared/system-sleep-breadcrumb.ts b/src/shared/system-sleep-breadcrumb.ts new file mode 100644 index 00000000000..5ff79058111 --- /dev/null +++ b/src/shared/system-sleep-breadcrumb.ts @@ -0,0 +1,5 @@ +/** + * Stamped on resume with the span the machine was suspended for. Shared so + * silence math can subtract OS sleep instead of reading it as a wedge. + */ +export const SYSTEM_SLEPT_BREADCRUMB = 'system_slept'