fix(crash-reporting): stamp renderer heartbeat silence and a terminal minidump status on process-gone reports

A triager looking at the `killed`/exit-1 process-gone cluster cannot tell the
two populations apart: a healthy app the OS or the user force-quit produces the
same report as a renderer that had been wedged for thirteen minutes before it
died. Every field on the report describes the death, none of them describe how
long the renderer had been alive-but-silent leading up to it.

The renderer already emits a memory breadcrumb on a fixed 60s cadence, so the
gap between the last one and the report is exactly that missing signal. Fold it
into the report as `rendererHeartbeatSilenceMs` plus the missed-interval count,
read off the report's own breadcrumb snapshot so ring eviction shows up as
`rendererHeartbeatStatus: 'none'` rather than a silently wrong duration.

Two ways the raw gap would lie to a triager, both handled:
- OS sleep stops renderer timers, so an overnight suspend would read as a
  479-interval wedge. Recorded `system_slept` spans are subtracted and reported
  separately, and the missed-interval count is taken off the awake span.
- The breadcrumb ring is shared across renderers, so a sibling's heartbeat would
  be credited to the dead one. Attribution is stamped (`crashed-renderer` vs
  `unattributed`), never assumed, and child-process deaths get no heartbeat
  fields at all because they have no origin to scope the ring by.

Also closes a related read-the-report gap: `minidumpStatus` was written only
once the Crashpad wait resolved, so a report where the main process exited first
carried no status field at all -- indistinguishable from an older build. Reports
now record `pending` up front and are amended to `captured`, `absent`, or
`attach-failed`, and the absent case gets its own span so the outcome is
countable in the diagnostics bundle. The pending value is stripped from the
process-gone span, which serializes before the wait starts and would otherwise
show `pending` on every crash.
This commit is contained in:
Neil
2026-08-25 02:13:06 -07:00
parent e7c5263a93
commit 67be2d036c
9 changed files with 777 additions and 78 deletions
@@ -0,0 +1,288 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
vi.mock('electron', () => ({
app: {
getVersion: () => '1.2.3-test',
getAppMetrics: (): unknown[] => []
}
}))
import { rendererCrashBreadcrumbOrigin } from '../../shared/crash-breadcrumb-origin'
import { clearCrashBreadcrumbsForTest, recordCrashBreadcrumb } from './crash-breadcrumb-store'
import { ProcessGoneDedupe } from './process-gone-dedupe'
import { recordProcessGoneCrash, type ProcessGoneCrashEvent } from './process-gone-recorder'
import { _resetTracerForTests, setActiveSink, type TracerSink } from '../observability/tracer'
const CRASHED_WEB_CONTENTS_ID = 11
const SIBLING_WEB_CONTENTS_ID = 22
const noMinidump = async () => null
function killedEvent(overrides: Partial<ProcessGoneCrashEvent> = {}): ProcessGoneCrashEvent {
return {
source: 'renderer',
processType: 'renderer',
reason: 'killed',
exitCode: 1,
expectedTeardown: 'none',
details: { processType: 'renderer' },
webContentsId: CRASHED_WEB_CONTENTS_ID,
...overrides
}
}
function recordedDetails(record: ReturnType<typeof vi.fn>): Record<string, unknown> {
const input = record.mock.calls[0]?.[0] as { details?: Record<string, unknown> } | undefined
return input?.details ?? {}
}
type SpanRecord = { name?: string; attributes?: Record<string, unknown> }
let spans: SpanRecord[]
function spanNamed(name: string): SpanRecord | undefined {
return spans.find((span) => span.name === name)
}
beforeEach(() => {
spans = []
const sink: TracerSink = {
push: (record) => spans.push(record as SpanRecord),
flush: vi.fn(),
close: vi.fn()
}
setActiveSink(sink)
clearCrashBreadcrumbsForTest()
})
afterEach(() => {
vi.useRealTimers()
vi.restoreAllMocks()
_resetTracerForTests()
clearCrashBreadcrumbsForTest()
})
describe('renderer silence on process-gone reports', () => {
it('stamps how long the crashing renderer had been silent before a killed/1 death', () => {
vi.useFakeTimers()
vi.setSystemTime(Date.parse('2026-08-20T11:01:20.000Z'))
recordCrashBreadcrumb(
'renderer_memory',
{ reason: 'interval', usedHeapMB: 120 },
rendererCrashBreadcrumbOrigin(CRASHED_WEB_CONTENTS_ID)
)
vi.setSystemTime(Date.parse('2026-08-20T11:14:50.000Z'))
const record = vi.fn().mockResolvedValue({ id: 'report-1' })
recordProcessGoneCrash(
{ record, attachDetails: async () => null } as never,
killedEvent(),
new ProcessGoneDedupe(),
noMinidump
)
expect(recordedDetails(record)).toMatchObject({
rendererHeartbeatStatus: 'observed',
rendererHeartbeatAttribution: 'crashed-renderer',
rendererHeartbeatSilenceMs: 810_000,
rendererHeartbeatMissedIntervals: 13
})
})
it('does not credit the crashing renderer with a sibling renderer heartbeat', () => {
vi.useFakeTimers()
vi.setSystemTime(Date.parse('2026-08-20T11:01:20.000Z'))
recordCrashBreadcrumb(
'renderer_memory',
{ reason: 'interval' },
rendererCrashBreadcrumbOrigin(CRASHED_WEB_CONTENTS_ID)
)
vi.setSystemTime(Date.parse('2026-08-20T11:14:20.000Z'))
recordCrashBreadcrumb(
'renderer_memory',
{ reason: 'interval' },
rendererCrashBreadcrumbOrigin(SIBLING_WEB_CONTENTS_ID)
)
vi.setSystemTime(Date.parse('2026-08-20T11:14:50.000Z'))
const record = vi.fn().mockResolvedValue({ id: 'report-1' })
recordProcessGoneCrash(
{ record, attachDetails: async () => null } as never,
killedEvent(),
new ProcessGoneDedupe(),
noMinidump
)
expect(recordedDetails(record)).toMatchObject({
rendererHeartbeatSilenceMs: 810_000,
rendererHeartbeatAttribution: 'crashed-renderer'
})
})
it('does not read an OS sleep as a wedged renderer', () => {
vi.useFakeTimers()
vi.setSystemTime(Date.parse('2026-08-20T03:00:20.000Z'))
recordCrashBreadcrumb(
'renderer_memory',
{ reason: 'interval', usedHeapMB: 120 },
rendererCrashBreadcrumbOrigin(CRASHED_WEB_CONTENTS_ID)
)
vi.setSystemTime(Date.parse('2026-08-20T11:14:20.000Z'))
recordCrashBreadcrumb('system_slept', { suspendedForMs: 29_580_000 })
vi.setSystemTime(Date.parse('2026-08-20T11:14:50.000Z'))
const record = vi.fn().mockResolvedValue({ id: 'report-1' })
recordProcessGoneCrash(
{ record, attachDetails: async () => null } as never,
killedEvent(),
new ProcessGoneDedupe(),
noMinidump
)
expect(recordedDetails(record)).toMatchObject({
rendererHeartbeatSilenceMs: 29_670_000,
rendererHeartbeatSuspendedMs: 29_580_000,
rendererHeartbeatAwakeSilenceMs: 90_000,
rendererHeartbeatMissedIntervals: 1
})
})
it('records that no heartbeat was seen rather than omitting the field', () => {
const record = vi.fn().mockResolvedValue({ id: 'report-1' })
recordProcessGoneCrash(
{ record, attachDetails: async () => null } as never,
killedEvent(),
new ProcessGoneDedupe(),
noMinidump
)
expect(recordedDetails(record)).toMatchObject({
rendererHeartbeatStatus: 'none'
})
})
it('does not stamp a renderer heartbeat verdict on a child-process death', () => {
vi.useFakeTimers()
vi.setSystemTime(Date.parse('2026-08-20T11:00:20.000Z'))
recordCrashBreadcrumb(
'renderer_memory',
{ reason: 'interval', usedHeapMB: 120 },
rendererCrashBreadcrumbOrigin(CRASHED_WEB_CONTENTS_ID)
)
vi.setSystemTime(Date.parse('2026-08-20T11:14:50.000Z'))
const record = vi.fn().mockResolvedValue({ id: 'report-1' })
recordProcessGoneCrash(
{ record, attachDetails: async () => null } as never,
killedEvent({
source: 'child',
processType: 'utility',
reason: 'crashed',
exitCode: 133,
details: { processType: 'utility', serviceName: 'storage.mojom.StorageService' },
webContentsId: undefined
}),
new ProcessGoneDedupe(),
noMinidump
)
const details = recordedDetails(record)
expect(details).not.toHaveProperty('rendererHeartbeatStatus')
expect(details).not.toHaveProperty('rendererHeartbeatSilenceMs')
expect(details).not.toHaveProperty('rendererHeartbeatMissedIntervals')
})
})
describe('minidump status on process-gone reports', () => {
// Why: the dump wait runs up to 8s after the record is written, so a main
// process that exits with the renderer (the killed/1 shape) never attaches.
it('stamps a pending minidump status at record time', () => {
const record = vi.fn().mockResolvedValue({ id: 'report-1' })
recordProcessGoneCrash(
{ record, attachDetails: async () => null } as never,
killedEvent(),
new ProcessGoneDedupe(),
() => new Promise(() => {})
)
expect(recordedDetails(record)).toMatchObject({
minidumpStatus: 'pending'
})
})
it('replaces the pending status once the dump wait resolves', async () => {
const record = vi.fn().mockResolvedValue({ id: 'report-1' })
const attach = vi.fn().mockResolvedValue(null)
recordProcessGoneCrash(
{ record, attachDetails: attach } as never,
killedEvent(),
new ProcessGoneDedupe(),
noMinidump
)
await vi.waitFor(() =>
expect(attach).toHaveBeenCalledWith('report-1', {
minidumpStatus: 'absent'
})
)
})
it('replaces the pending status when the terminal attach write fails', async () => {
vi.spyOn(console, 'error').mockImplementation(() => {})
const record = vi.fn().mockResolvedValue({ id: 'report-1' })
const attach = vi.fn().mockRejectedValueOnce(new Error('EPERM')).mockResolvedValue(null)
recordProcessGoneCrash(
{ record, attachDetails: attach } as never,
killedEvent(),
new ProcessGoneDedupe(),
noMinidump
)
await vi.waitFor(() =>
expect(attach).toHaveBeenCalledWith('report-1', {
minidumpStatus: 'attach-failed'
})
)
})
})
describe('minidump status on the process-gone span', () => {
it('does not stamp a dump status the span is serialized too early to know', () => {
const record = vi.fn().mockResolvedValue({ id: 'report-1' })
recordProcessGoneCrash(
{ record, attachDetails: async () => null } as never,
killedEvent(),
new ProcessGoneDedupe(),
noMinidump
)
const details = spanNamed('electron.process_gone')?.attributes?.details as Record<
string,
unknown
>
expect(details).toBeDefined()
expect(details).not.toHaveProperty('minidumpStatus')
})
it('emits the terminal dump status on its own span once the wait resolves', async () => {
const record = vi.fn().mockResolvedValue({ id: 'report-1' })
recordProcessGoneCrash(
{ record, attachDetails: async () => null } as never,
killedEvent(),
new ProcessGoneDedupe(),
noMinidump
)
await vi.waitFor(() =>
expect(spanNamed('electron.minidump_signature')?.attributes).toMatchObject({
'crash.report_id': 'report-1',
'crash.minidump_status': 'absent'
})
)
})
})
@@ -0,0 +1,98 @@
import { sanitizeCrashReportDetails } from '../../shared/crash-reporting'
import type { CrashReportStore } from './crash-report-store'
import { captureMinidumpSignature, type CapturedMinidump } from './crashpad-capture'
import { minidumpSignatureDetails } from './minidump-crash-signature'
import type { ProcessGoneSource } from './process-gone-classification'
import { flushActiveSink, startSpan } from '../observability/tracer'
type MinidumpDetailStore = Pick<CrashReportStore, 'attachDetails'>
/** Injectable so tests can drive the pairing without a Crashpad handler. */
export type MinidumpCapture = (
crashedAtMs: number,
expectedProcessType: string
) => Promise<CapturedMinidump | null>
const CHILD_CRASHPAD_PROCESS_TYPES: Readonly<Record<string, string>> = {
gpu: 'gpu-process',
utility: 'utility',
zygote: 'zygote'
}
export function expectedCrashpadProcessType(
source: ProcessGoneSource,
processType: string
): string | null {
return source === 'renderer'
? 'renderer'
: (CHILD_CRASHPAD_PROCESS_TYPES[processType.trim().toLowerCase()] ?? null)
}
export const captureProcessMinidump: MinidumpCapture = (crashedAtMs, expectedProcessType) =>
captureMinidumpSignature(crashedAtMs, { expectedProcessType })
// Why: the crash-report record is capped at 5 entries and is user-facing; the span
// is what makes the outcome countable in the diagnostics bundle. Emitted for the
// absent case too, so a trace can tell "no dump was written" from a process-gone
// span with no status at all -- the main process exited first (the killed/1 shape).
function recordMinidumpStatusSpan(
reportId: string,
status: 'captured' | 'absent',
attributes: Record<string, unknown> = {}
): void {
const span = startSpan('electron.minidump_signature', {
attributes: { 'crash.report_id': reportId, 'crash.minidump_status': status, ...attributes }
})
span.end()
flushActiveSink()
}
/**
* Best effort, so a store that is failing writes cannot leave `pending` on disk —
* which reads as "the dump wait never ran" rather than "the attach write failed".
*/
export async function markMinidumpAttachFailed(
store: MinidumpDetailStore,
reportId: string
): Promise<void> {
try {
await store.attachDetails(reportId, { minidumpStatus: 'attach-failed' })
} catch {
// The caller already logged and breadcrumbed the original failure.
}
}
/**
* Folds the Crashpad signature into a report that is already on disk.
*
* Why separate from the record write: an exit code of 0x80000003 only says "a
* CHECK fired"; the name, file and line live in the dump, which Crashpad is
* still writing when process-gone fires. Waiting inline would stall recovery.
* The record carries `minidumpStatus: 'pending'` until this lands, so a wait that
* never completes leaves that state visible instead of no field.
*/
export async function attachMinidumpSignature(
store: MinidumpDetailStore,
reportId: string,
crashedAtMs: number,
expectedProcessType: string | null,
capture: MinidumpCapture
): Promise<void> {
const captured = expectedProcessType ? await capture(crashedAtMs, expectedProcessType) : null
if (!captured) {
await store.attachDetails(reportId, { minidumpStatus: 'absent' })
recordMinidumpStatusSpan(reportId, 'absent')
return
}
const signatureDetails = sanitizeCrashReportDetails(minidumpSignatureDetails(captured.signature))
await store.attachDetails(reportId, {
...signatureDetails,
minidumpStatus: 'captured',
minidumpPath: captured.filePath,
minidumpBytes: captured.sizeBytes
})
recordMinidumpStatusSpan(reportId, 'captured', {
'crash.minidump_bytes': captured.sizeBytes,
...signatureDetails
})
}
@@ -2,7 +2,6 @@ import os from 'node:os'
import { app } from 'electron'
import {
isCrashReportReason,
sanitizeCrashReportDetails,
sanitizeCrashReportString,
type CrashReportBreadcrumbData
} from '../../shared/crash-reporting'
@@ -24,6 +23,7 @@ import {
type ProcessGoneSource
} from './process-gone-classification'
import { buildProcessGoneCrashDetails } from './process-gone-diagnostics'
import { rendererHeartbeatSilenceDetails } from './renderer-heartbeat-silence'
import { buildSuppressedProcessGoneBreadcrumbData } from './suppressed-process-gone-breadcrumb'
import {
getProcessGoneDedupeKey,
@@ -35,12 +35,14 @@ import {
siblingProcessDeathDetails
} from './process-gone-sibling-correlation'
import { getMainProcessLifecycleIdentity } from './main-process-lifecycle-identity'
import { scheduleCrashpadDumpPrune } from './crashpad-capture'
import {
captureMinidumpSignature,
scheduleCrashpadDumpPrune,
type CapturedMinidump
} from './crashpad-capture'
import { minidumpSignatureDetails } from './minidump-crash-signature'
attachMinidumpSignature,
captureProcessMinidump,
expectedCrashpadProcessType,
markMinidumpAttachFailed,
type MinidumpCapture
} from './process-gone-minidump-attachment'
import { flushActiveSink, startSpan } from '../observability/tracer'
export type ProcessGoneCrashEvent = {
@@ -55,27 +57,6 @@ export type ProcessGoneCrashEvent = {
type CrashReportRecorderStore = Pick<CrashReportStore, 'record' | 'attachDetails'>
/** Injectable so tests can drive the pairing without a Crashpad handler. */
export type MinidumpCapture = (
crashedAtMs: number,
expectedProcessType: string
) => Promise<CapturedMinidump | null>
const CHILD_CRASHPAD_PROCESS_TYPES: Readonly<Record<string, string>> = {
gpu: 'gpu-process',
utility: 'utility',
zygote: 'zygote'
}
function expectedCrashpadProcessType(event: ProcessGoneCrashEvent): string | null {
return event.source === 'renderer'
? 'renderer'
: (CHILD_CRASHPAD_PROCESS_TYPES[event.processType.trim().toLowerCase()] ?? null)
}
const captureProcessMinidump: MinidumpCapture = (crashedAtMs, expectedProcessType) =>
captureMinidumpSignature(crashedAtMs, { expectedProcessType })
// Why: the coalesce map prunes every key against the calling window, so a shorter
// one here would weaken the other 30s coalescers. Stay uniform with them.
const SUPPRESSED_PROCESS_GONE_COALESCE_MS = 30_000
@@ -129,45 +110,6 @@ function persistFailureData(event: ProcessGoneCrashEvent, error: unknown) {
}
}
/**
* Folds the Crashpad signature into a report that is already on disk.
*
* Why separate from the record write: an exit code of 0x80000003 only says "a
* CHECK fired"; the name, file and line live in the dump, which Crashpad is
* still writing when process-gone fires. Waiting inline would stall recovery.
*/
async function attachMinidumpSignature(
store: CrashReportRecorderStore,
reportId: string,
crashedAtMs: number,
expectedProcessType: string | null,
capture: MinidumpCapture
): Promise<void> {
const captured = expectedProcessType ? await capture(crashedAtMs, expectedProcessType) : null
if (!captured) {
await store.attachDetails(reportId, { minidumpStatus: 'absent' })
return
}
const signatureDetails = sanitizeCrashReportDetails(minidumpSignatureDetails(captured.signature))
await store.attachDetails(reportId, {
...signatureDetails,
minidumpStatus: 'captured',
minidumpPath: captured.filePath,
minidumpBytes: captured.sizeBytes
})
// Why: the crash-report record is capped at 5 entries and is user-facing;
// the span is what makes the signature countable in the diagnostics bundle.
const span = startSpan('electron.minidump_signature', {
attributes: {
'crash.report_id': reportId,
'crash.minidump_bytes': captured.sizeBytes,
...signatureDetails
}
})
span.end()
flushActiveSink()
}
export function recordProcessGoneCrash(
store: CrashReportRecorderStore | null,
event: ProcessGoneCrashEvent,
@@ -246,16 +188,29 @@ export function recordProcessGoneCrash(
: []
const siblingDetails =
siblingDeaths.length > 0 ? siblingProcessDeathDetails(siblingDeaths, goneAt) : {}
const reporterOrigin = processGoneRendererOrigin(event)
const breadcrumbs = getCrashBreadcrumbSnapshot(reporterOrigin)
const reportBreadcrumbs = breadcrumbs?.map(({ origin: _origin, ...breadcrumb }) => breadcrumb)
const crashDetails = buildProcessGoneCrashDetails(
{
...event.details,
...mainProcessLifecycle,
...siblingDetails
...siblingDetails,
// Why: only a renderer emits the heartbeat, and a child death has no origin to
// scope the ring by, so it would be stamped with an unrelated renderer's crumb.
...(event.source === 'renderer'
? rendererHeartbeatSilenceDetails(breadcrumbs, reporterOrigin, goneAt)
: {}),
// Overwritten by attachMinidumpSignature; surviving as `pending` means the dump
// wait never completed -- most often the main process exited first (killed/1).
minidumpStatus: 'pending'
},
event.processType
)
const breadcrumbs = getCrashBreadcrumbSnapshot(processGoneRendererOrigin(event))
const reportBreadcrumbs = breadcrumbs?.map(({ origin: _origin, ...breadcrumb }) => breadcrumb)
// Why: this span is serialized at fail(), before the dump wait even starts, so a
// `pending` on it would read as terminal on every crash. The terminal value goes
// out on the minidump span; only the on-disk record carries the pending state.
const { minidumpStatus: _pendingMinidumpStatus, ...spanDetails } = crashDetails
const span = startSpan('electron.process_gone', {
attributes: {
'crash.source': event.source,
@@ -272,7 +227,7 @@ export function recordProcessGoneCrash(
'app.main_process.pid': mainProcessLifecycle.mainProcessPid,
'app.main_process.launch_id': mainProcessLifecycle.mainProcessLaunchId,
'app.main_process.started_at': mainProcessLifecycle.mainProcessStartedAt,
details: crashDetails,
details: spanDetails,
breadcrumbs: reportBreadcrumbs
}
})
@@ -284,7 +239,7 @@ export function recordProcessGoneCrash(
flushActiveSink()
const crashedAtMs = Date.now()
const expectedProcessType = expectedCrashpadProcessType(event)
const expectedProcessType = expectedCrashpadProcessType(event.source, event.processType)
const recorded = store.record({
source: event.source,
processType: event.processType,
@@ -306,7 +261,7 @@ export function recordProcessGoneCrash(
(reportId, details) => store.attachDetails(reportId, details),
recorded,
processGoneBreadcrumbData(event),
processGoneRendererOrigin(event)
reporterOrigin
)
void recorded
.then((report) => {
@@ -320,11 +275,14 @@ export function recordProcessGoneCrash(
capture
).catch((error) => {
console.error('[crash-reporting] Failed to attach minidump signature:', error)
// Why: without a terminal status the report keeps `pending`, which reads as
// evidence the main process died first even when the dump was captured.
void markMinidumpAttachFailed(store, report.id)
recordDurableCrashBreadcrumb(
'minidump_signature_attach_failed',
processGoneBreadcrumbData(event),
error instanceof Error ? error.message : String(error),
processGoneRendererOrigin(event)
reporterOrigin
)
})
})
@@ -336,7 +294,7 @@ export function recordProcessGoneCrash(
'crash_report_persist_failed',
data,
`${String(data.errorName)}: ${String(data.errorMessage)}`,
processGoneRendererOrigin(event)
reporterOrigin
)
})
}
@@ -0,0 +1,203 @@
import { describe, expect, it } from 'vitest'
import type { CrashReportBreadcrumb } from '../../shared/crash-reporting'
import { rendererHeartbeatSilenceDetails } from './renderer-heartbeat-silence'
const CRASHED_ORIGIN = 'renderer:11'
const GONE_AT = Date.parse('2026-08-20T11:14:50.000Z')
function heartbeat(createdAt: string, origin?: string): CrashReportBreadcrumb {
return {
createdAt,
name: 'renderer_memory',
data: { reason: 'interval', usedHeapMB: 120 },
...(origin ? { origin } : {})
}
}
/** Stamped at resume, so `createdAt` is the wake instant and the span runs backwards from it. */
function slept(resumedAt: string, suspendedForMs: unknown): CrashReportBreadcrumb {
return {
createdAt: resumedAt,
name: 'system_slept',
data: { suspendedForMs } as CrashReportBreadcrumb['data']
}
}
describe('rendererHeartbeatSilenceDetails', () => {
it('measures how long the crashing renderer had been silent before it died', () => {
const breadcrumbs = [
heartbeat('2026-08-20T11:00:20.000Z', CRASHED_ORIGIN),
heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN)
]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toEqual({
rendererHeartbeatStatus: 'observed',
rendererHeartbeatAttribution: 'crashed-renderer',
rendererHeartbeatLastAt: '2026-08-20T11:01:20.000Z',
rendererHeartbeatSilenceMs: 810_000,
rendererHeartbeatIntervalMs: 60_000,
rendererHeartbeatMissedIntervals: 13
})
})
it('reports no missed intervals for a renderer that was beating up to the crash', () => {
const breadcrumbs = [heartbeat('2026-08-20T11:14:20.000Z', CRASHED_ORIGIN)]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({
rendererHeartbeatSilenceMs: 30_000,
rendererHeartbeatMissedIntervals: 0
})
})
it('marks the value unattributed when the heartbeat names no emitting renderer', () => {
const breadcrumbs = [heartbeat('2026-08-20T11:01:20.000Z')]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({
rendererHeartbeatAttribution: 'unattributed',
rendererHeartbeatSilenceMs: 810_000
})
})
it('has no attributable heartbeat for a child-process death', () => {
const breadcrumbs = [heartbeat('2026-08-20T11:14:20.000Z', CRASHED_ORIGIN)]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, undefined, GONE_AT)).toMatchObject({
rendererHeartbeatAttribution: 'unattributed'
})
})
it('prefers the crashed renderer own heartbeat over a newer unattributed one', () => {
const breadcrumbs = [
heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN),
heartbeat('2026-08-20T11:14:20.000Z')
]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({
rendererHeartbeatAttribution: 'crashed-renderer',
rendererHeartbeatLastAt: '2026-08-20T11:01:20.000Z',
rendererHeartbeatSilenceMs: 810_000
})
})
it('records the absence explicitly instead of omitting the field', () => {
expect(
rendererHeartbeatSilenceDetails(
[{ createdAt: '2026-08-20T11:01:20.000Z', name: 'app_started' }],
CRASHED_ORIGIN,
GONE_AT
)
).toEqual({ rendererHeartbeatStatus: 'none' })
})
it('ignores the one-shot highwater profile, which is not on the sampling cadence', () => {
const breadcrumbs: CrashReportBreadcrumb[] = [
heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN),
{
createdAt: '2026-08-20T11:14:40.000Z',
name: 'renderer_memory_highwater',
data: { thresholdPct: 80 },
origin: CRASHED_ORIGIN
}
]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({
rendererHeartbeatLastAt: '2026-08-20T11:01:20.000Z'
})
})
it('clamps a backwards clock correction instead of reporting negative silence', () => {
const breadcrumbs = [heartbeat('2026-08-20T11:20:00.000Z', CRASHED_ORIGIN)]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({
rendererHeartbeatSilenceMs: 0,
rendererHeartbeatMissedIntervals: 0
})
})
it('skips a breadcrumb whose timestamp cannot be parsed', () => {
const breadcrumbs = [
heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN),
heartbeat('not-a-timestamp', CRASHED_ORIGIN)
]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({
rendererHeartbeatLastAt: '2026-08-20T11:01:20.000Z'
})
})
it('discounts an OS sleep span instead of calling it a renderer wedge', () => {
const breadcrumbs = [
heartbeat('2026-08-20T03:00:20.000Z', CRASHED_ORIGIN),
slept('2026-08-20T11:14:20.000Z', 29_580_000)
]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toEqual({
rendererHeartbeatStatus: 'observed',
rendererHeartbeatAttribution: 'crashed-renderer',
rendererHeartbeatLastAt: '2026-08-20T03:00:20.000Z',
rendererHeartbeatSilenceMs: 29_670_000,
rendererHeartbeatSuspendedMs: 29_580_000,
rendererHeartbeatAwakeSilenceMs: 90_000,
rendererHeartbeatIntervalMs: 60_000,
rendererHeartbeatMissedIntervals: 1
})
})
it('counts only the part of a sleep that overlaps the silence window', () => {
const breadcrumbs = [
heartbeat('2026-08-20T11:04:50.000Z', CRASHED_ORIGIN),
// Slept 11:00:50 -> 11:10:50, so only the 6m after the heartbeat is explained.
slept('2026-08-20T11:10:50.000Z', 600_000)
]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({
rendererHeartbeatSilenceMs: 600_000,
rendererHeartbeatSuspendedMs: 360_000,
rendererHeartbeatAwakeSilenceMs: 240_000,
rendererHeartbeatMissedIntervals: 4
})
})
it('leaves silence undiscounted when the sleep ended before the last heartbeat', () => {
const breadcrumbs = [
slept('2026-08-20T11:00:20.000Z', 600_000),
heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN)
]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toEqual({
rendererHeartbeatStatus: 'observed',
rendererHeartbeatAttribution: 'crashed-renderer',
rendererHeartbeatLastAt: '2026-08-20T11:01:20.000Z',
rendererHeartbeatSilenceMs: 810_000,
rendererHeartbeatIntervalMs: 60_000,
rendererHeartbeatMissedIntervals: 13
})
})
it('sums repeated sleeps inside one silence window', () => {
const breadcrumbs = [
heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN),
slept('2026-08-20T11:05:20.000Z', 120_000),
slept('2026-08-20T11:12:20.000Z', 180_000)
]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({
rendererHeartbeatSilenceMs: 810_000,
rendererHeartbeatSuspendedMs: 300_000,
rendererHeartbeatAwakeSilenceMs: 510_000,
rendererHeartbeatMissedIntervals: 8
})
})
it('ignores a sleep crumb whose span is not a usable number', () => {
const breadcrumbs = [
heartbeat('2026-08-20T11:01:20.000Z', CRASHED_ORIGIN),
slept('2026-08-20T11:12:20.000Z', 'a while')
]
expect(rendererHeartbeatSilenceDetails(breadcrumbs, CRASHED_ORIGIN, GONE_AT)).toMatchObject({
rendererHeartbeatSilenceMs: 810_000,
rendererHeartbeatMissedIntervals: 13
})
})
})
@@ -0,0 +1,135 @@
import type { CrashReportBreadcrumb, CrashReportDetailValue } from '../../shared/crash-reporting'
import {
RENDERER_MEMORY_HEARTBEAT_BREADCRUMB,
RENDERER_MEMORY_HEARTBEAT_INTERVAL_MS
} from '../../shared/renderer-memory-heartbeat'
import { SYSTEM_SLEPT_BREADCRUMB } from '../../shared/system-sleep-breadcrumb'
// ─── Renderer silence before a process-gone death ───────────────────
// Why: the renderer emits a memory breadcrumb on a fixed interval, so the wall
// time between the last one and the report separates "a healthy app was force
// killed" from "the renderer had been wedged for 13 minutes" — the distinction
// the killed/1 cluster otherwise cannot make.
//
// Honesty limits, deliberately left to the reader instead of a verdict field:
// - Attribution is stamped, never assumed. `crashed-renderer` means the crumb
// carried the dead renderer's origin; `unattributed` means it could be any
// renderer (child-process deaths, or an event with no webContentsId).
// - Silence is not proof of a wedge. Chromium throttles background timers, so a
// hidden or occluded window stretches the cadence on its own.
// - OS sleep stops renderer timers outright, so recorded suspend spans are
// subtracted and reported separately; a sleep too short to record, a dark wake
// that never fires resume, or a resume crumb evicted from the ring still lands
// in the awake figure.
// - It is read off the report's own breadcrumb snapshot, so ring eviction shows
// up as `none` rather than as a silently wrong duration.
type CrashReportDetails = Record<string, CrashReportDetailValue>
type HeartbeatMatch = {
createdAt: string
atMs: number
attributed: boolean
}
function heartbeatMatch(
breadcrumb: CrashReportBreadcrumb,
reporterOrigin: string | undefined
): HeartbeatMatch | null {
if (breadcrumb.name !== RENDERER_MEMORY_HEARTBEAT_BREADCRUMB) {
return null
}
const atMs = Date.parse(breadcrumb.createdAt)
if (!Number.isFinite(atMs)) {
return null
}
return {
createdAt: breadcrumb.createdAt,
atMs,
attributed: reporterOrigin !== undefined && breadcrumb.origin === reporterOrigin
}
}
/** Newest heartbeat, preferring a provably-attributed one over a newer anonymous one. */
function lastHeartbeat(
breadcrumbs: readonly CrashReportBreadcrumb[],
reporterOrigin: string | undefined
): HeartbeatMatch | null {
let best: HeartbeatMatch | null = null
for (const breadcrumb of breadcrumbs) {
const match = heartbeatMatch(breadcrumb, reporterOrigin)
if (!match) {
continue
}
if (
!best ||
(match.attributed === best.attributed ? match.atMs > best.atMs : match.attributed)
) {
best = match
}
}
return best
}
/** Sleep is stamped at resume, so the crumb's own time is the wake edge of the span. */
function suspendedMsWithin(
breadcrumbs: readonly CrashReportBreadcrumb[],
fromMs: number,
toMs: number
): number {
let total = 0
for (const breadcrumb of breadcrumbs) {
if (breadcrumb.name !== SYSTEM_SLEPT_BREADCRUMB) {
continue
}
const resumedAtMs = Date.parse(breadcrumb.createdAt)
const suspendedForMs = breadcrumb.data?.suspendedForMs
if (
!Number.isFinite(resumedAtMs) ||
typeof suspendedForMs !== 'number' ||
!Number.isFinite(suspendedForMs) ||
suspendedForMs <= 0
) {
continue
}
total += Math.max(
0,
Math.min(toMs, resumedAtMs) - Math.max(fromMs, resumedAtMs - suspendedForMs)
)
}
return total
}
export function rendererHeartbeatSilenceDetails(
breadcrumbs: readonly CrashReportBreadcrumb[],
reporterOrigin: string | undefined,
nowMs: number
): CrashReportDetails {
const last = lastHeartbeat(breadcrumbs, reporterOrigin)
if (!last) {
// Explicit, so a reader can tell "no heartbeat in evidence" from a build
// that never stamped the field.
return { rendererHeartbeatStatus: 'none' }
}
const silenceMs = Math.max(0, nowMs - last.atMs)
// Clamped: overlapping suspend crumbs must never explain more than the gap itself.
const suspendedMs = Math.min(silenceMs, suspendedMsWithin(breadcrumbs, last.atMs, nowMs))
const awakeSilenceMs = silenceMs - suspendedMs
return {
rendererHeartbeatStatus: 'observed',
rendererHeartbeatAttribution: last.attributed ? 'crashed-renderer' : 'unattributed',
rendererHeartbeatLastAt: last.createdAt,
rendererHeartbeatSilenceMs: silenceMs,
...(suspendedMs > 0
? {
rendererHeartbeatSuspendedMs: suspendedMs,
rendererHeartbeatAwakeSilenceMs: awakeSilenceMs
}
: {}),
rendererHeartbeatIntervalMs: RENDERER_MEMORY_HEARTBEAT_INTERVAL_MS,
// Why: counted off the awake span, so an overnight sleep is not a 479-interval wedge.
rendererHeartbeatMissedIntervals: Math.floor(
awakeSilenceMs / RENDERER_MEMORY_HEARTBEAT_INTERVAL_MS
)
}
}
+2 -1
View File
@@ -1,5 +1,6 @@
import { BrowserWindow, powerMonitor } from 'electron'
import { recordCrashBreadcrumb } from './crash-reporting/crash-breadcrumb-store'
import { SYSTEM_SLEPT_BREADCRUMB } from '../shared/system-sleep-breadcrumb'
import { publishSystemResume, publishSystemSuspend } from './system-power-lifecycle'
export const SYSTEM_RESUMED_CHANNEL = 'system:resumed'
@@ -53,7 +54,7 @@ export function registerSystemResumeBroadcast(
const suspendedForMs = suspendedAt === null ? null : Math.max(0, now() - suspendedAt)
suspendedAt = null
if (suspendedForMs !== null && suspendedForMs >= MIN_REPORTABLE_SUSPEND_MS) {
recordCrashBreadcrumb('system_slept', { suspendedForMs })
recordCrashBreadcrumb(SYSTEM_SLEPT_BREADCRUMB, { suspendedForMs })
}
publishSystemResume()
for (const window of getWindows()) {
+6 -3
View File
@@ -2,6 +2,10 @@ import type {
CrashReportBreadcrumbData,
CrashReportDetailValue
} from '../../../shared/crash-reporting'
import {
RENDERER_MEMORY_HEARTBEAT_BREADCRUMB,
RENDERER_MEMORY_HEARTBEAT_INTERVAL_MS
} from '../../../shared/renderer-memory-heartbeat'
import {
getBrowserWebviewMemoryProfile,
type BrowserWebviewMemoryProfile
@@ -9,7 +13,6 @@ import {
import { recordRendererCrashBreadcrumb } from './crash-breadcrumb-recorder'
import { collectRendererMemoryProfileCounts } from './renderer-memory-profile'
const RENDERER_MEMORY_SAMPLE_INTERVAL_MS = 60_000
const BYTES_PER_MEGABYTE = 1024 * 1024
const BYTES_PER_KILOBYTE = 1024
// Why: one detailed breadcrumb per threshold names what grew before an OOM.
@@ -54,7 +57,7 @@ export function installRendererCrashDiagnostics(surface: RendererSurface = 'main
recordRendererMemory('startup')
rendererMemoryInterval = window.setInterval(
() => recordRendererMemory('interval'),
RENDERER_MEMORY_SAMPLE_INTERVAL_MS
RENDERER_MEMORY_HEARTBEAT_INTERVAL_MS
)
}
}
@@ -119,7 +122,7 @@ function recordRendererMemory(reason: string): void {
const browserWebviews = getBrowserWebviewMemoryProfile()
recordRendererCrashBreadcrumb(
'renderer_memory',
RENDERER_MEMORY_HEARTBEAT_BREADCRUMB,
compactBreadcrumbData({
reason,
usedHeapMB: toMegabytes(memory.usedJSHeapSize),
+8
View File
@@ -0,0 +1,8 @@
/**
* The renderer's periodic memory sample doubles as a liveness heartbeat: the
* gap between the last one and a crash report measures how long the renderer
* had been silent before it died. Shared so main reads the cadence it is
* measuring against instead of hardcoding a copy of the renderer's interval.
*/
export const RENDERER_MEMORY_HEARTBEAT_BREADCRUMB = 'renderer_memory'
export const RENDERER_MEMORY_HEARTBEAT_INTERVAL_MS = 60_000
+5
View File
@@ -0,0 +1,5 @@
/**
* Stamped on resume with the span the machine was suspended for. Shared so
* silence math can subtract OS sleep instead of reading it as a wedge.
*/
export const SYSTEM_SLEPT_BREADCRUMB = 'system_slept'