Preserve hang capture progress through late cleanup and reporting

This commit is contained in:
Brennan Benson
2026-11-11 13:40:45 -08:00
parent 495d7730c2
commit a861cb642c
12 changed files with 371 additions and 51 deletions
+1
View File
@@ -265,6 +265,7 @@
4 src/main/hang-watchdog/hang-incident-snapshot.ts
1 src/main/hang-watchdog/hang-watchdog-worker-path.ts
1 src/main/hang-watchdog/previous-hang-detection.ts
9 src/main/hang-watchdog/renderer-hang-records.ts
1 src/main/hermes/hermes-default-home.ts
10 src/main/hermes/hermes-home-filesystem.ts
2 src/main/hermes/hook-service.ts
@@ -39,11 +39,17 @@ export function reduceHangProfile(profile: Profiler.Profile): HangStacks {
counts.set(id, (counts.get(id) ?? 0) + 1)
validSamples++
}
const top: HangStack[] = []
const selected: { id: number; count: number }[] = []
for (const [id, count] of counts) {
if (top.length === 3 && count <= top[2].count) {
if (selected.length === 3 && count <= selected[2].count) {
continue
}
selected.push({ id, count })
selected.sort((a, b) => b.count - a.count)
selected.splice(3)
}
const top: HangStack[] = []
for (const { id, count } of selected) {
const frames: HangFrame[] = []
const seen = new Set<number>()
let current: number | undefined = id
@@ -2,10 +2,11 @@ import { reduceHangProfile } from './hang-profile-reducer'
import type { Profiler, Session } from 'node:inspector'
import { emptyHangStacks, type MainJsCapture } from './hang-stacks'
type ProfilerCommandParameters = { interval?: number }
type CaptureSession = Pick<Session, 'connectToMainThread' | 'disconnect'> & {
post: (
method: string,
params: object,
params: ProfilerCommandParameters,
callback: (error: Error | null, result?: unknown) => void
) => void
}
@@ -66,7 +67,10 @@ export function captureMainJsHang(options: {
}
const deadline = setTimeout(() => stop('deadline'), Math.max(0, 5_000 - elapsed()))
publish({})
const post = (method: string, params: object = {}): Promise<{ ok: boolean; result?: unknown }> =>
const post = (
method: string,
params: ProfilerCommandParameters = {}
): Promise<{ ok: boolean; result?: unknown }> =>
new Promise((resolve) => {
// Exactly one caller awaits each response before the next command or disconnect.
try {
@@ -0,0 +1,135 @@
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import type { MainToHangWatchdogWorkerMessage } from './hang-watchdog-worker-protocol'
const state = vi.hoisted(() => ({
clock: 0,
delayedWrite: false,
commands: new Array<string>(),
terminations: new Array<() => void>(),
records: new Array<unknown>()
}))
vi.mock('node:worker_threads', () => ({ isMainThread: false, parentPort: null, workerData: null }))
vi.mock('node:inspector', () => ({
Session: class {
connectToMainThread() {}
disconnect() {}
post(method: string, _params: unknown, response: (error: null, result?: unknown) => void) {
state.commands.push(method)
response(
null,
method === 'Profiler.stop'
? { profile: { nodes: [], samples: [], startTime: 0, endTime: 1 } }
: undefined
)
}
}
}))
vi.mock('@orca/process-host', () => ({
runProcess: async (options: { onChildTerminated: () => void }) => {
state.terminations.push(options.onChildTerminated)
return { stdout: '', stderr: '', code: null, timedOut: true }
}
}))
vi.mock('./hang-detection-marker', () => ({
readAvailableHangDetectionMarker: () => null,
writeHangDetectionMarker: (_path: string, record: unknown) => {
state.records.push(structuredClone(record))
if (state.delayedWrite) {
state.clock = 6000
}
}
}))
vi.mock('./hang-incident-snapshot', () => ({
readCompletedHangContext: () => null,
collectHangMachineSnapshot: async () => null
}))
import { runWatchdog } from './main-thread-hang-watchdog-entry'
beforeEach(() => {
state.clock = 0
state.delayedWrite = false
state.commands = []
state.terminations = []
state.records = []
vi.spyOn(process, 'platform', 'get').mockReturnValue('darwin')
vi.useFakeTimers()
vi.setSystemTime(0)
vi.spyOn(performance, 'now').mockImplementation(() => state.clock)
})
afterEach(() => {
for (const terminate of state.terminations.splice(0)) {
terminate()
}
vi.clearAllTimers()
vi.useRealTimers()
vi.restoreAllMocks()
})
function watchdog() {
let message: (value: MainToHangWatchdogWorkerMessage) => void = () => {}
runWatchdog(
{
parentPid: 123,
markerPath: '/qa/main-thread-hang.json',
timeoutMs: 100,
checkIntervalMs: 25
},
{
on: (_event, listener) => {
message = listener
},
close: () => {},
postMessage: (value) => message({ type: 'capture-drain-ack', token: value.token })
}
)
return { send: (value: MainToHangWatchdogWorkerMessage) => message(value) }
}
it('does not start a profiler after baseline publication has exhausted the detection budget', async () => {
state.delayedWrite = true
const owner = watchdog()
await vi.advanceTimersByTimeAsync(125)
await vi.dynamicImportSettled()
expect(state.commands).toEqual(['Profiler.enable', 'Profiler.disable'])
expect(state.records.at(-1)).toMatchObject({
observations: [
expect.objectContaining({
mainJs: expect.objectContaining({
state: 'not-started',
stopReason: 'deadline',
stopRequestedAfterDetectionMs: 6000
})
})
]
})
owner.send({ type: 'shutdown' })
})
it('admits a later incident after a late native termination receipt releases the physical owner', async () => {
const owner = watchdog()
await vi.advanceTimersByTimeAsync(125)
await vi.dynamicImportSettled()
owner.send({ type: 'heartbeat' })
await vi.dynamicImportSettled()
expect(state.records.at(-1)).toMatchObject({
observations: [
expect.objectContaining({ native: expect.objectContaining({ cleanup: 'unverified' }) })
]
})
state.terminations.shift()?.()
await vi.dynamicImportSettled()
expect(state.records.at(-1)).toMatchObject({
observations: [
expect.objectContaining({ native: expect.objectContaining({ cleanup: 'terminated' }) })
]
})
await vi.advanceTimersByTimeAsync(125)
await vi.dynamicImportSettled()
expect(state.records.at(-1)).toMatchObject({
observations: [
expect.anything(),
expect.objectContaining({
mainJs: expect.objectContaining({ state: 'collecting' }),
native: expect.objectContaining({ state: 'completed', cleanup: 'unverified' })
})
]
})
owner.send({ type: 'shutdown' })
})
@@ -80,11 +80,10 @@ export function runWatchdog(
observations = retainHangObservation(observations, { ...record, ...change })
save()
}
const capture = (record: HangObservation): void => {
const capture = (record: HangObservation, detectedAt: number): void => {
if (isMainThread || captureOwner || stopped || record.kind !== 'heartbeat-stall') {
return
}
const detectedAt = performance.now()
const owner: CaptureOwner = { id: record.id }
captureOwner = owner
owner.done = (async () => {
@@ -166,6 +165,7 @@ export function runWatchdog(
})
}
const observe = (kind: HangObservation['kind'], durationMs: number, startedAt: number): void => {
const detectedAt = performance.now()
const snapshot = captureHangSnapshot()
const now = snapshot.wallMs ?? Date.now()
const suspend = sleepPeriod?.suspend
@@ -204,7 +204,7 @@ export function runWatchdog(
}
openReceiptWindow(record.id)
save()
capture(record)
capture(record, detectedAt)
enrich(record)
}
const loop = createHangWatchdogDetectionLoop({
+14 -3
View File
@@ -119,6 +119,10 @@ export function captureNativeHang(options: {
}): { stop: () => void; done: Promise<void> } {
const abort = new AbortController()
let terminated = false
let release: () => void = () => {}
const termination = new Promise<void>((resolve) => {
release = resolve
})
const baseline: NativeHangCapture = {
...emptyHangStacks(),
state: 'collecting',
@@ -130,6 +134,11 @@ export function captureNativeHang(options: {
cleanup: 'pending'
}
options.publish(baseline)
let latest = baseline
const publish = (capture: NativeHangCapture): void => {
latest = capture
options.publish(capture)
}
const done = (async (): Promise<void> => {
try {
const { runProcess } = await import('@orca/process-host')
@@ -142,9 +151,10 @@ export function captureNativeHang(options: {
signal: abort.signal,
onChildTerminated: () => {
terminated = true
release()
}
})
options.publish({
publish({
...baseline,
...reduceNativeHangSample(result.stdout),
state: 'completed',
@@ -155,7 +165,7 @@ export function captureNativeHang(options: {
...(result.code !== null ? { exitCode: result.code } : {})
})
} catch {
options.publish({
publish({
...baseline,
state: 'failed',
completedAt: Date.now(),
@@ -163,7 +173,8 @@ export function captureNativeHang(options: {
})
}
if (!terminated) {
await new Promise<void>(() => {})
await termination
publish({ ...latest, cleanup: 'terminated' })
}
})()
return { stop: () => abort.abort(), done }
@@ -44,6 +44,7 @@ beforeEach(() => {
afterEach(() => {
setActiveSink(null)
vi.unstubAllEnvs()
vi.useRealTimers()
rmSync(dir, { recursive: true, force: true })
})
@@ -224,9 +225,11 @@ describe('previous hang detection', () => {
})
})
it('replays an old renderer capture into the current trace before consuming it so the default bundle includes it', () => {
const detectedAt = Date.now() - 2 * 60 * 60 * 1000
publishRendererHang(dir, {
it('replays an old renderer capture into the current trace before consuming it so the default bundle includes it', async () => {
vi.useFakeTimers()
vi.setSystemTime(new Date('2026-11-11T12:00:00Z'))
const detectedAt = 1794391200000
await publishRendererHang(dir, {
id: `456:${detectedAt}:3`,
parentPid: 456,
windowId: 3,
@@ -279,3 +282,56 @@ it('replays an old renderer capture into the current trace before consuming it s
sink.close()
}
})
it('puts renderer completion into the current bundle while retaining next-launch evidence', async () => {
vi.useFakeTimers()
vi.setSystemTime(new Date('2026-11-11T12:00:00Z'))
const traceFilePath = join(dir, 'logs', 'main.trace.ndjson')
const sink = createLocalFileSink({ filePath: traceFilePath, maxFiles: 1 })
try {
setActiveSink(sink)
await publishRendererHang(dir, {
id: '123:1000:3',
parentPid: 123,
windowId: 3,
detectedAt: 1000,
completedAt: 2000,
status: 'captured',
samples: 1,
validSamples: 1,
truncated: false,
stacks: [
{
count: 1,
hot: true,
frames: [{ name: 'rendererBusy', module: 'app.js', line: 9, column: 2 }]
}
]
})
const bundle = collectBundle({
traceFilePath,
maxFiles: 1,
appVersion: 'test',
platform: 'darwin',
arch: 'arm64',
osRelease: 'test',
orcaChannel: 'dev'
})
const capture = bundle.payload
.trim()
.split('\n')
.map((line) => JSON.parse(line))
.find((record) => record.attributes?.['breadcrumb.name'] === 'renderer_hang_capture')
expect(capture?.attributes['breadcrumb.data']).toMatchObject({
detectedAt: 1000,
completedAt: 2000,
rendererJs0_frame0: 'rendererBusy [app.js:9:2]'
})
expect(JSON.parse(readFileSync(join(dir, 'renderer-hang.json'), 'utf8'))).toMatchObject({
records: [expect.objectContaining({ completedAt: 2000 })]
})
} finally {
setActiveSink(null)
sink.close()
}
})
@@ -126,3 +126,12 @@ it('drops a cut input field before redaction while preserving complete frames',
truncated: true
})
})
it('records an unanswered renderer request explicitly while retaining its physical owner', () => {
const blocked = window(6)
blocked.start()?.stop()
expect(state.records.at(-1)).toMatchObject({ windowId: 6, status: 'request-pending', samples: 0 })
const shared = window(7)
shared.start()
expect(state.records.at(-1)).toMatchObject({ windowId: 7, status: 'process-capture-pending' })
})
@@ -73,9 +73,11 @@ function stopEpisode(contentsId: number, id: string): void {
episode,
episode.record.stacks.length
? 'captured'
: episode.record.status === 'collecting'
? 'stopped'
: episode.record.status
: [...pending.values()].some((owner) => owner.episodeId === id)
? 'request-pending'
: episode.record.status === 'collecting'
? 'stopped'
: episode.record.status
)
episodes.delete(contentsId)
}
+99 -34
View File
@@ -8,6 +8,7 @@ import {
statSync,
writeFileSync
} from 'node:fs'
import { open, readFile, rename, stat } from 'node:fs/promises'
import { join } from 'node:path'
import { parseHangStacks, type HangStacks } from './hang-stacks'
import { resolveObservabilityConsent } from '../observability'
@@ -28,6 +29,7 @@ export type RendererHangRecord = HangStacks & {
| 'policy-missing'
| 'failed'
| 'process-capture-pending'
| 'request-pending'
| 'stopped'
}
const validStatus = [
@@ -37,9 +39,17 @@ const validStatus = [
'policy-missing',
'failed',
'process-capture-pending',
'request-pending',
'stopped'
]
let current: RendererHangRecord[] = []
type RendererHangWriter = {
path: string
records: RendererHangRecord[]
initialized: boolean
revision: number
running?: Promise<void>
}
let writer: RendererHangWriter | undefined
function pathFor(userDataPath: string, previous = false): string {
return join(userDataPath, `renderer-hang.json${previous ? '.previous' : ''}`)
@@ -83,24 +93,27 @@ function parseRecord(value: unknown): RendererHangRecord | null {
: {})
}
}
function parseRecords(data: unknown): RendererHangRecord[] {
if (
!data ||
typeof data !== 'object' ||
!('version' in data) ||
data.version !== 1 ||
!('records' in data) ||
!Array.isArray(data.records) ||
data.records.length > 4
) {
return []
}
return data.records.map(parseRecord).filter((item): item is RendererHangRecord => item !== null)
}
function read(path: string): RendererHangRecord[] | undefined {
try {
if (statSync(path).size > 16 * 1024) {
return []
}
const data: unknown = JSON.parse(readFileSync(path, 'utf8'))
if (
!data ||
typeof data !== 'object' ||
!('version' in data) ||
data.version !== 1 ||
!('records' in data) ||
!Array.isArray(data.records) ||
data.records.length > 4
) {
return []
}
return data.records.map(parseRecord).filter((item): item is RendererHangRecord => item !== null)
return parseRecords(data)
} catch (error) {
return error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT'
? []
@@ -153,25 +166,83 @@ function write(path: string, records: RendererHangRecord[]): void {
renameSync(`${path}.tmp`, path)
}
export function publishRendererHang(userDataPath: string, record: RendererHangRecord): void {
if (!resolveObservabilityConsent().localFileEnabled) {
return
}
async function persist(owner: RendererHangWriter): Promise<void> {
try {
const normalized = parseRecord(record)
if (!normalized) {
return
if (!owner.initialized) {
let disk: RendererHangRecord[] = []
try {
if ((await stat(owner.path)).size <= 16 * 1024) {
disk = parseRecords(JSON.parse(await readFile(owner.path, 'utf8')))
}
} catch (error) {
if (!error || typeof error !== 'object' || !('code' in error) || error.code !== 'ENOENT') {
return
}
}
owner.records = merge(disk, owner.records)
owner.initialized = true
}
const disk = read(pathFor(userDataPath))
if (!disk) {
return
let committed = -1
while (committed !== owner.revision) {
const revision = owner.revision
const text = JSON.stringify({ version: 1, records: owner.records })
if (Buffer.byteLength(text) > 16 * 1024) {
return
}
const file = await open(`${owner.path}.tmp`, 'w', 0o600)
try {
await file.chmod(0o600)
await file.writeFile(text)
} finally {
await file.close()
}
await rename(`${owner.path}.tmp`, owner.path)
committed = revision
}
current = merge(merge(disk, current), [normalized])
write(pathFor(userDataPath), current)
} catch {
/* Optional diagnostics never block a window lifecycle event. */
// Keep the admitted snapshot for a later incident publication to retry.
} finally {
owner.running = undefined
}
}
function recordCapture(record: RendererHangRecord): void {
recordDurableCrashBreadcrumb('renderer_hang_capture', {
previousPid: record.parentPid,
windowId: record.windowId,
detectedAt: record.detectedAt,
completedAt: record.completedAt ?? null,
...hangCaptureDetails('rendererJs', record)
})
}
export function publishRendererHang(
userDataPath: string,
record: RendererHangRecord
): Promise<void> {
if (!resolveObservabilityConsent().localFileEnabled) {
return Promise.resolve()
}
const normalized = parseRecord(record)
if (!normalized) {
return Promise.resolve()
}
const path = pathFor(userDataPath)
if (!writer || writer.path !== path) {
writer = { path, records: [], initialized: false, revision: 0 }
}
const admitted = admit(normalized)
writer.records = merge(writer.records, [admitted])
writer.revision++
if (admitted.completedAt !== undefined) {
try {
recordCapture(admitted)
} catch {
// A failed trace flush must not prevent the independent pending-file write.
}
}
const owner = writer
owner.running ??= Promise.resolve().then(() => persist(owner))
return owner.running
}
export function preservePreviousRendererHangs(userDataPath: string): void {
if (!resolveObservabilityConsent().localFileEnabled) {
@@ -188,7 +259,7 @@ export function preservePreviousRendererHangs(userDataPath: string): void {
}
write(pathFor(userDataPath, true), merge(previous, records))
rmSync(pathFor(userDataPath), { force: true })
current = []
writer = undefined
} catch {
/* Keep the original pending file if preservation failed. */
}
@@ -204,13 +275,7 @@ export function reportPreviousRendererHangs(userDataPath: string): void {
return
}
for (const record of records) {
recordDurableCrashBreadcrumb('renderer_hang_capture', {
previousPid: record.parentPid,
windowId: record.windowId,
detectedAt: record.detectedAt,
completedAt: record.completedAt ?? null,
...hangCaptureDetails('rendererJs', record)
})
recordCapture(record)
}
// Each breadcrumb synchronously requests a trace flush before pending data is consumed.
rmSync(pathFor(userDataPath, true), { force: true })
@@ -42,3 +42,34 @@ it('opts in only the owned main document and preserves unrelated response policy
listener({ url: 'file:///app/index.html', resourceType: 'mainFrame', webContentsId: 4 }, answer)
expect(answer).toHaveBeenLastCalledWith({})
})
it('normalizes the development document URL for both the filter and response identity', () => {
let listener: (details: unknown, callback: (value: unknown) => void) => void = () => {}
const onHeadersReceived = vi.fn((_filter, callback) => {
listener = callback
})
const contents = Object.assign(new EventEmitter(), {
id: 8,
session: { webRequest: { onHeadersReceived } }
})
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this Electron test double supplies the members used by the policy; unused native members are never accessed.
installRendererHangStackPolicy(contents as unknown as WebContents, 'http://localhost:5173')
expect(onHeadersReceived).toHaveBeenCalledWith(
{ urls: ['http://localhost:5173/'], types: ['mainFrame'] },
expect.any(Function)
)
const answer = vi.fn()
listener(
{
url: 'http://localhost:5173/',
resourceType: 'mainFrame',
webContentsId: 8,
responseHeaders: {}
},
answer
)
expect(answer).toHaveBeenLastCalledWith({
responseHeaders: { 'Document-Policy': ['include-js-call-stacks-in-crash-reports'] }
})
contents.emit('destroyed')
})
@@ -5,7 +5,7 @@ const owners = new WeakMap<Session, Map<number, string>>()
export function installRendererHangStackPolicy(contents: WebContents, documentUrl: string): void {
try {
installPolicy(contents, documentUrl)
installPolicy(contents, new URL(documentUrl).href)
} catch {
// A rejected diagnostic policy must not prevent the main document from loading.
}