From 5642e283b3a3a050ef5dfae86fb5d53eeb02c826 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:01:07 -0800 Subject: [PATCH] Preserve bounded diagnostics for main-thread stalls and observer gaps --- .../hang-detection-marker.test.ts | 27 +- .../hang-watchdog/hang-detection-marker.ts | 89 +++--- .../hang-watchdog/hang-incident-snapshot.ts | 149 ++++++++++ .../hang-observation-ownership.test.ts | 190 +++++++++++++ .../hang-watchdog/hang-observation-schema.ts | 58 ++++ .../hang-watchdog/hang-observations.test.ts | 125 ++++++++ src/main/hang-watchdog/hang-observations.ts | 267 ++++++++++++++++++ src/main/hang-watchdog/hang-snapshot.ts | 23 ++ .../hang-watchdog-detection-loop.ts | 14 +- .../hang-watchdog-worker-protocol.ts | 7 +- .../main-thread-hang-watchdog-entry.ts | 233 +++++++++++---- .../main-thread-hang-watchdog.ts | 22 +- .../previous-hang-detection.test.ts | 133 ++++++++- .../hang-watchdog/previous-hang-detection.ts | 121 +++++++- 14 files changed, 1339 insertions(+), 119 deletions(-) create mode 100644 src/main/hang-watchdog/hang-incident-snapshot.ts create mode 100644 src/main/hang-watchdog/hang-observation-ownership.test.ts create mode 100644 src/main/hang-watchdog/hang-observation-schema.ts create mode 100644 src/main/hang-watchdog/hang-observations.test.ts create mode 100644 src/main/hang-watchdog/hang-observations.ts create mode 100644 src/main/hang-watchdog/hang-snapshot.ts diff --git a/src/main/hang-watchdog/hang-detection-marker.test.ts b/src/main/hang-watchdog/hang-detection-marker.test.ts index 462035c146f..cc34fa55617 100644 --- a/src/main/hang-watchdog/hang-detection-marker.test.ts +++ b/src/main/hang-watchdog/hang-detection-marker.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync, writeFileSync, existsSync, readFileSync, statSync } from 'node:fs' +import { mkdtempSync, rmSync, existsSync, readFileSync, statSync, renameSync } from 'node:fs' import type * as FileSystem from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' @@ -14,7 +14,8 @@ vi.mock('node:fs', async (importOriginal) => { return { ...actual, readFileSync: vi.fn(actual.readFileSync), - statSync: vi.fn(actual.statSync) + statSync: vi.fn(actual.statSync), + renameSync: vi.fn(actual.renameSync) } }) @@ -29,6 +30,19 @@ describe('hang detection marker', () => { rmSync(dir, { recursive: true, force: true }) }) + it('retains the complete baseline if publishing enrichment is interrupted', () => { + const markerPath = hangDetectionMarkerPath(dir) + const baseline = { detectedAt: 1, parentPid: 2, unresponsiveMs: 45_000, selfRecovered: false } + writeHangDetectionMarker(markerPath, baseline) + vi.mocked(renameSync).mockImplementationOnce(() => { + throw new Error('interrupted replacement') + }) + expect(() => + writeHangDetectionMarker(markerPath, { ...baseline, selfRecovered: true }) + ).toThrow() + expect(consumeHangDetectionMarker(markerPath)).toEqual(baseline) + }) + it.each(['stat', 'read'])('retains a real marker after a transient %s failure', (operation) => { const markerPath = hangDetectionMarkerPath(dir) const marker = { detectedAt: 1, parentPid: 2, unresponsiveMs: 45000, selfRecovered: false } @@ -46,13 +60,4 @@ describe('hang detection marker', () => { expect(consumeHangDetectionMarker(markerPath)).toEqual(marker) expect(existsSync(markerPath)).toBe(false) }) - - it('deletes an oversized marker without reading its contents', () => { - const markerPath = hangDetectionMarkerPath(dir) - writeFileSync(markerPath, 'x'.repeat(64 * 1024 + 1)) - vi.mocked(readFileSync).mockClear() - expect(consumeHangDetectionMarker(markerPath)).toBeNull() - expect(readFileSync).not.toHaveBeenCalled() - expect(existsSync(markerPath)).toBe(false) - }) }) diff --git a/src/main/hang-watchdog/hang-detection-marker.ts b/src/main/hang-watchdog/hang-detection-marker.ts index c01dc4dc948..c79eac7645f 100644 --- a/src/main/hang-watchdog/hang-detection-marker.ts +++ b/src/main/hang-watchdog/hang-detection-marker.ts @@ -1,66 +1,79 @@ -import { readFileSync, rmSync, statSync, writeFileSync } from 'node:fs' +import { + closeSync, + fchmodSync, + openSync, + readFileSync, + renameSync, + rmSync, + statSync, + writeFileSync +} from 'node:fs' import { join } from 'node:path' +import { + asHangObservations, + latestHangMarker, + parseLegacyHangMarker, + type HangDetectionMarker, + type HangObservationCollection +} from './hang-observations' -// The next launch consumes detection/resolution markers; selfRecovered distinguishes cleared stalls from unresolved hangs. -export type HangDetectionMarker = { - detectedAt: number - parentPid: number - unresponsiveMs: number - selfRecovered: boolean -} +export type { HangDetectionMarker } from './hang-observations' +export type StoredHangMarker = HangDetectionMarker | HangObservationCollection export function hangDetectionMarkerPath(userDataPath: string): string { return join(userDataPath, 'main-thread-hang.json') } -export function writeHangDetectionMarker(markerPath: string, marker: HangDetectionMarker): void { - writeFileSync(markerPath, JSON.stringify(marker), { mode: 0o600 }) +export function writeHangDetectionMarker(markerPath: string, marker: StoredHangMarker): void { + const collection = 'version' in marker ? asHangObservations(marker) : null + if ( + 'version' in marker && + (!collection || collection.observations.length !== marker.observations.length) + ) { + throw new Error('Invalid hang observations') + } + const text = JSON.stringify( + collection ? { ...latestHangMarker(collection), ...collection } : marker + ) + if (Buffer.byteLength(text) > 32 * 1024) { + throw new Error('Hang observations exceed byte bound') + } + const temporaryPath = `${markerPath}.tmp` + const fd = openSync(temporaryPath, 'w', 0o600) + try { + fchmodSync(fd, 0o600) + writeFileSync(fd, text) + } finally { + closeSync(fd) + } + renameSync(temporaryPath, markerPath) } -function readAvailableHangDetectionMarker( +export function readAvailableHangDetectionMarker( markerPath: string -): HangDetectionMarker | null | undefined { +): StoredHangMarker | null | undefined { let raw: string try { if (statSync(markerPath).size > 64 * 1024) { return null } raw = readFileSync(markerPath, 'utf8') - } catch { - return undefined + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'ENOENT' ? null : undefined } try { const parsed: unknown = JSON.parse(raw) - if ( - typeof parsed !== 'object' || - !parsed || - !('detectedAt' in parsed) || - typeof parsed.detectedAt !== 'number' || - !Number.isFinite(parsed.detectedAt) || - !('parentPid' in parsed) || - typeof parsed.parentPid !== 'number' || - !Number.isInteger(parsed.parentPid) || - parsed.parentPid <= 0 || - !('unresponsiveMs' in parsed) || - typeof parsed.unresponsiveMs !== 'number' || - !Number.isFinite(parsed.unresponsiveMs) || - parsed.unresponsiveMs < 0 - ) { - return null - } - return { - detectedAt: parsed.detectedAt, - parentPid: parsed.parentPid, - unresponsiveMs: parsed.unresponsiveMs, - // Missing recovery evidence leaves the stall unresolved. - selfRecovered: 'selfRecovered' in parsed && parsed.selfRecovered === true + if (typeof parsed === 'object' && parsed && 'version' in parsed && parsed.version === 1) { + const collection = Buffer.byteLength(raw) <= 32 * 1024 ? asHangObservations(parsed) : null + return collection ? { ...latestHangMarker(collection), ...collection } : null } + return parseLegacyHangMarker(parsed) } catch { return null } } -export function consumeHangDetectionMarker(markerPath: string): HangDetectionMarker | null { +export function consumeHangDetectionMarker(markerPath: string): StoredHangMarker | null { const marker = readAvailableHangDetectionMarker(markerPath) // Retry unavailable reads next startup; only successfully inspected files are consumable. if (marker === undefined) { diff --git a/src/main/hang-watchdog/hang-incident-snapshot.ts b/src/main/hang-watchdog/hang-incident-snapshot.ts new file mode 100644 index 00000000000..76c20ced683 --- /dev/null +++ b/src/main/hang-watchdog/hang-incident-snapshot.ts @@ -0,0 +1,149 @@ +import { closeSync, fstatSync, openSync, readSync } from 'node:fs' +import { freemem, loadavg, totalmem } from 'node:os' +import { sanitizeCrashReportString } from '../../shared/crash-report-redaction' +import { redactString } from '../observability/redactor' +import type { CompletedHangContext, HangMachineSnapshot } from './hang-observations' + +function numberMatch(text: string, expression: RegExp): number | null { + const match = expression.exec(text) + const value = match ? Number(match[1]) : Number.NaN + return Number.isFinite(value) && value >= 0 ? value : null +} +export function parseMacHangPressure( + sysctl: string, + vmStat: string +): Pick< + HangMachineSnapshot, + | 'systemMemorySwapTotalMB' + | 'systemMemorySwapUsedMB' + | 'systemMemoryCompressorMB' + | 'systemMemoryPressureLevel' + | 'systemMemoryStatusLevel' +> { + const swap = (key: string): number | null => { + const match = new RegExp(`\\b${key} = ([\\d.]+)([KMGT])\\b`).exec(sysctl) + if (!match) { + return null + } + const factor = { K: 1 / 1024, M: 1, G: 1024, T: 1024 * 1024 }[match[2]] + const value = Number(match[1]) * (factor ?? Number.NaN) + return Number.isFinite(value) && value >= 0 ? value : null + } + const pageSize = numberMatch(vmStat, /page size of (\d+) bytes/) + const compressor = numberMatch(vmStat, /^Pages occupied by compressor:\s+(\d+)\.?\s*$/m) + const compressorMB = + pageSize !== null && compressor !== null ? (pageSize * compressor) / 1024 / 1024 : null + return { + systemMemorySwapTotalMB: swap('total'), + systemMemorySwapUsedMB: swap('used'), + systemMemoryCompressorMB: + compressorMB !== null && Number.isFinite(compressorMB) ? compressorMB : null, + systemMemoryPressureLevel: numberMatch( + sysctl, + /^kern\.memorystatus_vm_pressure_level:\s+(\d+)\s*$/m + ), + systemMemoryStatusLevel: numberMatch(sysctl, /^kern\.memorystatus_level:\s+(\d+)\s*$/m) + } +} + +export async function collectHangMachineSnapshot( + signal: AbortSignal +): Promise { + const loads = loadavg() + const snapshot: HangMachineSnapshot = { + startedAt: Date.now(), + completedAt: null, + systemMemoryTotalMB: totalmem() / 1024 / 1024, + systemMemoryFreeMB: freemem() / 1024 / 1024, + systemMemorySwapTotalMB: null, + systemMemorySwapUsedMB: null, + systemMemoryCompressorMB: null, + systemMemoryPressureLevel: null, + systemMemoryStatusLevel: null, + load1: process.platform === 'win32' ? null : loads[0], + load5: process.platform === 'win32' ? null : loads[1], + load15: process.platform === 'win32' ? null : loads[2] + } + if (process.platform === 'darwin' && !signal.aborted) { + const { runProcess } = await import('@orca/process-host') + const commands = await Promise.allSettled([ + runProcess({ + program: '/usr/sbin/sysctl', + args: ['vm.swapusage', 'kern.memorystatus_level', 'kern.memorystatus_vm_pressure_level'], + timeoutMs: 2_000, + maxOutputBytes: 32 * 1024, + signal + }), + runProcess({ + program: '/usr/bin/vm_stat', + timeoutMs: 2_000, + maxOutputBytes: 32 * 1024, + signal + }) + ]) + const outputs = commands.map((result) => + result.status === 'fulfilled' && !result.value.timedOut && !result.value.outputTruncated + ? result.value.stdout + : '' + ) + Object.assign(snapshot, parseMacHangPressure(outputs[0], outputs[1])) + } + snapshot.completedAt = Date.now() + return snapshot +} + +export function readCompletedHangContext( + tracePath: string | undefined +): CompletedHangContext[] | null { + if (!tracePath) { + return null + } + let fd: number | undefined + try { + fd = openSync(tracePath, 'r') + const size = fstatSync(fd).size + const offset = Math.max(0, size - 32 * 1024) + const buffer = Buffer.allocUnsafe(Math.min(size, 32 * 1024)) + const bytes = readSync(fd, buffer, 0, buffer.length, offset) + const lines = buffer.subarray(0, bytes).toString('utf8').split('\n') + lines.pop() + if (offset > 0) { + lines.shift() + } + const spans: CompletedHangContext[] = [] + for (let index = lines.length - 1; index >= 0 && spans.length < 8; index--) { + try { + const data = JSON.parse(lines[index]) + if ( + data?.type !== 'effect-span' || + typeof data.name !== 'string' || + typeof data.startTimeUnixNano !== 'string' || + !/^\d{1,24}$/.test(data.startTimeUnixNano) || + typeof data.endTimeUnixNano !== 'string' || + !/^\d{1,24}$/.test(data.endTimeUnixNano) + ) { + continue + } + const startedAt = Number(data.startTimeUnixNano) / 1_000_000 + const completedAt = Number(data.endTimeUnixNano) / 1_000_000 + if (startedAt > completedAt) { + continue + } + spans.push({ + name: sanitizeCrashReportString(redactString(data.name), 120).slice(0, 120), + startedAt, + completedAt + }) + } catch { + // Concurrent appends and interrupted writes can leave incomplete records. + } + } + return spans.toReversed() + } catch { + return null + } finally { + if (fd !== undefined) { + closeSync(fd) + } + } +} diff --git a/src/main/hang-watchdog/hang-observation-ownership.test.ts b/src/main/hang-watchdog/hang-observation-ownership.test.ts new file mode 100644 index 00000000000..8be5aac9ad3 --- /dev/null +++ b/src/main/hang-watchdog/hang-observation-ownership.test.ts @@ -0,0 +1,190 @@ +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import type * as FileSystem from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, expect, it, vi } from 'vitest' +import type { HangMachineSnapshot } from './hang-observations' +import type { MainToHangWatchdogWorkerMessage } from './hang-watchdog-worker-protocol' +import { asHangObservations } from './hang-observations' +import { consumeHangDetectionMarker, writeHangDetectionMarker } from './hang-detection-marker' +import { runWatchdog } from './main-thread-hang-watchdog-entry' +import { preservePreviousHangDetection } from './previous-hang-detection' + +const probes = vi.hoisted(() => ({ pending: [] as ((value: HangMachineSnapshot) => void)[] })) +const files = vi.hoisted(() => ({ unavailablePath: null as string | null })) +vi.mock('node:fs', async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + readFileSync: new Proxy(actual.readFileSync, { + apply(target, thisArg, args) { + if (args[0] === files.unavailablePath) { + throw Object.assign(new Error('temporarily unreadable'), { code: 'EIO' }) + } + return Reflect.apply(target, thisArg, args) + } + }) + } +}) +vi.mock('../observability', () => ({ + resolveObservabilityConsent: () => ({ localFileEnabled: true }) +})) +vi.mock('./hang-incident-snapshot', () => ({ + readCompletedHangContext: () => [], + collectHangMachineSnapshot: () => + new Promise((resolve) => probes.pending.push(resolve)) +})) +const staleMachine: HangMachineSnapshot = { + startedAt: 125, + completedAt: 250, + systemMemoryTotalMB: 64_000, + systemMemoryFreeMB: 8_000, + systemMemorySwapTotalMB: 1024, + systemMemorySwapUsedMB: 100, + systemMemoryCompressorMB: 500, + systemMemoryPressureLevel: 2, + systemMemoryStatusLevel: 40, + load1: 2, + load5: 3, + load15: 4 +} +beforeEach(() => { + probes.pending = [] + files.unavailablePath = null + vi.useFakeTimers() + vi.setSystemTime(0) +}) +afterEach(() => { + files.unavailablePath = null + for (const resolve of probes.pending) { + resolve(staleMachine) + } + vi.useRealTimers() +}) + +it('cannot let a retired pressure read replace a newer stall or undo recovery', async () => { + const dir = mkdtempSync(join(tmpdir(), 'hang-ownership-')) + const markerPath = join(dir, 'marker.json') + let send: (message: MainToHangWatchdogWorkerMessage) => void = () => {} + runWatchdog( + { parentPid: process.pid, markerPath, timeoutMs: 100, checkIntervalMs: 25 }, + { + on: (_event, listener) => { + send = listener + }, + close() {} + } + ) + try { + vi.advanceTimersByTime(125) + await vi.dynamicImportSettled() + send({ type: 'heartbeat' }) + vi.advanceTimersByTime(125) + probes.pending[0](staleMachine) + await vi.dynamicImportSettled() + const records = asHangObservations(consumeHangDetectionMarker(markerPath))?.observations + expect( + records?.map(({ kind, selfRecovered, machine }) => ({ kind, selfRecovered, machine })) + ).toEqual([ + { kind: 'heartbeat-stall', selfRecovered: true, machine: null }, + { kind: 'heartbeat-stall', selfRecovered: false, machine: null } + ]) + expect(records?.[0].snapshot).not.toBeNull() + expect(records?.[0].resolution).not.toBeNull() + expect(records?.[1].resolution).toBeNull() + } finally { + send({ type: 'shutdown' }) + rmSync(dir, { recursive: true, force: true }) + } +}) + +it.each([ + { scenario: 'open suspend across a normal tick', suspendAt: 0, resumeAt: null, expected: true }, + { scenario: 'completed sleep before a normal tick', suspendAt: 0, resumeAt: 0, expected: false }, + { scenario: 'resume before the overdue tick', suspendAt: 0, resumeAt: 1000, expected: true }, + { scenario: 'suspend at the gap end', suspendAt: 1000, resumeAt: null, expected: false } +])('retains honest sleep association for $scenario', ({ suspendAt, resumeAt, expected }) => { + const dir = mkdtempSync(join(tmpdir(), 'hang-power-gap-')) + const markerPath = join(dir, 'marker.json') + let send: (message: MainToHangWatchdogWorkerMessage) => void = () => {} + runWatchdog( + { parentPid: process.pid, markerPath, timeoutMs: 100, checkIntervalMs: 25 }, + { + on: (_event, listener) => { + send = listener + }, + close() {} + } + ) + try { + if (suspendAt === 0) { + send({ type: 'power-event', event: 'suspend', observedAt: 0 }) + } + if (resumeAt === 0) { + send({ type: 'power-event', event: 'resume', observedAt: 0 }) + } + vi.advanceTimersByTime(25) + send({ type: 'heartbeat' }) + vi.setSystemTime(1000) + if (suspendAt === 1000) { + send({ type: 'power-event', event: 'suspend', observedAt: 1000 }) + } + if (resumeAt === 1000) { + send({ type: 'power-event', event: 'resume', observedAt: 1000 }) + } + vi.advanceTimersByTime(25) + const collection = asHangObservations(consumeHangDetectionMarker(markerPath)) + expect(collection?.observations).toMatchObject([ + { kind: 'watchdog-gap', initialSleepAssociated: expected, resolution: null } + ]) + if (suspendAt === 1000) { + for (let index = 0; index < 5; index++) { + send({ type: 'power-event', event: 'resume', observedAt: Date.now() }) + send({ type: 'heartbeat' }) + send({ type: 'power-event', event: 'suspend', observedAt: Date.now() }) + vi.advanceTimersByTime(25) + vi.setSystemTime(Date.now() + 1000) + vi.advanceTimersByTime(25) + } + expect( + asHangObservations(consumeHangDetectionMarker(markerPath))?.observations + ).toContainEqual(expect.objectContaining({ detectedAt: 1025, initialSleepAssociated: false })) + } + } finally { + send({ type: 'shutdown' }) + rmSync(dir, { recursive: true, force: true }) + } +}) + +it('keeps unavailable startup bytes until a later observation can safely merge them', () => { + const dir = mkdtempSync(join(tmpdir(), 'hang-unavailable-')) + const markerPath = join(dir, 'main-thread-hang.json') + const original = { detectedAt: 1, parentPid: 999, unresponsiveMs: 45_000, selfRecovered: false } + writeHangDetectionMarker(markerPath, original) + const originalBytes = readFileSync(markerPath, 'utf8') + files.unavailablePath = markerPath + preservePreviousHangDetection(dir) + let send: (message: MainToHangWatchdogWorkerMessage) => void = () => {} + runWatchdog( + { parentPid: process.pid, markerPath, timeoutMs: 100, checkIntervalMs: 25 }, + { + on: (_event, listener) => { + send = listener + }, + close() {} + } + ) + try { + vi.advanceTimersByTime(125) + files.unavailablePath = null + expect(readFileSync(markerPath, 'utf8')).toBe(originalBytes) + send({ type: 'heartbeat' }) + expect(asHangObservations(consumeHangDetectionMarker(markerPath))?.observations).toMatchObject([ + { parentPid: 999, detectedAt: 1, selfRecovered: false }, + { parentPid: process.pid, detectedAt: 125, selfRecovered: true } + ]) + } finally { + send({ type: 'shutdown' }) + rmSync(dir, { recursive: true, force: true }) + } +}) diff --git a/src/main/hang-watchdog/hang-observation-schema.ts b/src/main/hang-watchdog/hang-observation-schema.ts new file mode 100644 index 00000000000..d309ec0b1fb --- /dev/null +++ b/src/main/hang-watchdog/hang-observation-schema.ts @@ -0,0 +1,58 @@ +export const SNAPSHOT_FIELDS = [ + 'wallMs', + 'monotonicMs', + 'uptimeSeconds', + 'processUserUs', + 'processSystemUs', + 'watchdogUserUs', + 'watchdogSystemUs' +] as const +export type HangSnapshot = Record<(typeof SNAPSHOT_FIELDS)[number], number | null> + +export const MACHINE_FIELDS = [ + 'startedAt', + 'completedAt', + 'systemMemoryTotalMB', + 'systemMemoryFreeMB', + 'systemMemorySwapTotalMB', + 'systemMemorySwapUsedMB', + 'systemMemoryCompressorMB', + 'systemMemoryPressureLevel', + 'systemMemoryStatusLevel', + 'load1', + 'load5', + 'load15' +] as const +export type HangMachineSnapshot = Record<(typeof MACHINE_FIELDS)[number], number | null> +export type HangPowerReceipt = { + event: 'suspend' | 'resume' | 'lock-screen' | 'unlock-screen' + observedAt: number + receivedAt: number +} +export type CompletedHangContext = { name: string; startedAt: number; completedAt: number } +export type HangObservation = { + id: string + kind: 'heartbeat-stall' | 'watchdog-gap' + detectedAt: number + parentPid: number + durationMs: number + selfRecovered: boolean + initialSleepAssociated: boolean + snapshot: HangSnapshot | null + resolution: HangSnapshot | null + machine: HangMachineSnapshot | null + completedSpans: CompletedHangContext[] | null + powerEvents: HangPowerReceipt[] + watchdogGap?: { observedAt: number; durationMs: number; snapshot: HangSnapshot | null } +} +export type HangObservationCollection = { + version: 1 + observations: HangObservation[] + omitted: number +} +export type HangDetectionMarker = { + detectedAt: number + parentPid: number + unresponsiveMs: number + selfRecovered: boolean +} diff --git a/src/main/hang-watchdog/hang-observations.test.ts b/src/main/hang-watchdog/hang-observations.test.ts new file mode 100644 index 00000000000..77a2aeb1cf5 --- /dev/null +++ b/src/main/hang-watchdog/hang-observations.test.ts @@ -0,0 +1,125 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, it } from 'vitest' +import { consumeHangDetectionMarker, writeHangDetectionMarker } from './hang-detection-marker' +import { + asHangObservations, + emptyHangObservations, + mergeHangObservations, + retainHangObservation, + type HangObservation +} from './hang-observations' +import { parseMacHangPressure, readCompletedHangContext } from './hang-incident-snapshot' + +function observation(id: string, kind: HangObservation['kind'], sleep = false): HangObservation { + return { + id, + kind, + detectedAt: 10_000, + parentPid: 7, + durationMs: 162_000, + selfRecovered: false, + initialSleepAssociated: sleep, + snapshot: null, + resolution: null, + machine: null, + completedSpans: null, + powerEvents: [] + } +} + +it('keeps the original unexplained gap and recovered stall through subsequent ordinary sleeps', () => { + const dir = mkdtempSync(join(tmpdir(), 'hang-retention-')) + try { + const gap = observation('7:1:1', 'watchdog-gap') + const stall = observation('7:2:1', 'heartbeat-stall') + let collection = retainHangObservation( + retainHangObservation(emptyHangObservations(), gap), + stall + ) + collection = retainHangObservation(collection, { + ...gap, + initialSleepAssociated: true, + powerEvents: [{ event: 'resume', observedAt: 172_000, receivedAt: 172_001 }] + }) + collection = retainHangObservation(collection, { ...stall, selfRecovered: true }) + for (const id of ['7:3:1', '7:4:1', '7:5:1', '7:6:1', '7:7:1']) { + collection = retainHangObservation(collection, observation(id, 'watchdog-gap', true)) + } + collection = mergeHangObservations( + collection, + retainHangObservation(emptyHangObservations(), stall) + ) + const markerPath = join(dir, 'marker.json') + writeHangDetectionMarker(markerPath, collection) + const persisted = asHangObservations(consumeHangDetectionMarker(markerPath)) + expect( + persisted?.observations.map(({ id, selfRecovered, initialSleepAssociated }) => ({ + id, + selfRecovered, + initialSleepAssociated + })) + ).toEqual([ + { id: '7:1:1', selfRecovered: false, initialSleepAssociated: false }, + { id: '7:2:1', selfRecovered: true, initialSleepAssociated: false }, + { id: '7:6:1', selfRecovered: false, initialSleepAssociated: true }, + { id: '7:7:1', selfRecovered: false, initialSleepAssociated: true } + ]) + expect(persisted?.observations[0].powerEvents).toEqual([ + { event: 'resume', observedAt: 172_000, receivedAt: 172_001 } + ]) + expect(persisted?.omitted).toBe(3) + } finally { + rmSync(dir, { recursive: true, force: true }) + } +}) + +it('parses actual macOS units and page size without inventing missing measurements', () => { + expect( + parseMacHangPressure( + 'vm.swapusage: total = 1.50G used = 512.00M free = 1024.00M\nkern.memorystatus_level: 48\nkern.memorystatus_vm_pressure_level: 2\n', + 'Mach Virtual Memory Statistics: (page size of 16384 bytes)\nPages occupied by compressor: 42.\n' + ) + ).toEqual({ + systemMemorySwapTotalMB: 1536, + systemMemorySwapUsedMB: 512, + systemMemoryCompressorMB: 0.65625, + systemMemoryPressureLevel: 2, + systemMemoryStatusLevel: 48 + }) + expect(parseMacHangPressure('', 'Pages occupied by compressor: 42.')).toEqual({ + systemMemorySwapTotalMB: null, + systemMemorySwapUsedMB: null, + systemMemoryCompressorMB: null, + systemMemoryPressureLevel: null, + systemMemoryStatusLevel: null + }) +}) + +it('keeps only complete recent context and sanitizes complete names before truncating', () => { + const dir = mkdtempSync(join(tmpdir(), 'hang-context-')) + try { + const path = join(dir, 'trace.ndjson') + writeFileSync( + path, + `${JSON.stringify({ + type: 'effect-span', + name: 'old-span', + startTimeUnixNano: '1000000', + endTimeUnixNano: '2000000' + })}\n${'x'.repeat(40_000)}\n${JSON.stringify({ + type: 'effect-span', + name: `git.exec password=${'a'.repeat(200)}`, + startTimeUnixNano: '3000000', + endTimeUnixNano: '4000000', + attributes: { private: 'discard' } + })}\n{"type":"effect-span","name":"incomplete` + ) + expect(readCompletedHangContext(path)).toEqual([ + { name: 'git.exec [redacted:labeled-kv]', startedAt: 3, completedAt: 4 } + ]) + } finally { + rmSync(dir, { recursive: true, force: true }) + } +}) diff --git a/src/main/hang-watchdog/hang-observations.ts b/src/main/hang-watchdog/hang-observations.ts new file mode 100644 index 00000000000..45c53ed9d31 --- /dev/null +++ b/src/main/hang-watchdog/hang-observations.ts @@ -0,0 +1,267 @@ +import { + MACHINE_FIELDS, + SNAPSHOT_FIELDS, + type HangPowerReceipt, + type CompletedHangContext, + type HangObservation, + type HangObservationCollection, + type HangDetectionMarker +} from './hang-observation-schema' +export { MACHINE_FIELDS, SNAPSHOT_FIELDS } from './hang-observation-schema' +export type { + HangSnapshot, + HangMachineSnapshot, + HangPowerReceipt, + CompletedHangContext, + HangObservation, + HangObservationCollection, + HangDetectionMarker +} from './hang-observation-schema' + +function object(value: unknown): Record | null { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : null +} +function finite(value: unknown): value is number { + return typeof value === 'number' && Number.isFinite(value) && value >= 0 +} +function numericFields( + value: unknown, + keys: readonly K[] +): Record | null { + const data = object(value) + if (!data) { + return null + } + return Object.fromEntries( + keys.map((key) => [key, finite(data[key]) ? data[key] : null]) + ) as Record +} +export function parsePowerReceipt(value: unknown): HangPowerReceipt | null { + const data = object(value) + if ( + !data || + !['suspend', 'resume', 'lock-screen', 'unlock-screen'].includes(String(data.event)) || + !finite(data.observedAt) || + !finite(data.receivedAt) + ) { + return null + } + return { + event: data.event as HangPowerReceipt['event'], + observedAt: data.observedAt, + receivedAt: data.receivedAt + } +} +export function parseLegacyHangMarker(value: unknown): HangDetectionMarker | null { + const data = object(value) + if ( + !data || + !finite(data.detectedAt) || + !finite(data.parentPid) || + !Number.isInteger(data.parentPid) || + data.parentPid <= 0 || + !finite(data.unresponsiveMs) + ) { + return null + } + return { + detectedAt: data.detectedAt, + parentPid: data.parentPid, + unresponsiveMs: data.unresponsiveMs, + selfRecovered: data.selfRecovered === true + } +} +function parseObservation(value: unknown): HangObservation | null { + const data = object(value) + if ( + !data || + typeof data.id !== 'string' || + !/^[\d:.-]{1,80}$/.test(data.id) || + !['heartbeat-stall', 'watchdog-gap'].includes(String(data.kind)) || + !finite(data.detectedAt) || + !finite(data.parentPid) || + !Number.isInteger(data.parentPid) || + data.parentPid <= 0 || + !finite(data.durationMs) + ) { + return null + } + const completedSpans: CompletedHangContext[] = [] + if (Array.isArray(data.completedSpans) && data.completedSpans.length <= 8) { + for (const item of data.completedSpans) { + const span = object(item) + if ( + span && + typeof span.name === 'string' && + span.name.length <= 120 && + finite(span.startedAt) && + finite(span.completedAt) && + span.startedAt <= span.completedAt + ) { + completedSpans.push({ + name: span.name, + startedAt: span.startedAt, + completedAt: span.completedAt + }) + } + } + } + const gap = object(data.watchdogGap) + return { + id: data.id, + kind: data.kind as HangObservation['kind'], + detectedAt: data.detectedAt, + parentPid: data.parentPid, + durationMs: data.durationMs, + selfRecovered: data.selfRecovered === true, + initialSleepAssociated: data.initialSleepAssociated === true, + snapshot: numericFields(data.snapshot, SNAPSHOT_FIELDS), + resolution: numericFields(data.resolution, SNAPSHOT_FIELDS), + machine: numericFields(data.machine, MACHINE_FIELDS), + completedSpans: Array.isArray(data.completedSpans) ? completedSpans : null, + powerEvents: Array.isArray(data.powerEvents) + ? data.powerEvents + .slice(-4) + .map(parsePowerReceipt) + .filter((v): v is HangPowerReceipt => v !== null) + : [], + ...(gap && finite(gap.observedAt) && finite(gap.durationMs) + ? { + watchdogGap: { + observedAt: gap.observedAt, + durationMs: gap.durationMs, + snapshot: numericFields(gap.snapshot, SNAPSHOT_FIELDS) + } + } + : {}) + } +} +export function emptyHangObservations(): HangObservationCollection { + return { version: 1, observations: [], omitted: 0 } +} +export function asHangObservations(value: unknown): HangObservationCollection | null { + const data = object(value) + if (data?.version === 1) { + if (!Array.isArray(data.observations) || data.observations.length > 4) { + return null + } + return { + version: 1, + observations: data.observations + .map(parseObservation) + .filter((v): v is HangObservation => v !== null), + omitted: finite(data.omitted) ? Math.min(2_147_483_647, Math.floor(data.omitted)) : 0 + } + } + const legacy = parseLegacyHangMarker(value) + if (!legacy) { + return null + } + return { + version: 1, + omitted: 0, + observations: [ + { + id: `${legacy.parentPid}:${legacy.detectedAt}:0`, + kind: 'heartbeat-stall', + detectedAt: legacy.detectedAt, + parentPid: legacy.parentPid, + durationMs: legacy.unresponsiveMs, + selfRecovered: legacy.selfRecovered, + initialSleepAssociated: false, + snapshot: null, + resolution: null, + machine: null, + completedSpans: null, + powerEvents: [] + } + ] + } +} +function priority(record: HangObservation): number { + return record.kind === 'heartbeat-stall' ? 3 : record.initialSleepAssociated ? 1 : 2 +} +export function retainHangObservation( + collection: HangObservationCollection, + record: HangObservation +): HangObservationCollection { + const existing = collection.observations.findIndex((item) => item.id === record.id) + if (existing !== -1) { + const old = collection.observations[existing] + const observations = collection.observations.slice() + const receipts = new Map( + [...old.powerEvents, ...record.powerEvents].map( + (item) => [`${item.event}:${item.observedAt}:${item.receivedAt}`, item] as const + ) + ) + observations[existing] = { + ...record, + kind: old.kind, + id: old.id, + detectedAt: old.detectedAt, + parentPid: old.parentPid, + durationMs: old.durationMs, + initialSleepAssociated: old.initialSleepAssociated, + snapshot: old.snapshot, + selfRecovered: old.selfRecovered || record.selfRecovered, + resolution: old.resolution ?? record.resolution, + machine: old.machine ?? record.machine, + completedSpans: old.completedSpans ?? record.completedSpans, + powerEvents: [...receipts.values()].slice(-4), + watchdogGap: + old.watchdogGap && old.watchdogGap.durationMs > (record.watchdogGap?.durationMs ?? -1) + ? old.watchdogGap + : (record.watchdogGap ?? old.watchdogGap) + } + return { ...collection, observations } + } + const candidates = [...collection.observations, record] + const anchors = new Set( + [ + candidates.find((item) => item.kind === 'heartbeat-stall'), + candidates.find((item) => item.kind === 'watchdog-gap' && !item.initialSleepAssociated) + ].filter((item): item is HangObservation => !!item) + ) + const selected = new Set(anchors) + const ranked = candidates + .map((item, index) => ({ item, index })) + .filter(({ item }) => !anchors.has(item)) + .sort((a, b) => priority(b.item) - priority(a.item) || b.index - a.index) + for (const { item } of ranked) { + if (selected.size < 4) { + selected.add(item) + } + } + return { + version: 1, + observations: candidates.filter((item) => selected.has(item)), + omitted: Math.min(2_147_483_647, collection.omitted + candidates.length - selected.size) + } +} +export function mergeHangObservations( + first: HangObservationCollection, + second: HangObservationCollection +): HangObservationCollection { + let result = { ...first, omitted: Math.min(2_147_483_647, first.omitted + second.omitted) } + for (const record of second.observations) { + result = retainHangObservation(result, record) + } + return result +} +export function latestHangMarker( + collection: HangObservationCollection +): HangDetectionMarker | null { + const record = collection.observations.findLast((item) => item.kind === 'heartbeat-stall') + return record + ? { + detectedAt: record.detectedAt, + parentPid: record.parentPid, + unresponsiveMs: + record.durationMs + + Math.max(0, (record.resolution?.wallMs ?? record.detectedAt) - record.detectedAt), + selfRecovered: record.selfRecovered + } + : null +} diff --git a/src/main/hang-watchdog/hang-snapshot.ts b/src/main/hang-watchdog/hang-snapshot.ts new file mode 100644 index 00000000000..9ea13aae689 --- /dev/null +++ b/src/main/hang-watchdog/hang-snapshot.ts @@ -0,0 +1,23 @@ +import { uptime } from 'node:os' +import type { HangSnapshot } from './hang-observations' + +export function captureHangSnapshot(): HangSnapshot { + const cpu = process.cpuUsage() + let watchdog: NodeJS.CpuUsage | null = null + try { + const read = (process as NodeJS.Process & { threadCpuUsage?: () => NodeJS.CpuUsage }) + .threadCpuUsage + watchdog = read?.call(process) ?? null + } catch { + // Some Electron runtimes do not expose per-thread CPU counters. + } + return { + wallMs: Date.now(), + monotonicMs: performance.now(), + uptimeSeconds: uptime(), + processUserUs: cpu.user, + processSystemUs: cpu.system, + watchdogUserUs: watchdog?.user ?? null, + watchdogSystemUs: watchdog?.system ?? null + } +} diff --git a/src/main/hang-watchdog/hang-watchdog-detection-loop.ts b/src/main/hang-watchdog/hang-watchdog-detection-loop.ts index a91b8297532..52904fe1887 100644 --- a/src/main/hang-watchdog/hang-watchdog-detection-loop.ts +++ b/src/main/hang-watchdog/hang-watchdog-detection-loop.ts @@ -5,6 +5,7 @@ export type HangWatchdogDetectionLoopConfig = { onHangDetected: (unresponsiveMs: number) => void /** Heartbeats resumed after a detected hang — the main thread was stalled, not deadlocked. */ onHangResolved: (unresponsiveMs: number) => void + onWatchdogGap?: (gapMs: number, startedAt: number) => void } export type HangWatchdogDetectionLoop = { @@ -32,12 +33,17 @@ export function createHangWatchdogDetectionLoop( const tickGap = now - lastTickAt // Advance while hung so recovery cannot be misclassified as a long system-sleep gap. lastTickAt = now - if (detected) { + // Sleep, process suspension and observer starvation are indistinguishable here. + if (tickGap > config.checkIntervalMs * 3) { + if (tickGap > config.timeoutMs) { + config.onWatchdogGap?.(tickGap, now - tickGap) + } + if (!detected) { + lastHeartbeatAt = now + } return } - // System sleep suspends the watchdog too; restart the silence window after large tick gaps. - if (tickGap > config.checkIntervalMs * 3) { - lastHeartbeatAt = now + if (detected) { return } const unresponsiveMs = now - lastHeartbeatAt diff --git a/src/main/hang-watchdog/hang-watchdog-worker-protocol.ts b/src/main/hang-watchdog/hang-watchdog-worker-protocol.ts index 14e6d007796..f5e39662910 100644 --- a/src/main/hang-watchdog/hang-watchdog-worker-protocol.ts +++ b/src/main/hang-watchdog/hang-watchdog-worker-protocol.ts @@ -7,6 +7,11 @@ export type HangWatchdogWorkerData = { markerPath: string timeoutMs: number checkIntervalMs: number + tracePath?: string } -export type MainToHangWatchdogWorkerMessage = { type: 'heartbeat' } | { type: 'shutdown' } +export type MainToHangWatchdogWorkerMessage = + | { type: 'heartbeat' } + | { type: 'shutdown' } + | { type: 'power-event'; event: HangPowerReceipt['event']; observedAt: number } +import type { HangPowerReceipt } from './hang-observations' diff --git a/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts b/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts index 0d95d3ac6f9..2643ae42a11 100644 --- a/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts +++ b/src/main/hang-watchdog/main-thread-hang-watchdog-entry.ts @@ -1,6 +1,16 @@ import { isMainThread, parentPort, workerData } from 'node:worker_threads' import { createHangWatchdogDetectionLoop } from './hang-watchdog-detection-loop' -import { writeHangDetectionMarker } from './hang-detection-marker' +import { readAvailableHangDetectionMarker, writeHangDetectionMarker } from './hang-detection-marker' +import { + asHangObservations, + emptyHangObservations, + mergeHangObservations, + parsePowerReceipt, + retainHangObservation, + type HangObservation, + type HangPowerReceipt +} from './hang-observations' +import { captureHangSnapshot } from './hang-snapshot' import type { HangWatchdogWorkerData, MainToHangWatchdogWorkerMessage @@ -11,28 +21,6 @@ type HangWatchdogPort = { close: () => void } -// Observation only: a false positive must never kill a live main thread mid-write. -export function recordHangObservation(options: { - parentPid: number - markerPath: string - unresponsiveMs: number - selfRecovered: boolean -}): void { - if (!options.markerPath) { - return - } - try { - writeHangDetectionMarker(options.markerPath, { - detectedAt: Date.now(), - parentPid: options.parentPid, - unresponsiveMs: options.unresponsiveMs, - selfRecovered: options.selfRecovered - }) - } catch { - // Diagnostic write failures must not terminate the watchdog. - } -} - export function runWatchdog( config: HangWatchdogWorkerData, port: HangWatchdogPort | null = parentPort @@ -40,39 +28,187 @@ export function runWatchdog( if (!port) { return } + const initial = readAvailableHangDetectionMarker(config.markerPath) + let observations = asHangObservations(initial) ?? emptyHangObservations() + let existingUnavailable = initial === undefined + let sequence = 0 + let activeStallId: string | null = null + let stopped = false + let collector: AbortController | null = null + let recentPower: HangPowerReceipt[] = [] + let sleepPeriod: { suspend: HangPowerReceipt; resumeAt: number | null } | null = null + let receiptWindow: { id: string; until: number; remaining: number } | null = null + const save = (): void => { + if (stopped || !config.markerPath) { + return + } + try { + if (existingUnavailable) { + const existing = readAvailableHangDetectionMarker(config.markerPath) + if (existing === undefined) { + return + } + observations = mergeHangObservations( + asHangObservations(existing) ?? emptyHangObservations(), + observations + ) + existingUnavailable = false + } + writeHangDetectionMarker(config.markerPath, observations) + } catch { + // Optional diagnostics must not terminate the independent observer. + } + } + const find = (id: string): HangObservation | undefined => + observations.observations.find((item) => item.id === id) + const update = (id: string, change: Partial): void => { + const record = find(id) + if (!record || stopped) { + return + } + observations = retainHangObservation(observations, { ...record, ...change }) + save() + } + const openReceiptWindow = (id: string): void => { + receiptWindow = { id, until: Date.now() + config.checkIntervalMs, remaining: 4 } + } + const enrich = (record: HangObservation): void => { + if (!find(record.id)) { + return + } + if (collector) { + collector.abort() + return + } + const controller = new AbortController() + collector = controller + void import('./hang-incident-snapshot') + .then(async (incident) => { + if (controller.signal.aborted || stopped) { + return + } + if (record.kind === 'heartbeat-stall') { + update(record.id, { completedSpans: incident.readCompletedHangContext(config.tracePath) }) + } + const machine = await incident.collectHangMachineSnapshot(controller.signal) + if (!controller.signal.aborted && !stopped) { + update(record.id, { machine }) + } + }) + .catch(() => { + // The durable baseline already records unavailable optional evidence as unknown. + }) + .finally(() => { + if (collector === controller) { + collector = null + } + }) + } + const observe = (kind: HangObservation['kind'], durationMs: number, startedAt: number): void => { + const snapshot = captureHangSnapshot() + const now = snapshot.wallMs ?? Date.now() + const suspend = sleepPeriod?.suspend + const initialSleepAssociated = + kind === 'watchdog-gap' && + !!suspend && + suspend.observedAt <= startedAt + config.checkIntervalMs && + suspend.receivedAt <= startedAt + config.checkIntervalMs && + suspend.receivedAt < now - config.checkIntervalMs && + suspend.receivedAt >= suspend.observedAt && + suspend.receivedAt - suspend.observedAt <= config.checkIntervalMs && + (sleepPeriod?.resumeAt ?? Infinity) > startedAt + let powerEvents = recentPower + .filter((item) => item.observedAt >= startedAt && item.receivedAt <= now) + .slice(-4) + if (initialSleepAssociated && suspend) { + powerEvents = [suspend, ...powerEvents.filter((item) => item !== suspend).slice(-3)] + } + const record: HangObservation = { + id: `${config.parentPid}:${now}:${++sequence}`, + kind, + detectedAt: now, + parentPid: config.parentPid, + durationMs, + selfRecovered: false, + snapshot, + resolution: null, + machine: null, + completedSpans: null, + powerEvents, + initialSleepAssociated + } + observations = retainHangObservation(observations, record) + if (kind === 'heartbeat-stall') { + activeStallId = record.id + } + openReceiptWindow(record.id) + save() + enrich(record) + } const loop = createHangWatchdogDetectionLoop({ timeoutMs: config.timeoutMs, checkIntervalMs: config.checkIntervalMs, now: () => Date.now(), - onHangDetected: (unresponsiveMs) => - recordHangObservation({ - parentPid: config.parentPid, - markerPath: config.markerPath, - unresponsiveMs, - selfRecovered: false - }), - // Update one marker per stall so resolution does not require reconciling a second record. - onHangResolved: (unresponsiveMs) => - recordHangObservation({ - parentPid: config.parentPid, - markerPath: config.markerPath, - unresponsiveMs, - selfRecovered: true - }) + onHangDetected: (durationMs) => observe('heartbeat-stall', durationMs, Date.now() - durationMs), + onHangResolved: () => { + if (!activeStallId) { + return + } + const id = activeStallId + activeStallId = null + update(id, { selfRecovered: true, resolution: captureHangSnapshot() }) + openReceiptWindow(id) + }, + onWatchdogGap: (durationMs, startedAt) => { + const overlapping = observations.observations.findLast( + (item) => + item.kind === 'heartbeat-stall' && + (item.id === activeStallId || (item.resolution?.wallMs ?? -1) >= startedAt) + ) + if (overlapping) { + update(overlapping.id, { + watchdogGap: { observedAt: Date.now(), durationMs, snapshot: captureHangSnapshot() } + }) + openReceiptWindow(overlapping.id) + } else { + observe('watchdog-gap', durationMs, startedAt) + } + } }) - - let checkTimer: ReturnType | null = setInterval( - () => loop.tick(), - config.checkIntervalMs - ) - port.on('message', (message: MainToHangWatchdogWorkerMessage) => { + const checkTimer = setInterval(() => loop.tick(), config.checkIntervalMs) + port.on('message', (message) => { + if (stopped) { + return + } if (message.type === 'heartbeat') { loop.recordHeartbeat() - } else if (message.type === 'shutdown') { - if (checkTimer) { - clearInterval(checkTimer) - checkTimer = null + } else if (message.type === 'power-event') { + const receipt = parsePowerReceipt({ ...message, receivedAt: Date.now() }) + if (!receipt) { + return } + recentPower = [...recentPower, receipt].slice(-4) + if (receipt.event === 'suspend') { + sleepPeriod = { suspend: receipt, resumeAt: null } + } else if (receipt.event === 'resume' && sleepPeriod) { + sleepPeriod.resumeAt = receipt.observedAt + } + if ( + receiptWindow && + receiptWindow.remaining > 0 && + receipt.receivedAt <= receiptWindow.until + ) { + const record = find(receiptWindow.id) + if (record) { + receiptWindow.remaining-- + update(record.id, { powerEvents: [...record.powerEvents, receipt].slice(-4) }) + } + } + } else if (message.type === 'shutdown') { + stopped = true + receiptWindow = null + clearInterval(checkTimer) + collector?.abort() port.close() } }) @@ -85,6 +221,7 @@ export function isHangWatchdogWorkerData(value: unknown): value is HangWatchdogW Number.isInteger(data.parentPid) && (data.parentPid ?? 0) > 0 && typeof data.markerPath === 'string' && + (data.tracePath === undefined || typeof data.tracePath === 'string') && Number.isFinite(data.timeoutMs) && (data.timeoutMs ?? 0) > 0 && Number.isFinite(data.checkIntervalMs) && diff --git a/src/main/hang-watchdog/main-thread-hang-watchdog.ts b/src/main/hang-watchdog/main-thread-hang-watchdog.ts index cbe3ee374c6..909687e6842 100644 --- a/src/main/hang-watchdog/main-thread-hang-watchdog.ts +++ b/src/main/hang-watchdog/main-thread-hang-watchdog.ts @@ -1,9 +1,9 @@ import type { Worker } from 'node:worker_threads' import { spawnWorkerThread } from '../worker-threads/worker-thread-spawn' -import { app } from 'electron' +import { app, powerMonitor } from 'electron' import { hangDetectionMarkerPath } from './hang-detection-marker' import { resolveHangWatchdogWorkerPath } from './hang-watchdog-worker-path' -import { resolveObservabilityConsent } from '../observability' +import { getTraceFilePath, resolveObservabilityConsent } from '../observability' import { HANG_WATCHDOG_CHECK_INTERVAL_MS, HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS, @@ -39,6 +39,7 @@ export function installMainThreadHangWatchdog(options: { const workerData: HangWatchdogWorkerData = { parentPid: process.pid, markerPath: hangDetectionMarkerPath(options.userDataPath), + tracePath: getTraceFilePath(), timeoutMs: positiveTiming(process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS, HANG_WATCHDOG_TIMEOUT_MS), checkIntervalMs: positiveTiming( process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS, @@ -57,7 +58,7 @@ export function installMainThreadHangWatchdog(options: { } let stopped = false const postMessage = (message: MainToHangWatchdogWorkerMessage): void => { - if (stopped && message.type === 'heartbeat') { + if (stopped && message.type !== 'shutdown') { return } try { @@ -66,6 +67,19 @@ export function installMainThreadHangWatchdog(options: { // The worker already exited. } } + const powerListeners = (['suspend', 'resume', 'lock-screen', 'unlock-screen'] as const).map( + (event) => { + const listener = (): void => + postMessage({ type: 'power-event', event, observedAt: Date.now() }) + powerMonitor.on(event, listener) + return { event, listener } + } + ) + const removePowerListeners = (): void => { + for (const { event, listener } of powerListeners) { + powerMonitor.off(event, listener) + } + } const heartbeatTimer = setInterval(() => { postMessage({ type: 'heartbeat' }) }, HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS) @@ -76,6 +90,7 @@ export function installMainThreadHangWatchdog(options: { stopped = true // Drop the app listener immediately so retired workers are not retained by its closure. app.off('will-quit', stop) + removePowerListeners() clearInterval(heartbeatTimer) postMessage({ type: 'shutdown' }) } @@ -86,6 +101,7 @@ export function installMainThreadHangWatchdog(options: { worker.once('exit', () => { stopped = true app.off('will-quit', stop) + removePowerListeners() clearInterval(heartbeatTimer) }) worker.unref() diff --git a/src/main/hang-watchdog/previous-hang-detection.test.ts b/src/main/hang-watchdog/previous-hang-detection.test.ts index 59825f93704..707d1f393f7 100644 --- a/src/main/hang-watchdog/previous-hang-detection.test.ts +++ b/src/main/hang-watchdog/previous-hang-detection.test.ts @@ -3,6 +3,13 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { setActiveSink } from '../observability/tracer' +import { createLocalFileSink } from '../observability/local-file-sink' +import { collectBundle } from '../observability/bundle' +import { + emptyHangObservations, + retainHangObservation, + type HangObservation +} from './hang-observations' import { hangDetectionMarkerPath, writeHangDetectionMarker } from './hang-detection-marker' import { preservePreviousHangDetection, @@ -41,6 +48,28 @@ const marker = { unresponsiveMs: 10_000, selfRecovered: true } +const gap: HangObservation = { + id: '789:200:1', + kind: 'watchdog-gap', + detectedAt: 200, + parentPid: 789, + durationMs: 162_000, + selfRecovered: false, + initialSleepAssociated: false, + snapshot: { + wallMs: 200, + monotonicMs: 100, + uptimeSeconds: 1000, + processUserUs: 500, + processSystemUs: 200, + watchdogUserUs: null, + watchdogSystemUs: null + }, + resolution: null, + machine: null, + completedSpans: null, + powerEvents: [] +} describe('previous hang detection', () => { it('retains the previous run across startup and writes it only after a sink exists', () => { @@ -52,8 +81,7 @@ describe('previous hang detection', () => { expect(existsSync(`${markerPath}.previous`)).toBe(true) writeHangDetectionMarker(markerPath, { ...marker, parentPid: 789 }) const records: unknown[] = [] - const flush = vi.fn() - setActiveSink({ push: (record) => records.push(record), flush, close() {} }) + setActiveSink({ push: (record) => records.push(record), flush() {}, close() {} }) expect(reportPreviousHangDetection(dir)).toEqual(marker) expect(records).toEqual([ expect.objectContaining({ @@ -65,7 +93,6 @@ describe('previous hang detection', () => { }) }) ]) - expect(flush).toHaveBeenCalledOnce() expect(existsSync(`${markerPath}.previous`)).toBe(false) expect(JSON.parse(readFileSync(markerPath, 'utf8')).parentPid).toBe(789) expect(reportPreviousHangDetection(dir)).toBeNull() @@ -75,12 +102,112 @@ describe('previous hang detection', () => { const markerPath = hangDetectionMarkerPath(dir) writeHangDetectionMarker(markerPath, marker) preservePreviousHangDetection(dir) + writeHangDetectionMarker(markerPath, retainHangObservation(emptyHangObservations(), gap)) preservePreviousHangDetection(dir) expect(existsSync(`${markerPath}.previous`)).toBe(true) setActiveSink({ push() {}, flush() {}, close() {} }) expect(reportPreviousHangDetection(dir)).toEqual(marker) }) + it('carries gap and overlapping-stall evidence through the real trace and bundle', () => { + const traceFilePath = join(dir, 'logs', 'main.trace.ndjson') + const sink = createLocalFileSink({ filePath: traceFilePath, maxFiles: 1 }) + try { + writeHangDetectionMarker( + hangDetectionMarkerPath(dir), + retainHangObservation(emptyHangObservations(), gap) + ) + preservePreviousHangDetection(dir) + setActiveSink(sink) + expect(reportPreviousHangDetection(dir)).toBeNull() + const stall: HangObservation = { + ...gap, + id: '789:300:2', + kind: 'heartbeat-stall', + detectedAt: 300, + durationMs: 45_000, + selfRecovered: true, + snapshot: { ...gap.snapshot!, wallMs: 300, processUserUs: 100 }, + resolution: { ...gap.snapshot!, wallMs: 500, processUserUs: 900 }, + watchdogGap: { + observedAt: 400, + durationMs: 162_000, + snapshot: { ...gap.snapshot!, wallMs: 400, processUserUs: 600, watchdogUserUs: 3 } + }, + machine: { + startedAt: 301, + completedAt: 302, + systemMemoryTotalMB: 64_000, + systemMemoryFreeMB: 8_000, + systemMemorySwapTotalMB: 1024, + systemMemorySwapUsedMB: 100, + systemMemoryCompressorMB: 500, + systemMemoryPressureLevel: 2, + systemMemoryStatusLevel: 40, + load1: 2, + load5: 3, + load15: 4 + } + } + writeHangDetectionMarker( + hangDetectionMarkerPath(dir), + retainHangObservation(emptyHangObservations(), stall) + ) + preservePreviousHangDetection(dir) + expect(reportPreviousHangDetection(dir)).toEqual({ + detectedAt: 300, + parentPid: 789, + unresponsiveMs: 45_200, + selfRecovered: true + }) + const bundle = collectBundle({ + traceFilePath, + maxFiles: 1, + appVersion: 'test', + platform: 'darwin', + arch: 'arm64', + osRelease: 'test', + orcaChannel: 'dev' + }) + const records = bundle.payload + .trim() + .split('\n') + .map((line) => JSON.parse(line)) + .filter((record) => record.type === 'effect-span') + const gapRecord = records.find( + (record) => record.attributes['breadcrumb.name'] === 'main_thread_watchdog_gap' + ) + const stallRecord = records.find( + (record) => record.attributes['breadcrumb.name'] === 'main_thread_hang_detected' + ) + expect(gapRecord?.attributes['breadcrumb.data']).toMatchObject({ + watchdogGapMs: 162_000, + cause: 'unknown', + activeMainActivity: 'unknown', + detection_processUserUs: 500, + preGapCpuDeltaUs: null, + watchdogCpuDetectionToResolutionUs: null, + machineEvidence: 'unknown' + }) + expect(stallRecord?.attributes['breadcrumb.data']).toMatchObject({ + detection_wallMs: 300, + detection_processUserUs: 100, + gapEnd_wallMs: 400, + gapEnd_processUserUs: 600, + gapEnd_watchdogUserUs: 3, + resolution_wallMs: 500, + resolution_processUserUs: 900, + machine_startedAt: 301, + machine_completedAt: 302, + machine_systemMemorySwapUsedMB: 100 + }) + expect(stallRecord?.attributes['breadcrumb.data']).not.toHaveProperty('startedAt') + } finally { + setActiveSink(null) + sink.close() + } + }) + it('does not mutate marker files when local diagnostics are disabled', () => { const markerPath = hangDetectionMarkerPath(dir) writeHangDetectionMarker(markerPath, marker) diff --git a/src/main/hang-watchdog/previous-hang-detection.ts b/src/main/hang-watchdog/previous-hang-detection.ts index 2137499ae6e..b8282cb199d 100644 --- a/src/main/hang-watchdog/previous-hang-detection.ts +++ b/src/main/hang-watchdog/previous-hang-detection.ts @@ -1,12 +1,22 @@ -import { renameSync } from 'node:fs' +import { rmSync } from 'node:fs' +import type { CrashReportBreadcrumbData } from '../../shared/crash-reporting' import { resolveObservabilityConsent } from '../observability' import { hasActiveTraceSink } from '../observability/tracer' import { recordDurableCrashBreadcrumb } from '../crash-reporting/durable-crash-breadcrumb' import { consumeHangDetectionMarker, hangDetectionMarkerPath, + readAvailableHangDetectionMarker, + writeHangDetectionMarker, type HangDetectionMarker } from './hang-detection-marker' +import { + asHangObservations, + emptyHangObservations, + latestHangMarker, + mergeHangObservations, + type HangObservation +} from './hang-observations' function previousMarkerPath(userDataPath: string): string { return `${hangDetectionMarkerPath(userDataPath)}.previous` @@ -17,24 +27,113 @@ export function preservePreviousHangDetection(userDataPath: string): void { return } try { - // Keep the previous run's note safe from a new startup hang until its trace sink opens. - renameSync(hangDetectionMarkerPath(userDataPath), previousMarkerPath(userDataPath)) + const currentPath = hangDetectionMarkerPath(userDataPath) + const pendingPath = previousMarkerPath(userDataPath) + const current = asHangObservations(readAvailableHangDetectionMarker(currentPath)) + if (!current) { + return + } + const pending = readAvailableHangDetectionMarker(pendingPath) + if (pending === undefined) { + return + } + const merged = mergeHangObservations( + asHangObservations(pending) ?? emptyHangObservations(), + current + ) + writeHangDetectionMarker(pendingPath, merged) + rmSync(currentPath, { force: true }) } catch { // Missing or inaccessible markers must not prevent startup; retain any earlier pending note. } } +function observationDetails(record: HangObservation, omitted: number): CrashReportBreadcrumbData { + const details: CrashReportBreadcrumbData = { + previousPid: record.parentPid, + detectedAt: record.detectedAt, + observationKind: record.kind, + omittedObservations: omitted, + activeMainActivity: 'unknown', + cause: 'unknown', + cpuScope: 'process-all-threads', + machineEvidence: record.machine ? 'sampled' : 'unknown' + } + if (record.kind === 'heartbeat-stall') { + details.unresponsiveMs = + latestHangMarker({ version: 1, observations: [record], omitted: 0 })?.unresponsiveMs ?? + record.durationMs + details.detectedUnresponsiveMs = record.durationMs + details.selfRecovered = record.selfRecovered + } else { + details.watchdogGapMs = record.durationMs + details.preGapCpuDeltaUs = null + } + for (const [prefix, snapshot] of [ + ['detection', record.snapshot], + ['resolution', record.resolution], + ['gapEnd', record.watchdogGap?.snapshot] + ] as const) { + if (snapshot) { + for (const [key, value] of Object.entries(snapshot)) { + details[`${prefix}_${key}`] = value + } + } + } + const before = record.snapshot + const after = record.resolution + for (const scope of ['process', 'watchdog'] as const) { + const firstUser = before?.[`${scope}UserUs`] + const firstSystem = before?.[`${scope}SystemUs`] + const lastUser = after?.[`${scope}UserUs`] + const lastSystem = after?.[`${scope}SystemUs`] + const delta = + typeof firstUser === 'number' && + typeof firstSystem === 'number' && + typeof lastUser === 'number' && + typeof lastSystem === 'number' + ? lastUser + lastSystem - firstUser - firstSystem + : null + details[`${scope}CpuDetectionToResolutionUs`] = delta !== null && delta >= 0 ? delta : null + } + if (record.machine) { + for (const [key, value] of Object.entries(record.machine)) { + details[`machine_${key}`] = value + } + } + if (record.watchdogGap) { + details.overlappingWatchdogGapMs = record.watchdogGap.durationMs + details.overlappingWatchdogGapAt = record.watchdogGap.observedAt + } + for (const [index, receipt] of record.powerEvents.entries()) { + details[`power${index}_event`] = receipt.event + details[`power${index}_mainReceiptAt`] = receipt.observedAt + details[`power${index}_watchdogReceiptAt`] = receipt.receivedAt + } + details.completedContextEvidence = + record.completedSpans === null ? 'unknown' : 'completed-spans-on-disk' + for (const [index, span] of (record.completedSpans ?? []).entries()) { + details[`completed${index}_name`] = span.name + details[`completed${index}_startedAt`] = span.startedAt + details[`completed${index}_endedAt`] = span.completedAt + } + return details +} + export function reportPreviousHangDetection(userDataPath: string): HangDetectionMarker | null { - if (!hasActiveTraceSink()) { + if (!resolveObservabilityConsent().localFileEnabled || !hasActiveTraceSink()) { return null } const marker = consumeHangDetectionMarker(previousMarkerPath(userDataPath)) - if (marker) { - recordDurableCrashBreadcrumb('main_thread_hang_detected', { - unresponsiveMs: marker.unresponsiveMs, - previousPid: marker.parentPid, - selfRecovered: marker.selfRecovered - }) + const collection = asHangObservations(marker) + if (!collection) { + return null } - return marker + for (const record of collection.observations) { + recordDurableCrashBreadcrumb( + record.kind === 'heartbeat-stall' ? 'main_thread_hang_detected' : 'main_thread_watchdog_gap', + observationDetails(record, collection.omitted) + ) + } + return latestHangMarker(collection) }