Preserve bounded diagnostics for main-thread stalls and observer gaps

This commit is contained in:
Brennan Benson
2026-11-10 17:01:07 -08:00
parent 6192c9b8d5
commit 5642e283b3
14 changed files with 1339 additions and 119 deletions
@@ -1,4 +1,4 @@
import { mkdtempSync, rmSync, writeFileSync, existsSync, readFileSync, statSync } from 'node:fs'
import { mkdtempSync, rmSync, existsSync, readFileSync, statSync, renameSync } from 'node:fs'
import type * as FileSystem from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
@@ -14,7 +14,8 @@ vi.mock('node:fs', async (importOriginal) => {
return {
...actual,
readFileSync: vi.fn(actual.readFileSync),
statSync: vi.fn(actual.statSync)
statSync: vi.fn(actual.statSync),
renameSync: vi.fn(actual.renameSync)
}
})
@@ -29,6 +30,19 @@ describe('hang detection marker', () => {
rmSync(dir, { recursive: true, force: true })
})
it('retains the complete baseline if publishing enrichment is interrupted', () => {
const markerPath = hangDetectionMarkerPath(dir)
const baseline = { detectedAt: 1, parentPid: 2, unresponsiveMs: 45_000, selfRecovered: false }
writeHangDetectionMarker(markerPath, baseline)
vi.mocked(renameSync).mockImplementationOnce(() => {
throw new Error('interrupted replacement')
})
expect(() =>
writeHangDetectionMarker(markerPath, { ...baseline, selfRecovered: true })
).toThrow()
expect(consumeHangDetectionMarker(markerPath)).toEqual(baseline)
})
it.each(['stat', 'read'])('retains a real marker after a transient %s failure', (operation) => {
const markerPath = hangDetectionMarkerPath(dir)
const marker = { detectedAt: 1, parentPid: 2, unresponsiveMs: 45000, selfRecovered: false }
@@ -46,13 +60,4 @@ describe('hang detection marker', () => {
expect(consumeHangDetectionMarker(markerPath)).toEqual(marker)
expect(existsSync(markerPath)).toBe(false)
})
it('deletes an oversized marker without reading its contents', () => {
const markerPath = hangDetectionMarkerPath(dir)
writeFileSync(markerPath, 'x'.repeat(64 * 1024 + 1))
vi.mocked(readFileSync).mockClear()
expect(consumeHangDetectionMarker(markerPath)).toBeNull()
expect(readFileSync).not.toHaveBeenCalled()
expect(existsSync(markerPath)).toBe(false)
})
})
+51 -38
View File
@@ -1,66 +1,79 @@
import { readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
import {
closeSync,
fchmodSync,
openSync,
readFileSync,
renameSync,
rmSync,
statSync,
writeFileSync
} from 'node:fs'
import { join } from 'node:path'
import {
asHangObservations,
latestHangMarker,
parseLegacyHangMarker,
type HangDetectionMarker,
type HangObservationCollection
} from './hang-observations'
// The next launch consumes detection/resolution markers; selfRecovered distinguishes cleared stalls from unresolved hangs.
export type HangDetectionMarker = {
detectedAt: number
parentPid: number
unresponsiveMs: number
selfRecovered: boolean
}
export type { HangDetectionMarker } from './hang-observations'
export type StoredHangMarker = HangDetectionMarker | HangObservationCollection
export function hangDetectionMarkerPath(userDataPath: string): string {
return join(userDataPath, 'main-thread-hang.json')
}
export function writeHangDetectionMarker(markerPath: string, marker: HangDetectionMarker): void {
writeFileSync(markerPath, JSON.stringify(marker), { mode: 0o600 })
export function writeHangDetectionMarker(markerPath: string, marker: StoredHangMarker): void {
const collection = 'version' in marker ? asHangObservations(marker) : null
if (
'version' in marker &&
(!collection || collection.observations.length !== marker.observations.length)
) {
throw new Error('Invalid hang observations')
}
const text = JSON.stringify(
collection ? { ...latestHangMarker(collection), ...collection } : marker
)
if (Buffer.byteLength(text) > 32 * 1024) {
throw new Error('Hang observations exceed byte bound')
}
const temporaryPath = `${markerPath}.tmp`
const fd = openSync(temporaryPath, 'w', 0o600)
try {
fchmodSync(fd, 0o600)
writeFileSync(fd, text)
} finally {
closeSync(fd)
}
renameSync(temporaryPath, markerPath)
}
function readAvailableHangDetectionMarker(
export function readAvailableHangDetectionMarker(
markerPath: string
): HangDetectionMarker | null | undefined {
): StoredHangMarker | null | undefined {
let raw: string
try {
if (statSync(markerPath).size > 64 * 1024) {
return null
}
raw = readFileSync(markerPath, 'utf8')
} catch {
return undefined
} catch (error) {
return (error as NodeJS.ErrnoException).code === 'ENOENT' ? null : undefined
}
try {
const parsed: unknown = JSON.parse(raw)
if (
typeof parsed !== 'object' ||
!parsed ||
!('detectedAt' in parsed) ||
typeof parsed.detectedAt !== 'number' ||
!Number.isFinite(parsed.detectedAt) ||
!('parentPid' in parsed) ||
typeof parsed.parentPid !== 'number' ||
!Number.isInteger(parsed.parentPid) ||
parsed.parentPid <= 0 ||
!('unresponsiveMs' in parsed) ||
typeof parsed.unresponsiveMs !== 'number' ||
!Number.isFinite(parsed.unresponsiveMs) ||
parsed.unresponsiveMs < 0
) {
return null
}
return {
detectedAt: parsed.detectedAt,
parentPid: parsed.parentPid,
unresponsiveMs: parsed.unresponsiveMs,
// Missing recovery evidence leaves the stall unresolved.
selfRecovered: 'selfRecovered' in parsed && parsed.selfRecovered === true
if (typeof parsed === 'object' && parsed && 'version' in parsed && parsed.version === 1) {
const collection = Buffer.byteLength(raw) <= 32 * 1024 ? asHangObservations(parsed) : null
return collection ? { ...latestHangMarker(collection), ...collection } : null
}
return parseLegacyHangMarker(parsed)
} catch {
return null
}
}
export function consumeHangDetectionMarker(markerPath: string): HangDetectionMarker | null {
export function consumeHangDetectionMarker(markerPath: string): StoredHangMarker | null {
const marker = readAvailableHangDetectionMarker(markerPath)
// Retry unavailable reads next startup; only successfully inspected files are consumable.
if (marker === undefined) {
@@ -0,0 +1,149 @@
import { closeSync, fstatSync, openSync, readSync } from 'node:fs'
import { freemem, loadavg, totalmem } from 'node:os'
import { sanitizeCrashReportString } from '../../shared/crash-report-redaction'
import { redactString } from '../observability/redactor'
import type { CompletedHangContext, HangMachineSnapshot } from './hang-observations'
function numberMatch(text: string, expression: RegExp): number | null {
const match = expression.exec(text)
const value = match ? Number(match[1]) : Number.NaN
return Number.isFinite(value) && value >= 0 ? value : null
}
export function parseMacHangPressure(
sysctl: string,
vmStat: string
): Pick<
HangMachineSnapshot,
| 'systemMemorySwapTotalMB'
| 'systemMemorySwapUsedMB'
| 'systemMemoryCompressorMB'
| 'systemMemoryPressureLevel'
| 'systemMemoryStatusLevel'
> {
const swap = (key: string): number | null => {
const match = new RegExp(`\\b${key} = ([\\d.]+)([KMGT])\\b`).exec(sysctl)
if (!match) {
return null
}
const factor = { K: 1 / 1024, M: 1, G: 1024, T: 1024 * 1024 }[match[2]]
const value = Number(match[1]) * (factor ?? Number.NaN)
return Number.isFinite(value) && value >= 0 ? value : null
}
const pageSize = numberMatch(vmStat, /page size of (\d+) bytes/)
const compressor = numberMatch(vmStat, /^Pages occupied by compressor:\s+(\d+)\.?\s*$/m)
const compressorMB =
pageSize !== null && compressor !== null ? (pageSize * compressor) / 1024 / 1024 : null
return {
systemMemorySwapTotalMB: swap('total'),
systemMemorySwapUsedMB: swap('used'),
systemMemoryCompressorMB:
compressorMB !== null && Number.isFinite(compressorMB) ? compressorMB : null,
systemMemoryPressureLevel: numberMatch(
sysctl,
/^kern\.memorystatus_vm_pressure_level:\s+(\d+)\s*$/m
),
systemMemoryStatusLevel: numberMatch(sysctl, /^kern\.memorystatus_level:\s+(\d+)\s*$/m)
}
}
export async function collectHangMachineSnapshot(
signal: AbortSignal
): Promise<HangMachineSnapshot> {
const loads = loadavg()
const snapshot: HangMachineSnapshot = {
startedAt: Date.now(),
completedAt: null,
systemMemoryTotalMB: totalmem() / 1024 / 1024,
systemMemoryFreeMB: freemem() / 1024 / 1024,
systemMemorySwapTotalMB: null,
systemMemorySwapUsedMB: null,
systemMemoryCompressorMB: null,
systemMemoryPressureLevel: null,
systemMemoryStatusLevel: null,
load1: process.platform === 'win32' ? null : loads[0],
load5: process.platform === 'win32' ? null : loads[1],
load15: process.platform === 'win32' ? null : loads[2]
}
if (process.platform === 'darwin' && !signal.aborted) {
const { runProcess } = await import('@orca/process-host')
const commands = await Promise.allSettled([
runProcess({
program: '/usr/sbin/sysctl',
args: ['vm.swapusage', 'kern.memorystatus_level', 'kern.memorystatus_vm_pressure_level'],
timeoutMs: 2_000,
maxOutputBytes: 32 * 1024,
signal
}),
runProcess({
program: '/usr/bin/vm_stat',
timeoutMs: 2_000,
maxOutputBytes: 32 * 1024,
signal
})
])
const outputs = commands.map((result) =>
result.status === 'fulfilled' && !result.value.timedOut && !result.value.outputTruncated
? result.value.stdout
: ''
)
Object.assign(snapshot, parseMacHangPressure(outputs[0], outputs[1]))
}
snapshot.completedAt = Date.now()
return snapshot
}
export function readCompletedHangContext(
tracePath: string | undefined
): CompletedHangContext[] | null {
if (!tracePath) {
return null
}
let fd: number | undefined
try {
fd = openSync(tracePath, 'r')
const size = fstatSync(fd).size
const offset = Math.max(0, size - 32 * 1024)
const buffer = Buffer.allocUnsafe(Math.min(size, 32 * 1024))
const bytes = readSync(fd, buffer, 0, buffer.length, offset)
const lines = buffer.subarray(0, bytes).toString('utf8').split('\n')
lines.pop()
if (offset > 0) {
lines.shift()
}
const spans: CompletedHangContext[] = []
for (let index = lines.length - 1; index >= 0 && spans.length < 8; index--) {
try {
const data = JSON.parse(lines[index])
if (
data?.type !== 'effect-span' ||
typeof data.name !== 'string' ||
typeof data.startTimeUnixNano !== 'string' ||
!/^\d{1,24}$/.test(data.startTimeUnixNano) ||
typeof data.endTimeUnixNano !== 'string' ||
!/^\d{1,24}$/.test(data.endTimeUnixNano)
) {
continue
}
const startedAt = Number(data.startTimeUnixNano) / 1_000_000
const completedAt = Number(data.endTimeUnixNano) / 1_000_000
if (startedAt > completedAt) {
continue
}
spans.push({
name: sanitizeCrashReportString(redactString(data.name), 120).slice(0, 120),
startedAt,
completedAt
})
} catch {
// Concurrent appends and interrupted writes can leave incomplete records.
}
}
return spans.toReversed()
} catch {
return null
} finally {
if (fd !== undefined) {
closeSync(fd)
}
}
}
@@ -0,0 +1,190 @@
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
import type * as FileSystem from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import type { HangMachineSnapshot } from './hang-observations'
import type { MainToHangWatchdogWorkerMessage } from './hang-watchdog-worker-protocol'
import { asHangObservations } from './hang-observations'
import { consumeHangDetectionMarker, writeHangDetectionMarker } from './hang-detection-marker'
import { runWatchdog } from './main-thread-hang-watchdog-entry'
import { preservePreviousHangDetection } from './previous-hang-detection'
const probes = vi.hoisted(() => ({ pending: [] as ((value: HangMachineSnapshot) => void)[] }))
const files = vi.hoisted(() => ({ unavailablePath: null as string | null }))
vi.mock('node:fs', async (importOriginal) => {
const actual = await importOriginal<typeof FileSystem>()
return {
...actual,
readFileSync: new Proxy(actual.readFileSync, {
apply(target, thisArg, args) {
if (args[0] === files.unavailablePath) {
throw Object.assign(new Error('temporarily unreadable'), { code: 'EIO' })
}
return Reflect.apply(target, thisArg, args)
}
})
}
})
vi.mock('../observability', () => ({
resolveObservabilityConsent: () => ({ localFileEnabled: true })
}))
vi.mock('./hang-incident-snapshot', () => ({
readCompletedHangContext: () => [],
collectHangMachineSnapshot: () =>
new Promise<HangMachineSnapshot>((resolve) => probes.pending.push(resolve))
}))
const staleMachine: HangMachineSnapshot = {
startedAt: 125,
completedAt: 250,
systemMemoryTotalMB: 64_000,
systemMemoryFreeMB: 8_000,
systemMemorySwapTotalMB: 1024,
systemMemorySwapUsedMB: 100,
systemMemoryCompressorMB: 500,
systemMemoryPressureLevel: 2,
systemMemoryStatusLevel: 40,
load1: 2,
load5: 3,
load15: 4
}
beforeEach(() => {
probes.pending = []
files.unavailablePath = null
vi.useFakeTimers()
vi.setSystemTime(0)
})
afterEach(() => {
files.unavailablePath = null
for (const resolve of probes.pending) {
resolve(staleMachine)
}
vi.useRealTimers()
})
it('cannot let a retired pressure read replace a newer stall or undo recovery', async () => {
const dir = mkdtempSync(join(tmpdir(), 'hang-ownership-'))
const markerPath = join(dir, 'marker.json')
let send: (message: MainToHangWatchdogWorkerMessage) => void = () => {}
runWatchdog(
{ parentPid: process.pid, markerPath, timeoutMs: 100, checkIntervalMs: 25 },
{
on: (_event, listener) => {
send = listener
},
close() {}
}
)
try {
vi.advanceTimersByTime(125)
await vi.dynamicImportSettled()
send({ type: 'heartbeat' })
vi.advanceTimersByTime(125)
probes.pending[0](staleMachine)
await vi.dynamicImportSettled()
const records = asHangObservations(consumeHangDetectionMarker(markerPath))?.observations
expect(
records?.map(({ kind, selfRecovered, machine }) => ({ kind, selfRecovered, machine }))
).toEqual([
{ kind: 'heartbeat-stall', selfRecovered: true, machine: null },
{ kind: 'heartbeat-stall', selfRecovered: false, machine: null }
])
expect(records?.[0].snapshot).not.toBeNull()
expect(records?.[0].resolution).not.toBeNull()
expect(records?.[1].resolution).toBeNull()
} finally {
send({ type: 'shutdown' })
rmSync(dir, { recursive: true, force: true })
}
})
it.each([
{ scenario: 'open suspend across a normal tick', suspendAt: 0, resumeAt: null, expected: true },
{ scenario: 'completed sleep before a normal tick', suspendAt: 0, resumeAt: 0, expected: false },
{ scenario: 'resume before the overdue tick', suspendAt: 0, resumeAt: 1000, expected: true },
{ scenario: 'suspend at the gap end', suspendAt: 1000, resumeAt: null, expected: false }
])('retains honest sleep association for $scenario', ({ suspendAt, resumeAt, expected }) => {
const dir = mkdtempSync(join(tmpdir(), 'hang-power-gap-'))
const markerPath = join(dir, 'marker.json')
let send: (message: MainToHangWatchdogWorkerMessage) => void = () => {}
runWatchdog(
{ parentPid: process.pid, markerPath, timeoutMs: 100, checkIntervalMs: 25 },
{
on: (_event, listener) => {
send = listener
},
close() {}
}
)
try {
if (suspendAt === 0) {
send({ type: 'power-event', event: 'suspend', observedAt: 0 })
}
if (resumeAt === 0) {
send({ type: 'power-event', event: 'resume', observedAt: 0 })
}
vi.advanceTimersByTime(25)
send({ type: 'heartbeat' })
vi.setSystemTime(1000)
if (suspendAt === 1000) {
send({ type: 'power-event', event: 'suspend', observedAt: 1000 })
}
if (resumeAt === 1000) {
send({ type: 'power-event', event: 'resume', observedAt: 1000 })
}
vi.advanceTimersByTime(25)
const collection = asHangObservations(consumeHangDetectionMarker(markerPath))
expect(collection?.observations).toMatchObject([
{ kind: 'watchdog-gap', initialSleepAssociated: expected, resolution: null }
])
if (suspendAt === 1000) {
for (let index = 0; index < 5; index++) {
send({ type: 'power-event', event: 'resume', observedAt: Date.now() })
send({ type: 'heartbeat' })
send({ type: 'power-event', event: 'suspend', observedAt: Date.now() })
vi.advanceTimersByTime(25)
vi.setSystemTime(Date.now() + 1000)
vi.advanceTimersByTime(25)
}
expect(
asHangObservations(consumeHangDetectionMarker(markerPath))?.observations
).toContainEqual(expect.objectContaining({ detectedAt: 1025, initialSleepAssociated: false }))
}
} finally {
send({ type: 'shutdown' })
rmSync(dir, { recursive: true, force: true })
}
})
it('keeps unavailable startup bytes until a later observation can safely merge them', () => {
const dir = mkdtempSync(join(tmpdir(), 'hang-unavailable-'))
const markerPath = join(dir, 'main-thread-hang.json')
const original = { detectedAt: 1, parentPid: 999, unresponsiveMs: 45_000, selfRecovered: false }
writeHangDetectionMarker(markerPath, original)
const originalBytes = readFileSync(markerPath, 'utf8')
files.unavailablePath = markerPath
preservePreviousHangDetection(dir)
let send: (message: MainToHangWatchdogWorkerMessage) => void = () => {}
runWatchdog(
{ parentPid: process.pid, markerPath, timeoutMs: 100, checkIntervalMs: 25 },
{
on: (_event, listener) => {
send = listener
},
close() {}
}
)
try {
vi.advanceTimersByTime(125)
files.unavailablePath = null
expect(readFileSync(markerPath, 'utf8')).toBe(originalBytes)
send({ type: 'heartbeat' })
expect(asHangObservations(consumeHangDetectionMarker(markerPath))?.observations).toMatchObject([
{ parentPid: 999, detectedAt: 1, selfRecovered: false },
{ parentPid: process.pid, detectedAt: 125, selfRecovered: true }
])
} finally {
send({ type: 'shutdown' })
rmSync(dir, { recursive: true, force: true })
}
})
@@ -0,0 +1,58 @@
export const SNAPSHOT_FIELDS = [
'wallMs',
'monotonicMs',
'uptimeSeconds',
'processUserUs',
'processSystemUs',
'watchdogUserUs',
'watchdogSystemUs'
] as const
export type HangSnapshot = Record<(typeof SNAPSHOT_FIELDS)[number], number | null>
export const MACHINE_FIELDS = [
'startedAt',
'completedAt',
'systemMemoryTotalMB',
'systemMemoryFreeMB',
'systemMemorySwapTotalMB',
'systemMemorySwapUsedMB',
'systemMemoryCompressorMB',
'systemMemoryPressureLevel',
'systemMemoryStatusLevel',
'load1',
'load5',
'load15'
] as const
export type HangMachineSnapshot = Record<(typeof MACHINE_FIELDS)[number], number | null>
export type HangPowerReceipt = {
event: 'suspend' | 'resume' | 'lock-screen' | 'unlock-screen'
observedAt: number
receivedAt: number
}
export type CompletedHangContext = { name: string; startedAt: number; completedAt: number }
export type HangObservation = {
id: string
kind: 'heartbeat-stall' | 'watchdog-gap'
detectedAt: number
parentPid: number
durationMs: number
selfRecovered: boolean
initialSleepAssociated: boolean
snapshot: HangSnapshot | null
resolution: HangSnapshot | null
machine: HangMachineSnapshot | null
completedSpans: CompletedHangContext[] | null
powerEvents: HangPowerReceipt[]
watchdogGap?: { observedAt: number; durationMs: number; snapshot: HangSnapshot | null }
}
export type HangObservationCollection = {
version: 1
observations: HangObservation[]
omitted: number
}
export type HangDetectionMarker = {
detectedAt: number
parentPid: number
unresponsiveMs: number
selfRecovered: boolean
}
@@ -0,0 +1,125 @@
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { expect, it } from 'vitest'
import { consumeHangDetectionMarker, writeHangDetectionMarker } from './hang-detection-marker'
import {
asHangObservations,
emptyHangObservations,
mergeHangObservations,
retainHangObservation,
type HangObservation
} from './hang-observations'
import { parseMacHangPressure, readCompletedHangContext } from './hang-incident-snapshot'
function observation(id: string, kind: HangObservation['kind'], sleep = false): HangObservation {
return {
id,
kind,
detectedAt: 10_000,
parentPid: 7,
durationMs: 162_000,
selfRecovered: false,
initialSleepAssociated: sleep,
snapshot: null,
resolution: null,
machine: null,
completedSpans: null,
powerEvents: []
}
}
it('keeps the original unexplained gap and recovered stall through subsequent ordinary sleeps', () => {
const dir = mkdtempSync(join(tmpdir(), 'hang-retention-'))
try {
const gap = observation('7:1:1', 'watchdog-gap')
const stall = observation('7:2:1', 'heartbeat-stall')
let collection = retainHangObservation(
retainHangObservation(emptyHangObservations(), gap),
stall
)
collection = retainHangObservation(collection, {
...gap,
initialSleepAssociated: true,
powerEvents: [{ event: 'resume', observedAt: 172_000, receivedAt: 172_001 }]
})
collection = retainHangObservation(collection, { ...stall, selfRecovered: true })
for (const id of ['7:3:1', '7:4:1', '7:5:1', '7:6:1', '7:7:1']) {
collection = retainHangObservation(collection, observation(id, 'watchdog-gap', true))
}
collection = mergeHangObservations(
collection,
retainHangObservation(emptyHangObservations(), stall)
)
const markerPath = join(dir, 'marker.json')
writeHangDetectionMarker(markerPath, collection)
const persisted = asHangObservations(consumeHangDetectionMarker(markerPath))
expect(
persisted?.observations.map(({ id, selfRecovered, initialSleepAssociated }) => ({
id,
selfRecovered,
initialSleepAssociated
}))
).toEqual([
{ id: '7:1:1', selfRecovered: false, initialSleepAssociated: false },
{ id: '7:2:1', selfRecovered: true, initialSleepAssociated: false },
{ id: '7:6:1', selfRecovered: false, initialSleepAssociated: true },
{ id: '7:7:1', selfRecovered: false, initialSleepAssociated: true }
])
expect(persisted?.observations[0].powerEvents).toEqual([
{ event: 'resume', observedAt: 172_000, receivedAt: 172_001 }
])
expect(persisted?.omitted).toBe(3)
} finally {
rmSync(dir, { recursive: true, force: true })
}
})
it('parses actual macOS units and page size without inventing missing measurements', () => {
expect(
parseMacHangPressure(
'vm.swapusage: total = 1.50G used = 512.00M free = 1024.00M\nkern.memorystatus_level: 48\nkern.memorystatus_vm_pressure_level: 2\n',
'Mach Virtual Memory Statistics: (page size of 16384 bytes)\nPages occupied by compressor: 42.\n'
)
).toEqual({
systemMemorySwapTotalMB: 1536,
systemMemorySwapUsedMB: 512,
systemMemoryCompressorMB: 0.65625,
systemMemoryPressureLevel: 2,
systemMemoryStatusLevel: 48
})
expect(parseMacHangPressure('', 'Pages occupied by compressor: 42.')).toEqual({
systemMemorySwapTotalMB: null,
systemMemorySwapUsedMB: null,
systemMemoryCompressorMB: null,
systemMemoryPressureLevel: null,
systemMemoryStatusLevel: null
})
})
it('keeps only complete recent context and sanitizes complete names before truncating', () => {
const dir = mkdtempSync(join(tmpdir(), 'hang-context-'))
try {
const path = join(dir, 'trace.ndjson')
writeFileSync(
path,
`${JSON.stringify({
type: 'effect-span',
name: 'old-span',
startTimeUnixNano: '1000000',
endTimeUnixNano: '2000000'
})}\n${'x'.repeat(40_000)}\n${JSON.stringify({
type: 'effect-span',
name: `git.exec password=${'a'.repeat(200)}`,
startTimeUnixNano: '3000000',
endTimeUnixNano: '4000000',
attributes: { private: 'discard' }
})}\n{"type":"effect-span","name":"incomplete`
)
expect(readCompletedHangContext(path)).toEqual([
{ name: 'git.exec [redacted:labeled-kv]', startedAt: 3, completedAt: 4 }
])
} finally {
rmSync(dir, { recursive: true, force: true })
}
})
+267
View File
@@ -0,0 +1,267 @@
import {
MACHINE_FIELDS,
SNAPSHOT_FIELDS,
type HangPowerReceipt,
type CompletedHangContext,
type HangObservation,
type HangObservationCollection,
type HangDetectionMarker
} from './hang-observation-schema'
export { MACHINE_FIELDS, SNAPSHOT_FIELDS } from './hang-observation-schema'
export type {
HangSnapshot,
HangMachineSnapshot,
HangPowerReceipt,
CompletedHangContext,
HangObservation,
HangObservationCollection,
HangDetectionMarker
} from './hang-observation-schema'
function object(value: unknown): Record<string, unknown> | null {
return value !== null && typeof value === 'object' && !Array.isArray(value)
? (value as Record<string, unknown>)
: null
}
function finite(value: unknown): value is number {
return typeof value === 'number' && Number.isFinite(value) && value >= 0
}
function numericFields<K extends string>(
value: unknown,
keys: readonly K[]
): Record<K, number | null> | null {
const data = object(value)
if (!data) {
return null
}
return Object.fromEntries(
keys.map((key) => [key, finite(data[key]) ? data[key] : null])
) as Record<K, number | null>
}
export function parsePowerReceipt(value: unknown): HangPowerReceipt | null {
const data = object(value)
if (
!data ||
!['suspend', 'resume', 'lock-screen', 'unlock-screen'].includes(String(data.event)) ||
!finite(data.observedAt) ||
!finite(data.receivedAt)
) {
return null
}
return {
event: data.event as HangPowerReceipt['event'],
observedAt: data.observedAt,
receivedAt: data.receivedAt
}
}
export function parseLegacyHangMarker(value: unknown): HangDetectionMarker | null {
const data = object(value)
if (
!data ||
!finite(data.detectedAt) ||
!finite(data.parentPid) ||
!Number.isInteger(data.parentPid) ||
data.parentPid <= 0 ||
!finite(data.unresponsiveMs)
) {
return null
}
return {
detectedAt: data.detectedAt,
parentPid: data.parentPid,
unresponsiveMs: data.unresponsiveMs,
selfRecovered: data.selfRecovered === true
}
}
function parseObservation(value: unknown): HangObservation | null {
const data = object(value)
if (
!data ||
typeof data.id !== 'string' ||
!/^[\d:.-]{1,80}$/.test(data.id) ||
!['heartbeat-stall', 'watchdog-gap'].includes(String(data.kind)) ||
!finite(data.detectedAt) ||
!finite(data.parentPid) ||
!Number.isInteger(data.parentPid) ||
data.parentPid <= 0 ||
!finite(data.durationMs)
) {
return null
}
const completedSpans: CompletedHangContext[] = []
if (Array.isArray(data.completedSpans) && data.completedSpans.length <= 8) {
for (const item of data.completedSpans) {
const span = object(item)
if (
span &&
typeof span.name === 'string' &&
span.name.length <= 120 &&
finite(span.startedAt) &&
finite(span.completedAt) &&
span.startedAt <= span.completedAt
) {
completedSpans.push({
name: span.name,
startedAt: span.startedAt,
completedAt: span.completedAt
})
}
}
}
const gap = object(data.watchdogGap)
return {
id: data.id,
kind: data.kind as HangObservation['kind'],
detectedAt: data.detectedAt,
parentPid: data.parentPid,
durationMs: data.durationMs,
selfRecovered: data.selfRecovered === true,
initialSleepAssociated: data.initialSleepAssociated === true,
snapshot: numericFields(data.snapshot, SNAPSHOT_FIELDS),
resolution: numericFields(data.resolution, SNAPSHOT_FIELDS),
machine: numericFields(data.machine, MACHINE_FIELDS),
completedSpans: Array.isArray(data.completedSpans) ? completedSpans : null,
powerEvents: Array.isArray(data.powerEvents)
? data.powerEvents
.slice(-4)
.map(parsePowerReceipt)
.filter((v): v is HangPowerReceipt => v !== null)
: [],
...(gap && finite(gap.observedAt) && finite(gap.durationMs)
? {
watchdogGap: {
observedAt: gap.observedAt,
durationMs: gap.durationMs,
snapshot: numericFields(gap.snapshot, SNAPSHOT_FIELDS)
}
}
: {})
}
}
export function emptyHangObservations(): HangObservationCollection {
return { version: 1, observations: [], omitted: 0 }
}
export function asHangObservations(value: unknown): HangObservationCollection | null {
const data = object(value)
if (data?.version === 1) {
if (!Array.isArray(data.observations) || data.observations.length > 4) {
return null
}
return {
version: 1,
observations: data.observations
.map(parseObservation)
.filter((v): v is HangObservation => v !== null),
omitted: finite(data.omitted) ? Math.min(2_147_483_647, Math.floor(data.omitted)) : 0
}
}
const legacy = parseLegacyHangMarker(value)
if (!legacy) {
return null
}
return {
version: 1,
omitted: 0,
observations: [
{
id: `${legacy.parentPid}:${legacy.detectedAt}:0`,
kind: 'heartbeat-stall',
detectedAt: legacy.detectedAt,
parentPid: legacy.parentPid,
durationMs: legacy.unresponsiveMs,
selfRecovered: legacy.selfRecovered,
initialSleepAssociated: false,
snapshot: null,
resolution: null,
machine: null,
completedSpans: null,
powerEvents: []
}
]
}
}
function priority(record: HangObservation): number {
return record.kind === 'heartbeat-stall' ? 3 : record.initialSleepAssociated ? 1 : 2
}
export function retainHangObservation(
collection: HangObservationCollection,
record: HangObservation
): HangObservationCollection {
const existing = collection.observations.findIndex((item) => item.id === record.id)
if (existing !== -1) {
const old = collection.observations[existing]
const observations = collection.observations.slice()
const receipts = new Map(
[...old.powerEvents, ...record.powerEvents].map(
(item) => [`${item.event}:${item.observedAt}:${item.receivedAt}`, item] as const
)
)
observations[existing] = {
...record,
kind: old.kind,
id: old.id,
detectedAt: old.detectedAt,
parentPid: old.parentPid,
durationMs: old.durationMs,
initialSleepAssociated: old.initialSleepAssociated,
snapshot: old.snapshot,
selfRecovered: old.selfRecovered || record.selfRecovered,
resolution: old.resolution ?? record.resolution,
machine: old.machine ?? record.machine,
completedSpans: old.completedSpans ?? record.completedSpans,
powerEvents: [...receipts.values()].slice(-4),
watchdogGap:
old.watchdogGap && old.watchdogGap.durationMs > (record.watchdogGap?.durationMs ?? -1)
? old.watchdogGap
: (record.watchdogGap ?? old.watchdogGap)
}
return { ...collection, observations }
}
const candidates = [...collection.observations, record]
const anchors = new Set(
[
candidates.find((item) => item.kind === 'heartbeat-stall'),
candidates.find((item) => item.kind === 'watchdog-gap' && !item.initialSleepAssociated)
].filter((item): item is HangObservation => !!item)
)
const selected = new Set(anchors)
const ranked = candidates
.map((item, index) => ({ item, index }))
.filter(({ item }) => !anchors.has(item))
.sort((a, b) => priority(b.item) - priority(a.item) || b.index - a.index)
for (const { item } of ranked) {
if (selected.size < 4) {
selected.add(item)
}
}
return {
version: 1,
observations: candidates.filter((item) => selected.has(item)),
omitted: Math.min(2_147_483_647, collection.omitted + candidates.length - selected.size)
}
}
export function mergeHangObservations(
first: HangObservationCollection,
second: HangObservationCollection
): HangObservationCollection {
let result = { ...first, omitted: Math.min(2_147_483_647, first.omitted + second.omitted) }
for (const record of second.observations) {
result = retainHangObservation(result, record)
}
return result
}
export function latestHangMarker(
collection: HangObservationCollection
): HangDetectionMarker | null {
const record = collection.observations.findLast((item) => item.kind === 'heartbeat-stall')
return record
? {
detectedAt: record.detectedAt,
parentPid: record.parentPid,
unresponsiveMs:
record.durationMs +
Math.max(0, (record.resolution?.wallMs ?? record.detectedAt) - record.detectedAt),
selfRecovered: record.selfRecovered
}
: null
}
+23
View File
@@ -0,0 +1,23 @@
import { uptime } from 'node:os'
import type { HangSnapshot } from './hang-observations'
export function captureHangSnapshot(): HangSnapshot {
const cpu = process.cpuUsage()
let watchdog: NodeJS.CpuUsage | null = null
try {
const read = (process as NodeJS.Process & { threadCpuUsage?: () => NodeJS.CpuUsage })
.threadCpuUsage
watchdog = read?.call(process) ?? null
} catch {
// Some Electron runtimes do not expose per-thread CPU counters.
}
return {
wallMs: Date.now(),
monotonicMs: performance.now(),
uptimeSeconds: uptime(),
processUserUs: cpu.user,
processSystemUs: cpu.system,
watchdogUserUs: watchdog?.user ?? null,
watchdogSystemUs: watchdog?.system ?? null
}
}
@@ -5,6 +5,7 @@ export type HangWatchdogDetectionLoopConfig = {
onHangDetected: (unresponsiveMs: number) => void
/** Heartbeats resumed after a detected hang — the main thread was stalled, not deadlocked. */
onHangResolved: (unresponsiveMs: number) => void
onWatchdogGap?: (gapMs: number, startedAt: number) => void
}
export type HangWatchdogDetectionLoop = {
@@ -32,12 +33,17 @@ export function createHangWatchdogDetectionLoop(
const tickGap = now - lastTickAt
// Advance while hung so recovery cannot be misclassified as a long system-sleep gap.
lastTickAt = now
if (detected) {
// Sleep, process suspension and observer starvation are indistinguishable here.
if (tickGap > config.checkIntervalMs * 3) {
if (tickGap > config.timeoutMs) {
config.onWatchdogGap?.(tickGap, now - tickGap)
}
if (!detected) {
lastHeartbeatAt = now
}
return
}
// System sleep suspends the watchdog too; restart the silence window after large tick gaps.
if (tickGap > config.checkIntervalMs * 3) {
lastHeartbeatAt = now
if (detected) {
return
}
const unresponsiveMs = now - lastHeartbeatAt
@@ -7,6 +7,11 @@ export type HangWatchdogWorkerData = {
markerPath: string
timeoutMs: number
checkIntervalMs: number
tracePath?: string
}
export type MainToHangWatchdogWorkerMessage = { type: 'heartbeat' } | { type: 'shutdown' }
export type MainToHangWatchdogWorkerMessage =
| { type: 'heartbeat' }
| { type: 'shutdown' }
| { type: 'power-event'; event: HangPowerReceipt['event']; observedAt: number }
import type { HangPowerReceipt } from './hang-observations'
@@ -1,6 +1,16 @@
import { isMainThread, parentPort, workerData } from 'node:worker_threads'
import { createHangWatchdogDetectionLoop } from './hang-watchdog-detection-loop'
import { writeHangDetectionMarker } from './hang-detection-marker'
import { readAvailableHangDetectionMarker, writeHangDetectionMarker } from './hang-detection-marker'
import {
asHangObservations,
emptyHangObservations,
mergeHangObservations,
parsePowerReceipt,
retainHangObservation,
type HangObservation,
type HangPowerReceipt
} from './hang-observations'
import { captureHangSnapshot } from './hang-snapshot'
import type {
HangWatchdogWorkerData,
MainToHangWatchdogWorkerMessage
@@ -11,28 +21,6 @@ type HangWatchdogPort = {
close: () => void
}
// Observation only: a false positive must never kill a live main thread mid-write.
export function recordHangObservation(options: {
parentPid: number
markerPath: string
unresponsiveMs: number
selfRecovered: boolean
}): void {
if (!options.markerPath) {
return
}
try {
writeHangDetectionMarker(options.markerPath, {
detectedAt: Date.now(),
parentPid: options.parentPid,
unresponsiveMs: options.unresponsiveMs,
selfRecovered: options.selfRecovered
})
} catch {
// Diagnostic write failures must not terminate the watchdog.
}
}
export function runWatchdog(
config: HangWatchdogWorkerData,
port: HangWatchdogPort | null = parentPort
@@ -40,39 +28,187 @@ export function runWatchdog(
if (!port) {
return
}
const initial = readAvailableHangDetectionMarker(config.markerPath)
let observations = asHangObservations(initial) ?? emptyHangObservations()
let existingUnavailable = initial === undefined
let sequence = 0
let activeStallId: string | null = null
let stopped = false
let collector: AbortController | null = null
let recentPower: HangPowerReceipt[] = []
let sleepPeriod: { suspend: HangPowerReceipt; resumeAt: number | null } | null = null
let receiptWindow: { id: string; until: number; remaining: number } | null = null
const save = (): void => {
if (stopped || !config.markerPath) {
return
}
try {
if (existingUnavailable) {
const existing = readAvailableHangDetectionMarker(config.markerPath)
if (existing === undefined) {
return
}
observations = mergeHangObservations(
asHangObservations(existing) ?? emptyHangObservations(),
observations
)
existingUnavailable = false
}
writeHangDetectionMarker(config.markerPath, observations)
} catch {
// Optional diagnostics must not terminate the independent observer.
}
}
const find = (id: string): HangObservation | undefined =>
observations.observations.find((item) => item.id === id)
const update = (id: string, change: Partial<HangObservation>): void => {
const record = find(id)
if (!record || stopped) {
return
}
observations = retainHangObservation(observations, { ...record, ...change })
save()
}
const openReceiptWindow = (id: string): void => {
receiptWindow = { id, until: Date.now() + config.checkIntervalMs, remaining: 4 }
}
const enrich = (record: HangObservation): void => {
if (!find(record.id)) {
return
}
if (collector) {
collector.abort()
return
}
const controller = new AbortController()
collector = controller
void import('./hang-incident-snapshot')
.then(async (incident) => {
if (controller.signal.aborted || stopped) {
return
}
if (record.kind === 'heartbeat-stall') {
update(record.id, { completedSpans: incident.readCompletedHangContext(config.tracePath) })
}
const machine = await incident.collectHangMachineSnapshot(controller.signal)
if (!controller.signal.aborted && !stopped) {
update(record.id, { machine })
}
})
.catch(() => {
// The durable baseline already records unavailable optional evidence as unknown.
})
.finally(() => {
if (collector === controller) {
collector = null
}
})
}
const observe = (kind: HangObservation['kind'], durationMs: number, startedAt: number): void => {
const snapshot = captureHangSnapshot()
const now = snapshot.wallMs ?? Date.now()
const suspend = sleepPeriod?.suspend
const initialSleepAssociated =
kind === 'watchdog-gap' &&
!!suspend &&
suspend.observedAt <= startedAt + config.checkIntervalMs &&
suspend.receivedAt <= startedAt + config.checkIntervalMs &&
suspend.receivedAt < now - config.checkIntervalMs &&
suspend.receivedAt >= suspend.observedAt &&
suspend.receivedAt - suspend.observedAt <= config.checkIntervalMs &&
(sleepPeriod?.resumeAt ?? Infinity) > startedAt
let powerEvents = recentPower
.filter((item) => item.observedAt >= startedAt && item.receivedAt <= now)
.slice(-4)
if (initialSleepAssociated && suspend) {
powerEvents = [suspend, ...powerEvents.filter((item) => item !== suspend).slice(-3)]
}
const record: HangObservation = {
id: `${config.parentPid}:${now}:${++sequence}`,
kind,
detectedAt: now,
parentPid: config.parentPid,
durationMs,
selfRecovered: false,
snapshot,
resolution: null,
machine: null,
completedSpans: null,
powerEvents,
initialSleepAssociated
}
observations = retainHangObservation(observations, record)
if (kind === 'heartbeat-stall') {
activeStallId = record.id
}
openReceiptWindow(record.id)
save()
enrich(record)
}
const loop = createHangWatchdogDetectionLoop({
timeoutMs: config.timeoutMs,
checkIntervalMs: config.checkIntervalMs,
now: () => Date.now(),
onHangDetected: (unresponsiveMs) =>
recordHangObservation({
parentPid: config.parentPid,
markerPath: config.markerPath,
unresponsiveMs,
selfRecovered: false
}),
// Update one marker per stall so resolution does not require reconciling a second record.
onHangResolved: (unresponsiveMs) =>
recordHangObservation({
parentPid: config.parentPid,
markerPath: config.markerPath,
unresponsiveMs,
selfRecovered: true
})
onHangDetected: (durationMs) => observe('heartbeat-stall', durationMs, Date.now() - durationMs),
onHangResolved: () => {
if (!activeStallId) {
return
}
const id = activeStallId
activeStallId = null
update(id, { selfRecovered: true, resolution: captureHangSnapshot() })
openReceiptWindow(id)
},
onWatchdogGap: (durationMs, startedAt) => {
const overlapping = observations.observations.findLast(
(item) =>
item.kind === 'heartbeat-stall' &&
(item.id === activeStallId || (item.resolution?.wallMs ?? -1) >= startedAt)
)
if (overlapping) {
update(overlapping.id, {
watchdogGap: { observedAt: Date.now(), durationMs, snapshot: captureHangSnapshot() }
})
openReceiptWindow(overlapping.id)
} else {
observe('watchdog-gap', durationMs, startedAt)
}
}
})
let checkTimer: ReturnType<typeof setInterval> | null = setInterval(
() => loop.tick(),
config.checkIntervalMs
)
port.on('message', (message: MainToHangWatchdogWorkerMessage) => {
const checkTimer = setInterval(() => loop.tick(), config.checkIntervalMs)
port.on('message', (message) => {
if (stopped) {
return
}
if (message.type === 'heartbeat') {
loop.recordHeartbeat()
} else if (message.type === 'shutdown') {
if (checkTimer) {
clearInterval(checkTimer)
checkTimer = null
} else if (message.type === 'power-event') {
const receipt = parsePowerReceipt({ ...message, receivedAt: Date.now() })
if (!receipt) {
return
}
recentPower = [...recentPower, receipt].slice(-4)
if (receipt.event === 'suspend') {
sleepPeriod = { suspend: receipt, resumeAt: null }
} else if (receipt.event === 'resume' && sleepPeriod) {
sleepPeriod.resumeAt = receipt.observedAt
}
if (
receiptWindow &&
receiptWindow.remaining > 0 &&
receipt.receivedAt <= receiptWindow.until
) {
const record = find(receiptWindow.id)
if (record) {
receiptWindow.remaining--
update(record.id, { powerEvents: [...record.powerEvents, receipt].slice(-4) })
}
}
} else if (message.type === 'shutdown') {
stopped = true
receiptWindow = null
clearInterval(checkTimer)
collector?.abort()
port.close()
}
})
@@ -85,6 +221,7 @@ export function isHangWatchdogWorkerData(value: unknown): value is HangWatchdogW
Number.isInteger(data.parentPid) &&
(data.parentPid ?? 0) > 0 &&
typeof data.markerPath === 'string' &&
(data.tracePath === undefined || typeof data.tracePath === 'string') &&
Number.isFinite(data.timeoutMs) &&
(data.timeoutMs ?? 0) > 0 &&
Number.isFinite(data.checkIntervalMs) &&
@@ -1,9 +1,9 @@
import type { Worker } from 'node:worker_threads'
import { spawnWorkerThread } from '../worker-threads/worker-thread-spawn'
import { app } from 'electron'
import { app, powerMonitor } from 'electron'
import { hangDetectionMarkerPath } from './hang-detection-marker'
import { resolveHangWatchdogWorkerPath } from './hang-watchdog-worker-path'
import { resolveObservabilityConsent } from '../observability'
import { getTraceFilePath, resolveObservabilityConsent } from '../observability'
import {
HANG_WATCHDOG_CHECK_INTERVAL_MS,
HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS,
@@ -39,6 +39,7 @@ export function installMainThreadHangWatchdog(options: {
const workerData: HangWatchdogWorkerData = {
parentPid: process.pid,
markerPath: hangDetectionMarkerPath(options.userDataPath),
tracePath: getTraceFilePath(),
timeoutMs: positiveTiming(process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS, HANG_WATCHDOG_TIMEOUT_MS),
checkIntervalMs: positiveTiming(
process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS,
@@ -57,7 +58,7 @@ export function installMainThreadHangWatchdog(options: {
}
let stopped = false
const postMessage = (message: MainToHangWatchdogWorkerMessage): void => {
if (stopped && message.type === 'heartbeat') {
if (stopped && message.type !== 'shutdown') {
return
}
try {
@@ -66,6 +67,19 @@ export function installMainThreadHangWatchdog(options: {
// The worker already exited.
}
}
const powerListeners = (['suspend', 'resume', 'lock-screen', 'unlock-screen'] as const).map(
(event) => {
const listener = (): void =>
postMessage({ type: 'power-event', event, observedAt: Date.now() })
powerMonitor.on(event, listener)
return { event, listener }
}
)
const removePowerListeners = (): void => {
for (const { event, listener } of powerListeners) {
powerMonitor.off(event, listener)
}
}
const heartbeatTimer = setInterval(() => {
postMessage({ type: 'heartbeat' })
}, HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS)
@@ -76,6 +90,7 @@ export function installMainThreadHangWatchdog(options: {
stopped = true
// Drop the app listener immediately so retired workers are not retained by its closure.
app.off('will-quit', stop)
removePowerListeners()
clearInterval(heartbeatTimer)
postMessage({ type: 'shutdown' })
}
@@ -86,6 +101,7 @@ export function installMainThreadHangWatchdog(options: {
worker.once('exit', () => {
stopped = true
app.off('will-quit', stop)
removePowerListeners()
clearInterval(heartbeatTimer)
})
worker.unref()
@@ -3,6 +3,13 @@ import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { setActiveSink } from '../observability/tracer'
import { createLocalFileSink } from '../observability/local-file-sink'
import { collectBundle } from '../observability/bundle'
import {
emptyHangObservations,
retainHangObservation,
type HangObservation
} from './hang-observations'
import { hangDetectionMarkerPath, writeHangDetectionMarker } from './hang-detection-marker'
import {
preservePreviousHangDetection,
@@ -41,6 +48,28 @@ const marker = {
unresponsiveMs: 10_000,
selfRecovered: true
}
const gap: HangObservation = {
id: '789:200:1',
kind: 'watchdog-gap',
detectedAt: 200,
parentPid: 789,
durationMs: 162_000,
selfRecovered: false,
initialSleepAssociated: false,
snapshot: {
wallMs: 200,
monotonicMs: 100,
uptimeSeconds: 1000,
processUserUs: 500,
processSystemUs: 200,
watchdogUserUs: null,
watchdogSystemUs: null
},
resolution: null,
machine: null,
completedSpans: null,
powerEvents: []
}
describe('previous hang detection', () => {
it('retains the previous run across startup and writes it only after a sink exists', () => {
@@ -52,8 +81,7 @@ describe('previous hang detection', () => {
expect(existsSync(`${markerPath}.previous`)).toBe(true)
writeHangDetectionMarker(markerPath, { ...marker, parentPid: 789 })
const records: unknown[] = []
const flush = vi.fn()
setActiveSink({ push: (record) => records.push(record), flush, close() {} })
setActiveSink({ push: (record) => records.push(record), flush() {}, close() {} })
expect(reportPreviousHangDetection(dir)).toEqual(marker)
expect(records).toEqual([
expect.objectContaining({
@@ -65,7 +93,6 @@ describe('previous hang detection', () => {
})
})
])
expect(flush).toHaveBeenCalledOnce()
expect(existsSync(`${markerPath}.previous`)).toBe(false)
expect(JSON.parse(readFileSync(markerPath, 'utf8')).parentPid).toBe(789)
expect(reportPreviousHangDetection(dir)).toBeNull()
@@ -75,12 +102,112 @@ describe('previous hang detection', () => {
const markerPath = hangDetectionMarkerPath(dir)
writeHangDetectionMarker(markerPath, marker)
preservePreviousHangDetection(dir)
writeHangDetectionMarker(markerPath, retainHangObservation(emptyHangObservations(), gap))
preservePreviousHangDetection(dir)
expect(existsSync(`${markerPath}.previous`)).toBe(true)
setActiveSink({ push() {}, flush() {}, close() {} })
expect(reportPreviousHangDetection(dir)).toEqual(marker)
})
it('carries gap and overlapping-stall evidence through the real trace and bundle', () => {
const traceFilePath = join(dir, 'logs', 'main.trace.ndjson')
const sink = createLocalFileSink({ filePath: traceFilePath, maxFiles: 1 })
try {
writeHangDetectionMarker(
hangDetectionMarkerPath(dir),
retainHangObservation(emptyHangObservations(), gap)
)
preservePreviousHangDetection(dir)
setActiveSink(sink)
expect(reportPreviousHangDetection(dir)).toBeNull()
const stall: HangObservation = {
...gap,
id: '789:300:2',
kind: 'heartbeat-stall',
detectedAt: 300,
durationMs: 45_000,
selfRecovered: true,
snapshot: { ...gap.snapshot!, wallMs: 300, processUserUs: 100 },
resolution: { ...gap.snapshot!, wallMs: 500, processUserUs: 900 },
watchdogGap: {
observedAt: 400,
durationMs: 162_000,
snapshot: { ...gap.snapshot!, wallMs: 400, processUserUs: 600, watchdogUserUs: 3 }
},
machine: {
startedAt: 301,
completedAt: 302,
systemMemoryTotalMB: 64_000,
systemMemoryFreeMB: 8_000,
systemMemorySwapTotalMB: 1024,
systemMemorySwapUsedMB: 100,
systemMemoryCompressorMB: 500,
systemMemoryPressureLevel: 2,
systemMemoryStatusLevel: 40,
load1: 2,
load5: 3,
load15: 4
}
}
writeHangDetectionMarker(
hangDetectionMarkerPath(dir),
retainHangObservation(emptyHangObservations(), stall)
)
preservePreviousHangDetection(dir)
expect(reportPreviousHangDetection(dir)).toEqual({
detectedAt: 300,
parentPid: 789,
unresponsiveMs: 45_200,
selfRecovered: true
})
const bundle = collectBundle({
traceFilePath,
maxFiles: 1,
appVersion: 'test',
platform: 'darwin',
arch: 'arm64',
osRelease: 'test',
orcaChannel: 'dev'
})
const records = bundle.payload
.trim()
.split('\n')
.map((line) => JSON.parse(line))
.filter((record) => record.type === 'effect-span')
const gapRecord = records.find(
(record) => record.attributes['breadcrumb.name'] === 'main_thread_watchdog_gap'
)
const stallRecord = records.find(
(record) => record.attributes['breadcrumb.name'] === 'main_thread_hang_detected'
)
expect(gapRecord?.attributes['breadcrumb.data']).toMatchObject({
watchdogGapMs: 162_000,
cause: 'unknown',
activeMainActivity: 'unknown',
detection_processUserUs: 500,
preGapCpuDeltaUs: null,
watchdogCpuDetectionToResolutionUs: null,
machineEvidence: 'unknown'
})
expect(stallRecord?.attributes['breadcrumb.data']).toMatchObject({
detection_wallMs: 300,
detection_processUserUs: 100,
gapEnd_wallMs: 400,
gapEnd_processUserUs: 600,
gapEnd_watchdogUserUs: 3,
resolution_wallMs: 500,
resolution_processUserUs: 900,
machine_startedAt: 301,
machine_completedAt: 302,
machine_systemMemorySwapUsedMB: 100
})
expect(stallRecord?.attributes['breadcrumb.data']).not.toHaveProperty('startedAt')
} finally {
setActiveSink(null)
sink.close()
}
})
it('does not mutate marker files when local diagnostics are disabled', () => {
const markerPath = hangDetectionMarkerPath(dir)
writeHangDetectionMarker(markerPath, marker)
+110 -11
View File
@@ -1,12 +1,22 @@
import { renameSync } from 'node:fs'
import { rmSync } from 'node:fs'
import type { CrashReportBreadcrumbData } from '../../shared/crash-reporting'
import { resolveObservabilityConsent } from '../observability'
import { hasActiveTraceSink } from '../observability/tracer'
import { recordDurableCrashBreadcrumb } from '../crash-reporting/durable-crash-breadcrumb'
import {
consumeHangDetectionMarker,
hangDetectionMarkerPath,
readAvailableHangDetectionMarker,
writeHangDetectionMarker,
type HangDetectionMarker
} from './hang-detection-marker'
import {
asHangObservations,
emptyHangObservations,
latestHangMarker,
mergeHangObservations,
type HangObservation
} from './hang-observations'
function previousMarkerPath(userDataPath: string): string {
return `${hangDetectionMarkerPath(userDataPath)}.previous`
@@ -17,24 +27,113 @@ export function preservePreviousHangDetection(userDataPath: string): void {
return
}
try {
// Keep the previous run's note safe from a new startup hang until its trace sink opens.
renameSync(hangDetectionMarkerPath(userDataPath), previousMarkerPath(userDataPath))
const currentPath = hangDetectionMarkerPath(userDataPath)
const pendingPath = previousMarkerPath(userDataPath)
const current = asHangObservations(readAvailableHangDetectionMarker(currentPath))
if (!current) {
return
}
const pending = readAvailableHangDetectionMarker(pendingPath)
if (pending === undefined) {
return
}
const merged = mergeHangObservations(
asHangObservations(pending) ?? emptyHangObservations(),
current
)
writeHangDetectionMarker(pendingPath, merged)
rmSync(currentPath, { force: true })
} catch {
// Missing or inaccessible markers must not prevent startup; retain any earlier pending note.
}
}
function observationDetails(record: HangObservation, omitted: number): CrashReportBreadcrumbData {
const details: CrashReportBreadcrumbData = {
previousPid: record.parentPid,
detectedAt: record.detectedAt,
observationKind: record.kind,
omittedObservations: omitted,
activeMainActivity: 'unknown',
cause: 'unknown',
cpuScope: 'process-all-threads',
machineEvidence: record.machine ? 'sampled' : 'unknown'
}
if (record.kind === 'heartbeat-stall') {
details.unresponsiveMs =
latestHangMarker({ version: 1, observations: [record], omitted: 0 })?.unresponsiveMs ??
record.durationMs
details.detectedUnresponsiveMs = record.durationMs
details.selfRecovered = record.selfRecovered
} else {
details.watchdogGapMs = record.durationMs
details.preGapCpuDeltaUs = null
}
for (const [prefix, snapshot] of [
['detection', record.snapshot],
['resolution', record.resolution],
['gapEnd', record.watchdogGap?.snapshot]
] as const) {
if (snapshot) {
for (const [key, value] of Object.entries(snapshot)) {
details[`${prefix}_${key}`] = value
}
}
}
const before = record.snapshot
const after = record.resolution
for (const scope of ['process', 'watchdog'] as const) {
const firstUser = before?.[`${scope}UserUs`]
const firstSystem = before?.[`${scope}SystemUs`]
const lastUser = after?.[`${scope}UserUs`]
const lastSystem = after?.[`${scope}SystemUs`]
const delta =
typeof firstUser === 'number' &&
typeof firstSystem === 'number' &&
typeof lastUser === 'number' &&
typeof lastSystem === 'number'
? lastUser + lastSystem - firstUser - firstSystem
: null
details[`${scope}CpuDetectionToResolutionUs`] = delta !== null && delta >= 0 ? delta : null
}
if (record.machine) {
for (const [key, value] of Object.entries(record.machine)) {
details[`machine_${key}`] = value
}
}
if (record.watchdogGap) {
details.overlappingWatchdogGapMs = record.watchdogGap.durationMs
details.overlappingWatchdogGapAt = record.watchdogGap.observedAt
}
for (const [index, receipt] of record.powerEvents.entries()) {
details[`power${index}_event`] = receipt.event
details[`power${index}_mainReceiptAt`] = receipt.observedAt
details[`power${index}_watchdogReceiptAt`] = receipt.receivedAt
}
details.completedContextEvidence =
record.completedSpans === null ? 'unknown' : 'completed-spans-on-disk'
for (const [index, span] of (record.completedSpans ?? []).entries()) {
details[`completed${index}_name`] = span.name
details[`completed${index}_startedAt`] = span.startedAt
details[`completed${index}_endedAt`] = span.completedAt
}
return details
}
export function reportPreviousHangDetection(userDataPath: string): HangDetectionMarker | null {
if (!hasActiveTraceSink()) {
if (!resolveObservabilityConsent().localFileEnabled || !hasActiveTraceSink()) {
return null
}
const marker = consumeHangDetectionMarker(previousMarkerPath(userDataPath))
if (marker) {
recordDurableCrashBreadcrumb('main_thread_hang_detected', {
unresponsiveMs: marker.unresponsiveMs,
previousPid: marker.parentPid,
selfRecovered: marker.selfRecovered
})
const collection = asHangObservations(marker)
if (!collection) {
return null
}
return marker
for (const record of collection.observations) {
recordDurableCrashBreadcrumb(
record.kind === 'heartbeat-stall' ? 'main_thread_hang_detected' : 'main_thread_watchdog_gap',
observationDetails(record, collection.omitted)
)
}
return latestHangMarker(collection)
}