mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
Preserve bounded diagnostics for main-thread stalls and observer gaps
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync, existsSync, readFileSync, statSync } from 'node:fs'
|
||||
import { mkdtempSync, rmSync, existsSync, readFileSync, statSync, renameSync } from 'node:fs'
|
||||
import type * as FileSystem from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
@@ -14,7 +14,8 @@ vi.mock('node:fs', async (importOriginal) => {
|
||||
return {
|
||||
...actual,
|
||||
readFileSync: vi.fn(actual.readFileSync),
|
||||
statSync: vi.fn(actual.statSync)
|
||||
statSync: vi.fn(actual.statSync),
|
||||
renameSync: vi.fn(actual.renameSync)
|
||||
}
|
||||
})
|
||||
|
||||
@@ -29,6 +30,19 @@ describe('hang detection marker', () => {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
it('retains the complete baseline if publishing enrichment is interrupted', () => {
|
||||
const markerPath = hangDetectionMarkerPath(dir)
|
||||
const baseline = { detectedAt: 1, parentPid: 2, unresponsiveMs: 45_000, selfRecovered: false }
|
||||
writeHangDetectionMarker(markerPath, baseline)
|
||||
vi.mocked(renameSync).mockImplementationOnce(() => {
|
||||
throw new Error('interrupted replacement')
|
||||
})
|
||||
expect(() =>
|
||||
writeHangDetectionMarker(markerPath, { ...baseline, selfRecovered: true })
|
||||
).toThrow()
|
||||
expect(consumeHangDetectionMarker(markerPath)).toEqual(baseline)
|
||||
})
|
||||
|
||||
it.each(['stat', 'read'])('retains a real marker after a transient %s failure', (operation) => {
|
||||
const markerPath = hangDetectionMarkerPath(dir)
|
||||
const marker = { detectedAt: 1, parentPid: 2, unresponsiveMs: 45000, selfRecovered: false }
|
||||
@@ -46,13 +60,4 @@ describe('hang detection marker', () => {
|
||||
expect(consumeHangDetectionMarker(markerPath)).toEqual(marker)
|
||||
expect(existsSync(markerPath)).toBe(false)
|
||||
})
|
||||
|
||||
it('deletes an oversized marker without reading its contents', () => {
|
||||
const markerPath = hangDetectionMarkerPath(dir)
|
||||
writeFileSync(markerPath, 'x'.repeat(64 * 1024 + 1))
|
||||
vi.mocked(readFileSync).mockClear()
|
||||
expect(consumeHangDetectionMarker(markerPath)).toBeNull()
|
||||
expect(readFileSync).not.toHaveBeenCalled()
|
||||
expect(existsSync(markerPath)).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,66 +1,79 @@
|
||||
import { readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'
|
||||
import {
|
||||
closeSync,
|
||||
fchmodSync,
|
||||
openSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import {
|
||||
asHangObservations,
|
||||
latestHangMarker,
|
||||
parseLegacyHangMarker,
|
||||
type HangDetectionMarker,
|
||||
type HangObservationCollection
|
||||
} from './hang-observations'
|
||||
|
||||
// The next launch consumes detection/resolution markers; selfRecovered distinguishes cleared stalls from unresolved hangs.
|
||||
export type HangDetectionMarker = {
|
||||
detectedAt: number
|
||||
parentPid: number
|
||||
unresponsiveMs: number
|
||||
selfRecovered: boolean
|
||||
}
|
||||
export type { HangDetectionMarker } from './hang-observations'
|
||||
export type StoredHangMarker = HangDetectionMarker | HangObservationCollection
|
||||
|
||||
export function hangDetectionMarkerPath(userDataPath: string): string {
|
||||
return join(userDataPath, 'main-thread-hang.json')
|
||||
}
|
||||
|
||||
export function writeHangDetectionMarker(markerPath: string, marker: HangDetectionMarker): void {
|
||||
writeFileSync(markerPath, JSON.stringify(marker), { mode: 0o600 })
|
||||
export function writeHangDetectionMarker(markerPath: string, marker: StoredHangMarker): void {
|
||||
const collection = 'version' in marker ? asHangObservations(marker) : null
|
||||
if (
|
||||
'version' in marker &&
|
||||
(!collection || collection.observations.length !== marker.observations.length)
|
||||
) {
|
||||
throw new Error('Invalid hang observations')
|
||||
}
|
||||
const text = JSON.stringify(
|
||||
collection ? { ...latestHangMarker(collection), ...collection } : marker
|
||||
)
|
||||
if (Buffer.byteLength(text) > 32 * 1024) {
|
||||
throw new Error('Hang observations exceed byte bound')
|
||||
}
|
||||
const temporaryPath = `${markerPath}.tmp`
|
||||
const fd = openSync(temporaryPath, 'w', 0o600)
|
||||
try {
|
||||
fchmodSync(fd, 0o600)
|
||||
writeFileSync(fd, text)
|
||||
} finally {
|
||||
closeSync(fd)
|
||||
}
|
||||
renameSync(temporaryPath, markerPath)
|
||||
}
|
||||
|
||||
function readAvailableHangDetectionMarker(
|
||||
export function readAvailableHangDetectionMarker(
|
||||
markerPath: string
|
||||
): HangDetectionMarker | null | undefined {
|
||||
): StoredHangMarker | null | undefined {
|
||||
let raw: string
|
||||
try {
|
||||
if (statSync(markerPath).size > 64 * 1024) {
|
||||
return null
|
||||
}
|
||||
raw = readFileSync(markerPath, 'utf8')
|
||||
} catch {
|
||||
return undefined
|
||||
} catch (error) {
|
||||
return (error as NodeJS.ErrnoException).code === 'ENOENT' ? null : undefined
|
||||
}
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw)
|
||||
if (
|
||||
typeof parsed !== 'object' ||
|
||||
!parsed ||
|
||||
!('detectedAt' in parsed) ||
|
||||
typeof parsed.detectedAt !== 'number' ||
|
||||
!Number.isFinite(parsed.detectedAt) ||
|
||||
!('parentPid' in parsed) ||
|
||||
typeof parsed.parentPid !== 'number' ||
|
||||
!Number.isInteger(parsed.parentPid) ||
|
||||
parsed.parentPid <= 0 ||
|
||||
!('unresponsiveMs' in parsed) ||
|
||||
typeof parsed.unresponsiveMs !== 'number' ||
|
||||
!Number.isFinite(parsed.unresponsiveMs) ||
|
||||
parsed.unresponsiveMs < 0
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
detectedAt: parsed.detectedAt,
|
||||
parentPid: parsed.parentPid,
|
||||
unresponsiveMs: parsed.unresponsiveMs,
|
||||
// Missing recovery evidence leaves the stall unresolved.
|
||||
selfRecovered: 'selfRecovered' in parsed && parsed.selfRecovered === true
|
||||
if (typeof parsed === 'object' && parsed && 'version' in parsed && parsed.version === 1) {
|
||||
const collection = Buffer.byteLength(raw) <= 32 * 1024 ? asHangObservations(parsed) : null
|
||||
return collection ? { ...latestHangMarker(collection), ...collection } : null
|
||||
}
|
||||
return parseLegacyHangMarker(parsed)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export function consumeHangDetectionMarker(markerPath: string): HangDetectionMarker | null {
|
||||
export function consumeHangDetectionMarker(markerPath: string): StoredHangMarker | null {
|
||||
const marker = readAvailableHangDetectionMarker(markerPath)
|
||||
// Retry unavailable reads next startup; only successfully inspected files are consumable.
|
||||
if (marker === undefined) {
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import { closeSync, fstatSync, openSync, readSync } from 'node:fs'
|
||||
import { freemem, loadavg, totalmem } from 'node:os'
|
||||
import { sanitizeCrashReportString } from '../../shared/crash-report-redaction'
|
||||
import { redactString } from '../observability/redactor'
|
||||
import type { CompletedHangContext, HangMachineSnapshot } from './hang-observations'
|
||||
|
||||
function numberMatch(text: string, expression: RegExp): number | null {
|
||||
const match = expression.exec(text)
|
||||
const value = match ? Number(match[1]) : Number.NaN
|
||||
return Number.isFinite(value) && value >= 0 ? value : null
|
||||
}
|
||||
export function parseMacHangPressure(
|
||||
sysctl: string,
|
||||
vmStat: string
|
||||
): Pick<
|
||||
HangMachineSnapshot,
|
||||
| 'systemMemorySwapTotalMB'
|
||||
| 'systemMemorySwapUsedMB'
|
||||
| 'systemMemoryCompressorMB'
|
||||
| 'systemMemoryPressureLevel'
|
||||
| 'systemMemoryStatusLevel'
|
||||
> {
|
||||
const swap = (key: string): number | null => {
|
||||
const match = new RegExp(`\\b${key} = ([\\d.]+)([KMGT])\\b`).exec(sysctl)
|
||||
if (!match) {
|
||||
return null
|
||||
}
|
||||
const factor = { K: 1 / 1024, M: 1, G: 1024, T: 1024 * 1024 }[match[2]]
|
||||
const value = Number(match[1]) * (factor ?? Number.NaN)
|
||||
return Number.isFinite(value) && value >= 0 ? value : null
|
||||
}
|
||||
const pageSize = numberMatch(vmStat, /page size of (\d+) bytes/)
|
||||
const compressor = numberMatch(vmStat, /^Pages occupied by compressor:\s+(\d+)\.?\s*$/m)
|
||||
const compressorMB =
|
||||
pageSize !== null && compressor !== null ? (pageSize * compressor) / 1024 / 1024 : null
|
||||
return {
|
||||
systemMemorySwapTotalMB: swap('total'),
|
||||
systemMemorySwapUsedMB: swap('used'),
|
||||
systemMemoryCompressorMB:
|
||||
compressorMB !== null && Number.isFinite(compressorMB) ? compressorMB : null,
|
||||
systemMemoryPressureLevel: numberMatch(
|
||||
sysctl,
|
||||
/^kern\.memorystatus_vm_pressure_level:\s+(\d+)\s*$/m
|
||||
),
|
||||
systemMemoryStatusLevel: numberMatch(sysctl, /^kern\.memorystatus_level:\s+(\d+)\s*$/m)
|
||||
}
|
||||
}
|
||||
|
||||
export async function collectHangMachineSnapshot(
|
||||
signal: AbortSignal
|
||||
): Promise<HangMachineSnapshot> {
|
||||
const loads = loadavg()
|
||||
const snapshot: HangMachineSnapshot = {
|
||||
startedAt: Date.now(),
|
||||
completedAt: null,
|
||||
systemMemoryTotalMB: totalmem() / 1024 / 1024,
|
||||
systemMemoryFreeMB: freemem() / 1024 / 1024,
|
||||
systemMemorySwapTotalMB: null,
|
||||
systemMemorySwapUsedMB: null,
|
||||
systemMemoryCompressorMB: null,
|
||||
systemMemoryPressureLevel: null,
|
||||
systemMemoryStatusLevel: null,
|
||||
load1: process.platform === 'win32' ? null : loads[0],
|
||||
load5: process.platform === 'win32' ? null : loads[1],
|
||||
load15: process.platform === 'win32' ? null : loads[2]
|
||||
}
|
||||
if (process.platform === 'darwin' && !signal.aborted) {
|
||||
const { runProcess } = await import('@orca/process-host')
|
||||
const commands = await Promise.allSettled([
|
||||
runProcess({
|
||||
program: '/usr/sbin/sysctl',
|
||||
args: ['vm.swapusage', 'kern.memorystatus_level', 'kern.memorystatus_vm_pressure_level'],
|
||||
timeoutMs: 2_000,
|
||||
maxOutputBytes: 32 * 1024,
|
||||
signal
|
||||
}),
|
||||
runProcess({
|
||||
program: '/usr/bin/vm_stat',
|
||||
timeoutMs: 2_000,
|
||||
maxOutputBytes: 32 * 1024,
|
||||
signal
|
||||
})
|
||||
])
|
||||
const outputs = commands.map((result) =>
|
||||
result.status === 'fulfilled' && !result.value.timedOut && !result.value.outputTruncated
|
||||
? result.value.stdout
|
||||
: ''
|
||||
)
|
||||
Object.assign(snapshot, parseMacHangPressure(outputs[0], outputs[1]))
|
||||
}
|
||||
snapshot.completedAt = Date.now()
|
||||
return snapshot
|
||||
}
|
||||
|
||||
export function readCompletedHangContext(
|
||||
tracePath: string | undefined
|
||||
): CompletedHangContext[] | null {
|
||||
if (!tracePath) {
|
||||
return null
|
||||
}
|
||||
let fd: number | undefined
|
||||
try {
|
||||
fd = openSync(tracePath, 'r')
|
||||
const size = fstatSync(fd).size
|
||||
const offset = Math.max(0, size - 32 * 1024)
|
||||
const buffer = Buffer.allocUnsafe(Math.min(size, 32 * 1024))
|
||||
const bytes = readSync(fd, buffer, 0, buffer.length, offset)
|
||||
const lines = buffer.subarray(0, bytes).toString('utf8').split('\n')
|
||||
lines.pop()
|
||||
if (offset > 0) {
|
||||
lines.shift()
|
||||
}
|
||||
const spans: CompletedHangContext[] = []
|
||||
for (let index = lines.length - 1; index >= 0 && spans.length < 8; index--) {
|
||||
try {
|
||||
const data = JSON.parse(lines[index])
|
||||
if (
|
||||
data?.type !== 'effect-span' ||
|
||||
typeof data.name !== 'string' ||
|
||||
typeof data.startTimeUnixNano !== 'string' ||
|
||||
!/^\d{1,24}$/.test(data.startTimeUnixNano) ||
|
||||
typeof data.endTimeUnixNano !== 'string' ||
|
||||
!/^\d{1,24}$/.test(data.endTimeUnixNano)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
const startedAt = Number(data.startTimeUnixNano) / 1_000_000
|
||||
const completedAt = Number(data.endTimeUnixNano) / 1_000_000
|
||||
if (startedAt > completedAt) {
|
||||
continue
|
||||
}
|
||||
spans.push({
|
||||
name: sanitizeCrashReportString(redactString(data.name), 120).slice(0, 120),
|
||||
startedAt,
|
||||
completedAt
|
||||
})
|
||||
} catch {
|
||||
// Concurrent appends and interrupted writes can leave incomplete records.
|
||||
}
|
||||
}
|
||||
return spans.toReversed()
|
||||
} catch {
|
||||
return null
|
||||
} finally {
|
||||
if (fd !== undefined) {
|
||||
closeSync(fd)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
import { mkdtempSync, readFileSync, rmSync } from 'node:fs'
|
||||
import type * as FileSystem from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
|
||||
import type { HangMachineSnapshot } from './hang-observations'
|
||||
import type { MainToHangWatchdogWorkerMessage } from './hang-watchdog-worker-protocol'
|
||||
import { asHangObservations } from './hang-observations'
|
||||
import { consumeHangDetectionMarker, writeHangDetectionMarker } from './hang-detection-marker'
|
||||
import { runWatchdog } from './main-thread-hang-watchdog-entry'
|
||||
import { preservePreviousHangDetection } from './previous-hang-detection'
|
||||
|
||||
const probes = vi.hoisted(() => ({ pending: [] as ((value: HangMachineSnapshot) => void)[] }))
|
||||
const files = vi.hoisted(() => ({ unavailablePath: null as string | null }))
|
||||
vi.mock('node:fs', async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof FileSystem>()
|
||||
return {
|
||||
...actual,
|
||||
readFileSync: new Proxy(actual.readFileSync, {
|
||||
apply(target, thisArg, args) {
|
||||
if (args[0] === files.unavailablePath) {
|
||||
throw Object.assign(new Error('temporarily unreadable'), { code: 'EIO' })
|
||||
}
|
||||
return Reflect.apply(target, thisArg, args)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
vi.mock('../observability', () => ({
|
||||
resolveObservabilityConsent: () => ({ localFileEnabled: true })
|
||||
}))
|
||||
vi.mock('./hang-incident-snapshot', () => ({
|
||||
readCompletedHangContext: () => [],
|
||||
collectHangMachineSnapshot: () =>
|
||||
new Promise<HangMachineSnapshot>((resolve) => probes.pending.push(resolve))
|
||||
}))
|
||||
const staleMachine: HangMachineSnapshot = {
|
||||
startedAt: 125,
|
||||
completedAt: 250,
|
||||
systemMemoryTotalMB: 64_000,
|
||||
systemMemoryFreeMB: 8_000,
|
||||
systemMemorySwapTotalMB: 1024,
|
||||
systemMemorySwapUsedMB: 100,
|
||||
systemMemoryCompressorMB: 500,
|
||||
systemMemoryPressureLevel: 2,
|
||||
systemMemoryStatusLevel: 40,
|
||||
load1: 2,
|
||||
load5: 3,
|
||||
load15: 4
|
||||
}
|
||||
beforeEach(() => {
|
||||
probes.pending = []
|
||||
files.unavailablePath = null
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(0)
|
||||
})
|
||||
afterEach(() => {
|
||||
files.unavailablePath = null
|
||||
for (const resolve of probes.pending) {
|
||||
resolve(staleMachine)
|
||||
}
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('cannot let a retired pressure read replace a newer stall or undo recovery', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'hang-ownership-'))
|
||||
const markerPath = join(dir, 'marker.json')
|
||||
let send: (message: MainToHangWatchdogWorkerMessage) => void = () => {}
|
||||
runWatchdog(
|
||||
{ parentPid: process.pid, markerPath, timeoutMs: 100, checkIntervalMs: 25 },
|
||||
{
|
||||
on: (_event, listener) => {
|
||||
send = listener
|
||||
},
|
||||
close() {}
|
||||
}
|
||||
)
|
||||
try {
|
||||
vi.advanceTimersByTime(125)
|
||||
await vi.dynamicImportSettled()
|
||||
send({ type: 'heartbeat' })
|
||||
vi.advanceTimersByTime(125)
|
||||
probes.pending[0](staleMachine)
|
||||
await vi.dynamicImportSettled()
|
||||
const records = asHangObservations(consumeHangDetectionMarker(markerPath))?.observations
|
||||
expect(
|
||||
records?.map(({ kind, selfRecovered, machine }) => ({ kind, selfRecovered, machine }))
|
||||
).toEqual([
|
||||
{ kind: 'heartbeat-stall', selfRecovered: true, machine: null },
|
||||
{ kind: 'heartbeat-stall', selfRecovered: false, machine: null }
|
||||
])
|
||||
expect(records?.[0].snapshot).not.toBeNull()
|
||||
expect(records?.[0].resolution).not.toBeNull()
|
||||
expect(records?.[1].resolution).toBeNull()
|
||||
} finally {
|
||||
send({ type: 'shutdown' })
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ scenario: 'open suspend across a normal tick', suspendAt: 0, resumeAt: null, expected: true },
|
||||
{ scenario: 'completed sleep before a normal tick', suspendAt: 0, resumeAt: 0, expected: false },
|
||||
{ scenario: 'resume before the overdue tick', suspendAt: 0, resumeAt: 1000, expected: true },
|
||||
{ scenario: 'suspend at the gap end', suspendAt: 1000, resumeAt: null, expected: false }
|
||||
])('retains honest sleep association for $scenario', ({ suspendAt, resumeAt, expected }) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'hang-power-gap-'))
|
||||
const markerPath = join(dir, 'marker.json')
|
||||
let send: (message: MainToHangWatchdogWorkerMessage) => void = () => {}
|
||||
runWatchdog(
|
||||
{ parentPid: process.pid, markerPath, timeoutMs: 100, checkIntervalMs: 25 },
|
||||
{
|
||||
on: (_event, listener) => {
|
||||
send = listener
|
||||
},
|
||||
close() {}
|
||||
}
|
||||
)
|
||||
try {
|
||||
if (suspendAt === 0) {
|
||||
send({ type: 'power-event', event: 'suspend', observedAt: 0 })
|
||||
}
|
||||
if (resumeAt === 0) {
|
||||
send({ type: 'power-event', event: 'resume', observedAt: 0 })
|
||||
}
|
||||
vi.advanceTimersByTime(25)
|
||||
send({ type: 'heartbeat' })
|
||||
vi.setSystemTime(1000)
|
||||
if (suspendAt === 1000) {
|
||||
send({ type: 'power-event', event: 'suspend', observedAt: 1000 })
|
||||
}
|
||||
if (resumeAt === 1000) {
|
||||
send({ type: 'power-event', event: 'resume', observedAt: 1000 })
|
||||
}
|
||||
vi.advanceTimersByTime(25)
|
||||
const collection = asHangObservations(consumeHangDetectionMarker(markerPath))
|
||||
expect(collection?.observations).toMatchObject([
|
||||
{ kind: 'watchdog-gap', initialSleepAssociated: expected, resolution: null }
|
||||
])
|
||||
if (suspendAt === 1000) {
|
||||
for (let index = 0; index < 5; index++) {
|
||||
send({ type: 'power-event', event: 'resume', observedAt: Date.now() })
|
||||
send({ type: 'heartbeat' })
|
||||
send({ type: 'power-event', event: 'suspend', observedAt: Date.now() })
|
||||
vi.advanceTimersByTime(25)
|
||||
vi.setSystemTime(Date.now() + 1000)
|
||||
vi.advanceTimersByTime(25)
|
||||
}
|
||||
expect(
|
||||
asHangObservations(consumeHangDetectionMarker(markerPath))?.observations
|
||||
).toContainEqual(expect.objectContaining({ detectedAt: 1025, initialSleepAssociated: false }))
|
||||
}
|
||||
} finally {
|
||||
send({ type: 'shutdown' })
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps unavailable startup bytes until a later observation can safely merge them', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'hang-unavailable-'))
|
||||
const markerPath = join(dir, 'main-thread-hang.json')
|
||||
const original = { detectedAt: 1, parentPid: 999, unresponsiveMs: 45_000, selfRecovered: false }
|
||||
writeHangDetectionMarker(markerPath, original)
|
||||
const originalBytes = readFileSync(markerPath, 'utf8')
|
||||
files.unavailablePath = markerPath
|
||||
preservePreviousHangDetection(dir)
|
||||
let send: (message: MainToHangWatchdogWorkerMessage) => void = () => {}
|
||||
runWatchdog(
|
||||
{ parentPid: process.pid, markerPath, timeoutMs: 100, checkIntervalMs: 25 },
|
||||
{
|
||||
on: (_event, listener) => {
|
||||
send = listener
|
||||
},
|
||||
close() {}
|
||||
}
|
||||
)
|
||||
try {
|
||||
vi.advanceTimersByTime(125)
|
||||
files.unavailablePath = null
|
||||
expect(readFileSync(markerPath, 'utf8')).toBe(originalBytes)
|
||||
send({ type: 'heartbeat' })
|
||||
expect(asHangObservations(consumeHangDetectionMarker(markerPath))?.observations).toMatchObject([
|
||||
{ parentPid: 999, detectedAt: 1, selfRecovered: false },
|
||||
{ parentPid: process.pid, detectedAt: 125, selfRecovered: true }
|
||||
])
|
||||
} finally {
|
||||
send({ type: 'shutdown' })
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,58 @@
|
||||
export const SNAPSHOT_FIELDS = [
|
||||
'wallMs',
|
||||
'monotonicMs',
|
||||
'uptimeSeconds',
|
||||
'processUserUs',
|
||||
'processSystemUs',
|
||||
'watchdogUserUs',
|
||||
'watchdogSystemUs'
|
||||
] as const
|
||||
export type HangSnapshot = Record<(typeof SNAPSHOT_FIELDS)[number], number | null>
|
||||
|
||||
export const MACHINE_FIELDS = [
|
||||
'startedAt',
|
||||
'completedAt',
|
||||
'systemMemoryTotalMB',
|
||||
'systemMemoryFreeMB',
|
||||
'systemMemorySwapTotalMB',
|
||||
'systemMemorySwapUsedMB',
|
||||
'systemMemoryCompressorMB',
|
||||
'systemMemoryPressureLevel',
|
||||
'systemMemoryStatusLevel',
|
||||
'load1',
|
||||
'load5',
|
||||
'load15'
|
||||
] as const
|
||||
export type HangMachineSnapshot = Record<(typeof MACHINE_FIELDS)[number], number | null>
|
||||
export type HangPowerReceipt = {
|
||||
event: 'suspend' | 'resume' | 'lock-screen' | 'unlock-screen'
|
||||
observedAt: number
|
||||
receivedAt: number
|
||||
}
|
||||
export type CompletedHangContext = { name: string; startedAt: number; completedAt: number }
|
||||
export type HangObservation = {
|
||||
id: string
|
||||
kind: 'heartbeat-stall' | 'watchdog-gap'
|
||||
detectedAt: number
|
||||
parentPid: number
|
||||
durationMs: number
|
||||
selfRecovered: boolean
|
||||
initialSleepAssociated: boolean
|
||||
snapshot: HangSnapshot | null
|
||||
resolution: HangSnapshot | null
|
||||
machine: HangMachineSnapshot | null
|
||||
completedSpans: CompletedHangContext[] | null
|
||||
powerEvents: HangPowerReceipt[]
|
||||
watchdogGap?: { observedAt: number; durationMs: number; snapshot: HangSnapshot | null }
|
||||
}
|
||||
export type HangObservationCollection = {
|
||||
version: 1
|
||||
observations: HangObservation[]
|
||||
omitted: number
|
||||
}
|
||||
export type HangDetectionMarker = {
|
||||
detectedAt: number
|
||||
parentPid: number
|
||||
unresponsiveMs: number
|
||||
selfRecovered: boolean
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { expect, it } from 'vitest'
|
||||
import { consumeHangDetectionMarker, writeHangDetectionMarker } from './hang-detection-marker'
|
||||
import {
|
||||
asHangObservations,
|
||||
emptyHangObservations,
|
||||
mergeHangObservations,
|
||||
retainHangObservation,
|
||||
type HangObservation
|
||||
} from './hang-observations'
|
||||
import { parseMacHangPressure, readCompletedHangContext } from './hang-incident-snapshot'
|
||||
|
||||
function observation(id: string, kind: HangObservation['kind'], sleep = false): HangObservation {
|
||||
return {
|
||||
id,
|
||||
kind,
|
||||
detectedAt: 10_000,
|
||||
parentPid: 7,
|
||||
durationMs: 162_000,
|
||||
selfRecovered: false,
|
||||
initialSleepAssociated: sleep,
|
||||
snapshot: null,
|
||||
resolution: null,
|
||||
machine: null,
|
||||
completedSpans: null,
|
||||
powerEvents: []
|
||||
}
|
||||
}
|
||||
|
||||
it('keeps the original unexplained gap and recovered stall through subsequent ordinary sleeps', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'hang-retention-'))
|
||||
try {
|
||||
const gap = observation('7:1:1', 'watchdog-gap')
|
||||
const stall = observation('7:2:1', 'heartbeat-stall')
|
||||
let collection = retainHangObservation(
|
||||
retainHangObservation(emptyHangObservations(), gap),
|
||||
stall
|
||||
)
|
||||
collection = retainHangObservation(collection, {
|
||||
...gap,
|
||||
initialSleepAssociated: true,
|
||||
powerEvents: [{ event: 'resume', observedAt: 172_000, receivedAt: 172_001 }]
|
||||
})
|
||||
collection = retainHangObservation(collection, { ...stall, selfRecovered: true })
|
||||
for (const id of ['7:3:1', '7:4:1', '7:5:1', '7:6:1', '7:7:1']) {
|
||||
collection = retainHangObservation(collection, observation(id, 'watchdog-gap', true))
|
||||
}
|
||||
collection = mergeHangObservations(
|
||||
collection,
|
||||
retainHangObservation(emptyHangObservations(), stall)
|
||||
)
|
||||
const markerPath = join(dir, 'marker.json')
|
||||
writeHangDetectionMarker(markerPath, collection)
|
||||
const persisted = asHangObservations(consumeHangDetectionMarker(markerPath))
|
||||
expect(
|
||||
persisted?.observations.map(({ id, selfRecovered, initialSleepAssociated }) => ({
|
||||
id,
|
||||
selfRecovered,
|
||||
initialSleepAssociated
|
||||
}))
|
||||
).toEqual([
|
||||
{ id: '7:1:1', selfRecovered: false, initialSleepAssociated: false },
|
||||
{ id: '7:2:1', selfRecovered: true, initialSleepAssociated: false },
|
||||
{ id: '7:6:1', selfRecovered: false, initialSleepAssociated: true },
|
||||
{ id: '7:7:1', selfRecovered: false, initialSleepAssociated: true }
|
||||
])
|
||||
expect(persisted?.observations[0].powerEvents).toEqual([
|
||||
{ event: 'resume', observedAt: 172_000, receivedAt: 172_001 }
|
||||
])
|
||||
expect(persisted?.omitted).toBe(3)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('parses actual macOS units and page size without inventing missing measurements', () => {
|
||||
expect(
|
||||
parseMacHangPressure(
|
||||
'vm.swapusage: total = 1.50G used = 512.00M free = 1024.00M\nkern.memorystatus_level: 48\nkern.memorystatus_vm_pressure_level: 2\n',
|
||||
'Mach Virtual Memory Statistics: (page size of 16384 bytes)\nPages occupied by compressor: 42.\n'
|
||||
)
|
||||
).toEqual({
|
||||
systemMemorySwapTotalMB: 1536,
|
||||
systemMemorySwapUsedMB: 512,
|
||||
systemMemoryCompressorMB: 0.65625,
|
||||
systemMemoryPressureLevel: 2,
|
||||
systemMemoryStatusLevel: 48
|
||||
})
|
||||
expect(parseMacHangPressure('', 'Pages occupied by compressor: 42.')).toEqual({
|
||||
systemMemorySwapTotalMB: null,
|
||||
systemMemorySwapUsedMB: null,
|
||||
systemMemoryCompressorMB: null,
|
||||
systemMemoryPressureLevel: null,
|
||||
systemMemoryStatusLevel: null
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps only complete recent context and sanitizes complete names before truncating', () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'hang-context-'))
|
||||
try {
|
||||
const path = join(dir, 'trace.ndjson')
|
||||
writeFileSync(
|
||||
path,
|
||||
`${JSON.stringify({
|
||||
type: 'effect-span',
|
||||
name: 'old-span',
|
||||
startTimeUnixNano: '1000000',
|
||||
endTimeUnixNano: '2000000'
|
||||
})}\n${'x'.repeat(40_000)}\n${JSON.stringify({
|
||||
type: 'effect-span',
|
||||
name: `git.exec password=${'a'.repeat(200)}`,
|
||||
startTimeUnixNano: '3000000',
|
||||
endTimeUnixNano: '4000000',
|
||||
attributes: { private: 'discard' }
|
||||
})}\n{"type":"effect-span","name":"incomplete`
|
||||
)
|
||||
expect(readCompletedHangContext(path)).toEqual([
|
||||
{ name: 'git.exec [redacted:labeled-kv]', startedAt: 3, completedAt: 4 }
|
||||
])
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,267 @@
|
||||
import {
|
||||
MACHINE_FIELDS,
|
||||
SNAPSHOT_FIELDS,
|
||||
type HangPowerReceipt,
|
||||
type CompletedHangContext,
|
||||
type HangObservation,
|
||||
type HangObservationCollection,
|
||||
type HangDetectionMarker
|
||||
} from './hang-observation-schema'
|
||||
export { MACHINE_FIELDS, SNAPSHOT_FIELDS } from './hang-observation-schema'
|
||||
export type {
|
||||
HangSnapshot,
|
||||
HangMachineSnapshot,
|
||||
HangPowerReceipt,
|
||||
CompletedHangContext,
|
||||
HangObservation,
|
||||
HangObservationCollection,
|
||||
HangDetectionMarker
|
||||
} from './hang-observation-schema'
|
||||
|
||||
function object(value: unknown): Record<string, unknown> | null {
|
||||
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: null
|
||||
}
|
||||
function finite(value: unknown): value is number {
|
||||
return typeof value === 'number' && Number.isFinite(value) && value >= 0
|
||||
}
|
||||
function numericFields<K extends string>(
|
||||
value: unknown,
|
||||
keys: readonly K[]
|
||||
): Record<K, number | null> | null {
|
||||
const data = object(value)
|
||||
if (!data) {
|
||||
return null
|
||||
}
|
||||
return Object.fromEntries(
|
||||
keys.map((key) => [key, finite(data[key]) ? data[key] : null])
|
||||
) as Record<K, number | null>
|
||||
}
|
||||
export function parsePowerReceipt(value: unknown): HangPowerReceipt | null {
|
||||
const data = object(value)
|
||||
if (
|
||||
!data ||
|
||||
!['suspend', 'resume', 'lock-screen', 'unlock-screen'].includes(String(data.event)) ||
|
||||
!finite(data.observedAt) ||
|
||||
!finite(data.receivedAt)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
event: data.event as HangPowerReceipt['event'],
|
||||
observedAt: data.observedAt,
|
||||
receivedAt: data.receivedAt
|
||||
}
|
||||
}
|
||||
export function parseLegacyHangMarker(value: unknown): HangDetectionMarker | null {
|
||||
const data = object(value)
|
||||
if (
|
||||
!data ||
|
||||
!finite(data.detectedAt) ||
|
||||
!finite(data.parentPid) ||
|
||||
!Number.isInteger(data.parentPid) ||
|
||||
data.parentPid <= 0 ||
|
||||
!finite(data.unresponsiveMs)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
detectedAt: data.detectedAt,
|
||||
parentPid: data.parentPid,
|
||||
unresponsiveMs: data.unresponsiveMs,
|
||||
selfRecovered: data.selfRecovered === true
|
||||
}
|
||||
}
|
||||
function parseObservation(value: unknown): HangObservation | null {
|
||||
const data = object(value)
|
||||
if (
|
||||
!data ||
|
||||
typeof data.id !== 'string' ||
|
||||
!/^[\d:.-]{1,80}$/.test(data.id) ||
|
||||
!['heartbeat-stall', 'watchdog-gap'].includes(String(data.kind)) ||
|
||||
!finite(data.detectedAt) ||
|
||||
!finite(data.parentPid) ||
|
||||
!Number.isInteger(data.parentPid) ||
|
||||
data.parentPid <= 0 ||
|
||||
!finite(data.durationMs)
|
||||
) {
|
||||
return null
|
||||
}
|
||||
const completedSpans: CompletedHangContext[] = []
|
||||
if (Array.isArray(data.completedSpans) && data.completedSpans.length <= 8) {
|
||||
for (const item of data.completedSpans) {
|
||||
const span = object(item)
|
||||
if (
|
||||
span &&
|
||||
typeof span.name === 'string' &&
|
||||
span.name.length <= 120 &&
|
||||
finite(span.startedAt) &&
|
||||
finite(span.completedAt) &&
|
||||
span.startedAt <= span.completedAt
|
||||
) {
|
||||
completedSpans.push({
|
||||
name: span.name,
|
||||
startedAt: span.startedAt,
|
||||
completedAt: span.completedAt
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
const gap = object(data.watchdogGap)
|
||||
return {
|
||||
id: data.id,
|
||||
kind: data.kind as HangObservation['kind'],
|
||||
detectedAt: data.detectedAt,
|
||||
parentPid: data.parentPid,
|
||||
durationMs: data.durationMs,
|
||||
selfRecovered: data.selfRecovered === true,
|
||||
initialSleepAssociated: data.initialSleepAssociated === true,
|
||||
snapshot: numericFields(data.snapshot, SNAPSHOT_FIELDS),
|
||||
resolution: numericFields(data.resolution, SNAPSHOT_FIELDS),
|
||||
machine: numericFields(data.machine, MACHINE_FIELDS),
|
||||
completedSpans: Array.isArray(data.completedSpans) ? completedSpans : null,
|
||||
powerEvents: Array.isArray(data.powerEvents)
|
||||
? data.powerEvents
|
||||
.slice(-4)
|
||||
.map(parsePowerReceipt)
|
||||
.filter((v): v is HangPowerReceipt => v !== null)
|
||||
: [],
|
||||
...(gap && finite(gap.observedAt) && finite(gap.durationMs)
|
||||
? {
|
||||
watchdogGap: {
|
||||
observedAt: gap.observedAt,
|
||||
durationMs: gap.durationMs,
|
||||
snapshot: numericFields(gap.snapshot, SNAPSHOT_FIELDS)
|
||||
}
|
||||
}
|
||||
: {})
|
||||
}
|
||||
}
|
||||
export function emptyHangObservations(): HangObservationCollection {
|
||||
return { version: 1, observations: [], omitted: 0 }
|
||||
}
|
||||
export function asHangObservations(value: unknown): HangObservationCollection | null {
|
||||
const data = object(value)
|
||||
if (data?.version === 1) {
|
||||
if (!Array.isArray(data.observations) || data.observations.length > 4) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
version: 1,
|
||||
observations: data.observations
|
||||
.map(parseObservation)
|
||||
.filter((v): v is HangObservation => v !== null),
|
||||
omitted: finite(data.omitted) ? Math.min(2_147_483_647, Math.floor(data.omitted)) : 0
|
||||
}
|
||||
}
|
||||
const legacy = parseLegacyHangMarker(value)
|
||||
if (!legacy) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
version: 1,
|
||||
omitted: 0,
|
||||
observations: [
|
||||
{
|
||||
id: `${legacy.parentPid}:${legacy.detectedAt}:0`,
|
||||
kind: 'heartbeat-stall',
|
||||
detectedAt: legacy.detectedAt,
|
||||
parentPid: legacy.parentPid,
|
||||
durationMs: legacy.unresponsiveMs,
|
||||
selfRecovered: legacy.selfRecovered,
|
||||
initialSleepAssociated: false,
|
||||
snapshot: null,
|
||||
resolution: null,
|
||||
machine: null,
|
||||
completedSpans: null,
|
||||
powerEvents: []
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
function priority(record: HangObservation): number {
|
||||
return record.kind === 'heartbeat-stall' ? 3 : record.initialSleepAssociated ? 1 : 2
|
||||
}
|
||||
export function retainHangObservation(
|
||||
collection: HangObservationCollection,
|
||||
record: HangObservation
|
||||
): HangObservationCollection {
|
||||
const existing = collection.observations.findIndex((item) => item.id === record.id)
|
||||
if (existing !== -1) {
|
||||
const old = collection.observations[existing]
|
||||
const observations = collection.observations.slice()
|
||||
const receipts = new Map(
|
||||
[...old.powerEvents, ...record.powerEvents].map(
|
||||
(item) => [`${item.event}:${item.observedAt}:${item.receivedAt}`, item] as const
|
||||
)
|
||||
)
|
||||
observations[existing] = {
|
||||
...record,
|
||||
kind: old.kind,
|
||||
id: old.id,
|
||||
detectedAt: old.detectedAt,
|
||||
parentPid: old.parentPid,
|
||||
durationMs: old.durationMs,
|
||||
initialSleepAssociated: old.initialSleepAssociated,
|
||||
snapshot: old.snapshot,
|
||||
selfRecovered: old.selfRecovered || record.selfRecovered,
|
||||
resolution: old.resolution ?? record.resolution,
|
||||
machine: old.machine ?? record.machine,
|
||||
completedSpans: old.completedSpans ?? record.completedSpans,
|
||||
powerEvents: [...receipts.values()].slice(-4),
|
||||
watchdogGap:
|
||||
old.watchdogGap && old.watchdogGap.durationMs > (record.watchdogGap?.durationMs ?? -1)
|
||||
? old.watchdogGap
|
||||
: (record.watchdogGap ?? old.watchdogGap)
|
||||
}
|
||||
return { ...collection, observations }
|
||||
}
|
||||
const candidates = [...collection.observations, record]
|
||||
const anchors = new Set(
|
||||
[
|
||||
candidates.find((item) => item.kind === 'heartbeat-stall'),
|
||||
candidates.find((item) => item.kind === 'watchdog-gap' && !item.initialSleepAssociated)
|
||||
].filter((item): item is HangObservation => !!item)
|
||||
)
|
||||
const selected = new Set(anchors)
|
||||
const ranked = candidates
|
||||
.map((item, index) => ({ item, index }))
|
||||
.filter(({ item }) => !anchors.has(item))
|
||||
.sort((a, b) => priority(b.item) - priority(a.item) || b.index - a.index)
|
||||
for (const { item } of ranked) {
|
||||
if (selected.size < 4) {
|
||||
selected.add(item)
|
||||
}
|
||||
}
|
||||
return {
|
||||
version: 1,
|
||||
observations: candidates.filter((item) => selected.has(item)),
|
||||
omitted: Math.min(2_147_483_647, collection.omitted + candidates.length - selected.size)
|
||||
}
|
||||
}
|
||||
export function mergeHangObservations(
|
||||
first: HangObservationCollection,
|
||||
second: HangObservationCollection
|
||||
): HangObservationCollection {
|
||||
let result = { ...first, omitted: Math.min(2_147_483_647, first.omitted + second.omitted) }
|
||||
for (const record of second.observations) {
|
||||
result = retainHangObservation(result, record)
|
||||
}
|
||||
return result
|
||||
}
|
||||
export function latestHangMarker(
|
||||
collection: HangObservationCollection
|
||||
): HangDetectionMarker | null {
|
||||
const record = collection.observations.findLast((item) => item.kind === 'heartbeat-stall')
|
||||
return record
|
||||
? {
|
||||
detectedAt: record.detectedAt,
|
||||
parentPid: record.parentPid,
|
||||
unresponsiveMs:
|
||||
record.durationMs +
|
||||
Math.max(0, (record.resolution?.wallMs ?? record.detectedAt) - record.detectedAt),
|
||||
selfRecovered: record.selfRecovered
|
||||
}
|
||||
: null
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import { uptime } from 'node:os'
|
||||
import type { HangSnapshot } from './hang-observations'
|
||||
|
||||
export function captureHangSnapshot(): HangSnapshot {
|
||||
const cpu = process.cpuUsage()
|
||||
let watchdog: NodeJS.CpuUsage | null = null
|
||||
try {
|
||||
const read = (process as NodeJS.Process & { threadCpuUsage?: () => NodeJS.CpuUsage })
|
||||
.threadCpuUsage
|
||||
watchdog = read?.call(process) ?? null
|
||||
} catch {
|
||||
// Some Electron runtimes do not expose per-thread CPU counters.
|
||||
}
|
||||
return {
|
||||
wallMs: Date.now(),
|
||||
monotonicMs: performance.now(),
|
||||
uptimeSeconds: uptime(),
|
||||
processUserUs: cpu.user,
|
||||
processSystemUs: cpu.system,
|
||||
watchdogUserUs: watchdog?.user ?? null,
|
||||
watchdogSystemUs: watchdog?.system ?? null
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ export type HangWatchdogDetectionLoopConfig = {
|
||||
onHangDetected: (unresponsiveMs: number) => void
|
||||
/** Heartbeats resumed after a detected hang — the main thread was stalled, not deadlocked. */
|
||||
onHangResolved: (unresponsiveMs: number) => void
|
||||
onWatchdogGap?: (gapMs: number, startedAt: number) => void
|
||||
}
|
||||
|
||||
export type HangWatchdogDetectionLoop = {
|
||||
@@ -32,12 +33,17 @@ export function createHangWatchdogDetectionLoop(
|
||||
const tickGap = now - lastTickAt
|
||||
// Advance while hung so recovery cannot be misclassified as a long system-sleep gap.
|
||||
lastTickAt = now
|
||||
if (detected) {
|
||||
// Sleep, process suspension and observer starvation are indistinguishable here.
|
||||
if (tickGap > config.checkIntervalMs * 3) {
|
||||
if (tickGap > config.timeoutMs) {
|
||||
config.onWatchdogGap?.(tickGap, now - tickGap)
|
||||
}
|
||||
if (!detected) {
|
||||
lastHeartbeatAt = now
|
||||
}
|
||||
return
|
||||
}
|
||||
// System sleep suspends the watchdog too; restart the silence window after large tick gaps.
|
||||
if (tickGap > config.checkIntervalMs * 3) {
|
||||
lastHeartbeatAt = now
|
||||
if (detected) {
|
||||
return
|
||||
}
|
||||
const unresponsiveMs = now - lastHeartbeatAt
|
||||
|
||||
@@ -7,6 +7,11 @@ export type HangWatchdogWorkerData = {
|
||||
markerPath: string
|
||||
timeoutMs: number
|
||||
checkIntervalMs: number
|
||||
tracePath?: string
|
||||
}
|
||||
|
||||
export type MainToHangWatchdogWorkerMessage = { type: 'heartbeat' } | { type: 'shutdown' }
|
||||
export type MainToHangWatchdogWorkerMessage =
|
||||
| { type: 'heartbeat' }
|
||||
| { type: 'shutdown' }
|
||||
| { type: 'power-event'; event: HangPowerReceipt['event']; observedAt: number }
|
||||
import type { HangPowerReceipt } from './hang-observations'
|
||||
|
||||
@@ -1,6 +1,16 @@
|
||||
import { isMainThread, parentPort, workerData } from 'node:worker_threads'
|
||||
import { createHangWatchdogDetectionLoop } from './hang-watchdog-detection-loop'
|
||||
import { writeHangDetectionMarker } from './hang-detection-marker'
|
||||
import { readAvailableHangDetectionMarker, writeHangDetectionMarker } from './hang-detection-marker'
|
||||
import {
|
||||
asHangObservations,
|
||||
emptyHangObservations,
|
||||
mergeHangObservations,
|
||||
parsePowerReceipt,
|
||||
retainHangObservation,
|
||||
type HangObservation,
|
||||
type HangPowerReceipt
|
||||
} from './hang-observations'
|
||||
import { captureHangSnapshot } from './hang-snapshot'
|
||||
import type {
|
||||
HangWatchdogWorkerData,
|
||||
MainToHangWatchdogWorkerMessage
|
||||
@@ -11,28 +21,6 @@ type HangWatchdogPort = {
|
||||
close: () => void
|
||||
}
|
||||
|
||||
// Observation only: a false positive must never kill a live main thread mid-write.
|
||||
export function recordHangObservation(options: {
|
||||
parentPid: number
|
||||
markerPath: string
|
||||
unresponsiveMs: number
|
||||
selfRecovered: boolean
|
||||
}): void {
|
||||
if (!options.markerPath) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
writeHangDetectionMarker(options.markerPath, {
|
||||
detectedAt: Date.now(),
|
||||
parentPid: options.parentPid,
|
||||
unresponsiveMs: options.unresponsiveMs,
|
||||
selfRecovered: options.selfRecovered
|
||||
})
|
||||
} catch {
|
||||
// Diagnostic write failures must not terminate the watchdog.
|
||||
}
|
||||
}
|
||||
|
||||
export function runWatchdog(
|
||||
config: HangWatchdogWorkerData,
|
||||
port: HangWatchdogPort | null = parentPort
|
||||
@@ -40,39 +28,187 @@ export function runWatchdog(
|
||||
if (!port) {
|
||||
return
|
||||
}
|
||||
const initial = readAvailableHangDetectionMarker(config.markerPath)
|
||||
let observations = asHangObservations(initial) ?? emptyHangObservations()
|
||||
let existingUnavailable = initial === undefined
|
||||
let sequence = 0
|
||||
let activeStallId: string | null = null
|
||||
let stopped = false
|
||||
let collector: AbortController | null = null
|
||||
let recentPower: HangPowerReceipt[] = []
|
||||
let sleepPeriod: { suspend: HangPowerReceipt; resumeAt: number | null } | null = null
|
||||
let receiptWindow: { id: string; until: number; remaining: number } | null = null
|
||||
const save = (): void => {
|
||||
if (stopped || !config.markerPath) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
if (existingUnavailable) {
|
||||
const existing = readAvailableHangDetectionMarker(config.markerPath)
|
||||
if (existing === undefined) {
|
||||
return
|
||||
}
|
||||
observations = mergeHangObservations(
|
||||
asHangObservations(existing) ?? emptyHangObservations(),
|
||||
observations
|
||||
)
|
||||
existingUnavailable = false
|
||||
}
|
||||
writeHangDetectionMarker(config.markerPath, observations)
|
||||
} catch {
|
||||
// Optional diagnostics must not terminate the independent observer.
|
||||
}
|
||||
}
|
||||
const find = (id: string): HangObservation | undefined =>
|
||||
observations.observations.find((item) => item.id === id)
|
||||
const update = (id: string, change: Partial<HangObservation>): void => {
|
||||
const record = find(id)
|
||||
if (!record || stopped) {
|
||||
return
|
||||
}
|
||||
observations = retainHangObservation(observations, { ...record, ...change })
|
||||
save()
|
||||
}
|
||||
const openReceiptWindow = (id: string): void => {
|
||||
receiptWindow = { id, until: Date.now() + config.checkIntervalMs, remaining: 4 }
|
||||
}
|
||||
const enrich = (record: HangObservation): void => {
|
||||
if (!find(record.id)) {
|
||||
return
|
||||
}
|
||||
if (collector) {
|
||||
collector.abort()
|
||||
return
|
||||
}
|
||||
const controller = new AbortController()
|
||||
collector = controller
|
||||
void import('./hang-incident-snapshot')
|
||||
.then(async (incident) => {
|
||||
if (controller.signal.aborted || stopped) {
|
||||
return
|
||||
}
|
||||
if (record.kind === 'heartbeat-stall') {
|
||||
update(record.id, { completedSpans: incident.readCompletedHangContext(config.tracePath) })
|
||||
}
|
||||
const machine = await incident.collectHangMachineSnapshot(controller.signal)
|
||||
if (!controller.signal.aborted && !stopped) {
|
||||
update(record.id, { machine })
|
||||
}
|
||||
})
|
||||
.catch(() => {
|
||||
// The durable baseline already records unavailable optional evidence as unknown.
|
||||
})
|
||||
.finally(() => {
|
||||
if (collector === controller) {
|
||||
collector = null
|
||||
}
|
||||
})
|
||||
}
|
||||
const observe = (kind: HangObservation['kind'], durationMs: number, startedAt: number): void => {
|
||||
const snapshot = captureHangSnapshot()
|
||||
const now = snapshot.wallMs ?? Date.now()
|
||||
const suspend = sleepPeriod?.suspend
|
||||
const initialSleepAssociated =
|
||||
kind === 'watchdog-gap' &&
|
||||
!!suspend &&
|
||||
suspend.observedAt <= startedAt + config.checkIntervalMs &&
|
||||
suspend.receivedAt <= startedAt + config.checkIntervalMs &&
|
||||
suspend.receivedAt < now - config.checkIntervalMs &&
|
||||
suspend.receivedAt >= suspend.observedAt &&
|
||||
suspend.receivedAt - suspend.observedAt <= config.checkIntervalMs &&
|
||||
(sleepPeriod?.resumeAt ?? Infinity) > startedAt
|
||||
let powerEvents = recentPower
|
||||
.filter((item) => item.observedAt >= startedAt && item.receivedAt <= now)
|
||||
.slice(-4)
|
||||
if (initialSleepAssociated && suspend) {
|
||||
powerEvents = [suspend, ...powerEvents.filter((item) => item !== suspend).slice(-3)]
|
||||
}
|
||||
const record: HangObservation = {
|
||||
id: `${config.parentPid}:${now}:${++sequence}`,
|
||||
kind,
|
||||
detectedAt: now,
|
||||
parentPid: config.parentPid,
|
||||
durationMs,
|
||||
selfRecovered: false,
|
||||
snapshot,
|
||||
resolution: null,
|
||||
machine: null,
|
||||
completedSpans: null,
|
||||
powerEvents,
|
||||
initialSleepAssociated
|
||||
}
|
||||
observations = retainHangObservation(observations, record)
|
||||
if (kind === 'heartbeat-stall') {
|
||||
activeStallId = record.id
|
||||
}
|
||||
openReceiptWindow(record.id)
|
||||
save()
|
||||
enrich(record)
|
||||
}
|
||||
const loop = createHangWatchdogDetectionLoop({
|
||||
timeoutMs: config.timeoutMs,
|
||||
checkIntervalMs: config.checkIntervalMs,
|
||||
now: () => Date.now(),
|
||||
onHangDetected: (unresponsiveMs) =>
|
||||
recordHangObservation({
|
||||
parentPid: config.parentPid,
|
||||
markerPath: config.markerPath,
|
||||
unresponsiveMs,
|
||||
selfRecovered: false
|
||||
}),
|
||||
// Update one marker per stall so resolution does not require reconciling a second record.
|
||||
onHangResolved: (unresponsiveMs) =>
|
||||
recordHangObservation({
|
||||
parentPid: config.parentPid,
|
||||
markerPath: config.markerPath,
|
||||
unresponsiveMs,
|
||||
selfRecovered: true
|
||||
})
|
||||
onHangDetected: (durationMs) => observe('heartbeat-stall', durationMs, Date.now() - durationMs),
|
||||
onHangResolved: () => {
|
||||
if (!activeStallId) {
|
||||
return
|
||||
}
|
||||
const id = activeStallId
|
||||
activeStallId = null
|
||||
update(id, { selfRecovered: true, resolution: captureHangSnapshot() })
|
||||
openReceiptWindow(id)
|
||||
},
|
||||
onWatchdogGap: (durationMs, startedAt) => {
|
||||
const overlapping = observations.observations.findLast(
|
||||
(item) =>
|
||||
item.kind === 'heartbeat-stall' &&
|
||||
(item.id === activeStallId || (item.resolution?.wallMs ?? -1) >= startedAt)
|
||||
)
|
||||
if (overlapping) {
|
||||
update(overlapping.id, {
|
||||
watchdogGap: { observedAt: Date.now(), durationMs, snapshot: captureHangSnapshot() }
|
||||
})
|
||||
openReceiptWindow(overlapping.id)
|
||||
} else {
|
||||
observe('watchdog-gap', durationMs, startedAt)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
let checkTimer: ReturnType<typeof setInterval> | null = setInterval(
|
||||
() => loop.tick(),
|
||||
config.checkIntervalMs
|
||||
)
|
||||
port.on('message', (message: MainToHangWatchdogWorkerMessage) => {
|
||||
const checkTimer = setInterval(() => loop.tick(), config.checkIntervalMs)
|
||||
port.on('message', (message) => {
|
||||
if (stopped) {
|
||||
return
|
||||
}
|
||||
if (message.type === 'heartbeat') {
|
||||
loop.recordHeartbeat()
|
||||
} else if (message.type === 'shutdown') {
|
||||
if (checkTimer) {
|
||||
clearInterval(checkTimer)
|
||||
checkTimer = null
|
||||
} else if (message.type === 'power-event') {
|
||||
const receipt = parsePowerReceipt({ ...message, receivedAt: Date.now() })
|
||||
if (!receipt) {
|
||||
return
|
||||
}
|
||||
recentPower = [...recentPower, receipt].slice(-4)
|
||||
if (receipt.event === 'suspend') {
|
||||
sleepPeriod = { suspend: receipt, resumeAt: null }
|
||||
} else if (receipt.event === 'resume' && sleepPeriod) {
|
||||
sleepPeriod.resumeAt = receipt.observedAt
|
||||
}
|
||||
if (
|
||||
receiptWindow &&
|
||||
receiptWindow.remaining > 0 &&
|
||||
receipt.receivedAt <= receiptWindow.until
|
||||
) {
|
||||
const record = find(receiptWindow.id)
|
||||
if (record) {
|
||||
receiptWindow.remaining--
|
||||
update(record.id, { powerEvents: [...record.powerEvents, receipt].slice(-4) })
|
||||
}
|
||||
}
|
||||
} else if (message.type === 'shutdown') {
|
||||
stopped = true
|
||||
receiptWindow = null
|
||||
clearInterval(checkTimer)
|
||||
collector?.abort()
|
||||
port.close()
|
||||
}
|
||||
})
|
||||
@@ -85,6 +221,7 @@ export function isHangWatchdogWorkerData(value: unknown): value is HangWatchdogW
|
||||
Number.isInteger(data.parentPid) &&
|
||||
(data.parentPid ?? 0) > 0 &&
|
||||
typeof data.markerPath === 'string' &&
|
||||
(data.tracePath === undefined || typeof data.tracePath === 'string') &&
|
||||
Number.isFinite(data.timeoutMs) &&
|
||||
(data.timeoutMs ?? 0) > 0 &&
|
||||
Number.isFinite(data.checkIntervalMs) &&
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import type { Worker } from 'node:worker_threads'
|
||||
import { spawnWorkerThread } from '../worker-threads/worker-thread-spawn'
|
||||
import { app } from 'electron'
|
||||
import { app, powerMonitor } from 'electron'
|
||||
import { hangDetectionMarkerPath } from './hang-detection-marker'
|
||||
import { resolveHangWatchdogWorkerPath } from './hang-watchdog-worker-path'
|
||||
import { resolveObservabilityConsent } from '../observability'
|
||||
import { getTraceFilePath, resolveObservabilityConsent } from '../observability'
|
||||
import {
|
||||
HANG_WATCHDOG_CHECK_INTERVAL_MS,
|
||||
HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS,
|
||||
@@ -39,6 +39,7 @@ export function installMainThreadHangWatchdog(options: {
|
||||
const workerData: HangWatchdogWorkerData = {
|
||||
parentPid: process.pid,
|
||||
markerPath: hangDetectionMarkerPath(options.userDataPath),
|
||||
tracePath: getTraceFilePath(),
|
||||
timeoutMs: positiveTiming(process.env.ORCA_HANG_WATCHDOG_TIMEOUT_MS, HANG_WATCHDOG_TIMEOUT_MS),
|
||||
checkIntervalMs: positiveTiming(
|
||||
process.env.ORCA_HANG_WATCHDOG_CHECK_INTERVAL_MS,
|
||||
@@ -57,7 +58,7 @@ export function installMainThreadHangWatchdog(options: {
|
||||
}
|
||||
let stopped = false
|
||||
const postMessage = (message: MainToHangWatchdogWorkerMessage): void => {
|
||||
if (stopped && message.type === 'heartbeat') {
|
||||
if (stopped && message.type !== 'shutdown') {
|
||||
return
|
||||
}
|
||||
try {
|
||||
@@ -66,6 +67,19 @@ export function installMainThreadHangWatchdog(options: {
|
||||
// The worker already exited.
|
||||
}
|
||||
}
|
||||
const powerListeners = (['suspend', 'resume', 'lock-screen', 'unlock-screen'] as const).map(
|
||||
(event) => {
|
||||
const listener = (): void =>
|
||||
postMessage({ type: 'power-event', event, observedAt: Date.now() })
|
||||
powerMonitor.on(event, listener)
|
||||
return { event, listener }
|
||||
}
|
||||
)
|
||||
const removePowerListeners = (): void => {
|
||||
for (const { event, listener } of powerListeners) {
|
||||
powerMonitor.off(event, listener)
|
||||
}
|
||||
}
|
||||
const heartbeatTimer = setInterval(() => {
|
||||
postMessage({ type: 'heartbeat' })
|
||||
}, HANG_WATCHDOG_HEARTBEAT_INTERVAL_MS)
|
||||
@@ -76,6 +90,7 @@ export function installMainThreadHangWatchdog(options: {
|
||||
stopped = true
|
||||
// Drop the app listener immediately so retired workers are not retained by its closure.
|
||||
app.off('will-quit', stop)
|
||||
removePowerListeners()
|
||||
clearInterval(heartbeatTimer)
|
||||
postMessage({ type: 'shutdown' })
|
||||
}
|
||||
@@ -86,6 +101,7 @@ export function installMainThreadHangWatchdog(options: {
|
||||
worker.once('exit', () => {
|
||||
stopped = true
|
||||
app.off('will-quit', stop)
|
||||
removePowerListeners()
|
||||
clearInterval(heartbeatTimer)
|
||||
})
|
||||
worker.unref()
|
||||
|
||||
@@ -3,6 +3,13 @@ import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { setActiveSink } from '../observability/tracer'
|
||||
import { createLocalFileSink } from '../observability/local-file-sink'
|
||||
import { collectBundle } from '../observability/bundle'
|
||||
import {
|
||||
emptyHangObservations,
|
||||
retainHangObservation,
|
||||
type HangObservation
|
||||
} from './hang-observations'
|
||||
import { hangDetectionMarkerPath, writeHangDetectionMarker } from './hang-detection-marker'
|
||||
import {
|
||||
preservePreviousHangDetection,
|
||||
@@ -41,6 +48,28 @@ const marker = {
|
||||
unresponsiveMs: 10_000,
|
||||
selfRecovered: true
|
||||
}
|
||||
const gap: HangObservation = {
|
||||
id: '789:200:1',
|
||||
kind: 'watchdog-gap',
|
||||
detectedAt: 200,
|
||||
parentPid: 789,
|
||||
durationMs: 162_000,
|
||||
selfRecovered: false,
|
||||
initialSleepAssociated: false,
|
||||
snapshot: {
|
||||
wallMs: 200,
|
||||
monotonicMs: 100,
|
||||
uptimeSeconds: 1000,
|
||||
processUserUs: 500,
|
||||
processSystemUs: 200,
|
||||
watchdogUserUs: null,
|
||||
watchdogSystemUs: null
|
||||
},
|
||||
resolution: null,
|
||||
machine: null,
|
||||
completedSpans: null,
|
||||
powerEvents: []
|
||||
}
|
||||
|
||||
describe('previous hang detection', () => {
|
||||
it('retains the previous run across startup and writes it only after a sink exists', () => {
|
||||
@@ -52,8 +81,7 @@ describe('previous hang detection', () => {
|
||||
expect(existsSync(`${markerPath}.previous`)).toBe(true)
|
||||
writeHangDetectionMarker(markerPath, { ...marker, parentPid: 789 })
|
||||
const records: unknown[] = []
|
||||
const flush = vi.fn()
|
||||
setActiveSink({ push: (record) => records.push(record), flush, close() {} })
|
||||
setActiveSink({ push: (record) => records.push(record), flush() {}, close() {} })
|
||||
expect(reportPreviousHangDetection(dir)).toEqual(marker)
|
||||
expect(records).toEqual([
|
||||
expect.objectContaining({
|
||||
@@ -65,7 +93,6 @@ describe('previous hang detection', () => {
|
||||
})
|
||||
})
|
||||
])
|
||||
expect(flush).toHaveBeenCalledOnce()
|
||||
expect(existsSync(`${markerPath}.previous`)).toBe(false)
|
||||
expect(JSON.parse(readFileSync(markerPath, 'utf8')).parentPid).toBe(789)
|
||||
expect(reportPreviousHangDetection(dir)).toBeNull()
|
||||
@@ -75,12 +102,112 @@ describe('previous hang detection', () => {
|
||||
const markerPath = hangDetectionMarkerPath(dir)
|
||||
writeHangDetectionMarker(markerPath, marker)
|
||||
preservePreviousHangDetection(dir)
|
||||
writeHangDetectionMarker(markerPath, retainHangObservation(emptyHangObservations(), gap))
|
||||
preservePreviousHangDetection(dir)
|
||||
expect(existsSync(`${markerPath}.previous`)).toBe(true)
|
||||
setActiveSink({ push() {}, flush() {}, close() {} })
|
||||
expect(reportPreviousHangDetection(dir)).toEqual(marker)
|
||||
})
|
||||
|
||||
it('carries gap and overlapping-stall evidence through the real trace and bundle', () => {
|
||||
const traceFilePath = join(dir, 'logs', 'main.trace.ndjson')
|
||||
const sink = createLocalFileSink({ filePath: traceFilePath, maxFiles: 1 })
|
||||
try {
|
||||
writeHangDetectionMarker(
|
||||
hangDetectionMarkerPath(dir),
|
||||
retainHangObservation(emptyHangObservations(), gap)
|
||||
)
|
||||
preservePreviousHangDetection(dir)
|
||||
setActiveSink(sink)
|
||||
expect(reportPreviousHangDetection(dir)).toBeNull()
|
||||
const stall: HangObservation = {
|
||||
...gap,
|
||||
id: '789:300:2',
|
||||
kind: 'heartbeat-stall',
|
||||
detectedAt: 300,
|
||||
durationMs: 45_000,
|
||||
selfRecovered: true,
|
||||
snapshot: { ...gap.snapshot!, wallMs: 300, processUserUs: 100 },
|
||||
resolution: { ...gap.snapshot!, wallMs: 500, processUserUs: 900 },
|
||||
watchdogGap: {
|
||||
observedAt: 400,
|
||||
durationMs: 162_000,
|
||||
snapshot: { ...gap.snapshot!, wallMs: 400, processUserUs: 600, watchdogUserUs: 3 }
|
||||
},
|
||||
machine: {
|
||||
startedAt: 301,
|
||||
completedAt: 302,
|
||||
systemMemoryTotalMB: 64_000,
|
||||
systemMemoryFreeMB: 8_000,
|
||||
systemMemorySwapTotalMB: 1024,
|
||||
systemMemorySwapUsedMB: 100,
|
||||
systemMemoryCompressorMB: 500,
|
||||
systemMemoryPressureLevel: 2,
|
||||
systemMemoryStatusLevel: 40,
|
||||
load1: 2,
|
||||
load5: 3,
|
||||
load15: 4
|
||||
}
|
||||
}
|
||||
writeHangDetectionMarker(
|
||||
hangDetectionMarkerPath(dir),
|
||||
retainHangObservation(emptyHangObservations(), stall)
|
||||
)
|
||||
preservePreviousHangDetection(dir)
|
||||
expect(reportPreviousHangDetection(dir)).toEqual({
|
||||
detectedAt: 300,
|
||||
parentPid: 789,
|
||||
unresponsiveMs: 45_200,
|
||||
selfRecovered: true
|
||||
})
|
||||
const bundle = collectBundle({
|
||||
traceFilePath,
|
||||
maxFiles: 1,
|
||||
appVersion: 'test',
|
||||
platform: 'darwin',
|
||||
arch: 'arm64',
|
||||
osRelease: 'test',
|
||||
orcaChannel: 'dev'
|
||||
})
|
||||
const records = bundle.payload
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((line) => JSON.parse(line))
|
||||
.filter((record) => record.type === 'effect-span')
|
||||
const gapRecord = records.find(
|
||||
(record) => record.attributes['breadcrumb.name'] === 'main_thread_watchdog_gap'
|
||||
)
|
||||
const stallRecord = records.find(
|
||||
(record) => record.attributes['breadcrumb.name'] === 'main_thread_hang_detected'
|
||||
)
|
||||
expect(gapRecord?.attributes['breadcrumb.data']).toMatchObject({
|
||||
watchdogGapMs: 162_000,
|
||||
cause: 'unknown',
|
||||
activeMainActivity: 'unknown',
|
||||
detection_processUserUs: 500,
|
||||
preGapCpuDeltaUs: null,
|
||||
watchdogCpuDetectionToResolutionUs: null,
|
||||
machineEvidence: 'unknown'
|
||||
})
|
||||
expect(stallRecord?.attributes['breadcrumb.data']).toMatchObject({
|
||||
detection_wallMs: 300,
|
||||
detection_processUserUs: 100,
|
||||
gapEnd_wallMs: 400,
|
||||
gapEnd_processUserUs: 600,
|
||||
gapEnd_watchdogUserUs: 3,
|
||||
resolution_wallMs: 500,
|
||||
resolution_processUserUs: 900,
|
||||
machine_startedAt: 301,
|
||||
machine_completedAt: 302,
|
||||
machine_systemMemorySwapUsedMB: 100
|
||||
})
|
||||
expect(stallRecord?.attributes['breadcrumb.data']).not.toHaveProperty('startedAt')
|
||||
} finally {
|
||||
setActiveSink(null)
|
||||
sink.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not mutate marker files when local diagnostics are disabled', () => {
|
||||
const markerPath = hangDetectionMarkerPath(dir)
|
||||
writeHangDetectionMarker(markerPath, marker)
|
||||
|
||||
@@ -1,12 +1,22 @@
|
||||
import { renameSync } from 'node:fs'
|
||||
import { rmSync } from 'node:fs'
|
||||
import type { CrashReportBreadcrumbData } from '../../shared/crash-reporting'
|
||||
import { resolveObservabilityConsent } from '../observability'
|
||||
import { hasActiveTraceSink } from '../observability/tracer'
|
||||
import { recordDurableCrashBreadcrumb } from '../crash-reporting/durable-crash-breadcrumb'
|
||||
import {
|
||||
consumeHangDetectionMarker,
|
||||
hangDetectionMarkerPath,
|
||||
readAvailableHangDetectionMarker,
|
||||
writeHangDetectionMarker,
|
||||
type HangDetectionMarker
|
||||
} from './hang-detection-marker'
|
||||
import {
|
||||
asHangObservations,
|
||||
emptyHangObservations,
|
||||
latestHangMarker,
|
||||
mergeHangObservations,
|
||||
type HangObservation
|
||||
} from './hang-observations'
|
||||
|
||||
function previousMarkerPath(userDataPath: string): string {
|
||||
return `${hangDetectionMarkerPath(userDataPath)}.previous`
|
||||
@@ -17,24 +27,113 @@ export function preservePreviousHangDetection(userDataPath: string): void {
|
||||
return
|
||||
}
|
||||
try {
|
||||
// Keep the previous run's note safe from a new startup hang until its trace sink opens.
|
||||
renameSync(hangDetectionMarkerPath(userDataPath), previousMarkerPath(userDataPath))
|
||||
const currentPath = hangDetectionMarkerPath(userDataPath)
|
||||
const pendingPath = previousMarkerPath(userDataPath)
|
||||
const current = asHangObservations(readAvailableHangDetectionMarker(currentPath))
|
||||
if (!current) {
|
||||
return
|
||||
}
|
||||
const pending = readAvailableHangDetectionMarker(pendingPath)
|
||||
if (pending === undefined) {
|
||||
return
|
||||
}
|
||||
const merged = mergeHangObservations(
|
||||
asHangObservations(pending) ?? emptyHangObservations(),
|
||||
current
|
||||
)
|
||||
writeHangDetectionMarker(pendingPath, merged)
|
||||
rmSync(currentPath, { force: true })
|
||||
} catch {
|
||||
// Missing or inaccessible markers must not prevent startup; retain any earlier pending note.
|
||||
}
|
||||
}
|
||||
|
||||
function observationDetails(record: HangObservation, omitted: number): CrashReportBreadcrumbData {
|
||||
const details: CrashReportBreadcrumbData = {
|
||||
previousPid: record.parentPid,
|
||||
detectedAt: record.detectedAt,
|
||||
observationKind: record.kind,
|
||||
omittedObservations: omitted,
|
||||
activeMainActivity: 'unknown',
|
||||
cause: 'unknown',
|
||||
cpuScope: 'process-all-threads',
|
||||
machineEvidence: record.machine ? 'sampled' : 'unknown'
|
||||
}
|
||||
if (record.kind === 'heartbeat-stall') {
|
||||
details.unresponsiveMs =
|
||||
latestHangMarker({ version: 1, observations: [record], omitted: 0 })?.unresponsiveMs ??
|
||||
record.durationMs
|
||||
details.detectedUnresponsiveMs = record.durationMs
|
||||
details.selfRecovered = record.selfRecovered
|
||||
} else {
|
||||
details.watchdogGapMs = record.durationMs
|
||||
details.preGapCpuDeltaUs = null
|
||||
}
|
||||
for (const [prefix, snapshot] of [
|
||||
['detection', record.snapshot],
|
||||
['resolution', record.resolution],
|
||||
['gapEnd', record.watchdogGap?.snapshot]
|
||||
] as const) {
|
||||
if (snapshot) {
|
||||
for (const [key, value] of Object.entries(snapshot)) {
|
||||
details[`${prefix}_${key}`] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
const before = record.snapshot
|
||||
const after = record.resolution
|
||||
for (const scope of ['process', 'watchdog'] as const) {
|
||||
const firstUser = before?.[`${scope}UserUs`]
|
||||
const firstSystem = before?.[`${scope}SystemUs`]
|
||||
const lastUser = after?.[`${scope}UserUs`]
|
||||
const lastSystem = after?.[`${scope}SystemUs`]
|
||||
const delta =
|
||||
typeof firstUser === 'number' &&
|
||||
typeof firstSystem === 'number' &&
|
||||
typeof lastUser === 'number' &&
|
||||
typeof lastSystem === 'number'
|
||||
? lastUser + lastSystem - firstUser - firstSystem
|
||||
: null
|
||||
details[`${scope}CpuDetectionToResolutionUs`] = delta !== null && delta >= 0 ? delta : null
|
||||
}
|
||||
if (record.machine) {
|
||||
for (const [key, value] of Object.entries(record.machine)) {
|
||||
details[`machine_${key}`] = value
|
||||
}
|
||||
}
|
||||
if (record.watchdogGap) {
|
||||
details.overlappingWatchdogGapMs = record.watchdogGap.durationMs
|
||||
details.overlappingWatchdogGapAt = record.watchdogGap.observedAt
|
||||
}
|
||||
for (const [index, receipt] of record.powerEvents.entries()) {
|
||||
details[`power${index}_event`] = receipt.event
|
||||
details[`power${index}_mainReceiptAt`] = receipt.observedAt
|
||||
details[`power${index}_watchdogReceiptAt`] = receipt.receivedAt
|
||||
}
|
||||
details.completedContextEvidence =
|
||||
record.completedSpans === null ? 'unknown' : 'completed-spans-on-disk'
|
||||
for (const [index, span] of (record.completedSpans ?? []).entries()) {
|
||||
details[`completed${index}_name`] = span.name
|
||||
details[`completed${index}_startedAt`] = span.startedAt
|
||||
details[`completed${index}_endedAt`] = span.completedAt
|
||||
}
|
||||
return details
|
||||
}
|
||||
|
||||
export function reportPreviousHangDetection(userDataPath: string): HangDetectionMarker | null {
|
||||
if (!hasActiveTraceSink()) {
|
||||
if (!resolveObservabilityConsent().localFileEnabled || !hasActiveTraceSink()) {
|
||||
return null
|
||||
}
|
||||
const marker = consumeHangDetectionMarker(previousMarkerPath(userDataPath))
|
||||
if (marker) {
|
||||
recordDurableCrashBreadcrumb('main_thread_hang_detected', {
|
||||
unresponsiveMs: marker.unresponsiveMs,
|
||||
previousPid: marker.parentPid,
|
||||
selfRecovered: marker.selfRecovered
|
||||
})
|
||||
const collection = asHangObservations(marker)
|
||||
if (!collection) {
|
||||
return null
|
||||
}
|
||||
return marker
|
||||
for (const record of collection.observations) {
|
||||
recordDurableCrashBreadcrumb(
|
||||
record.kind === 'heartbeat-stall' ? 'main_thread_hang_detected' : 'main_thread_watchdog_gap',
|
||||
observationDetails(record, collection.omitted)
|
||||
)
|
||||
}
|
||||
return latestHangMarker(collection)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user