Files
orca/src/main/startup/main-process-error-guards.test.ts
T
Brennan Benson 6eb70d8370 fix(main): stop unhandled rejections from silently killing the app during startup restore (#9970)
* fix(main): log unhandled rejections durably and stop them killing the app (#9441)

Main had no unhandledRejection handler — Node's default killed the process
with a clean exit code and no macOS crash report, which is exactly the
silent startup-restore exit in #9441. Rejections now record a durable crash
breadcrumb (flushed to the trace sink) and the app stays alive; uncaught
exceptions still terminate but record the same breadcrumb first so field
exits stop being undiagnosable.

* fix(main): harden process error guard diagnostics

* fix(main): cap fatal-error records to stop a rejection storm stalling main

One broken resource can reject hundreds of concurrent restore chains, and
each durable record does a synchronous trace flush. Cap records at 20 per
60s window and carry the suppressed count into the next recorded event.

* fix(main): never rate-limit the fatal uncaught-exception breadcrumb

A rejection storm that exhausted the 60s record window would suppress the
one main_uncaught_exception record taken right before the re-throw kills
the process, recreating the undiagnosable death from issue #9441. Scope
the fixed window to main_unhandled_rejection only; the uncaught path
records at most once per install (handler removes itself, EPIPE/EIO never
record), so it cannot storm.

* fix(main): reset fatal-error window on backward clock jumps
2026-07-23 18:31:02 -07:00

267 lines
11 KiB
TypeScript

import { afterEach, describe, expect, it, vi } from 'vitest'
afterEach(() => {
vi.restoreAllMocks()
})
describe('main-process fatal error guards (issue #9441)', () => {
it('records unhandled rejections durably and keeps the process alive', async () => {
vi.resetModules()
const record = vi.fn()
vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
recordDurableCrashBreadcrumb: record
}))
const { installUnhandledRejectionLogging } = await import('./main-process-error-guards')
const before = process.listeners('unhandledRejection').length
installUnhandledRejectionLogging()
const listeners = process.listeners('unhandledRejection')
expect(listeners.length).toBe(before + 1)
const listener = listeners.at(-1) as (reason: unknown, promise: Promise<unknown>) => void
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
try {
// Why: invoking the listener directly must not throw — a throwing handler would still kill main.
expect(() =>
listener(Object.assign(new Error('spawn EAGAIN'), { code: 'EAGAIN' }), Promise.resolve())
).not.toThrow()
} finally {
process.removeListener('unhandledRejection', listener as never)
consoleError.mockRestore()
}
expect(record).toHaveBeenCalledWith(
'main_unhandled_rejection',
expect.objectContaining({ errorMessage: 'spawn EAGAIN', errorCode: 'EAGAIN' }),
'main_unhandled_rejection'
)
})
it('never throws when the breadcrumb sink fails', async () => {
vi.resetModules()
vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
recordDurableCrashBreadcrumb: vi.fn(() => {
throw new Error('sink offline')
})
}))
const { recordFatalMainProcessError } = await import('./main-process-error-guards')
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
try {
expect(() =>
recordFatalMainProcessError('main_uncaught_exception', 'not-an-error')
).not.toThrow()
} finally {
consoleError.mockRestore()
}
})
it('keeps absent optional error fields empty', async () => {
vi.resetModules()
const record = vi.fn()
vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
recordDurableCrashBreadcrumb: record
}))
const { recordFatalMainProcessError } = await import('./main-process-error-guards')
vi.spyOn(console, 'error').mockImplementation(() => {})
recordFatalMainProcessError('main_unhandled_rejection', new Error('boom'))
expect(record).toHaveBeenCalledWith(
'main_unhandled_rejection',
expect.objectContaining({ errorMessage: 'boom', errorCode: '' }),
'main_unhandled_rejection'
)
})
it('bounds and isolates console formatting for hostile rejection values', async () => {
vi.resetModules()
const record = vi.fn()
vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
recordDurableCrashBreadcrumb: record
}))
const { recordFatalMainProcessError } = await import('./main-process-error-guards')
const hostileReason = {
toString(): never {
throw new Error('toString failed')
},
[Symbol.for('nodejs.util.inspect.custom')](): never {
throw new Error('inspect failed')
}
}
const consoleError = vi.spyOn(console, 'error').mockImplementation((...values: unknown[]) => {
if (values.some((value) => typeof value !== 'string')) {
throw new Error('unsafe console formatting')
}
})
expect(() =>
recordFatalMainProcessError('main_unhandled_rejection', hostileReason)
).not.toThrow()
expect(record).toHaveBeenCalledWith(
'main_unhandled_rejection',
expect.objectContaining({ errorName: 'object', errorMessage: '[unprintable value]' }),
'main_unhandled_rejection'
)
expect(consoleError).toHaveBeenCalledWith(
expect.stringMatching(/^\[main_unhandled_rejection\]/)
)
expect(String(consoleError.mock.calls[0]?.[0]).length).toBeLessThan(5_000)
})
it('caps oversized rejection diagnostics before recording or logging', async () => {
vi.resetModules()
const record = vi.fn()
vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
recordDurableCrashBreadcrumb: record
}))
const { recordFatalMainProcessError } = await import('./main-process-error-guards')
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
const error = Object.assign(new Error('m'.repeat(100_000)), { code: 'c'.repeat(100_000) })
error.name = 'n'.repeat(100_000)
error.stack = Array.from({ length: 100 }, () => 's'.repeat(1_000)).join('\n')
recordFatalMainProcessError('main_unhandled_rejection', error)
const details = record.mock.calls[0]?.[1] as Record<string, string>
expect(details.errorName).toHaveLength(100)
expect(details.errorMessage).toHaveLength(500)
expect(details.errorStack.length).toBeLessThanOrEqual(4_000)
expect(details.errorCode).toHaveLength(100)
expect(String(consoleError.mock.calls[0]?.[0]).length).toBeLessThan(5_000)
})
it('caps a rejection storm and carries the suppressed count into the next window', async () => {
vi.resetModules()
const record = vi.fn()
vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
recordDurableCrashBreadcrumb: record
}))
const { recordFatalMainProcessError } = await import('./main-process-error-guards')
vi.spyOn(console, 'error').mockImplementation(() => {})
let now = 1_000_000
vi.spyOn(Date, 'now').mockImplementation(() => now)
for (let i = 0; i < 25; i++) {
recordFatalMainProcessError('main_unhandled_rejection', new Error(`storm ${i}`))
}
expect(record).toHaveBeenCalledTimes(20)
now += 60_000
recordFatalMainProcessError('main_unhandled_rejection', new Error('after window'))
expect(record).toHaveBeenCalledTimes(21)
expect(record).toHaveBeenLastCalledWith(
'main_unhandled_rejection',
expect.objectContaining({ errorMessage: 'after window', suppressedSinceLast: 5 }),
'main_unhandled_rejection'
)
})
it('reopens the window when the wall clock jumps backwards after exhaustion', async () => {
vi.resetModules()
const record = vi.fn()
vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
recordDurableCrashBreadcrumb: record
}))
const { recordFatalMainProcessError } = await import('./main-process-error-guards')
vi.spyOn(console, 'error').mockImplementation(() => {})
let now = 1_000_000
vi.spyOn(Date, 'now').mockImplementation(() => now)
for (let i = 0; i < 25; i++) {
recordFatalMainProcessError('main_unhandled_rejection', new Error(`storm ${i}`))
}
expect(record).toHaveBeenCalledTimes(20)
// Why: a backward jump must not trap the exhausted window and suppress every later breadcrumb.
now -= 3_600_000
recordFatalMainProcessError('main_unhandled_rejection', new Error('after backward jump'))
expect(record).toHaveBeenCalledTimes(21)
expect(record).toHaveBeenLastCalledWith(
'main_unhandled_rejection',
expect.objectContaining({ errorMessage: 'after backward jump', suppressedSinceLast: 5 }),
'main_unhandled_rejection'
)
})
it('never suppresses the fatal uncaught-exception record after a rejection storm', async () => {
vi.resetModules()
const record = vi.fn()
vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
recordDurableCrashBreadcrumb: record
}))
const { recordFatalMainProcessError } = await import('./main-process-error-guards')
vi.spyOn(console, 'error').mockImplementation(() => {})
vi.spyOn(Date, 'now').mockReturnValue(1_000_000)
for (let i = 0; i < 25; i++) {
recordFatalMainProcessError('main_unhandled_rejection', new Error(`storm ${i}`))
}
expect(record).toHaveBeenCalledTimes(20)
// Why: this record precedes the re-throw that kills main; losing it would recreate issue #9441.
recordFatalMainProcessError('main_uncaught_exception', new Error('fatal after storm'))
expect(record).toHaveBeenCalledTimes(21)
expect(record).toHaveBeenLastCalledWith(
'main_uncaught_exception',
expect.objectContaining({ errorMessage: 'fatal after storm', suppressedSinceLast: 5 }),
'main_uncaught_exception'
)
})
it('keeps uncaught pipe errors swallowed without a durable record', async () => {
vi.resetModules()
const record = vi.fn()
vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
recordDurableCrashBreadcrumb: record
}))
const { installUncaughtPipeErrorGuard } = await import('./main-process-error-guards')
const before = process.listeners('uncaughtException').length
installUncaughtPipeErrorGuard()
const listeners = process.listeners('uncaughtException')
expect(listeners.length).toBe(before + 1)
const listener = listeners.at(-1) as (error: unknown) => void
try {
listener(Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }))
} finally {
process.removeListener('uncaughtException', listener as never)
}
// Why: EPIPE/EIO are expected pipe churn; recording them would flood the breadcrumb store.
expect(record).not.toHaveBeenCalled()
})
it('rethrows non-pipe errors outside the uncaughtException handler', async () => {
vi.resetModules()
vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
recordDurableCrashBreadcrumb: vi.fn()
}))
const { installUncaughtPipeErrorGuard } = await import('./main-process-error-guards')
const originalOn = process.on.bind(process)
const originalOff = process.off.bind(process)
let handler: ((error: unknown) => void) | null = null
let scheduled: (() => void) | null = null
vi.spyOn(console, 'error').mockImplementation(() => {})
vi.spyOn(process, 'on').mockImplementation(((event, listener) => {
if (event === 'uncaughtException') {
handler = listener as (error: unknown) => void
return process
}
return originalOn(event, listener)
}) as typeof process.on)
const offSpy = vi.spyOn(process, 'off').mockImplementation(((event, listener) => {
if (event === 'uncaughtException') {
return process
}
return originalOff(event, listener)
}) as typeof process.off)
vi.spyOn(globalThis, 'setImmediate').mockImplementation(((callback) => {
scheduled = callback as () => void
return {} as NodeJS.Immediate
}) as typeof setImmediate)
installUncaughtPipeErrorGuard()
const error = new Error('boom')
expect(() => handler?.(error)).not.toThrow()
expect(offSpy).toHaveBeenCalledWith('uncaughtException', handler)
expect(scheduled).not.toBeNull()
expect(() => scheduled?.()).toThrow(error)
})
})