diff --git a/src/main/index.ts b/src/main/index.ts index 4b8339cdb22..d0fe7a89a31 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -80,11 +80,14 @@ import { installDevParentDisconnectQuit, installDevParentSignalQuit, installDevParentWatchdog, - installUncaughtPipeErrorGuard, isDevParentShutdownRequested, patchPackagedProcessPath, shouldInstallManagedHooks } from './startup/configure-process' +import { + installUncaughtPipeErrorGuard, + installUnhandledRejectionLogging +} from './startup/main-process-error-guards' import { enableRendererHeapHeadroom } from './startup/renderer-heap-headroom' import { ensureVirtualDisplayForHeadlessServe } from './startup/ensure-virtual-display' import { @@ -461,6 +464,8 @@ const devAgentHookEndpointNamespace = devInstanceIdentity.isDev : undefined installUncaughtPipeErrorGuard() +// Why (issue #9441): without this, one rejected background promise during startup restore kills main silently (exit 1, no crash report). +installUnhandledRejectionLogging() // Why: expose the app version via process.env so main and the forked daemon can set TERM_PROGRAM_VERSION without importing electron. process.env.ORCA_APP_VERSION = app.getVersion() configureRemoteServerUpdater({ diff --git a/src/main/startup/configure-process-pipe-error-guard.test.ts b/src/main/startup/configure-process-pipe-error-guard.test.ts deleted file mode 100644 index 08f42c357d7..00000000000 --- a/src/main/startup/configure-process-pipe-error-guard.test.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest' - -vi.mock('electron', () => { - return { - app: { - getPath: vi.fn(() => ''), - setPath: vi.fn(), - quit: vi.fn(), - exit: vi.fn(), - isPackaged: false, - disableHardwareAcceleration: vi.fn(), - commandLine: { - appendSwitch: vi.fn(), - getSwitchValue: vi.fn(() => '') - } - } - } -}) - -describe('installUncaughtPipeErrorGuard', () => { - afterEach(() => { - vi.restoreAllMocks() - }) - - it('suppresses uncaught pipe errors', async () => { - const { installUncaughtPipeErrorGuard } = await import('./configure-process') - const originalOn = process.on.bind(process) - let handler: ((error: unknown) => void) | null = null - const onSpy = vi.spyOn(process, 'on').mockImplementation(((event, listener) => { - if (event === 'uncaughtException') { - handler = listener as (error: unknown) => void - return process - } - return originalOn(event, listener) - }) as typeof process.on) - - installUncaughtPipeErrorGuard() - - const pipeError = new Error('broken pipe') as NodeJS.ErrnoException - pipeError.code = 'EPIPE' - expect(() => handler?.(pipeError)).not.toThrow() - expect(onSpy).toHaveBeenCalledWith('uncaughtException', expect.any(Function)) - }) - - it('rethrows non-pipe errors outside the uncaughtException handler', async () => { - const { installUncaughtPipeErrorGuard } = await import('./configure-process') - const originalOn = process.on.bind(process) - const originalOff = process.off.bind(process) - let handler: ((error: unknown) => void) | null = null - let scheduled: (() => void) | null = null - vi.spyOn(process, 'on').mockImplementation(((event, listener) => { - if (event === 'uncaughtException') { - handler = listener as (error: unknown) => void - return process - } - return originalOn(event, listener) - }) as typeof process.on) - const offSpy = vi.spyOn(process, 'off').mockImplementation(((event, listener) => { - if (event === 'uncaughtException') { - return process - } - return originalOff(event, listener) - }) as typeof process.off) - vi.spyOn(globalThis, 'setImmediate').mockImplementation(((callback) => { - scheduled = callback as () => void - return {} as NodeJS.Immediate - }) as typeof setImmediate) - - installUncaughtPipeErrorGuard() - - const error = new Error('boom') - expect(() => handler?.(error)).not.toThrow() - expect(offSpy).toHaveBeenCalledWith('uncaughtException', handler) - expect(scheduled).not.toBeNull() - expect(() => scheduled?.()).toThrow(error) - }) -}) diff --git a/src/main/startup/configure-process.ts b/src/main/startup/configure-process.ts index 5db98d1afbf..62ed97a36bf 100644 --- a/src/main/startup/configure-process.ts +++ b/src/main/startup/configure-process.ts @@ -91,28 +91,6 @@ export function resetDevParentShutdownRequestForTests(): void { devParentShutdownRequested = false } -export function installUncaughtPipeErrorGuard(): void { - const onUncaughtException = (error: unknown): void => { - if ( - error && - typeof error === 'object' && - 'code' in error && - ((error as NodeJS.ErrnoException).code === 'EIO' || - (error as NodeJS.ErrnoException).code === 'EPIPE') - ) { - return - } - - process.off('uncaughtException', onUncaughtException) - // Why: throwing inside an uncaughtException handler exits with status 7 and hides the fault; re-throw next tick for the real stack. - setImmediate(() => { - throw error - }) - } - - process.on('uncaughtException', onUncaughtException) -} - export function patchPackagedProcessPath(): void { if (!app.isPackaged) { return diff --git a/src/main/startup/main-process-error-guards.test.ts b/src/main/startup/main-process-error-guards.test.ts new file mode 100644 index 00000000000..a9fae48ecb4 --- /dev/null +++ b/src/main/startup/main-process-error-guards.test.ts @@ -0,0 +1,266 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +afterEach(() => { + vi.restoreAllMocks() +}) + +describe('main-process fatal error guards (issue #9441)', () => { + it('records unhandled rejections durably and keeps the process alive', async () => { + vi.resetModules() + const record = vi.fn() + vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({ + recordDurableCrashBreadcrumb: record + })) + const { installUnhandledRejectionLogging } = await import('./main-process-error-guards') + const before = process.listeners('unhandledRejection').length + installUnhandledRejectionLogging() + const listeners = process.listeners('unhandledRejection') + expect(listeners.length).toBe(before + 1) + const listener = listeners.at(-1) as (reason: unknown, promise: Promise) => void + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}) + try { + // Why: invoking the listener directly must not throw — a throwing handler would still kill main. + expect(() => + listener(Object.assign(new Error('spawn EAGAIN'), { code: 'EAGAIN' }), Promise.resolve()) + ).not.toThrow() + } finally { + process.removeListener('unhandledRejection', listener as never) + consoleError.mockRestore() + } + expect(record).toHaveBeenCalledWith( + 'main_unhandled_rejection', + expect.objectContaining({ errorMessage: 'spawn EAGAIN', errorCode: 'EAGAIN' }), + 'main_unhandled_rejection' + ) + }) + + it('never throws when the breadcrumb sink fails', async () => { + vi.resetModules() + vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({ + recordDurableCrashBreadcrumb: vi.fn(() => { + throw new Error('sink offline') + }) + })) + const { recordFatalMainProcessError } = await import('./main-process-error-guards') + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}) + try { + expect(() => + recordFatalMainProcessError('main_uncaught_exception', 'not-an-error') + ).not.toThrow() + } finally { + consoleError.mockRestore() + } + }) + + it('keeps absent optional error fields empty', async () => { + vi.resetModules() + const record = vi.fn() + vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({ + recordDurableCrashBreadcrumb: record + })) + const { recordFatalMainProcessError } = await import('./main-process-error-guards') + vi.spyOn(console, 'error').mockImplementation(() => {}) + + recordFatalMainProcessError('main_unhandled_rejection', new Error('boom')) + + expect(record).toHaveBeenCalledWith( + 'main_unhandled_rejection', + expect.objectContaining({ errorMessage: 'boom', errorCode: '' }), + 'main_unhandled_rejection' + ) + }) + + it('bounds and isolates console formatting for hostile rejection values', async () => { + vi.resetModules() + const record = vi.fn() + vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({ + recordDurableCrashBreadcrumb: record + })) + const { recordFatalMainProcessError } = await import('./main-process-error-guards') + const hostileReason = { + toString(): never { + throw new Error('toString failed') + }, + [Symbol.for('nodejs.util.inspect.custom')](): never { + throw new Error('inspect failed') + } + } + const consoleError = vi.spyOn(console, 'error').mockImplementation((...values: unknown[]) => { + if (values.some((value) => typeof value !== 'string')) { + throw new Error('unsafe console formatting') + } + }) + + expect(() => + recordFatalMainProcessError('main_unhandled_rejection', hostileReason) + ).not.toThrow() + expect(record).toHaveBeenCalledWith( + 'main_unhandled_rejection', + expect.objectContaining({ errorName: 'object', errorMessage: '[unprintable value]' }), + 'main_unhandled_rejection' + ) + expect(consoleError).toHaveBeenCalledWith( + expect.stringMatching(/^\[main_unhandled_rejection\]/) + ) + expect(String(consoleError.mock.calls[0]?.[0]).length).toBeLessThan(5_000) + }) + + it('caps oversized rejection diagnostics before recording or logging', async () => { + vi.resetModules() + const record = vi.fn() + vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({ + recordDurableCrashBreadcrumb: record + })) + const { recordFatalMainProcessError } = await import('./main-process-error-guards') + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}) + const error = Object.assign(new Error('m'.repeat(100_000)), { code: 'c'.repeat(100_000) }) + error.name = 'n'.repeat(100_000) + error.stack = Array.from({ length: 100 }, () => 's'.repeat(1_000)).join('\n') + + recordFatalMainProcessError('main_unhandled_rejection', error) + + const details = record.mock.calls[0]?.[1] as Record + expect(details.errorName).toHaveLength(100) + expect(details.errorMessage).toHaveLength(500) + expect(details.errorStack.length).toBeLessThanOrEqual(4_000) + expect(details.errorCode).toHaveLength(100) + expect(String(consoleError.mock.calls[0]?.[0]).length).toBeLessThan(5_000) + }) + + it('caps a rejection storm and carries the suppressed count into the next window', async () => { + vi.resetModules() + const record = vi.fn() + vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({ + recordDurableCrashBreadcrumb: record + })) + const { recordFatalMainProcessError } = await import('./main-process-error-guards') + vi.spyOn(console, 'error').mockImplementation(() => {}) + let now = 1_000_000 + vi.spyOn(Date, 'now').mockImplementation(() => now) + + for (let i = 0; i < 25; i++) { + recordFatalMainProcessError('main_unhandled_rejection', new Error(`storm ${i}`)) + } + expect(record).toHaveBeenCalledTimes(20) + + now += 60_000 + recordFatalMainProcessError('main_unhandled_rejection', new Error('after window')) + expect(record).toHaveBeenCalledTimes(21) + expect(record).toHaveBeenLastCalledWith( + 'main_unhandled_rejection', + expect.objectContaining({ errorMessage: 'after window', suppressedSinceLast: 5 }), + 'main_unhandled_rejection' + ) + }) + + it('reopens the window when the wall clock jumps backwards after exhaustion', async () => { + vi.resetModules() + const record = vi.fn() + vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({ + recordDurableCrashBreadcrumb: record + })) + const { recordFatalMainProcessError } = await import('./main-process-error-guards') + vi.spyOn(console, 'error').mockImplementation(() => {}) + let now = 1_000_000 + vi.spyOn(Date, 'now').mockImplementation(() => now) + + for (let i = 0; i < 25; i++) { + recordFatalMainProcessError('main_unhandled_rejection', new Error(`storm ${i}`)) + } + expect(record).toHaveBeenCalledTimes(20) + + // Why: a backward jump must not trap the exhausted window and suppress every later breadcrumb. + now -= 3_600_000 + recordFatalMainProcessError('main_unhandled_rejection', new Error('after backward jump')) + expect(record).toHaveBeenCalledTimes(21) + expect(record).toHaveBeenLastCalledWith( + 'main_unhandled_rejection', + expect.objectContaining({ errorMessage: 'after backward jump', suppressedSinceLast: 5 }), + 'main_unhandled_rejection' + ) + }) + + it('never suppresses the fatal uncaught-exception record after a rejection storm', async () => { + vi.resetModules() + const record = vi.fn() + vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({ + recordDurableCrashBreadcrumb: record + })) + const { recordFatalMainProcessError } = await import('./main-process-error-guards') + vi.spyOn(console, 'error').mockImplementation(() => {}) + vi.spyOn(Date, 'now').mockReturnValue(1_000_000) + + for (let i = 0; i < 25; i++) { + recordFatalMainProcessError('main_unhandled_rejection', new Error(`storm ${i}`)) + } + expect(record).toHaveBeenCalledTimes(20) + + // Why: this record precedes the re-throw that kills main; losing it would recreate issue #9441. + recordFatalMainProcessError('main_uncaught_exception', new Error('fatal after storm')) + expect(record).toHaveBeenCalledTimes(21) + expect(record).toHaveBeenLastCalledWith( + 'main_uncaught_exception', + expect.objectContaining({ errorMessage: 'fatal after storm', suppressedSinceLast: 5 }), + 'main_uncaught_exception' + ) + }) + + it('keeps uncaught pipe errors swallowed without a durable record', async () => { + vi.resetModules() + const record = vi.fn() + vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({ + recordDurableCrashBreadcrumb: record + })) + const { installUncaughtPipeErrorGuard } = await import('./main-process-error-guards') + const before = process.listeners('uncaughtException').length + installUncaughtPipeErrorGuard() + const listeners = process.listeners('uncaughtException') + expect(listeners.length).toBe(before + 1) + const listener = listeners.at(-1) as (error: unknown) => void + try { + listener(Object.assign(new Error('write EPIPE'), { code: 'EPIPE' })) + } finally { + process.removeListener('uncaughtException', listener as never) + } + // Why: EPIPE/EIO are expected pipe churn; recording them would flood the breadcrumb store. + expect(record).not.toHaveBeenCalled() + }) + + it('rethrows non-pipe errors outside the uncaughtException handler', async () => { + vi.resetModules() + vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({ + recordDurableCrashBreadcrumb: vi.fn() + })) + const { installUncaughtPipeErrorGuard } = await import('./main-process-error-guards') + const originalOn = process.on.bind(process) + const originalOff = process.off.bind(process) + let handler: ((error: unknown) => void) | null = null + let scheduled: (() => void) | null = null + vi.spyOn(console, 'error').mockImplementation(() => {}) + vi.spyOn(process, 'on').mockImplementation(((event, listener) => { + if (event === 'uncaughtException') { + handler = listener as (error: unknown) => void + return process + } + return originalOn(event, listener) + }) as typeof process.on) + const offSpy = vi.spyOn(process, 'off').mockImplementation(((event, listener) => { + if (event === 'uncaughtException') { + return process + } + return originalOff(event, listener) + }) as typeof process.off) + vi.spyOn(globalThis, 'setImmediate').mockImplementation(((callback) => { + scheduled = callback as () => void + return {} as NodeJS.Immediate + }) as typeof setImmediate) + + installUncaughtPipeErrorGuard() + + const error = new Error('boom') + expect(() => handler?.(error)).not.toThrow() + expect(offSpy).toHaveBeenCalledWith('uncaughtException', handler) + expect(scheduled).not.toBeNull() + expect(() => scheduled?.()).toThrow(error) + }) +}) diff --git a/src/main/startup/main-process-error-guards.ts b/src/main/startup/main-process-error-guards.ts new file mode 100644 index 00000000000..fcf178c67be --- /dev/null +++ b/src/main/startup/main-process-error-guards.ts @@ -0,0 +1,131 @@ +import { recordDurableCrashBreadcrumb } from '../crash-reporting/durable-crash-breadcrumb' + +type FatalMainProcessErrorKind = 'main_uncaught_exception' | 'main_unhandled_rejection' + +type FatalMainProcessErrorDetails = { + errorName: string + errorMessage: string + errorStack: string + errorCode: string +} + +function readErrorProperty(error: unknown, property: string): unknown { + try { + return error !== null && (typeof error === 'object' || typeof error === 'function') + ? (error as Record)[property] + : undefined + } catch { + return undefined + } +} + +function boundedString(value: unknown, maxLength: number, fallback = ''): string { + try { + return String(value).slice(0, maxLength) + } catch { + return fallback + } +} + +function fatalMainProcessErrorDetails(error: unknown): FatalMainProcessErrorDetails { + let isError = false + try { + isError = error instanceof Error + } catch { + // Why: a proxy can throw from instanceof; fatal diagnostics still need a safe fallback. + } + + return { + errorName: isError + ? boundedString(readErrorProperty(error, 'name') ?? 'Error', 100, 'Error') + : typeof error, + errorMessage: isError + ? boundedString(readErrorProperty(error, 'message') ?? '', 500) + : boundedString(error, 500, '[unprintable value]'), + errorStack: isError + ? boundedString(readErrorProperty(error, 'stack') ?? '', 4_000) + .split('\n') + .slice(0, 12) + .join('\n') + : '', + errorCode: boundedString(readErrorProperty(error, 'code') ?? '', 100) + } +} + +// Why: one broken resource can reject hundreds of concurrent restore chains; each record does a +// synchronous trace flush, so an uncapped storm stalls main and churns the trace-file rotation. +const RECORD_WINDOW_MS = 60_000 +const RECORD_WINDOW_MAX = 20 +let recordWindowStartedAt = 0 +let recordWindowCount = 0 +let recordsSuppressed = 0 + +/** Durably record a main-process fatal/near-fatal error before default handling runs. Exported for tests. */ +export function recordFatalMainProcessError(kind: FatalMainProcessErrorKind, error: unknown): void { + // Why: only rejections can storm; the one uncaught-exception record before the fatal re-throw + // must never be lost to a window a storm already exhausted. + if (kind === 'main_unhandled_rejection') { + const now = Date.now() + // Why: a backward clock jump (sleep/resume, NTP) would otherwise trap an exhausted window and suppress every breadcrumb until wall time catches up. + if (now < recordWindowStartedAt || now - recordWindowStartedAt >= RECORD_WINDOW_MS) { + recordWindowStartedAt = now + recordWindowCount = 0 + } + if (recordWindowCount >= RECORD_WINDOW_MAX) { + recordsSuppressed += 1 + return + } + recordWindowCount += 1 + } + const suppressedSinceLast = recordsSuppressed + recordsSuppressed = 0 + const details = fatalMainProcessErrorDetails(error) + try { + recordDurableCrashBreadcrumb( + kind, + suppressedSinceLast > 0 ? { ...details, suppressedSinceLast } : details, + kind + ) + } catch { + // Why: diagnostics must never turn a fatal-error report into a second fault. + } + try { + console.error( + `[${kind}] ${details.errorStack || `${details.errorName}: ${details.errorMessage}`}` + ) + } catch { + // Why: custom console sinks must not defeat the process-level safety guard. + } +} + +export function installUncaughtPipeErrorGuard(): void { + const onUncaughtException = (error: unknown): void => { + const errorCode = readErrorProperty(error, 'code') + if (errorCode === 'EIO' || errorCode === 'EPIPE') { + return + } + + // Why (issue #9441): the re-throw below exits with a clean code and no macOS crash report; record durably first or the death is undiagnosable in the field. + recordFatalMainProcessError('main_uncaught_exception', error) + process.off('uncaughtException', onUncaughtException) + // Why: throwing inside an uncaughtException handler exits with status 7 and hides the fault; re-throw next tick for the real stack. + setImmediate(() => { + throw error + }) + } + + process.on('uncaughtException', onUncaughtException) +} + +/** Keep one failed background promise from silently killing the whole app. + * + * Node's default kills the process on an unhandled rejection. Large-profile startup restore runs + * hundreds of concurrent async chains (worktree scans, terminal reconnects) in main; a single + * rejection in any of them exited the app with no crash report (issue #9441). Log it durably and + * stay alive — dying cannot be less disruptive than continuing with one failed background task. + */ +export function installUnhandledRejectionLogging(): void { + process.on('unhandledRejection', (reason) => { + recordFatalMainProcessError('main_unhandled_rejection', reason) + }) +}