fix(main): stop unhandled rejections from silently killing the app during startup restore (#9970)

* fix(main): log unhandled rejections durably and stop them killing the app (#9441)

Main had no unhandledRejection handler — Node's default killed the process
with a clean exit code and no macOS crash report, which is exactly the
silent startup-restore exit in #9441. Rejections now record a durable crash
breadcrumb (flushed to the trace sink) and the app stays alive; uncaught
exceptions still terminate but record the same breadcrumb first so field
exits stop being undiagnosable.

* fix(main): harden process error guard diagnostics

* fix(main): cap fatal-error records to stop a rejection storm stalling main

One broken resource can reject hundreds of concurrent restore chains, and
each durable record does a synchronous trace flush. Cap records at 20 per
60s window and carry the suppressed count into the next recorded event.

* fix(main): never rate-limit the fatal uncaught-exception breadcrumb

A rejection storm that exhausted the 60s record window would suppress the
one main_uncaught_exception record taken right before the re-throw kills
the process, recreating the undiagnosable death from issue #9441. Scope
the fixed window to main_unhandled_rejection only; the uncaught path
records at most once per install (handler removes itself, EPIPE/EIO never
record), so it cannot storm.

* fix(main): reset fatal-error window on backward clock jumps

src/main/index.ts

@@ -80,11 +80,14 @@ import {
   installDevParentDisconnectQuit,
   installDevParentSignalQuit,
   installDevParentWatchdog,
-  installUncaughtPipeErrorGuard,
   isDevParentShutdownRequested,
   patchPackagedProcessPath,
   shouldInstallManagedHooks
 } from './startup/configure-process'
+import {
+  installUncaughtPipeErrorGuard,
+  installUnhandledRejectionLogging
+} from './startup/main-process-error-guards'
 import { enableRendererHeapHeadroom } from './startup/renderer-heap-headroom'
 import { ensureVirtualDisplayForHeadlessServe } from './startup/ensure-virtual-display'
 import {
@@ -461,6 +464,8 @@ const devAgentHookEndpointNamespace = devInstanceIdentity.isDev
   : undefined
 
 installUncaughtPipeErrorGuard()
+// Why (issue #9441): without this, one rejected background promise during startup restore kills main silently (exit 1, no crash report).
+installUnhandledRejectionLogging()
 // Why: expose the app version via process.env so main and the forked daemon can set TERM_PROGRAM_VERSION without importing electron.
 process.env.ORCA_APP_VERSION = app.getVersion()
 configureRemoteServerUpdater({

src/main/startup/configure-process-pipe-error-guard.test.ts

@@ -1,77 +0,0 @@
-import { afterEach, describe, expect, it, vi } from 'vitest'
-
-vi.mock('electron', () => {
-  return {
-    app: {
-      getPath: vi.fn(() => ''),
-      setPath: vi.fn(),
-      quit: vi.fn(),
-      exit: vi.fn(),
-      isPackaged: false,
-      disableHardwareAcceleration: vi.fn(),
-      commandLine: {
-        appendSwitch: vi.fn(),
-        getSwitchValue: vi.fn(() => '')
-      }
-    }
-  }
-})
-
-describe('installUncaughtPipeErrorGuard', () => {
-  afterEach(() => {
-    vi.restoreAllMocks()
-  })
-
-  it('suppresses uncaught pipe errors', async () => {
-    const { installUncaughtPipeErrorGuard } = await import('./configure-process')
-    const originalOn = process.on.bind(process)
-    let handler: ((error: unknown) => void) | null = null
-    const onSpy = vi.spyOn(process, 'on').mockImplementation(((event, listener) => {
-      if (event === 'uncaughtException') {
-        handler = listener as (error: unknown) => void
-        return process
-      }
-      return originalOn(event, listener)
-    }) as typeof process.on)
-
-    installUncaughtPipeErrorGuard()
-
-    const pipeError = new Error('broken pipe') as NodeJS.ErrnoException
-    pipeError.code = 'EPIPE'
-    expect(() => handler?.(pipeError)).not.toThrow()
-    expect(onSpy).toHaveBeenCalledWith('uncaughtException', expect.any(Function))
-  })
-
-  it('rethrows non-pipe errors outside the uncaughtException handler', async () => {
-    const { installUncaughtPipeErrorGuard } = await import('./configure-process')
-    const originalOn = process.on.bind(process)
-    const originalOff = process.off.bind(process)
-    let handler: ((error: unknown) => void) | null = null
-    let scheduled: (() => void) | null = null
-    vi.spyOn(process, 'on').mockImplementation(((event, listener) => {
-      if (event === 'uncaughtException') {
-        handler = listener as (error: unknown) => void
-        return process
-      }
-      return originalOn(event, listener)
-    }) as typeof process.on)
-    const offSpy = vi.spyOn(process, 'off').mockImplementation(((event, listener) => {
-      if (event === 'uncaughtException') {
-        return process
-      }
-      return originalOff(event, listener)
-    }) as typeof process.off)
-    vi.spyOn(globalThis, 'setImmediate').mockImplementation(((callback) => {
-      scheduled = callback as () => void
-      return {} as NodeJS.Immediate
-    }) as typeof setImmediate)
-
-    installUncaughtPipeErrorGuard()
-
-    const error = new Error('boom')
-    expect(() => handler?.(error)).not.toThrow()
-    expect(offSpy).toHaveBeenCalledWith('uncaughtException', handler)
-    expect(scheduled).not.toBeNull()
-    expect(() => scheduled?.()).toThrow(error)
-  })
-})

src/main/startup/configure-process.ts

@@ -91,28 +91,6 @@ export function resetDevParentShutdownRequestForTests(): void {
   devParentShutdownRequested = false
 }
 
-export function installUncaughtPipeErrorGuard(): void {
-  const onUncaughtException = (error: unknown): void => {
-    if (
-      error &&
-      typeof error === 'object' &&
-      'code' in error &&
-      ((error as NodeJS.ErrnoException).code === 'EIO' ||
-        (error as NodeJS.ErrnoException).code === 'EPIPE')
-    ) {
-      return
-    }
-
-    process.off('uncaughtException', onUncaughtException)
-    // Why: throwing inside an uncaughtException handler exits with status 7 and hides the fault; re-throw next tick for the real stack.
-    setImmediate(() => {
-      throw error
-    })
-  }
-
-  process.on('uncaughtException', onUncaughtException)
-}
-
 export function patchPackagedProcessPath(): void {
   if (!app.isPackaged) {
     return

src/main/startup/main-process-error-guards.test.ts

@@ -0,0 +1,266 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+afterEach(() => {
+  vi.restoreAllMocks()
+})
+
+describe('main-process fatal error guards (issue #9441)', () => {
+  it('records unhandled rejections durably and keeps the process alive', async () => {
+    vi.resetModules()
+    const record = vi.fn()
+    vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
+      recordDurableCrashBreadcrumb: record
+    }))
+    const { installUnhandledRejectionLogging } = await import('./main-process-error-guards')
+    const before = process.listeners('unhandledRejection').length
+    installUnhandledRejectionLogging()
+    const listeners = process.listeners('unhandledRejection')
+    expect(listeners.length).toBe(before + 1)
+    const listener = listeners.at(-1) as (reason: unknown, promise: Promise<unknown>) => void
+    const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
+    try {
+      // Why: invoking the listener directly must not throw — a throwing handler would still kill main.
+      expect(() =>
+        listener(Object.assign(new Error('spawn EAGAIN'), { code: 'EAGAIN' }), Promise.resolve())
+      ).not.toThrow()
+    } finally {
+      process.removeListener('unhandledRejection', listener as never)
+      consoleError.mockRestore()
+    }
+    expect(record).toHaveBeenCalledWith(
+      'main_unhandled_rejection',
+      expect.objectContaining({ errorMessage: 'spawn EAGAIN', errorCode: 'EAGAIN' }),
+      'main_unhandled_rejection'
+    )
+  })
+
+  it('never throws when the breadcrumb sink fails', async () => {
+    vi.resetModules()
+    vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
+      recordDurableCrashBreadcrumb: vi.fn(() => {
+        throw new Error('sink offline')
+      })
+    }))
+    const { recordFatalMainProcessError } = await import('./main-process-error-guards')
+    const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
+    try {
+      expect(() =>
+        recordFatalMainProcessError('main_uncaught_exception', 'not-an-error')
+      ).not.toThrow()
+    } finally {
+      consoleError.mockRestore()
+    }
+  })
+
+  it('keeps absent optional error fields empty', async () => {
+    vi.resetModules()
+    const record = vi.fn()
+    vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
+      recordDurableCrashBreadcrumb: record
+    }))
+    const { recordFatalMainProcessError } = await import('./main-process-error-guards')
+    vi.spyOn(console, 'error').mockImplementation(() => {})
+
+    recordFatalMainProcessError('main_unhandled_rejection', new Error('boom'))
+
+    expect(record).toHaveBeenCalledWith(
+      'main_unhandled_rejection',
+      expect.objectContaining({ errorMessage: 'boom', errorCode: '' }),
+      'main_unhandled_rejection'
+    )
+  })
+
+  it('bounds and isolates console formatting for hostile rejection values', async () => {
+    vi.resetModules()
+    const record = vi.fn()
+    vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
+      recordDurableCrashBreadcrumb: record
+    }))
+    const { recordFatalMainProcessError } = await import('./main-process-error-guards')
+    const hostileReason = {
+      toString(): never {
+        throw new Error('toString failed')
+      },
+      [Symbol.for('nodejs.util.inspect.custom')](): never {
+        throw new Error('inspect failed')
+      }
+    }
+    const consoleError = vi.spyOn(console, 'error').mockImplementation((...values: unknown[]) => {
+      if (values.some((value) => typeof value !== 'string')) {
+        throw new Error('unsafe console formatting')
+      }
+    })
+
+    expect(() =>
+      recordFatalMainProcessError('main_unhandled_rejection', hostileReason)
+    ).not.toThrow()
+    expect(record).toHaveBeenCalledWith(
+      'main_unhandled_rejection',
+      expect.objectContaining({ errorName: 'object', errorMessage: '[unprintable value]' }),
+      'main_unhandled_rejection'
+    )
+    expect(consoleError).toHaveBeenCalledWith(
+      expect.stringMatching(/^\[main_unhandled_rejection\]/)
+    )
+    expect(String(consoleError.mock.calls[0]?.[0]).length).toBeLessThan(5_000)
+  })
+
+  it('caps oversized rejection diagnostics before recording or logging', async () => {
+    vi.resetModules()
+    const record = vi.fn()
+    vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
+      recordDurableCrashBreadcrumb: record
+    }))
+    const { recordFatalMainProcessError } = await import('./main-process-error-guards')
+    const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
+    const error = Object.assign(new Error('m'.repeat(100_000)), { code: 'c'.repeat(100_000) })
+    error.name = 'n'.repeat(100_000)
+    error.stack = Array.from({ length: 100 }, () => 's'.repeat(1_000)).join('\n')
+
+    recordFatalMainProcessError('main_unhandled_rejection', error)
+
+    const details = record.mock.calls[0]?.[1] as Record<string, string>
+    expect(details.errorName).toHaveLength(100)
+    expect(details.errorMessage).toHaveLength(500)
+    expect(details.errorStack.length).toBeLessThanOrEqual(4_000)
+    expect(details.errorCode).toHaveLength(100)
+    expect(String(consoleError.mock.calls[0]?.[0]).length).toBeLessThan(5_000)
+  })
+
+  it('caps a rejection storm and carries the suppressed count into the next window', async () => {
+    vi.resetModules()
+    const record = vi.fn()
+    vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
+      recordDurableCrashBreadcrumb: record
+    }))
+    const { recordFatalMainProcessError } = await import('./main-process-error-guards')
+    vi.spyOn(console, 'error').mockImplementation(() => {})
+    let now = 1_000_000
+    vi.spyOn(Date, 'now').mockImplementation(() => now)
+
+    for (let i = 0; i < 25; i++) {
+      recordFatalMainProcessError('main_unhandled_rejection', new Error(`storm ${i}`))
+    }
+    expect(record).toHaveBeenCalledTimes(20)
+
+    now += 60_000
+    recordFatalMainProcessError('main_unhandled_rejection', new Error('after window'))
+    expect(record).toHaveBeenCalledTimes(21)
+    expect(record).toHaveBeenLastCalledWith(
+      'main_unhandled_rejection',
+      expect.objectContaining({ errorMessage: 'after window', suppressedSinceLast: 5 }),
+      'main_unhandled_rejection'
+    )
+  })
+
+  it('reopens the window when the wall clock jumps backwards after exhaustion', async () => {
+    vi.resetModules()
+    const record = vi.fn()
+    vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
+      recordDurableCrashBreadcrumb: record
+    }))
+    const { recordFatalMainProcessError } = await import('./main-process-error-guards')
+    vi.spyOn(console, 'error').mockImplementation(() => {})
+    let now = 1_000_000
+    vi.spyOn(Date, 'now').mockImplementation(() => now)
+
+    for (let i = 0; i < 25; i++) {
+      recordFatalMainProcessError('main_unhandled_rejection', new Error(`storm ${i}`))
+    }
+    expect(record).toHaveBeenCalledTimes(20)
+
+    // Why: a backward jump must not trap the exhausted window and suppress every later breadcrumb.
+    now -= 3_600_000
+    recordFatalMainProcessError('main_unhandled_rejection', new Error('after backward jump'))
+    expect(record).toHaveBeenCalledTimes(21)
+    expect(record).toHaveBeenLastCalledWith(
+      'main_unhandled_rejection',
+      expect.objectContaining({ errorMessage: 'after backward jump', suppressedSinceLast: 5 }),
+      'main_unhandled_rejection'
+    )
+  })
+
+  it('never suppresses the fatal uncaught-exception record after a rejection storm', async () => {
+    vi.resetModules()
+    const record = vi.fn()
+    vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
+      recordDurableCrashBreadcrumb: record
+    }))
+    const { recordFatalMainProcessError } = await import('./main-process-error-guards')
+    vi.spyOn(console, 'error').mockImplementation(() => {})
+    vi.spyOn(Date, 'now').mockReturnValue(1_000_000)
+
+    for (let i = 0; i < 25; i++) {
+      recordFatalMainProcessError('main_unhandled_rejection', new Error(`storm ${i}`))
+    }
+    expect(record).toHaveBeenCalledTimes(20)
+
+    // Why: this record precedes the re-throw that kills main; losing it would recreate issue #9441.
+    recordFatalMainProcessError('main_uncaught_exception', new Error('fatal after storm'))
+    expect(record).toHaveBeenCalledTimes(21)
+    expect(record).toHaveBeenLastCalledWith(
+      'main_uncaught_exception',
+      expect.objectContaining({ errorMessage: 'fatal after storm', suppressedSinceLast: 5 }),
+      'main_uncaught_exception'
+    )
+  })
+
+  it('keeps uncaught pipe errors swallowed without a durable record', async () => {
+    vi.resetModules()
+    const record = vi.fn()
+    vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
+      recordDurableCrashBreadcrumb: record
+    }))
+    const { installUncaughtPipeErrorGuard } = await import('./main-process-error-guards')
+    const before = process.listeners('uncaughtException').length
+    installUncaughtPipeErrorGuard()
+    const listeners = process.listeners('uncaughtException')
+    expect(listeners.length).toBe(before + 1)
+    const listener = listeners.at(-1) as (error: unknown) => void
+    try {
+      listener(Object.assign(new Error('write EPIPE'), { code: 'EPIPE' }))
+    } finally {
+      process.removeListener('uncaughtException', listener as never)
+    }
+    // Why: EPIPE/EIO are expected pipe churn; recording them would flood the breadcrumb store.
+    expect(record).not.toHaveBeenCalled()
+  })
+
+  it('rethrows non-pipe errors outside the uncaughtException handler', async () => {
+    vi.resetModules()
+    vi.doMock('../crash-reporting/durable-crash-breadcrumb', () => ({
+      recordDurableCrashBreadcrumb: vi.fn()
+    }))
+    const { installUncaughtPipeErrorGuard } = await import('./main-process-error-guards')
+    const originalOn = process.on.bind(process)
+    const originalOff = process.off.bind(process)
+    let handler: ((error: unknown) => void) | null = null
+    let scheduled: (() => void) | null = null
+    vi.spyOn(console, 'error').mockImplementation(() => {})
+    vi.spyOn(process, 'on').mockImplementation(((event, listener) => {
+      if (event === 'uncaughtException') {
+        handler = listener as (error: unknown) => void
+        return process
+      }
+      return originalOn(event, listener)
+    }) as typeof process.on)
+    const offSpy = vi.spyOn(process, 'off').mockImplementation(((event, listener) => {
+      if (event === 'uncaughtException') {
+        return process
+      }
+      return originalOff(event, listener)
+    }) as typeof process.off)
+    vi.spyOn(globalThis, 'setImmediate').mockImplementation(((callback) => {
+      scheduled = callback as () => void
+      return {} as NodeJS.Immediate
+    }) as typeof setImmediate)
+
+    installUncaughtPipeErrorGuard()
+
+    const error = new Error('boom')
+    expect(() => handler?.(error)).not.toThrow()
+    expect(offSpy).toHaveBeenCalledWith('uncaughtException', handler)
+    expect(scheduled).not.toBeNull()
+    expect(() => scheduled?.()).toThrow(error)
+  })
+})

src/main/startup/main-process-error-guards.ts

@@ -0,0 +1,131 @@
+import { recordDurableCrashBreadcrumb } from '../crash-reporting/durable-crash-breadcrumb'
+
+type FatalMainProcessErrorKind = 'main_uncaught_exception' | 'main_unhandled_rejection'
+
+type FatalMainProcessErrorDetails = {
+  errorName: string
+  errorMessage: string
+  errorStack: string
+  errorCode: string
+}
+
+function readErrorProperty(error: unknown, property: string): unknown {
+  try {
+    return error !== null && (typeof error === 'object' || typeof error === 'function')
+      ? (error as Record<string, unknown>)[property]
+      : undefined
+  } catch {
+    return undefined
+  }
+}
+
+function boundedString(value: unknown, maxLength: number, fallback = ''): string {
+  try {
+    return String(value).slice(0, maxLength)
+  } catch {
+    return fallback
+  }
+}
+
+function fatalMainProcessErrorDetails(error: unknown): FatalMainProcessErrorDetails {
+  let isError = false
+  try {
+    isError = error instanceof Error
+  } catch {
+    // Why: a proxy can throw from instanceof; fatal diagnostics still need a safe fallback.
+  }
+
+  return {
+    errorName: isError
+      ? boundedString(readErrorProperty(error, 'name') ?? 'Error', 100, 'Error')
+      : typeof error,
+    errorMessage: isError
+      ? boundedString(readErrorProperty(error, 'message') ?? '', 500)
+      : boundedString(error, 500, '[unprintable value]'),
+    errorStack: isError
+      ? boundedString(readErrorProperty(error, 'stack') ?? '', 4_000)
+          .split('\n')
+          .slice(0, 12)
+          .join('\n')
+      : '',
+    errorCode: boundedString(readErrorProperty(error, 'code') ?? '', 100)
+  }
+}
+
+// Why: one broken resource can reject hundreds of concurrent restore chains; each record does a
+// synchronous trace flush, so an uncapped storm stalls main and churns the trace-file rotation.
+const RECORD_WINDOW_MS = 60_000
+const RECORD_WINDOW_MAX = 20
+let recordWindowStartedAt = 0
+let recordWindowCount = 0
+let recordsSuppressed = 0
+
+/** Durably record a main-process fatal/near-fatal error before default handling runs. Exported for tests. */
+export function recordFatalMainProcessError(kind: FatalMainProcessErrorKind, error: unknown): void {
+  // Why: only rejections can storm; the one uncaught-exception record before the fatal re-throw
+  // must never be lost to a window a storm already exhausted.
+  if (kind === 'main_unhandled_rejection') {
+    const now = Date.now()
+    // Why: a backward clock jump (sleep/resume, NTP) would otherwise trap an exhausted window and suppress every breadcrumb until wall time catches up.
+    if (now < recordWindowStartedAt || now - recordWindowStartedAt >= RECORD_WINDOW_MS) {
+      recordWindowStartedAt = now
+      recordWindowCount = 0
+    }
+    if (recordWindowCount >= RECORD_WINDOW_MAX) {
+      recordsSuppressed += 1
+      return
+    }
+    recordWindowCount += 1
+  }
+  const suppressedSinceLast = recordsSuppressed
+  recordsSuppressed = 0
+  const details = fatalMainProcessErrorDetails(error)
+  try {
+    recordDurableCrashBreadcrumb(
+      kind,
+      suppressedSinceLast > 0 ? { ...details, suppressedSinceLast } : details,
+      kind
+    )
+  } catch {
+    // Why: diagnostics must never turn a fatal-error report into a second fault.
+  }
+  try {
+    console.error(
+      `[${kind}] ${details.errorStack || `${details.errorName}: ${details.errorMessage}`}`
+    )
+  } catch {
+    // Why: custom console sinks must not defeat the process-level safety guard.
+  }
+}
+
+export function installUncaughtPipeErrorGuard(): void {
+  const onUncaughtException = (error: unknown): void => {
+    const errorCode = readErrorProperty(error, 'code')
+    if (errorCode === 'EIO' || errorCode === 'EPIPE') {
+      return
+    }
+
+    // Why (issue #9441): the re-throw below exits with a clean code and no macOS crash report; record durably first or the death is undiagnosable in the field.
+    recordFatalMainProcessError('main_uncaught_exception', error)
+    process.off('uncaughtException', onUncaughtException)
+    // Why: throwing inside an uncaughtException handler exits with status 7 and hides the fault; re-throw next tick for the real stack.
+    setImmediate(() => {
+      throw error
+    })
+  }
+
+  process.on('uncaughtException', onUncaughtException)
+}
+
+/** Keep one failed background promise from silently killing the whole app.
+ *
+ * Node's default kills the process on an unhandled rejection. Large-profile startup restore runs
+ * hundreds of concurrent async chains (worktree scans, terminal reconnects) in main; a single
+ * rejection in any of them exited the app with no crash report (issue #9441). Log it durably and
+ * stay alive — dying cannot be less disruptive than continuing with one failed background task.
+ */
+export function installUnhandledRejectionLogging(): void {
+  process.on('unhandledRejection', (reason) => {
+    recordFatalMainProcessError('main_unhandled_rejection', reason)
+  })
+}