Reduce renderer crash recovery loops

This commit is contained in:
brennanb2025
2026-06-23 15:53:42 -07:00
parent 30a1a52459
commit 5a352af80e
9 changed files with 603 additions and 24 deletions
@@ -201,6 +201,36 @@ describe('shouldRecordProcessGoneCrash', () => {
expectedTeardown: 'none'
})
).toBe(false)
expect(
shouldRecordProcessGoneCrash({
source: 'child',
processType: 'Utility',
serviceName: 'video_capture.mojom.VideoCaptureService',
reason: 'killed',
exitCode: 1,
expectedTeardown: 'none'
})
).toBe(false)
expect(
shouldRecordProcessGoneCrash({
source: 'child',
processType: 'Utility',
serviceName: 'video_capture.mojom.VideoCaptureService',
reason: 'crashed',
exitCode: -1,
expectedTeardown: 'none'
})
).toBe(false)
expect(
shouldRecordProcessGoneCrash({
source: 'child',
processType: 'Utility',
serviceName: 'video_capture.mojom.VideoCaptureService',
reason: 'abnormal-exit',
exitCode: 512,
expectedTeardown: 'none'
})
).toBe(false)
})
it('still records unknown child process crashes', () => {
@@ -236,6 +266,26 @@ describe('shouldRecordProcessGoneCrash', () => {
expectedTeardown: 'none'
})
).toBe(true)
expect(
shouldRecordProcessGoneCrash({
source: 'child',
processType: 'Utility',
serviceName: 'video_capture.mojom.VideoCaptureService',
reason: 'oom',
exitCode: 1,
expectedTeardown: 'none'
})
).toBe(true)
expect(
shouldRecordProcessGoneCrash({
source: 'child',
processType: 'Utility',
serviceName: 'video_capture.mojom.VideoCaptureService',
reason: 'launch-failed',
exitCode: 18,
expectedTeardown: 'none'
})
).toBe(true)
})
it('still records renderer kills from the recent Linux crash-report cluster', () => {
@@ -5,7 +5,8 @@ const WINDOWS_CONTROL_TERMINATION_EXIT_CODES = new Set([0xc000013a, 0x40010004])
const RECOVERABLE_CHILD_PROCESS_TYPES = new Set(['gpu'])
const RECOVERABLE_UTILITY_SERVICE_NAMES = new Set([
'audio.mojom.AudioService',
'network.mojom.NetworkService'
'network.mojom.NetworkService',
'video_capture.mojom.VideoCaptureService'
])
const RECOVERABLE_CHILD_PROCESS_REASONS = new Set(['abnormal-exit', 'crashed', 'killed'])
const NON_RECOVERABLE_RENDERER_REASONS = new Set(['integrity-failure', 'launch-failed'])
@@ -60,8 +61,8 @@ export function shouldRecordProcessGoneCrash({
exitCode: number | null
expectedTeardown: ExpectedTeardownScope
}): boolean {
// Why: GPU, Network Service, and Audio Service exits are recoverable Chromium
// child-process churn; treating them as app crashes creates noisy user prompts.
// Why: GPU and named Chromium utility service exits are recoverable child
// process churn; treating them as app crashes creates noisy user prompts.
if (isRecoverableChromiumChildProcess({ source, processType, serviceName, reason })) {
return false
}
@@ -64,6 +64,30 @@ describe('process gone diagnostics', () => {
})
})
it('preserves renderer recovery summary fields in persisted crash details', () => {
expect(
buildProcessGoneCrashDetails({
processType: 'renderer',
rendererRecoveryTotalProcessGoneCount: 3,
rendererRecoveryRecentProcessGoneCount: 3,
rendererRecoverySuppressedRecoveryCount: 1,
rendererRecoveryLastReason: 'crashed',
rendererRecoveryLastExitCode: 5,
rendererRecoveryDegraded: true,
rendererRecoveryDegradedUntil: 123_456
})
).toMatchObject({
processType: 'renderer',
rendererRecoveryTotalProcessGoneCount: 3,
rendererRecoveryRecentProcessGoneCount: 3,
rendererRecoverySuppressedRecoveryCount: 1,
rendererRecoveryLastReason: 'crashed',
rendererRecoveryLastExitCode: 5,
rendererRecoveryDegraded: true,
rendererRecoveryDegradedUntil: 123_456
})
})
it('preserves child process identity on suppressed breadcrumbs', () => {
expect(
buildSuppressedProcessGoneBreadcrumbData({
+21 -2
View File
@@ -79,6 +79,7 @@ import { getInitialClaudeRateLimitTarget } from './rate-limits/claude-rate-limit
import { getInitialCodexRateLimitTarget } from './rate-limits/codex-rate-limit-target'
import { attachMainWindowServices } from './window/attach-main-window-services'
import { createMainWindow, loadMainWindow } from './window/createMainWindow'
import type { RendererRecoverySummary } from './window/renderer-recovery-guard'
import { createSystemTray, destroySystemTray } from './tray/system-tray'
import { focusExistingMainWindow } from './window/focus-existing-window'
import { CodexAccountService } from './codex-accounts/service'
@@ -649,18 +650,22 @@ function openMainWindow(): BrowserWindow {
isQuitting = false
clearExpectedRendererReload()
},
onRendererProcessGone: (details, webContentsId) => {
onRendererProcessGone: (details, webContentsId, recoverySummary) => {
recordProcessGoneCrash(
'renderer',
'renderer',
details.reason,
details.exitCode ?? null,
{
processType: 'renderer'
processType: 'renderer',
...toRendererRecoveryCrashDetails(recoverySummary)
},
webContentsId
)
},
onRendererRecoveryEvent: (name, summary) => {
recordCrashBreadcrumb(name, toRendererRecoveryCrashDetails(summary))
},
shouldRecordRendererCrash: (details, webContentsId) =>
shouldRecordProcessGoneCrash({
source: 'renderer',
@@ -887,6 +892,20 @@ function sendOpenCrashReport(targetWindow?: BrowserWindow | null): void {
webContents?.send('ui:openCrashReport')
}
function toRendererRecoveryCrashDetails(
summary: RendererRecoverySummary
): Record<string, string | number | boolean | null> {
return {
rendererRecoveryTotalProcessGoneCount: summary.totalProcessGoneCount,
rendererRecoveryRecentProcessGoneCount: summary.recentProcessGoneCount,
rendererRecoverySuppressedRecoveryCount: summary.suppressedRecoveryCount,
rendererRecoveryLastReason: summary.lastReason,
rendererRecoveryLastExitCode: summary.lastExitCode,
rendererRecoveryDegraded: summary.degraded,
rendererRecoveryDegradedUntil: summary.degradedUntil
}
}
function recordProcessGoneCrash(
source: 'renderer' | 'child',
processType: string,
+185 -2
View File
@@ -2262,7 +2262,16 @@ describe('createMainWindow', () => {
const details = { reason: 'crashed', exitCode: 5 } as Electron.RenderProcessGoneDetails
windowHandlers['render-process-gone']?.({} as never, details)
expect(onRendererProcessGone).toHaveBeenCalledWith(details, 142)
expect(onRendererProcessGone).toHaveBeenCalledWith(
details,
142,
expect.objectContaining({
totalProcessGoneCount: 1,
recentProcessGoneCount: 1,
lastReason: 'crashed',
lastExitCode: 5
})
)
})
it('passes the renderer webContents id through crash classification callbacks', () => {
@@ -2313,7 +2322,14 @@ describe('createMainWindow', () => {
vi.advanceTimersByTime(250)
expect(shouldRecordRendererCrash).toHaveBeenCalledWith(details, 424)
expect(onRendererProcessGone).toHaveBeenCalledWith(details, 424)
expect(onRendererProcessGone).toHaveBeenCalledWith(
details,
424,
expect.objectContaining({
totalProcessGoneCount: 1,
recentProcessGoneCount: 1
})
)
expect(shouldRecoverRenderer).toHaveBeenCalledWith(details, 424)
} finally {
consoleError.mockRestore()
@@ -2499,6 +2515,173 @@ describe('createMainWindow', () => {
consoleError.mockRestore()
})
it('does not let expected renderer reload teardown consume automatic recovery budget', () => {
vi.useFakeTimers()
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
const { browserWindowInstance, windowHandlers } = createRendererRecoveryWindowHarness()
const shouldRecoverRenderer = vi.fn(
(details: Electron.RenderProcessGoneDetails) => details.reason !== 'killed'
)
createMainWindow(null, { shouldRecoverRenderer })
const expectedReloadDetails = {
reason: 'killed',
exitCode: 1
} as Electron.RenderProcessGoneDetails
windowHandlers['render-process-gone']?.({} as never, expectedReloadDetails)
windowHandlers['render-process-gone']?.({} as never, expectedReloadDetails)
vi.advanceTimersByTime(250)
const crashDetails = {
reason: 'crashed',
exitCode: 5
} as Electron.RenderProcessGoneDetails
windowHandlers['render-process-gone']?.({} as never, crashDetails)
vi.advanceTimersByTime(250)
expect(browserWindowInstance.loadFile).toHaveBeenCalledTimes(2)
expect(browserWindowInstance.loadURL).not.toHaveBeenCalled()
expect(shouldRecoverRenderer).toHaveBeenCalledWith(crashDetails, 143)
consoleError.mockRestore()
})
it('suppresses automatic recovery after repeated renderer losses in the loop window', () => {
vi.useFakeTimers()
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
const { browserWindowInstance, windowHandlers } = createRendererRecoveryWindowHarness()
const onRendererRecoveryEvent = vi.fn()
const onRendererProcessGone = vi.fn()
createMainWindow(null, { onRendererProcessGone, onRendererRecoveryEvent })
const details = {
reason: 'crashed',
exitCode: 5
} as Electron.RenderProcessGoneDetails
for (let i = 0; i < 3; i += 1) {
windowHandlers['render-process-gone']?.({} as never, details)
vi.advanceTimersByTime(250)
}
expect(browserWindowInstance.loadFile).toHaveBeenCalledTimes(3)
expect(browserWindowInstance.loadURL).not.toHaveBeenCalled()
expect(onRendererRecoveryEvent).toHaveBeenCalledWith(
'renderer_recovery_suppressed',
expect.objectContaining({
totalProcessGoneCount: 3,
recentProcessGoneCount: 3,
suppressedRecoveryCount: 1,
degraded: true,
lastReason: 'crashed',
lastExitCode: 5
})
)
expect(onRendererProcessGone).toHaveBeenLastCalledWith(
details,
143,
expect.objectContaining({
totalProcessGoneCount: 3,
recentProcessGoneCount: 3,
suppressedRecoveryCount: 1,
degraded: true,
lastReason: 'crashed',
lastExitCode: 5
})
)
consoleError.mockRestore()
})
it('allows automatic recovery again after the renderer stays stable', () => {
vi.useFakeTimers()
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
const { browserWindowInstance, windowHandlers } = createRendererRecoveryWindowHarness()
const onRendererRecoveryEvent = vi.fn()
createMainWindow(null, { onRendererRecoveryEvent })
const details = {
reason: 'crashed',
exitCode: 5
} as Electron.RenderProcessGoneDetails
windowHandlers['render-process-gone']?.({} as never, details)
vi.advanceTimersByTime(250)
windowHandlers['did-finish-load']?.()
vi.advanceTimersByTime(30_000)
windowHandlers['render-process-gone']?.({} as never, details)
vi.advanceTimersByTime(250)
windowHandlers['render-process-gone']?.({} as never, details)
vi.advanceTimersByTime(250)
expect(browserWindowInstance.loadFile).toHaveBeenCalledTimes(4)
expect(onRendererRecoveryEvent).toHaveBeenCalledWith(
'renderer_recovery_stable',
expect.objectContaining({
totalProcessGoneCount: 1,
recentProcessGoneCount: 0,
degraded: false
})
)
consoleError.mockRestore()
})
it('does not emit stable recovery breadcrumbs after the window closes', () => {
vi.useFakeTimers()
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
const { windowHandlers } = createRendererRecoveryWindowHarness()
const onRendererRecoveryEvent = vi.fn()
createMainWindow(null, { onRendererRecoveryEvent })
windowHandlers['render-process-gone']?.(
{} as never,
{
reason: 'crashed',
exitCode: 5
} as Electron.RenderProcessGoneDetails
)
windowHandlers['did-finish-load']?.()
onRendererRecoveryEvent.mockClear()
windowHandlers.closed?.()
vi.advanceTimersByTime(30_000)
expect(onRendererRecoveryEvent).not.toHaveBeenCalled()
consoleError.mockRestore()
})
it('cancels pending automatic recovery when renderer webContents is destroyed', () => {
vi.useFakeTimers()
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
const { browserWindowInstance, windowHandlers } = createRendererRecoveryWindowHarness()
createMainWindow(null)
windowHandlers['render-process-gone']?.(
{} as never,
{
reason: 'crashed',
exitCode: 5
} as Electron.RenderProcessGoneDetails
)
windowHandlers.destroyed?.()
vi.advanceTimersByTime(250)
expect(browserWindowInstance.loadFile).toHaveBeenCalledTimes(1)
expect(browserWindowInstance.loadURL).not.toHaveBeenCalled()
consoleError.mockRestore()
})
it('does not reload after a clean renderer exit', () => {
vi.useFakeTimers()
+57 -14
View File
@@ -41,6 +41,11 @@ import {
import { getMainE2EConfig } from '../e2e-config'
import { buildEditableContextMenuTemplate } from './editable-context-menu'
import { clearTrustedUIRendererWebContentsId, setTrustedUIRendererWebContentsId } from '../ipc/ui'
import {
createRendererRecoveryGuard,
type RendererRecoveryEventName,
type RendererRecoverySummary
} from './renderer-recovery-guard'
function forceRepaint(window: BrowserWindow): void {
if (window.isDestroyed()) {
@@ -121,7 +126,12 @@ type CreateMainWindowOptions = {
onQuitAborted?: () => void
onRendererProcessGone?: (
details: Electron.RenderProcessGoneDetails,
webContentsId: number
webContentsId: number,
recoverySummary: RendererRecoverySummary
) => void
onRendererRecoveryEvent?: (
name: RendererRecoveryEventName,
summary: RendererRecoverySummary
) => void
/** Returns true when a renderer loss should be reported as a crash. Why:
* intentional reload/update/quit paths can emit crash-like `killed`
@@ -608,23 +618,33 @@ export function createMainWindow(
}
let rendererProcessGone = false
let rendererRecoveryTimer: ReturnType<typeof setTimeout> | null = null
const rendererRecoveryGuard = createRendererRecoveryGuard({
onEvent: opts?.onRendererRecoveryEvent
})
const clearRendererRecoveryTimer = (): void => {
if (rendererRecoveryTimer) {
clearTimeout(rendererRecoveryTimer)
rendererRecoveryTimer = null
}
}
const scheduleRendererRecovery = (details: Electron.RenderProcessGoneDetails): void => {
if (
rendererRecoveryTimer ||
!details ||
!isCrashReportReason(details.reason) ||
windowClosing ||
opts?.getIsQuitting?.() ||
opts?.shouldRecoverRenderer?.(details, rendererWebContentsId) === false ||
mainWindow.isDestroyed()
) {
return
const shouldAttemptRendererRecovery = (details: Electron.RenderProcessGoneDetails): boolean =>
Boolean(
details &&
isCrashReportReason(details.reason) &&
!windowClosing &&
!opts?.getIsQuitting?.() &&
opts?.shouldRecoverRenderer?.(details, rendererWebContentsId) !== false &&
!mainWindow.isDestroyed()
)
const scheduleRendererRecovery = (
details: Electron.RenderProcessGoneDetails
): RendererRecoverySummary | null => {
if (rendererRecoveryTimer || !shouldAttemptRendererRecovery(details)) {
return null
}
const recoveryDecision = rendererRecoveryGuard.canScheduleAutomaticRecovery()
if (!recoveryDecision.allowed) {
return recoveryDecision.summary
}
rendererRecoveryTimer = setTimeout(() => {
rendererRecoveryTimer = null
@@ -641,6 +661,7 @@ export function createMainWindow(
// back to a usable window instead of needing a full relaunch.
loadMainWindow(mainWindow)
}, 250)
return recoveryDecision.summary
}
mainWindow.webContents.on('render-process-gone', (_event, details) => {
rendererProcessGone = true
@@ -648,24 +669,38 @@ export function createMainWindow(
resetTerminalInputFocus()
resetFloatingTerminalInputFocus()
resetShortcutRecorderFocus()
if (!details) {
return
}
// Why: expected reload/update teardown should not consume automatic
// recovery budget; otherwise manual reloads can suppress the next real crash.
let recoverySummary = rendererRecoveryGuard.getSummary()
if (shouldAttemptRendererRecovery(details)) {
rendererRecoveryGuard.recordProcessGone({
reason: details.reason,
exitCode: details.exitCode ?? null
})
recoverySummary = scheduleRendererRecovery(details) ?? rendererRecoveryGuard.getSummary()
}
// Why: macOS can report BrowserWindow teardown as renderer `killed`/SIGKILL
// after a confirmed close; that is window lifecycle noise, not a crash.
if (
!windowClosing &&
opts?.shouldRecordRendererCrash?.(details, rendererWebContentsId) !== false
) {
opts?.onRendererProcessGone?.(details, rendererWebContentsId)
opts?.onRendererProcessGone?.(details, rendererWebContentsId, recoverySummary)
}
if (!windowClosing) {
console.error('[window] Renderer process gone; close confirmation will be bypassed', details)
}
scheduleRendererRecovery(details)
})
mainWindow.webContents.on('destroyed', () => {
resetMarkdownEditorFocus()
resetTerminalInputFocus()
resetFloatingTerminalInputFocus()
resetShortcutRecorderFocus()
clearRendererRecoveryTimer()
rendererRecoveryGuard.dispose()
})
mainWindow.webContents.on('did-start-navigation', (_e, _url, _isInPlace, isMainFrame) => {
if (isMainFrame) {
@@ -678,6 +713,9 @@ export function createMainWindow(
mainWindow.webContents.on('did-finish-load', () => {
rendererProcessGone = false
clearRendererRecoveryTimer()
// Why: a load that crashes immediately is still part of the loop; only a
// stable main-frame period resets automatic recovery dampening.
rendererRecoveryGuard.onStableLoad()
})
const doubleTapDetector = new ModifierDoubleTapDetector()
@@ -1004,6 +1042,8 @@ export function createMainWindow(
// would otherwise make the post-update relaunch come up at minWidth ×
// minHeight (issue surfaced in v1.3.26-rc2).
windowClosing = true
clearRendererRecoveryTimer()
rendererRecoveryGuard.dispose()
if (boundsTimer) {
clearTimeout(boundsTimer)
boundsTimer = null
@@ -1015,6 +1055,8 @@ export function createMainWindow(
// window:close-requested. Let Cmd+Q / OS close complete instead of
// trapping the user in a blank, unquittable window.
windowClosing = true
clearRendererRecoveryTimer()
rendererRecoveryGuard.dispose()
if (boundsTimer) {
clearTimeout(boundsTimer)
boundsTimer = null
@@ -1121,6 +1163,7 @@ export function createMainWindow(
floatingTerminalInputFocused = false
shortcutRecorderFocused = false
clearRendererRecoveryTimer()
rendererRecoveryGuard.dispose()
ipcMain.removeListener(trafficLightChannel, onSyncTrafficLights)
ipcMain.removeListener(minimizeChannel, onMinimize)
ipcMain.removeListener(maximizeChannel, onMaximize)
@@ -0,0 +1,114 @@
import { describe, expect, it, vi } from 'vitest'
import {
createRendererRecoveryGuard,
type RendererRecoveryEventName
} from './renderer-recovery-guard'
describe('renderer recovery guard', () => {
it('permits the first automatic recoveries and suppresses later events in the TTL', () => {
let now = 1_000
const events: RendererRecoveryEventName[] = []
const guard = createRendererRecoveryGuard({
now: () => now,
loopWindowMs: 60_000,
maxAutomaticRecoveries: 2,
cooldownMs: 300_000,
onEvent: (name) => events.push(name)
})
guard.recordProcessGone({ reason: 'crashed', exitCode: 5 })
expect(guard.canScheduleAutomaticRecovery().allowed).toBe(true)
now += 1_000
guard.recordProcessGone({ reason: 'oom', exitCode: -1 })
expect(guard.canScheduleAutomaticRecovery().allowed).toBe(true)
now += 1_000
const summary = guard.recordProcessGone({ reason: 'killed', exitCode: 1 })
const decision = guard.canScheduleAutomaticRecovery()
expect(summary).toMatchObject({
totalProcessGoneCount: 3,
recentProcessGoneCount: 3,
lastReason: 'killed',
lastExitCode: 1
})
expect(decision.allowed).toBe(false)
expect(decision.summary).toMatchObject({
suppressedRecoveryCount: 1,
degraded: true,
degradedUntil: now + 300_000
})
expect(events).toEqual([
'renderer_recovery_scheduled',
'renderer_recovery_scheduled',
'renderer_recovery_suppressed'
])
})
it('allows recovery after the cooldown and loop window expire', () => {
let now = 10_000
const guard = createRendererRecoveryGuard({
now: () => now,
loopWindowMs: 60_000,
maxAutomaticRecoveries: 2,
cooldownMs: 300_000
})
for (const reason of ['crashed', 'oom', 'killed']) {
guard.recordProcessGone({ reason, exitCode: 1 })
guard.canScheduleAutomaticRecovery()
now += 1_000
}
now += 301_000
guard.recordProcessGone({ reason: 'crashed', exitCode: 2 })
expect(guard.canScheduleAutomaticRecovery()).toMatchObject({
allowed: true,
summary: { recentProcessGoneCount: 1, degraded: false }
})
})
it('resets recent loop state after a stable load period', () => {
vi.useFakeTimers()
const events: { name: RendererRecoveryEventName; recent: number; degraded: boolean }[] = []
const guard = createRendererRecoveryGuard({
stableLoadMs: 30_000,
onEvent: (name, summary) =>
events.push({ name, recent: summary.recentProcessGoneCount, degraded: summary.degraded })
})
guard.recordProcessGone({ reason: 'crashed', exitCode: 5 })
guard.canScheduleAutomaticRecovery()
guard.onStableLoad()
vi.advanceTimersByTime(29_999)
expect(guard.getSummary().recentProcessGoneCount).toBe(1)
vi.advanceTimersByTime(1)
expect(guard.getSummary()).toMatchObject({
totalProcessGoneCount: 1,
recentProcessGoneCount: 0,
degraded: false
})
expect(events.at(-1)).toEqual({
name: 'renderer_recovery_stable',
recent: 0,
degraded: false
})
})
it('clears the stable load timer on dispose', () => {
vi.useFakeTimers()
const onEvent = vi.fn()
const guard = createRendererRecoveryGuard({
stableLoadMs: 30_000,
onEvent
})
guard.recordProcessGone({ reason: 'crashed', exitCode: 5 })
guard.onStableLoad()
guard.dispose()
vi.advanceTimersByTime(30_000)
expect(onEvent).not.toHaveBeenCalledWith('renderer_recovery_stable', expect.anything())
})
})
+148
View File
@@ -0,0 +1,148 @@
export type RendererRecoveryEventName =
| 'renderer_recovery_scheduled'
| 'renderer_recovery_suppressed'
| 'renderer_recovery_stable'
export type RendererRecoverySummary = {
totalProcessGoneCount: number
recentProcessGoneCount: number
suppressedRecoveryCount: number
lastReason: string
lastExitCode: number | null
degradedUntil: number | null
degraded: boolean
}
export type RendererRecoveryGuardOptions = {
loopWindowMs?: number
maxAutomaticRecoveries?: number
cooldownMs?: number
stableLoadMs?: number
now?: () => number
setTimer?: (callback: () => void, delayMs: number) => NodeJS.Timeout
clearTimer?: (timer: NodeJS.Timeout) => void
onEvent?: (name: RendererRecoveryEventName, summary: RendererRecoverySummary) => void
}
export const RECOVERY_LOOP_WINDOW_MS = 60_000
export const MAX_AUTOMATIC_RECOVERIES_PER_WINDOW = 2
export const RECOVERY_COOLDOWN_MS = 5 * 60_000
export const STABLE_LOAD_MS = 30_000
export type RendererRecoveryGuard = ReturnType<typeof createRendererRecoveryGuard>
const EMPTY_LAST_REASON = 'none'
function cloneSummary(summary: RendererRecoverySummary): RendererRecoverySummary {
return { ...summary }
}
export function createRendererRecoveryGuard(options: RendererRecoveryGuardOptions = {}) {
const loopWindowMs = options.loopWindowMs ?? RECOVERY_LOOP_WINDOW_MS
const maxAutomaticRecoveries =
options.maxAutomaticRecoveries ?? MAX_AUTOMATIC_RECOVERIES_PER_WINDOW
const cooldownMs = options.cooldownMs ?? RECOVERY_COOLDOWN_MS
const stableLoadMs = options.stableLoadMs ?? STABLE_LOAD_MS
const now = options.now ?? Date.now
const setTimer =
options.setTimer ?? ((callback: () => void, delayMs: number) => setTimeout(callback, delayMs))
const clearTimer = options.clearTimer ?? clearTimeout
let stableLoadTimer: NodeJS.Timeout | null = null
let disposed = false
let totalProcessGoneCount = 0
let suppressedRecoveryCount = 0
let processGoneTimes: number[] = []
let lastReason = EMPTY_LAST_REASON
let lastExitCode: number | null = null
let degradedUntil: number | null = null
const clearStableLoadTimer = (): void => {
if (stableLoadTimer) {
clearTimer(stableLoadTimer)
stableLoadTimer = null
}
}
const pruneRecentProcessGoneTimes = (timestamp: number): void => {
processGoneTimes = processGoneTimes.filter((seenAt) => timestamp - seenAt <= loopWindowMs)
}
const getSummary = (timestamp = now()): RendererRecoverySummary => {
pruneRecentProcessGoneTimes(timestamp)
const degraded = degradedUntil !== null && timestamp < degradedUntil
return {
totalProcessGoneCount,
recentProcessGoneCount: processGoneTimes.length,
suppressedRecoveryCount,
lastReason,
lastExitCode,
degradedUntil,
degraded
}
}
const emit = (name: RendererRecoveryEventName, timestamp = now()): void => {
options.onEvent?.(name, cloneSummary(getSummary(timestamp)))
}
return {
recordProcessGone(details: {
reason: string
exitCode?: number | null
}): RendererRecoverySummary {
const timestamp = now()
clearStableLoadTimer()
pruneRecentProcessGoneTimes(timestamp)
totalProcessGoneCount += 1
processGoneTimes.push(timestamp)
lastReason = details.reason
lastExitCode = details.exitCode ?? null
return cloneSummary(getSummary(timestamp))
},
canScheduleAutomaticRecovery(): { allowed: boolean; summary: RendererRecoverySummary } {
const timestamp = now()
const summary = getSummary(timestamp)
if (summary.degraded || summary.recentProcessGoneCount > maxAutomaticRecoveries) {
if (!summary.degraded) {
degradedUntil = timestamp + cooldownMs
}
suppressedRecoveryCount += 1
const suppressedSummary = cloneSummary(getSummary(timestamp))
emit('renderer_recovery_suppressed', timestamp)
return { allowed: false, summary: suppressedSummary }
}
emit('renderer_recovery_scheduled', timestamp)
return { allowed: true, summary: cloneSummary(summary) }
},
onStableLoad(): void {
if (disposed) {
return
}
clearStableLoadTimer()
const hadRecoveryState = processGoneTimes.length > 0 || degradedUntil !== null
stableLoadTimer = setTimer(() => {
stableLoadTimer = null
if (disposed) {
return
}
processGoneTimes = []
degradedUntil = null
if (hadRecoveryState) {
emit('renderer_recovery_stable')
}
}, stableLoadMs)
},
getSummary(): RendererRecoverySummary {
return cloneSummary(getSummary())
},
dispose(): void {
disposed = true
clearStableLoadTimer()
}
}
}
@@ -65,8 +65,5 @@ export function getTaskPageGitHubPRIconTone(item: GitHubWorkItemStatusItem): str
return 'text-purple-600 dark:text-purple-300'
case 'closed':
return 'text-rose-600 dark:text-rose-300'
default:
// Fallback for any unexpected state
return 'text-muted-foreground'
}
}