fix(window): bound slow renderer recovery loops

This commit is contained in:
OrcaWin
2026-08-09 23:33:34 -07:00
parent 7dce0442ab
commit 572159369b
6 changed files with 202 additions and 14 deletions
@@ -50,6 +50,21 @@ describe('RendererRecoveryCircuitBreaker', () => {
expect(breaker.registerRecoveryAttempt(10_000_000).allowed).toBe(true)
})
it('opens for the b32b2865 recovery cadence after a GPU child crash', () => {
const breaker = new RendererRecoveryCircuitBreaker({
windowMs: DEFAULT_RENDERER_RECOVERY_WINDOW_MS,
maxRecoveries: DEFAULT_RENDERER_RECOVERY_MAX_RECOVERIES
})
for (const at of [0, 7_000, 64_000]) {
expect(breaker.registerRecoveryAttempt(at).allowed).toBe(true)
}
expect(breaker.registerRecoveryAttempt(70_000)).toEqual({
allowed: false,
recentRecoveryCount: 3
})
})
it('reset() clears history so the next recovery is allowed', () => {
const breaker = new RendererRecoveryCircuitBreaker({ windowMs: 60_000, maxRecoveries: 1 })
expect(breaker.registerRecoveryAttempt(0).allowed).toBe(true)
@@ -59,7 +74,7 @@ describe('RendererRecoveryCircuitBreaker', () => {
})
it('ships conservative defaults', () => {
expect(DEFAULT_RENDERER_RECOVERY_WINDOW_MS).toBe(60_000)
expect(DEFAULT_RENDERER_RECOVERY_WINDOW_MS).toBe(120_000)
expect(DEFAULT_RENDERER_RECOVERY_MAX_RECOVERIES).toBe(3)
})
})
@@ -9,9 +9,9 @@ export type RendererRecoveryCircuitBreakerOptions = {
// AV interference) crashes on every load. Orca auto-reloads recoverable
// renderer deaths, so without a breaker it reloads every ~0.25-1.3s forever,
// burning CPU and spamming breadcrumbs (Windows clusters F0BDRAZN55L /
// F0BDPCL93UM). 3 reloads in 60s is well above any single transient crash but
// far below a runaway loop.
export const DEFAULT_RENDERER_RECOVERY_WINDOW_MS = 60_000
// F0BDPCL93UM). Keep the window above slow process-start failures so attempts
// cannot age out while the same renderer fault is still recurring.
export const DEFAULT_RENDERER_RECOVERY_WINDOW_MS = 120_000
export const DEFAULT_RENDERER_RECOVERY_MAX_RECOVERIES = 3
/**
+23 -10
View File
@@ -193,6 +193,7 @@ import {
ensureAutoUpdaterConfigured
} from './window/attach-main-window-services'
import { createMainWindow, loadMainWindow } from './window/createMainWindow'
import { RendererRecoveryPromptController } from './window/renderer-recovery-prompt-controller'
import {
getDashboardPopoutWindow,
zoomDashboardPopoutIfFocused
@@ -1590,6 +1591,8 @@ function sendOpenCrashReport(targetWindow?: BrowserWindow | null): void {
webContents?.send('ui:openCrashReport')
}
const rendererRecoveryPromptController = new RendererRecoveryPromptController()
// Why: on renderer crash-loop the breaker stops auto-reloading and the window goes blank, so a main-process dialog is the only retry/quit surface.
async function presentRendererRecoveryPrompt(recentRecoveryCount: number): Promise<void> {
if (isQuitting) {
@@ -1605,16 +1608,26 @@ async function presentRendererRecoveryPrompt(recentRecoveryCount: number): Promi
message: 'The app window crashed repeatedly and stopped reloading automatically.',
detail: `Orca tried to recover ${recentRecoveryCount} times in a row without success. This is often a graphics-driver or installation problem. Reload to try again, or quit and relaunch Orca.`
}
const { response } = window
? await dialog.showMessageBox(window, options)
: await dialog.showMessageBox(options)
if (response === 0 && mainWindow && !mainWindow.isDestroyed()) {
recordDurableCrashBreadcrumb('renderer_recovery_manual_retry')
loadMainWindow(mainWindow)
} else if (response === 1) {
isQuitting = true
app.quit()
}
await rendererRecoveryPromptController.present({
showPrompt: () =>
window ? dialog.showMessageBox(window, options) : dialog.showMessageBox(options),
isQuitting: () => isQuitting,
reload: () => {
if (!mainWindow || mainWindow.isDestroyed()) {
return
}
markRecoveryReloadInFlight(mainWindow.webContents.id)
recordDurableCrashBreadcrumb('renderer_recovery_manual_retry')
loadMainWindow(mainWindow)
},
quit: () => {
isQuitting = true
app.quit()
},
onPromptError: (error) => {
console.warn('[renderer-recovery] failed to show recovery prompt; quitting:', error)
}
})
}
function getGpuFallbackEnvironment(): GpuFallbackEnvironment {
+42
View File
@@ -3425,6 +3425,48 @@ describe('createMainWindow', () => {
}
})
it('opens the breaker for the b32b2865 renderer recovery timestamps', () => {
vi.useFakeTimers()
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
const onRendererRecoveryExhausted = vi.fn()
const { browserWindowInstance, windowHandlers } = createRendererRecoveryWindowHarness()
try {
createMainWindow(null, { onRendererRecoveryExhausted })
const driveFailure = (
reason: Electron.RenderProcessGoneDetails['reason'],
exitCode: number
) => {
windowHandlers['render-process-gone']?.(
{} as never,
{ reason, exitCode } as Electron.RenderProcessGoneDetails
)
vi.advanceTimersByTime(250)
}
driveFailure('crashed', 1)
vi.advanceTimersByTime(6_750)
driveFailure('oom', -536_870_904)
vi.advanceTimersByTime(56_750)
driveFailure('oom', -536_870_904)
vi.advanceTimersByTime(5_750)
driveFailure('launch-failed', 18)
expect(browserWindowInstance.loadFile).toHaveBeenCalledTimes(4)
expect(onRendererRecoveryExhausted).toHaveBeenCalledOnce()
expect(onRendererRecoveryExhausted).toHaveBeenCalledWith(
expect.objectContaining({
details: expect.objectContaining({ reason: 'launch-failed', exitCode: 18 }),
recentRecoveryCount: 3
})
)
} finally {
consoleError.mockRestore()
}
})
function createStartupRevealWindowFixture() {
const windowHandlers: Record<string, (...args: any[]) => void> = {}
const webContents = {
@@ -0,0 +1,80 @@
import { describe, expect, it, vi } from 'vitest'
import { RendererRecoveryPromptController } from './renderer-recovery-prompt-controller'
function createOptions(showPrompt: () => Promise<{ response: number }>) {
return {
showPrompt: vi.fn(showPrompt),
isQuitting: vi.fn(() => false),
reload: vi.fn(),
quit: vi.fn(),
onPromptError: vi.fn()
}
}
describe('RendererRecoveryPromptController', () => {
it('routes the dialog choices', async () => {
const controller = new RendererRecoveryPromptController()
const reload = createOptions(async () => ({ response: 0 }))
const quit = createOptions(async () => ({ response: 1 }))
await controller.present(reload)
await controller.present(quit)
expect(reload.reload).toHaveBeenCalledOnce()
expect(reload.quit).not.toHaveBeenCalled()
expect(quit.quit).toHaveBeenCalledOnce()
expect(quit.reload).not.toHaveBeenCalled()
})
it('allows only one concurrent prompt', async () => {
const controller = new RendererRecoveryPromptController()
let resolveFirst: ((value: { response: number }) => void) | undefined
const first = createOptions(
() =>
new Promise((resolve) => {
resolveFirst = resolve
})
)
const second = createOptions(async () => ({ response: 0 }))
const pending = controller.present(first)
await controller.present(second)
expect(second.showPrompt).not.toHaveBeenCalled()
resolveFirst?.({ response: 0 })
await pending
})
it('quits cleanly when the native prompt fails', async () => {
const controller = new RendererRecoveryPromptController()
const error = new Error('dialog failed')
const options = createOptions(async () => {
throw error
})
await controller.present(options)
expect(options.onPromptError).toHaveBeenCalledWith(error)
expect(options.quit).toHaveBeenCalledOnce()
expect(options.reload).not.toHaveBeenCalled()
})
it('does nothing when teardown starts while the prompt is open', async () => {
const controller = new RendererRecoveryPromptController()
let resolvePrompt: ((value: { response: number }) => void) | undefined
const options = createOptions(
() =>
new Promise((resolve) => {
resolvePrompt = resolve
})
)
const pending = controller.present(options)
options.isQuitting.mockReturnValue(true)
resolvePrompt?.({ response: 0 })
await pending
expect(options.reload).not.toHaveBeenCalled()
expect(options.quit).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,38 @@
export type RendererRecoveryPromptControllerOptions = {
showPrompt: () => Promise<{ response: number }>
isQuitting: () => boolean
reload: () => void
quit: () => void
onPromptError: (error: unknown) => void
}
export class RendererRecoveryPromptController {
private showing = false
async present(options: RendererRecoveryPromptControllerOptions): Promise<void> {
if (this.showing) {
return
}
this.showing = true
let response: number
try {
response = (await options.showPrompt()).response
} catch (error) {
options.onPromptError(error)
if (!options.isQuitting()) {
options.quit()
}
return
} finally {
this.showing = false
}
if (options.isQuitting()) {
return
}
if (response === 0) {
options.reload()
} else if (response === 1) {
options.quit()
}
}
}