From 572159369b3db4849c2b60bb4278440dbdb9b700 Mon Sep 17 00:00:00 2001 From: OrcaWin <293788423+OrcaWin@users.noreply.github.com> Date: Sun, 9 Aug 2026 23:33:34 -0700 Subject: [PATCH] fix(window): bound slow renderer recovery loops --- .../renderer-recovery-circuit-breaker.test.ts | 17 +++- .../renderer-recovery-circuit-breaker.ts | 6 +- src/main/index.ts | 33 +++++--- src/main/window/createMainWindow.test.ts | 42 ++++++++++ ...enderer-recovery-prompt-controller.test.ts | 80 +++++++++++++++++++ .../renderer-recovery-prompt-controller.ts | 38 +++++++++ 6 files changed, 202 insertions(+), 14 deletions(-) create mode 100644 src/main/window/renderer-recovery-prompt-controller.test.ts create mode 100644 src/main/window/renderer-recovery-prompt-controller.ts diff --git a/src/main/crash-reporting/renderer-recovery-circuit-breaker.test.ts b/src/main/crash-reporting/renderer-recovery-circuit-breaker.test.ts index eab933e9b8f..96193ea95e5 100644 --- a/src/main/crash-reporting/renderer-recovery-circuit-breaker.test.ts +++ b/src/main/crash-reporting/renderer-recovery-circuit-breaker.test.ts @@ -50,6 +50,21 @@ describe('RendererRecoveryCircuitBreaker', () => { expect(breaker.registerRecoveryAttempt(10_000_000).allowed).toBe(true) }) + it('opens for the b32b2865 recovery cadence after a GPU child crash', () => { + const breaker = new RendererRecoveryCircuitBreaker({ + windowMs: DEFAULT_RENDERER_RECOVERY_WINDOW_MS, + maxRecoveries: DEFAULT_RENDERER_RECOVERY_MAX_RECOVERIES + }) + + for (const at of [0, 7_000, 64_000]) { + expect(breaker.registerRecoveryAttempt(at).allowed).toBe(true) + } + expect(breaker.registerRecoveryAttempt(70_000)).toEqual({ + allowed: false, + recentRecoveryCount: 3 + }) + }) + it('reset() clears history so the next recovery is allowed', () => { const breaker = new RendererRecoveryCircuitBreaker({ windowMs: 60_000, maxRecoveries: 1 }) expect(breaker.registerRecoveryAttempt(0).allowed).toBe(true) @@ -59,7 +74,7 @@ describe('RendererRecoveryCircuitBreaker', () => { }) it('ships conservative defaults', () => { - expect(DEFAULT_RENDERER_RECOVERY_WINDOW_MS).toBe(60_000) + expect(DEFAULT_RENDERER_RECOVERY_WINDOW_MS).toBe(120_000) expect(DEFAULT_RENDERER_RECOVERY_MAX_RECOVERIES).toBe(3) }) }) diff --git a/src/main/crash-reporting/renderer-recovery-circuit-breaker.ts b/src/main/crash-reporting/renderer-recovery-circuit-breaker.ts index 1440586773d..efc39b7b71b 100644 --- a/src/main/crash-reporting/renderer-recovery-circuit-breaker.ts +++ b/src/main/crash-reporting/renderer-recovery-circuit-breaker.ts @@ -9,9 +9,9 @@ export type RendererRecoveryCircuitBreakerOptions = { // AV interference) crashes on every load. Orca auto-reloads recoverable // renderer deaths, so without a breaker it reloads every ~0.25-1.3s forever, // burning CPU and spamming breadcrumbs (Windows clusters F0BDRAZN55L / -// F0BDPCL93UM). 3 reloads in 60s is well above any single transient crash but -// far below a runaway loop. -export const DEFAULT_RENDERER_RECOVERY_WINDOW_MS = 60_000 +// F0BDPCL93UM). Keep the window above slow process-start failures so attempts +// cannot age out while the same renderer fault is still recurring. +export const DEFAULT_RENDERER_RECOVERY_WINDOW_MS = 120_000 export const DEFAULT_RENDERER_RECOVERY_MAX_RECOVERIES = 3 /** diff --git a/src/main/index.ts b/src/main/index.ts index 0a4be2562c8..1a9ac28cb96 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -193,6 +193,7 @@ import { ensureAutoUpdaterConfigured } from './window/attach-main-window-services' import { createMainWindow, loadMainWindow } from './window/createMainWindow' +import { RendererRecoveryPromptController } from './window/renderer-recovery-prompt-controller' import { getDashboardPopoutWindow, zoomDashboardPopoutIfFocused @@ -1590,6 +1591,8 @@ function sendOpenCrashReport(targetWindow?: BrowserWindow | null): void { webContents?.send('ui:openCrashReport') } +const rendererRecoveryPromptController = new RendererRecoveryPromptController() + // Why: on renderer crash-loop the breaker stops auto-reloading and the window goes blank, so a main-process dialog is the only retry/quit surface. async function presentRendererRecoveryPrompt(recentRecoveryCount: number): Promise { if (isQuitting) { @@ -1605,16 +1608,26 @@ async function presentRendererRecoveryPrompt(recentRecoveryCount: number): Promi message: 'The app window crashed repeatedly and stopped reloading automatically.', detail: `Orca tried to recover ${recentRecoveryCount} times in a row without success. This is often a graphics-driver or installation problem. Reload to try again, or quit and relaunch Orca.` } - const { response } = window - ? await dialog.showMessageBox(window, options) - : await dialog.showMessageBox(options) - if (response === 0 && mainWindow && !mainWindow.isDestroyed()) { - recordDurableCrashBreadcrumb('renderer_recovery_manual_retry') - loadMainWindow(mainWindow) - } else if (response === 1) { - isQuitting = true - app.quit() - } + await rendererRecoveryPromptController.present({ + showPrompt: () => + window ? dialog.showMessageBox(window, options) : dialog.showMessageBox(options), + isQuitting: () => isQuitting, + reload: () => { + if (!mainWindow || mainWindow.isDestroyed()) { + return + } + markRecoveryReloadInFlight(mainWindow.webContents.id) + recordDurableCrashBreadcrumb('renderer_recovery_manual_retry') + loadMainWindow(mainWindow) + }, + quit: () => { + isQuitting = true + app.quit() + }, + onPromptError: (error) => { + console.warn('[renderer-recovery] failed to show recovery prompt; quitting:', error) + } + }) } function getGpuFallbackEnvironment(): GpuFallbackEnvironment { diff --git a/src/main/window/createMainWindow.test.ts b/src/main/window/createMainWindow.test.ts index b6d1ff70035..f873aa80c83 100644 --- a/src/main/window/createMainWindow.test.ts +++ b/src/main/window/createMainWindow.test.ts @@ -3425,6 +3425,48 @@ describe('createMainWindow', () => { } }) + it('opens the breaker for the b32b2865 renderer recovery timestamps', () => { + vi.useFakeTimers() + + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {}) + const onRendererRecoveryExhausted = vi.fn() + const { browserWindowInstance, windowHandlers } = createRendererRecoveryWindowHarness() + + try { + createMainWindow(null, { onRendererRecoveryExhausted }) + + const driveFailure = ( + reason: Electron.RenderProcessGoneDetails['reason'], + exitCode: number + ) => { + windowHandlers['render-process-gone']?.( + {} as never, + { reason, exitCode } as Electron.RenderProcessGoneDetails + ) + vi.advanceTimersByTime(250) + } + + driveFailure('crashed', 1) + vi.advanceTimersByTime(6_750) + driveFailure('oom', -536_870_904) + vi.advanceTimersByTime(56_750) + driveFailure('oom', -536_870_904) + vi.advanceTimersByTime(5_750) + driveFailure('launch-failed', 18) + + expect(browserWindowInstance.loadFile).toHaveBeenCalledTimes(4) + expect(onRendererRecoveryExhausted).toHaveBeenCalledOnce() + expect(onRendererRecoveryExhausted).toHaveBeenCalledWith( + expect.objectContaining({ + details: expect.objectContaining({ reason: 'launch-failed', exitCode: 18 }), + recentRecoveryCount: 3 + }) + ) + } finally { + consoleError.mockRestore() + } + }) + function createStartupRevealWindowFixture() { const windowHandlers: Record void> = {} const webContents = { diff --git a/src/main/window/renderer-recovery-prompt-controller.test.ts b/src/main/window/renderer-recovery-prompt-controller.test.ts new file mode 100644 index 00000000000..0a2473adcc4 --- /dev/null +++ b/src/main/window/renderer-recovery-prompt-controller.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it, vi } from 'vitest' +import { RendererRecoveryPromptController } from './renderer-recovery-prompt-controller' + +function createOptions(showPrompt: () => Promise<{ response: number }>) { + return { + showPrompt: vi.fn(showPrompt), + isQuitting: vi.fn(() => false), + reload: vi.fn(), + quit: vi.fn(), + onPromptError: vi.fn() + } +} + +describe('RendererRecoveryPromptController', () => { + it('routes the dialog choices', async () => { + const controller = new RendererRecoveryPromptController() + const reload = createOptions(async () => ({ response: 0 })) + const quit = createOptions(async () => ({ response: 1 })) + + await controller.present(reload) + await controller.present(quit) + + expect(reload.reload).toHaveBeenCalledOnce() + expect(reload.quit).not.toHaveBeenCalled() + expect(quit.quit).toHaveBeenCalledOnce() + expect(quit.reload).not.toHaveBeenCalled() + }) + + it('allows only one concurrent prompt', async () => { + const controller = new RendererRecoveryPromptController() + let resolveFirst: ((value: { response: number }) => void) | undefined + const first = createOptions( + () => + new Promise((resolve) => { + resolveFirst = resolve + }) + ) + const second = createOptions(async () => ({ response: 0 })) + + const pending = controller.present(first) + await controller.present(second) + + expect(second.showPrompt).not.toHaveBeenCalled() + resolveFirst?.({ response: 0 }) + await pending + }) + + it('quits cleanly when the native prompt fails', async () => { + const controller = new RendererRecoveryPromptController() + const error = new Error('dialog failed') + const options = createOptions(async () => { + throw error + }) + + await controller.present(options) + + expect(options.onPromptError).toHaveBeenCalledWith(error) + expect(options.quit).toHaveBeenCalledOnce() + expect(options.reload).not.toHaveBeenCalled() + }) + + it('does nothing when teardown starts while the prompt is open', async () => { + const controller = new RendererRecoveryPromptController() + let resolvePrompt: ((value: { response: number }) => void) | undefined + const options = createOptions( + () => + new Promise((resolve) => { + resolvePrompt = resolve + }) + ) + + const pending = controller.present(options) + options.isQuitting.mockReturnValue(true) + resolvePrompt?.({ response: 0 }) + await pending + + expect(options.reload).not.toHaveBeenCalled() + expect(options.quit).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/window/renderer-recovery-prompt-controller.ts b/src/main/window/renderer-recovery-prompt-controller.ts new file mode 100644 index 00000000000..3c3b5d1f5c7 --- /dev/null +++ b/src/main/window/renderer-recovery-prompt-controller.ts @@ -0,0 +1,38 @@ +export type RendererRecoveryPromptControllerOptions = { + showPrompt: () => Promise<{ response: number }> + isQuitting: () => boolean + reload: () => void + quit: () => void + onPromptError: (error: unknown) => void +} + +export class RendererRecoveryPromptController { + private showing = false + + async present(options: RendererRecoveryPromptControllerOptions): Promise { + if (this.showing) { + return + } + this.showing = true + let response: number + try { + response = (await options.showPrompt()).response + } catch (error) { + options.onPromptError(error) + if (!options.isQuitting()) { + options.quit() + } + return + } finally { + this.showing = false + } + if (options.isQuitting()) { + return + } + if (response === 0) { + options.reload() + } else if (response === 1) { + options.quit() + } + } +}