mirror of
https://github.com/stablyai/orca.git
synced 2026-10-21 00:03:46 +00:00
fix(window): bound slow renderer recovery loops
This commit is contained in:
@@ -50,6 +50,21 @@ describe('RendererRecoveryCircuitBreaker', () => {
|
||||
expect(breaker.registerRecoveryAttempt(10_000_000).allowed).toBe(true)
|
||||
})
|
||||
|
||||
it('opens for the b32b2865 recovery cadence after a GPU child crash', () => {
|
||||
const breaker = new RendererRecoveryCircuitBreaker({
|
||||
windowMs: DEFAULT_RENDERER_RECOVERY_WINDOW_MS,
|
||||
maxRecoveries: DEFAULT_RENDERER_RECOVERY_MAX_RECOVERIES
|
||||
})
|
||||
|
||||
for (const at of [0, 7_000, 64_000]) {
|
||||
expect(breaker.registerRecoveryAttempt(at).allowed).toBe(true)
|
||||
}
|
||||
expect(breaker.registerRecoveryAttempt(70_000)).toEqual({
|
||||
allowed: false,
|
||||
recentRecoveryCount: 3
|
||||
})
|
||||
})
|
||||
|
||||
it('reset() clears history so the next recovery is allowed', () => {
|
||||
const breaker = new RendererRecoveryCircuitBreaker({ windowMs: 60_000, maxRecoveries: 1 })
|
||||
expect(breaker.registerRecoveryAttempt(0).allowed).toBe(true)
|
||||
@@ -59,7 +74,7 @@ describe('RendererRecoveryCircuitBreaker', () => {
|
||||
})
|
||||
|
||||
it('ships conservative defaults', () => {
|
||||
expect(DEFAULT_RENDERER_RECOVERY_WINDOW_MS).toBe(60_000)
|
||||
expect(DEFAULT_RENDERER_RECOVERY_WINDOW_MS).toBe(120_000)
|
||||
expect(DEFAULT_RENDERER_RECOVERY_MAX_RECOVERIES).toBe(3)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -9,9 +9,9 @@ export type RendererRecoveryCircuitBreakerOptions = {
|
||||
// AV interference) crashes on every load. Orca auto-reloads recoverable
|
||||
// renderer deaths, so without a breaker it reloads every ~0.25-1.3s forever,
|
||||
// burning CPU and spamming breadcrumbs (Windows clusters F0BDRAZN55L /
|
||||
// F0BDPCL93UM). 3 reloads in 60s is well above any single transient crash but
|
||||
// far below a runaway loop.
|
||||
export const DEFAULT_RENDERER_RECOVERY_WINDOW_MS = 60_000
|
||||
// F0BDPCL93UM). Keep the window above slow process-start failures so attempts
|
||||
// cannot age out while the same renderer fault is still recurring.
|
||||
export const DEFAULT_RENDERER_RECOVERY_WINDOW_MS = 120_000
|
||||
export const DEFAULT_RENDERER_RECOVERY_MAX_RECOVERIES = 3
|
||||
|
||||
/**
|
||||
|
||||
+23
-10
@@ -193,6 +193,7 @@ import {
|
||||
ensureAutoUpdaterConfigured
|
||||
} from './window/attach-main-window-services'
|
||||
import { createMainWindow, loadMainWindow } from './window/createMainWindow'
|
||||
import { RendererRecoveryPromptController } from './window/renderer-recovery-prompt-controller'
|
||||
import {
|
||||
getDashboardPopoutWindow,
|
||||
zoomDashboardPopoutIfFocused
|
||||
@@ -1590,6 +1591,8 @@ function sendOpenCrashReport(targetWindow?: BrowserWindow | null): void {
|
||||
webContents?.send('ui:openCrashReport')
|
||||
}
|
||||
|
||||
const rendererRecoveryPromptController = new RendererRecoveryPromptController()
|
||||
|
||||
// Why: on renderer crash-loop the breaker stops auto-reloading and the window goes blank, so a main-process dialog is the only retry/quit surface.
|
||||
async function presentRendererRecoveryPrompt(recentRecoveryCount: number): Promise<void> {
|
||||
if (isQuitting) {
|
||||
@@ -1605,16 +1608,26 @@ async function presentRendererRecoveryPrompt(recentRecoveryCount: number): Promi
|
||||
message: 'The app window crashed repeatedly and stopped reloading automatically.',
|
||||
detail: `Orca tried to recover ${recentRecoveryCount} times in a row without success. This is often a graphics-driver or installation problem. Reload to try again, or quit and relaunch Orca.`
|
||||
}
|
||||
const { response } = window
|
||||
? await dialog.showMessageBox(window, options)
|
||||
: await dialog.showMessageBox(options)
|
||||
if (response === 0 && mainWindow && !mainWindow.isDestroyed()) {
|
||||
recordDurableCrashBreadcrumb('renderer_recovery_manual_retry')
|
||||
loadMainWindow(mainWindow)
|
||||
} else if (response === 1) {
|
||||
isQuitting = true
|
||||
app.quit()
|
||||
}
|
||||
await rendererRecoveryPromptController.present({
|
||||
showPrompt: () =>
|
||||
window ? dialog.showMessageBox(window, options) : dialog.showMessageBox(options),
|
||||
isQuitting: () => isQuitting,
|
||||
reload: () => {
|
||||
if (!mainWindow || mainWindow.isDestroyed()) {
|
||||
return
|
||||
}
|
||||
markRecoveryReloadInFlight(mainWindow.webContents.id)
|
||||
recordDurableCrashBreadcrumb('renderer_recovery_manual_retry')
|
||||
loadMainWindow(mainWindow)
|
||||
},
|
||||
quit: () => {
|
||||
isQuitting = true
|
||||
app.quit()
|
||||
},
|
||||
onPromptError: (error) => {
|
||||
console.warn('[renderer-recovery] failed to show recovery prompt; quitting:', error)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function getGpuFallbackEnvironment(): GpuFallbackEnvironment {
|
||||
|
||||
@@ -3425,6 +3425,48 @@ describe('createMainWindow', () => {
|
||||
}
|
||||
})
|
||||
|
||||
it('opens the breaker for the b32b2865 renderer recovery timestamps', () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const onRendererRecoveryExhausted = vi.fn()
|
||||
const { browserWindowInstance, windowHandlers } = createRendererRecoveryWindowHarness()
|
||||
|
||||
try {
|
||||
createMainWindow(null, { onRendererRecoveryExhausted })
|
||||
|
||||
const driveFailure = (
|
||||
reason: Electron.RenderProcessGoneDetails['reason'],
|
||||
exitCode: number
|
||||
) => {
|
||||
windowHandlers['render-process-gone']?.(
|
||||
{} as never,
|
||||
{ reason, exitCode } as Electron.RenderProcessGoneDetails
|
||||
)
|
||||
vi.advanceTimersByTime(250)
|
||||
}
|
||||
|
||||
driveFailure('crashed', 1)
|
||||
vi.advanceTimersByTime(6_750)
|
||||
driveFailure('oom', -536_870_904)
|
||||
vi.advanceTimersByTime(56_750)
|
||||
driveFailure('oom', -536_870_904)
|
||||
vi.advanceTimersByTime(5_750)
|
||||
driveFailure('launch-failed', 18)
|
||||
|
||||
expect(browserWindowInstance.loadFile).toHaveBeenCalledTimes(4)
|
||||
expect(onRendererRecoveryExhausted).toHaveBeenCalledOnce()
|
||||
expect(onRendererRecoveryExhausted).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
details: expect.objectContaining({ reason: 'launch-failed', exitCode: 18 }),
|
||||
recentRecoveryCount: 3
|
||||
})
|
||||
)
|
||||
} finally {
|
||||
consoleError.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
function createStartupRevealWindowFixture() {
|
||||
const windowHandlers: Record<string, (...args: any[]) => void> = {}
|
||||
const webContents = {
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { RendererRecoveryPromptController } from './renderer-recovery-prompt-controller'
|
||||
|
||||
function createOptions(showPrompt: () => Promise<{ response: number }>) {
|
||||
return {
|
||||
showPrompt: vi.fn(showPrompt),
|
||||
isQuitting: vi.fn(() => false),
|
||||
reload: vi.fn(),
|
||||
quit: vi.fn(),
|
||||
onPromptError: vi.fn()
|
||||
}
|
||||
}
|
||||
|
||||
describe('RendererRecoveryPromptController', () => {
|
||||
it('routes the dialog choices', async () => {
|
||||
const controller = new RendererRecoveryPromptController()
|
||||
const reload = createOptions(async () => ({ response: 0 }))
|
||||
const quit = createOptions(async () => ({ response: 1 }))
|
||||
|
||||
await controller.present(reload)
|
||||
await controller.present(quit)
|
||||
|
||||
expect(reload.reload).toHaveBeenCalledOnce()
|
||||
expect(reload.quit).not.toHaveBeenCalled()
|
||||
expect(quit.quit).toHaveBeenCalledOnce()
|
||||
expect(quit.reload).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('allows only one concurrent prompt', async () => {
|
||||
const controller = new RendererRecoveryPromptController()
|
||||
let resolveFirst: ((value: { response: number }) => void) | undefined
|
||||
const first = createOptions(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveFirst = resolve
|
||||
})
|
||||
)
|
||||
const second = createOptions(async () => ({ response: 0 }))
|
||||
|
||||
const pending = controller.present(first)
|
||||
await controller.present(second)
|
||||
|
||||
expect(second.showPrompt).not.toHaveBeenCalled()
|
||||
resolveFirst?.({ response: 0 })
|
||||
await pending
|
||||
})
|
||||
|
||||
it('quits cleanly when the native prompt fails', async () => {
|
||||
const controller = new RendererRecoveryPromptController()
|
||||
const error = new Error('dialog failed')
|
||||
const options = createOptions(async () => {
|
||||
throw error
|
||||
})
|
||||
|
||||
await controller.present(options)
|
||||
|
||||
expect(options.onPromptError).toHaveBeenCalledWith(error)
|
||||
expect(options.quit).toHaveBeenCalledOnce()
|
||||
expect(options.reload).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does nothing when teardown starts while the prompt is open', async () => {
|
||||
const controller = new RendererRecoveryPromptController()
|
||||
let resolvePrompt: ((value: { response: number }) => void) | undefined
|
||||
const options = createOptions(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolvePrompt = resolve
|
||||
})
|
||||
)
|
||||
|
||||
const pending = controller.present(options)
|
||||
options.isQuitting.mockReturnValue(true)
|
||||
resolvePrompt?.({ response: 0 })
|
||||
await pending
|
||||
|
||||
expect(options.reload).not.toHaveBeenCalled()
|
||||
expect(options.quit).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,38 @@
|
||||
export type RendererRecoveryPromptControllerOptions = {
|
||||
showPrompt: () => Promise<{ response: number }>
|
||||
isQuitting: () => boolean
|
||||
reload: () => void
|
||||
quit: () => void
|
||||
onPromptError: (error: unknown) => void
|
||||
}
|
||||
|
||||
export class RendererRecoveryPromptController {
|
||||
private showing = false
|
||||
|
||||
async present(options: RendererRecoveryPromptControllerOptions): Promise<void> {
|
||||
if (this.showing) {
|
||||
return
|
||||
}
|
||||
this.showing = true
|
||||
let response: number
|
||||
try {
|
||||
response = (await options.showPrompt()).response
|
||||
} catch (error) {
|
||||
options.onPromptError(error)
|
||||
if (!options.isQuitting()) {
|
||||
options.quit()
|
||||
}
|
||||
return
|
||||
} finally {
|
||||
this.showing = false
|
||||
}
|
||||
if (options.isQuitting()) {
|
||||
return
|
||||
}
|
||||
if (response === 0) {
|
||||
options.reload()
|
||||
} else if (response === 1) {
|
||||
options.quit()
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user