mirror of
https://github.com/stablyai/orca.git
synced 2026-10-21 00:03:46 +00:00
fix(window): escalate renderer launch-failed to a relaunch prompt
A render-process-gone with reason 'launch-failed' means the renderer never
spawned; reloading the same webContents cannot create it, so the circuit
breaker burned all three retries and left a permanently blank window whose
only prompt offered Reload/Quit. Escalate launch failures immediately and
offer 'Restart Orca' backed by relaunchApp('renderer-launch-failed').
Dedupe the dialog on "a prompt is on screen", not "we prompted once": the
field sequence answers the prompt with Reload and the fresh renderer fails
to launch ~15ms later, so a once-per-window latch would swallow the second
escalation and restore the blank window it exists to remove.
Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
@@ -7,6 +7,7 @@ export type AppRelaunchReason =
|
||||
| 'gpu-fallback'
|
||||
| 'profile-switch'
|
||||
| 'profile-transfer'
|
||||
| 'renderer-launch-failed'
|
||||
| 'renderer-request'
|
||||
|
||||
export function relaunchApp(reason: AppRelaunchReason, data?: CrashReportBreadcrumbData): void {
|
||||
|
||||
+35
-14
@@ -183,6 +183,11 @@ import {
|
||||
ensureAutoUpdaterConfigured
|
||||
} from './window/attach-main-window-services'
|
||||
import { createMainWindow, loadMainWindow } from './window/createMainWindow'
|
||||
import {
|
||||
planRendererRecoveryPrompt,
|
||||
RendererRecoveryPromptGate,
|
||||
type RendererRecoveryAction
|
||||
} from './window/renderer-recovery-escalation'
|
||||
import { zoomDashboardPopoutIfFocused } from './window/dashboard-popout-window'
|
||||
import {
|
||||
createSystemTray,
|
||||
@@ -1227,13 +1232,14 @@ function openMainWindow(): BrowserWindow {
|
||||
reason: details.reason,
|
||||
expectedTeardown: getExpectedTeardownScope(webContentsId)
|
||||
}),
|
||||
onRendererRecoveryExhausted: ({ details, recentRecoveryCount }) => {
|
||||
onRendererRecoveryExhausted: ({ details, recentRecoveryCount, recommendedAction }) => {
|
||||
recordDurableCrashBreadcrumb('renderer_recovery_circuit_breaker_open', {
|
||||
reason: details.reason,
|
||||
exitCode: details.exitCode ?? null,
|
||||
recentRecoveryCount
|
||||
recentRecoveryCount,
|
||||
recommendedAction
|
||||
})
|
||||
void presentRendererRecoveryPrompt(recentRecoveryCount)
|
||||
void presentRendererRecoveryPrompt(recentRecoveryCount, recommendedAction)
|
||||
},
|
||||
deferLoad: true,
|
||||
title: devInstanceIdentity.name,
|
||||
@@ -1508,28 +1514,43 @@ function sendOpenCrashReport(targetWindow?: BrowserWindow | null): void {
|
||||
webContents?.send('ui:openCrashReport')
|
||||
}
|
||||
|
||||
const rendererRecoveryPromptGate = new RendererRecoveryPromptGate()
|
||||
|
||||
// Why: on renderer crash-loop the breaker stops auto-reloading and the window goes blank, so a main-process dialog is the only retry/quit surface.
|
||||
async function presentRendererRecoveryPrompt(recentRecoveryCount: number): Promise<void> {
|
||||
async function presentRendererRecoveryPrompt(
|
||||
recentRecoveryCount: number,
|
||||
recommendedAction: RendererRecoveryAction
|
||||
): Promise<void> {
|
||||
if (isQuitting) {
|
||||
return
|
||||
}
|
||||
const window = mainWindow && !mainWindow.isDestroyed() ? mainWindow : undefined
|
||||
const plan = planRendererRecoveryPrompt({ action: recommendedAction, recentRecoveryCount })
|
||||
const options = {
|
||||
type: 'error' as const,
|
||||
buttons: ['Reload', 'Quit'],
|
||||
defaultId: 0,
|
||||
cancelId: 1,
|
||||
buttons: plan.buttons,
|
||||
defaultId: plan.defaultId,
|
||||
cancelId: plan.cancelId,
|
||||
title: 'Orca keeps failing to load',
|
||||
message: 'The app window crashed repeatedly and stopped reloading automatically.',
|
||||
detail: `Orca tried to recover ${recentRecoveryCount} times in a row without success. This is often a graphics-driver or installation problem. Reload to try again, or quit and relaunch Orca.`
|
||||
message: plan.message,
|
||||
detail: plan.detail
|
||||
}
|
||||
const { response } = window
|
||||
? await dialog.showMessageBox(window, options)
|
||||
: await dialog.showMessageBox(options)
|
||||
if (response === 0 && mainWindow && !mainWindow.isDestroyed()) {
|
||||
const result = await rendererRecoveryPromptGate.show(() =>
|
||||
window ? dialog.showMessageBox(window, options) : dialog.showMessageBox(options)
|
||||
)
|
||||
if (!result) {
|
||||
return
|
||||
}
|
||||
const choice = plan.responses[result.response]
|
||||
if (choice === 'relaunch') {
|
||||
recordDurableCrashBreadcrumb('renderer_recovery_relaunch')
|
||||
isQuitting = true
|
||||
relaunchApp('renderer-launch-failed')
|
||||
app.quit()
|
||||
} else if (choice === 'reload' && mainWindow && !mainWindow.isDestroyed()) {
|
||||
recordDurableCrashBreadcrumb('renderer_recovery_manual_retry')
|
||||
loadMainWindow(mainWindow)
|
||||
} else if (response === 1) {
|
||||
} else if (choice === 'quit') {
|
||||
isQuitting = true
|
||||
app.quit()
|
||||
}
|
||||
|
||||
@@ -3280,7 +3280,7 @@ describe('createMainWindow', () => {
|
||||
consoleError.mockRestore()
|
||||
})
|
||||
|
||||
it('bounds renderer launch-failed recovery with the crash-loop breaker', () => {
|
||||
it('escalates again when a launch failure follows the host-driven manual retry', () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
@@ -3307,15 +3307,52 @@ describe('createMainWindow', () => {
|
||||
}
|
||||
|
||||
driveLaunchFailure()
|
||||
driveLaunchFailure()
|
||||
driveLaunchFailure()
|
||||
expect(browserWindowInstance.loadFile).toHaveBeenCalledTimes(4)
|
||||
expect(onRendererRecoveryExhausted).toHaveBeenCalledOnce()
|
||||
|
||||
// The field sequence: user answers the prompt with Reload, and the fresh renderer fails to launch too.
|
||||
windowHandlers['did-finish-load']?.()
|
||||
driveLaunchFailure()
|
||||
expect(browserWindowInstance.loadFile).toHaveBeenCalledTimes(4)
|
||||
|
||||
expect(onRendererRecoveryExhausted).toHaveBeenCalledTimes(2)
|
||||
expect(onRendererRecoveryExhausted).toHaveBeenLastCalledWith(
|
||||
expect.objectContaining({ details, recommendedAction: 'relaunch' })
|
||||
)
|
||||
// Escalation never reloads the dead webContents; only the initial load ran.
|
||||
expect(browserWindowInstance.loadFile).toHaveBeenCalledTimes(1)
|
||||
} finally {
|
||||
consoleError.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('escalates renderer launch-failed to a relaunch instead of reloading the dead webContents', () => {
|
||||
vi.useFakeTimers()
|
||||
|
||||
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {})
|
||||
const onRendererRecoveryExhausted = vi.fn()
|
||||
const { browserWindowInstance, windowHandlers } = createRendererRecoveryWindowHarness()
|
||||
|
||||
try {
|
||||
createMainWindow(null, {
|
||||
onRendererRecoveryExhausted,
|
||||
shouldRecoverRenderer: (details) =>
|
||||
shouldRecoverRendererAfterProcessGone({
|
||||
reason: details.reason,
|
||||
expectedTeardown: 'none'
|
||||
})
|
||||
})
|
||||
|
||||
const details = {
|
||||
reason: 'launch-failed',
|
||||
exitCode: 18
|
||||
} as Electron.RenderProcessGoneDetails
|
||||
windowHandlers['render-process-gone']?.({} as never, details)
|
||||
vi.advanceTimersByTime(250)
|
||||
|
||||
// Only the initial load; a reload cannot spawn a process that failed to launch.
|
||||
expect(browserWindowInstance.loadFile).toHaveBeenCalledTimes(1)
|
||||
expect(onRendererRecoveryExhausted).toHaveBeenCalledOnce()
|
||||
expect(onRendererRecoveryExhausted).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ details, recentRecoveryCount: 3 })
|
||||
expect.objectContaining({ details, recommendedAction: 'relaunch' })
|
||||
)
|
||||
} finally {
|
||||
consoleError.mockRestore()
|
||||
|
||||
@@ -44,6 +44,10 @@ import {
|
||||
import { getMainE2EConfig } from '../e2e-config'
|
||||
import { buildEditableContextMenuTemplate } from './editable-context-menu'
|
||||
import { clearTrustedUIRendererWebContentsId, setTrustedUIRendererWebContentsId } from '../ipc/ui'
|
||||
import {
|
||||
getRendererRecoveryAction,
|
||||
type RendererRecoveryAction
|
||||
} from './renderer-recovery-escalation'
|
||||
import { resolveWindowCloseAction } from './window-close-decision'
|
||||
import { rectHasVisibleAreaOnAnyDisplay } from './window-bounds-validation'
|
||||
import { closeDashboardPopout } from './dashboard-popout-window'
|
||||
@@ -189,11 +193,12 @@ type CreateMainWindowOptions = {
|
||||
details: Electron.RenderProcessGoneDetails,
|
||||
webContentsId: number
|
||||
) => boolean
|
||||
/** Called when consecutive auto-recoveries hit the circuit-breaker limit so the host can prompt instead of crash-looping. */
|
||||
/** Called when auto-recovery stops — breaker limit hit, or a launch failure a reload can't fix — so the host can prompt instead of crash-looping. */
|
||||
onRendererRecoveryExhausted?: (info: {
|
||||
details: Electron.RenderProcessGoneDetails
|
||||
webContentsId: number
|
||||
recentRecoveryCount: number
|
||||
recommendedAction: RendererRecoveryAction
|
||||
}) => void
|
||||
/** Defer renderer load until IPC handlers are registered, or eager renderer calls race into missing channels. */
|
||||
deferLoad?: boolean
|
||||
@@ -586,13 +591,26 @@ export function createMainWindow(
|
||||
) {
|
||||
return
|
||||
}
|
||||
const recommendedAction = getRendererRecoveryAction(details.reason)
|
||||
if (recommendedAction === 'relaunch') {
|
||||
// Why: the renderer never spawned, so reloading the same webContents can only burn retries and leave a blank window.
|
||||
// Every launch failure escalates — the host dedupes the dialog, so a failure after the user retried still gets a prompt.
|
||||
opts?.onRendererRecoveryExhausted?.({
|
||||
details,
|
||||
webContentsId: rendererWebContentsId,
|
||||
recentRecoveryCount: rendererRecoveryCircuitBreaker.recentRecoveryCount(Date.now()),
|
||||
recommendedAction
|
||||
})
|
||||
return
|
||||
}
|
||||
const recovery = rendererRecoveryCircuitBreaker.registerRecoveryAttempt(Date.now())
|
||||
if (!recovery.allowed) {
|
||||
// Why: too many reloads means it will just crash again; stop and let the host surface a recovery prompt.
|
||||
opts?.onRendererRecoveryExhausted?.({
|
||||
details,
|
||||
webContentsId: rendererWebContentsId,
|
||||
recentRecoveryCount: recovery.recentRecoveryCount
|
||||
recentRecoveryCount: recovery.recentRecoveryCount,
|
||||
recommendedAction
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
getRendererRecoveryAction,
|
||||
planRendererRecoveryPrompt,
|
||||
RendererRecoveryPromptGate
|
||||
} from './renderer-recovery-escalation'
|
||||
|
||||
describe('getRendererRecoveryAction', () => {
|
||||
it('recommends a relaunch when the renderer process never launched', () => {
|
||||
expect(getRendererRecoveryAction('launch-failed')).toBe('relaunch')
|
||||
})
|
||||
|
||||
it('recommends a reload for renderer deaths a fresh document can fix', () => {
|
||||
expect(getRendererRecoveryAction('crashed')).toBe('reload')
|
||||
expect(getRendererRecoveryAction('oom')).toBe('reload')
|
||||
expect(getRendererRecoveryAction(undefined)).toBe('reload')
|
||||
})
|
||||
})
|
||||
|
||||
describe('planRendererRecoveryPrompt', () => {
|
||||
it('offers Restart Orca first for a launch failure', () => {
|
||||
const plan = planRendererRecoveryPrompt({ action: 'relaunch', recentRecoveryCount: 0 })
|
||||
|
||||
expect(plan.buttons).toEqual(['Restart Orca', 'Reload', 'Quit'])
|
||||
expect(plan.responses).toEqual(['relaunch', 'reload', 'quit'])
|
||||
expect(plan.responses[plan.defaultId]).toBe('relaunch')
|
||||
expect(plan.responses[plan.cancelId]).toBe('quit')
|
||||
})
|
||||
|
||||
it('keeps the reload/quit prompt for crash loops and reports the attempt count', () => {
|
||||
const plan = planRendererRecoveryPrompt({ action: 'reload', recentRecoveryCount: 3 })
|
||||
|
||||
expect(plan.buttons).toEqual(['Reload', 'Quit'])
|
||||
expect(plan.responses).toEqual(['reload', 'quit'])
|
||||
expect(plan.responses[plan.defaultId]).toBe('reload')
|
||||
expect(plan.responses[plan.cancelId]).toBe('quit')
|
||||
expect(plan.detail).toContain('3 times')
|
||||
})
|
||||
})
|
||||
|
||||
describe('RendererRecoveryPromptGate', () => {
|
||||
it('drops a prompt requested while one is already on screen', async () => {
|
||||
const gate = new RendererRecoveryPromptGate()
|
||||
let resolveFirst: ((value: string) => void) | undefined
|
||||
const first = gate.show(
|
||||
() =>
|
||||
new Promise<string>((resolve) => {
|
||||
resolveFirst = resolve
|
||||
})
|
||||
)
|
||||
const second = vi.fn(async () => 'second')
|
||||
|
||||
expect(await gate.show(second)).toBeUndefined()
|
||||
expect(second).not.toHaveBeenCalled()
|
||||
|
||||
resolveFirst?.('first')
|
||||
expect(await first).toBe('first')
|
||||
})
|
||||
|
||||
it('prompts again once the user has answered the previous dialog', async () => {
|
||||
const gate = new RendererRecoveryPromptGate()
|
||||
|
||||
expect(await gate.show(async () => 'first')).toBe('first')
|
||||
expect(await gate.show(async () => 'second')).toBe('second')
|
||||
})
|
||||
|
||||
it('reopens after a rejected prompt so a dialog failure cannot wedge the gate shut', async () => {
|
||||
const gate = new RendererRecoveryPromptGate()
|
||||
|
||||
await expect(
|
||||
gate.show(async () => {
|
||||
throw new Error('dialog failed')
|
||||
})
|
||||
).rejects.toThrow('dialog failed')
|
||||
expect(await gate.show(async () => 'next')).toBe('next')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,73 @@
|
||||
/** How Orca should answer a renderer loss once auto-recovery is off the table. */
|
||||
export type RendererRecoveryAction = 'relaunch' | 'reload'
|
||||
|
||||
/** What the user picked in the recovery dialog. */
|
||||
export type RendererRecoveryPromptResponse = RendererRecoveryAction | 'quit'
|
||||
|
||||
export type RendererRecoveryPromptPlan = {
|
||||
buttons: string[]
|
||||
defaultId: number
|
||||
cancelId: number
|
||||
message: string
|
||||
detail: string
|
||||
/** Response index -> action, so callers never hardcode button positions. */
|
||||
responses: RendererRecoveryPromptResponse[]
|
||||
}
|
||||
|
||||
/**
|
||||
* `launch-failed` means the renderer process never spawned (sandbox, AV, or
|
||||
* broken install). Reloading reuses the same webContents, so it can only burn
|
||||
* the circuit breaker and leave a permanently blank window — only a fresh app
|
||||
* process can recover.
|
||||
*/
|
||||
export function getRendererRecoveryAction(reason: string | undefined): RendererRecoveryAction {
|
||||
return reason === 'launch-failed' ? 'relaunch' : 'reload'
|
||||
}
|
||||
|
||||
export function planRendererRecoveryPrompt(input: {
|
||||
action: RendererRecoveryAction
|
||||
recentRecoveryCount: number
|
||||
}): RendererRecoveryPromptPlan {
|
||||
if (input.action === 'relaunch') {
|
||||
return {
|
||||
buttons: ['Restart Orca', 'Reload', 'Quit'],
|
||||
defaultId: 0,
|
||||
cancelId: 2,
|
||||
message: 'Orca could not start its window process.',
|
||||
detail:
|
||||
'The window process failed to launch, so reloading it cannot help. Restarting Orca gives it a fresh process. This is often caused by antivirus interference, a sandbox restriction, or a damaged installation.',
|
||||
responses: ['relaunch', 'reload', 'quit']
|
||||
}
|
||||
}
|
||||
return {
|
||||
buttons: ['Reload', 'Quit'],
|
||||
defaultId: 0,
|
||||
cancelId: 1,
|
||||
message: 'The app window crashed repeatedly and stopped reloading automatically.',
|
||||
detail: `Orca tried to recover ${input.recentRecoveryCount} times in a row without success. This is often a graphics-driver or installation problem. Reload to try again, or quit and relaunch Orca.`,
|
||||
responses: ['reload', 'quit']
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps at most one recovery dialog on screen. Scoped to "a prompt is up right
|
||||
* now", never "we prompted once": a launch failure that lands after the user
|
||||
* answered must get its own prompt, or the window stays blank with no recovery
|
||||
* affordance at all.
|
||||
*/
|
||||
export class RendererRecoveryPromptGate {
|
||||
private showing = false
|
||||
|
||||
/** Resolves undefined when a prompt is already on screen. */
|
||||
async show<T>(showPrompt: () => Promise<T>): Promise<T | undefined> {
|
||||
if (this.showing) {
|
||||
return undefined
|
||||
}
|
||||
this.showing = true
|
||||
try {
|
||||
return await showPrompt()
|
||||
} finally {
|
||||
this.showing = false
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user