mirror of
https://github.com/stablyai/orca.git
synced 2026-10-24 16:03:45 +00:00
fix(gpu): persist safe-graphics marker before the restart prompt
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
engageGpuFallbackAfterCrashBurst,
|
||||
type GpuFallbackEngagement,
|
||||
type GpuFallbackEngagementHandlers
|
||||
} from './gpu-fallback-engagement'
|
||||
import type { GpuFallbackRestartDecision } from './gpu-fallback-restart-prompt'
|
||||
|
||||
const ENGAGEMENT: GpuFallbackEngagement = {
|
||||
reason: 'crashed',
|
||||
// 0x80000003 STATUS_BREAKPOINT — the production Windows signature.
|
||||
exitCode: -2147483645,
|
||||
crashesInWindow: 3,
|
||||
engagedAt: 1_700_000_000_000
|
||||
}
|
||||
|
||||
function createHandlers(
|
||||
overrides: Partial<GpuFallbackEngagementHandlers> = {}
|
||||
): { handlers: GpuFallbackEngagementHandlers; order: string[] } {
|
||||
const order: string[] = []
|
||||
const handlers: GpuFallbackEngagementHandlers = {
|
||||
isQuitting: () => false,
|
||||
persistMarker: vi.fn(() => {
|
||||
order.push('persistMarker')
|
||||
return true
|
||||
}),
|
||||
clearMarker: vi.fn(() => order.push('clearMarker')),
|
||||
promptForRestart: vi.fn(async () => {
|
||||
order.push('prompt')
|
||||
return 'restart' as GpuFallbackRestartDecision
|
||||
}),
|
||||
onPromptFailed: vi.fn(),
|
||||
onEngaged: vi.fn(),
|
||||
onRestartDeferred: vi.fn(() => order.push('restartDeferred')),
|
||||
restartIntoSafeGraphics: vi.fn(() => order.push('restart')),
|
||||
...overrides
|
||||
}
|
||||
return { handlers, order }
|
||||
}
|
||||
|
||||
describe('engageGpuFallbackAfterCrashBurst', () => {
|
||||
// Repro for the Windows startup GPU cluster (v1.4.190, exit -2147483645).
|
||||
// Measured on Windows 11 26200 / Electron 43.1.0: Chromium's own ladder aborts
|
||||
// the whole browser ("GPU process isn't usable. Goodbye.") on the 6th GPU
|
||||
// crash, 1.285s after the 3rd — the crash that trips this threshold. If the
|
||||
// marker is only written after the user answers the modal, the app dies first
|
||||
// and the next launch retries hardware acceleration, looping forever.
|
||||
it('persists the safe-graphics marker before the restart prompt is answered', async () => {
|
||||
let resolvePrompt: ((decision: GpuFallbackRestartDecision) => void) | undefined
|
||||
const { handlers } = createHandlers({
|
||||
promptForRestart: vi.fn(
|
||||
() =>
|
||||
new Promise<GpuFallbackRestartDecision>((resolve) => {
|
||||
resolvePrompt = resolve
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
const engaging = engageGpuFallbackAfterCrashBurst(ENGAGEMENT, handlers)
|
||||
await Promise.resolve()
|
||||
|
||||
// Chromium kills the process here; whatever is on disk now is all the next launch gets.
|
||||
expect(handlers.persistMarker).toHaveBeenCalledWith(ENGAGEMENT)
|
||||
|
||||
resolvePrompt?.('restart')
|
||||
await engaging
|
||||
})
|
||||
|
||||
it('relaunches into safe graphics when the user accepts', async () => {
|
||||
const { handlers, order } = createHandlers()
|
||||
await engageGpuFallbackAfterCrashBurst(ENGAGEMENT, handlers)
|
||||
expect(order).toEqual(['persistMarker', 'prompt', 'restart'])
|
||||
expect(handlers.clearMarker).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('undoes the marker when the user chooses Keep Running', async () => {
|
||||
const { handlers, order } = createHandlers({
|
||||
promptForRestart: vi.fn(async () => 'continue' as GpuFallbackRestartDecision)
|
||||
})
|
||||
await engageGpuFallbackAfterCrashBurst(ENGAGEMENT, handlers)
|
||||
expect(order).toEqual(['persistMarker', 'clearMarker', 'restartDeferred'])
|
||||
expect(handlers.restartIntoSafeGraphics).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps the marker when the prompt itself fails, so the next launch is still safe', async () => {
|
||||
const error = new Error('no display')
|
||||
const { handlers } = createHandlers({
|
||||
promptForRestart: vi.fn(async () => {
|
||||
throw error
|
||||
})
|
||||
})
|
||||
await engageGpuFallbackAfterCrashBurst(ENGAGEMENT, handlers)
|
||||
expect(handlers.persistMarker).toHaveBeenCalledTimes(1)
|
||||
expect(handlers.clearMarker).not.toHaveBeenCalled()
|
||||
expect(handlers.onPromptFailed).toHaveBeenCalledWith(error)
|
||||
})
|
||||
|
||||
it('does not relaunch when a quit began while the prompt was open', async () => {
|
||||
const { handlers } = createHandlers({ isQuitting: () => true })
|
||||
await engageGpuFallbackAfterCrashBurst(ENGAGEMENT, handlers)
|
||||
expect(handlers.restartIntoSafeGraphics).not.toHaveBeenCalled()
|
||||
// The marker stays: a quit is not the user declining safe graphics.
|
||||
expect(handlers.clearMarker).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not claim a restart when the marker could not be written', async () => {
|
||||
const { handlers } = createHandlers({ persistMarker: vi.fn(() => false) })
|
||||
await engageGpuFallbackAfterCrashBurst(ENGAGEMENT, handlers)
|
||||
expect(handlers.restartIntoSafeGraphics).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,63 @@
|
||||
import type { GpuFallbackRestartDecision } from './gpu-fallback-restart-prompt'
|
||||
|
||||
/** The GPU-crash burst that tripped the software-rendering fallback. */
|
||||
export type GpuFallbackEngagement = {
|
||||
reason: string
|
||||
exitCode: number | null
|
||||
crashesInWindow: number
|
||||
/** Wall-clock ms stamped into the persisted marker. */
|
||||
engagedAt: number
|
||||
}
|
||||
|
||||
export type GpuFallbackEngagementHandlers = {
|
||||
isQuitting: () => boolean
|
||||
/** Writes the build-scoped safe-graphics marker; false when it could not be persisted. */
|
||||
persistMarker: (engagement: GpuFallbackEngagement) => boolean
|
||||
clearMarker: () => void
|
||||
promptForRestart: () => Promise<GpuFallbackRestartDecision>
|
||||
onPromptFailed: (error: unknown) => void
|
||||
onEngaged: (engagement: GpuFallbackEngagement) => void
|
||||
onRestartDeferred: (engagement: GpuFallbackEngagement) => void
|
||||
restartIntoSafeGraphics: (engagement: GpuFallbackEngagement) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Drives the safe-graphics handover once a GPU crash burst trips the threshold.
|
||||
*
|
||||
* Why the marker is written before the prompt: measured on Windows 11 26200 /
|
||||
* Electron 43.1.0, Chromium's own GPU ladder aborts the whole browser process
|
||||
* (`FATAL gpu_data_manager_impl_private.cc: GPU process isn't usable. Goodbye.`)
|
||||
* on the 6th GPU crash — 1.285s after the 3rd, which is the crash that trips
|
||||
* this threshold. Persisting only after the user answers a modal means the app
|
||||
* dies first, no marker survives, and the next launch retries hardware
|
||||
* acceleration: the crash loop never ends. An explicit "Keep Running" undoes it.
|
||||
*/
|
||||
export async function engageGpuFallbackAfterCrashBurst(
|
||||
engagement: GpuFallbackEngagement,
|
||||
handlers: GpuFallbackEngagementHandlers
|
||||
): Promise<void> {
|
||||
handlers.onEngaged(engagement)
|
||||
const persisted = handlers.persistMarker(engagement)
|
||||
let decision: GpuFallbackRestartDecision
|
||||
try {
|
||||
decision = await handlers.promptForRestart()
|
||||
} catch (error) {
|
||||
// Marker stays: the next launch is safe even though the user was never asked.
|
||||
handlers.onPromptFailed(error)
|
||||
return
|
||||
}
|
||||
if (handlers.isQuitting()) {
|
||||
return
|
||||
}
|
||||
if (decision !== 'restart') {
|
||||
if (persisted) {
|
||||
handlers.clearMarker()
|
||||
}
|
||||
handlers.onRestartDeferred(engagement)
|
||||
return
|
||||
}
|
||||
if (!persisted) {
|
||||
return
|
||||
}
|
||||
handlers.restartIntoSafeGraphics(engagement)
|
||||
}
|
||||
+48
-46
@@ -169,6 +169,7 @@ import { enableRendererHeapHeadroom } from './startup/renderer-heap-headroom'
|
||||
import { argvRequestsServeMode, normalizeServeModeArgv } from './startup/serve-mode-argv'
|
||||
import { ensureVirtualDisplayForHeadlessServe } from './startup/ensure-virtual-display'
|
||||
import {
|
||||
clearGpuFallbackMarker,
|
||||
readActiveGpuFallbackMarker,
|
||||
writeGpuFallbackMarker,
|
||||
type GpuFallbackEnvironment,
|
||||
@@ -181,10 +182,8 @@ import {
|
||||
GpuCrashFallbackTracker,
|
||||
isGpuFallbackCrashCandidate
|
||||
} from './crash-reporting/gpu-crash-fallback-decision'
|
||||
import {
|
||||
promptForGpuFallbackRestart,
|
||||
type GpuFallbackRestartDecision
|
||||
} from './crash-reporting/gpu-fallback-restart-prompt'
|
||||
import { promptForGpuFallbackRestart } from './crash-reporting/gpu-fallback-restart-prompt'
|
||||
import { engageGpuFallbackAfterCrashBurst } from './crash-reporting/gpu-fallback-engagement'
|
||||
import {
|
||||
shouldSuppressDevEducation,
|
||||
suppressDevEducationForStore
|
||||
@@ -1874,54 +1873,57 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro
|
||||
if (!result.shouldEngageFallback) {
|
||||
return
|
||||
}
|
||||
recordCrashBreadcrumb('gpu_fallback_engaged', {
|
||||
reason,
|
||||
exitCode,
|
||||
crashesInWindow: result.crashesInWindow
|
||||
})
|
||||
const engagedAt = Date.now()
|
||||
const window = mainWindow && !mainWindow.isDestroyed() ? mainWindow : undefined
|
||||
let restartDecision: GpuFallbackRestartDecision
|
||||
try {
|
||||
restartDecision = await promptForGpuFallbackRestart(window)
|
||||
} catch (error) {
|
||||
console.warn('[gpu-fallback] failed to show restart prompt:', error)
|
||||
return
|
||||
}
|
||||
const fallbackData = {
|
||||
processReason: reason,
|
||||
exitCode,
|
||||
crashesInWindow: result.crashesInWindow
|
||||
}
|
||||
if (isQuitting) {
|
||||
return
|
||||
}
|
||||
if (restartDecision !== 'restart') {
|
||||
recordDurableCrashBreadcrumb('gpu_fallback_restart_deferred', fallbackData)
|
||||
return
|
||||
}
|
||||
const environment = getWindowsGpuFallbackEnvironment()
|
||||
if (!environment) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
writeGpuFallbackMarker(
|
||||
app.getPath('userData'),
|
||||
{
|
||||
engagedAt,
|
||||
crashesInWindow: result.crashesInWindow
|
||||
const userDataPath = app.getPath('userData')
|
||||
await engageGpuFallbackAfterCrashBurst(
|
||||
{ reason, exitCode, crashesInWindow: result.crashesInWindow, engagedAt: Date.now() },
|
||||
{
|
||||
isQuitting: () => isQuitting,
|
||||
onEngaged: (engagement) =>
|
||||
recordCrashBreadcrumb('gpu_fallback_engaged', {
|
||||
reason: engagement.reason,
|
||||
exitCode: engagement.exitCode,
|
||||
crashesInWindow: engagement.crashesInWindow
|
||||
}),
|
||||
persistMarker: (engagement) => {
|
||||
const environment = getWindowsGpuFallbackEnvironment()
|
||||
if (!environment) {
|
||||
return false
|
||||
}
|
||||
try {
|
||||
writeGpuFallbackMarker(
|
||||
userDataPath,
|
||||
{ engagedAt: engagement.engagedAt, crashesInWindow: engagement.crashesInWindow },
|
||||
environment
|
||||
)
|
||||
return true
|
||||
} catch (error) {
|
||||
console.warn('[gpu-fallback] failed to persist marker:', error)
|
||||
return false
|
||||
}
|
||||
},
|
||||
environment
|
||||
)
|
||||
} catch (error) {
|
||||
console.warn('[gpu-fallback] failed to persist marker:', error)
|
||||
return
|
||||
}
|
||||
isQuitting = true
|
||||
relaunchApp('gpu-fallback', fallbackData)
|
||||
// Why: app.exit(0) skips before-quit, so destroy the Windows tray manually to avoid a stale icon.
|
||||
destroySystemTray()
|
||||
app.exit(0)
|
||||
clearMarker: () => clearGpuFallbackMarker(userDataPath),
|
||||
promptForRestart: () =>
|
||||
promptForGpuFallbackRestart(
|
||||
mainWindow && !mainWindow.isDestroyed() ? mainWindow : undefined
|
||||
),
|
||||
onPromptFailed: (error) =>
|
||||
console.warn('[gpu-fallback] failed to show restart prompt:', error),
|
||||
onRestartDeferred: () =>
|
||||
recordDurableCrashBreadcrumb('gpu_fallback_restart_deferred', fallbackData),
|
||||
restartIntoSafeGraphics: () => {
|
||||
isQuitting = true
|
||||
relaunchApp('gpu-fallback', fallbackData)
|
||||
// Why: app.exit(0) skips before-quit, so destroy the Windows tray manually to avoid a stale icon.
|
||||
destroySystemTray()
|
||||
app.exit(0)
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
function recordProcessGoneCrash(
|
||||
|
||||
Reference in New Issue
Block a user