wip: gpu-startup-recovery

This commit is contained in:
Neil
2026-08-27 22:20:55 -07:00
parent 91e7cd088a
commit 2d7efefa87
6 changed files with 435 additions and 65 deletions
@@ -0,0 +1,128 @@
import { readFileSync } from 'node:fs'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import {
DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD,
DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS,
GpuCrashFallbackTracker,
isGpuFallbackCrashCandidate
} from './gpu-crash-fallback-decision'
import { shouldRecordProcessGoneCrash } from './process-gone-classification'
/**
* Replays the win32 GPU-child deaths from the 1.4.190 'crashed' renderer cluster
* through the production decision path, so the reason no host was ever offered
* Safe Graphics Mode is pinned rather than argued about.
*
* Each session below is one crash report's `Recent activity`; a breadcrumb's
* `suppressedSinceLast=N` means N further identical GPU deaths were coalesced
* into it, so the session saw N+1 GPU crashes.
*/
type FieldSession = {
/** Crash-report id from the field bundle. */
report: string
/** GPU child crash times, ms since main-process start. */
gpuCrashesMsSinceLaunch: number[]
}
// crashed.txt: every win32 report in the cluster. Times are (breadcrumb ts -
// mainProcessStartedAt); coalesced repeats are placed inside the same second,
// which is the only interval the emitted breadcrumb pins them to.
const CRASHED_CLUSTER_SESSIONS: FieldSession[] = [
// 23:36:54.200 - 23:36:49.931, suppressedSinceLast=1 -> 2 crashes
{ report: 'db1f1ee2', gpuCrashesMsSinceLaunch: [4_269, 4_800] },
// 02:58:25.287 - 02:58:20.749, no suppression -> 1 crash
{ report: '66cc54d8', gpuCrashesMsSinceLaunch: [4_538] },
// 21:47:05.776 - 21:46:59.584, suppressedSinceLast=1 -> 2 crashes
{ report: '1f5564de', gpuCrashesMsSinceLaunch: [6_192, 6_240] },
// 00:17:17.840 - 00:17:15.732, no suppression -> 1 crash
{ report: '96d8c63b', gpuCrashesMsSinceLaunch: [2_108] }
]
/** index.ts's `child-process-gone` listener body — the wiring these claims rest on. */
function readChildProcessGoneListener(): string {
const source = readFileSync(join(__dirname, '..', 'index.ts'), 'utf8')
const start = source.indexOf("app.on('child-process-gone'")
expect(start).toBeGreaterThan(0)
return source.slice(start, source.indexOf('\n })', start))
}
function newTracker(): GpuCrashFallbackTracker {
return new GpuCrashFallbackTracker({
windowMs: DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS,
threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD
})
}
const FIELD_GPU_EVENT = {
source: 'child',
processType: 'GPU',
serviceName: 'GPU',
reason: 'crashed',
expectedTeardown: 'none'
} as const
describe('1.4.190 win32 GPU-child crash cluster', () => {
it('does not let process_gone_suppressed gate the fallback candidate check', () => {
// The GPU death is suppressed as recoverable churn (no user-facing report)...
expect(shouldRecordProcessGoneCrash({ ...FIELD_GPU_EVENT, exitCode: -2147483645 })).toBe(false)
// ...but the fallback path reads the raw child-process-gone event, so the
// suppression cannot hide a broken driver from recovery.
expect(
isGpuFallbackCrashCandidate({
platform: 'win32',
processType: 'GPU',
reason: 'crashed'
})
).toBe(true)
// Both assertions above still pass if the candidate check is moved behind the
// suppressed-report path, so pin that nothing branches ahead of it in index.ts.
const listener = readChildProcessGoneListener()
const guardStart = listener.indexOf('isGpuFallbackCrashCandidate(')
expect(guardStart).toBeGreaterThan(0)
expect(listener.slice(0, guardStart).match(/\bif\s*\(/g) ?? []).toHaveLength(1)
expect(listener).toMatch(/isGpuFallbackCrashCandidate\([\s\S]*?void handleGpuChildCrash\(/)
// The `if (` count alone still allows `recorded && isGpuFallbackCrashCandidate(...)`, which
// re-couples recovery to the suppression decision, so pin the guard to that check alone.
const recoveryGuard = listener.slice(
listener.lastIndexOf('if (', guardStart),
listener.indexOf('void handleGpuChildCrash(')
)
expect(recoveryGuard).not.toMatch(/&&|\|\|/)
})
it('never reaches the burst threshold in any observed cluster session', () => {
const outcomes = CRASHED_CLUSTER_SESSIONS.map((session) => {
// Each launch constructs a fresh tracker (src/main/index.ts), so evidence
// does not survive the relaunch these users performed after every crash.
const tracker = newTracker()
const engaged = session.gpuCrashesMsSinceLaunch.some(
(at) => tracker.recordGpuCrash(at).shouldEngageFallback
)
return {
report: session.report,
gpuCrashes: session.gpuCrashesMsSinceLaunch.length,
engagedSafeGraphicsPrompt: engaged
}
})
expect(outcomes).toEqual([
{ report: 'db1f1ee2', gpuCrashes: 2, engagedSafeGraphicsPrompt: false },
{ report: '66cc54d8', gpuCrashes: 1, engagedSafeGraphicsPrompt: false },
{ report: '1f5564de', gpuCrashes: 2, engagedSafeGraphicsPrompt: false },
{ report: '96d8c63b', gpuCrashes: 1, engagedSafeGraphicsPrompt: false }
])
})
it('engages on the session that did reach three crashes (field launch 51b9e93c)', () => {
// oom.txt, win32: GPU crashed/exitCode=34 with suppressedSinceLast=2, then
// `gpu_fallback_engaged (crashesInWindow=3)` and `gpu_fallback_restart_deferred`.
const tracker = newTracker()
expect(tracker.recordGpuCrash(3_600_000).shouldEngageFallback).toBe(false)
expect(tracker.recordGpuCrash(3_601_000).shouldEngageFallback).toBe(false)
expect(tracker.recordGpuCrash(3_601_890)).toEqual({
shouldEngageFallback: true,
crashesInWindow: 3
})
})
})
@@ -0,0 +1,112 @@
import { describe, expect, it, vi } from 'vitest'
import {
engageGpuFallbackAfterCrashBurst,
type GpuFallbackEngagement,
type GpuFallbackEngagementHandlers
} from './gpu-fallback-engagement'
import type { GpuFallbackRestartDecision } from './gpu-fallback-restart-prompt'
const ENGAGEMENT: GpuFallbackEngagement = {
reason: 'crashed',
// 0x80000003 STATUS_BREAKPOINT — the production Windows signature.
exitCode: -2147483645,
crashesInWindow: 3,
engagedAt: 1_700_000_000_000
}
function createHandlers(overrides: Partial<GpuFallbackEngagementHandlers> = {}): {
handlers: GpuFallbackEngagementHandlers
order: string[]
} {
const order: string[] = []
const handlers: GpuFallbackEngagementHandlers = {
isQuitting: () => false,
persistMarker: vi.fn(() => {
order.push('persistMarker')
return true
}),
clearMarker: vi.fn(() => order.push('clearMarker')),
promptForRestart: vi.fn(async () => {
order.push('prompt')
return 'restart' as GpuFallbackRestartDecision
}),
onPromptFailed: vi.fn(),
onEngaged: vi.fn(),
onRestartDeferred: vi.fn(() => order.push('restartDeferred')),
restartIntoSafeGraphics: vi.fn(() => order.push('restart')),
...overrides
}
return { handlers, order }
}
describe('engageGpuFallbackAfterCrashBurst', () => {
// Repro for the Windows startup GPU cluster (v1.4.190, exit -2147483645).
// Measured on Windows 11 26200 / Electron 43.1.0: Chromium's own ladder aborts
// the whole browser ("GPU process isn't usable. Goodbye.") on the 6th GPU
// crash, 1.285s after the 3rd — the crash that trips this threshold. If the
// marker is only written after the user answers the modal, the app dies first
// and the next launch retries hardware acceleration, looping forever.
it('persists the safe-graphics marker before the restart prompt is answered', async () => {
let resolvePrompt: ((decision: GpuFallbackRestartDecision) => void) | undefined
const { handlers } = createHandlers({
promptForRestart: vi.fn(
() =>
new Promise<GpuFallbackRestartDecision>((resolve) => {
resolvePrompt = resolve
})
)
})
const engaging = engageGpuFallbackAfterCrashBurst(ENGAGEMENT, handlers)
await Promise.resolve()
// Chromium kills the process here; whatever is on disk now is all the next launch gets.
expect(handlers.persistMarker).toHaveBeenCalledWith(ENGAGEMENT)
resolvePrompt?.('restart')
await engaging
})
it('relaunches into safe graphics when the user accepts', async () => {
const { handlers, order } = createHandlers()
await engageGpuFallbackAfterCrashBurst(ENGAGEMENT, handlers)
expect(order).toEqual(['persistMarker', 'prompt', 'restart'])
expect(handlers.clearMarker).not.toHaveBeenCalled()
})
it('undoes the marker when the user chooses Keep Running', async () => {
const { handlers, order } = createHandlers({
promptForRestart: vi.fn(async () => 'continue' as GpuFallbackRestartDecision)
})
await engageGpuFallbackAfterCrashBurst(ENGAGEMENT, handlers)
expect(order).toEqual(['persistMarker', 'clearMarker', 'restartDeferred'])
expect(handlers.restartIntoSafeGraphics).not.toHaveBeenCalled()
})
it('keeps the marker when the prompt itself fails, so the next launch is still safe', async () => {
const error = new Error('no display')
const { handlers } = createHandlers({
promptForRestart: vi.fn(async () => {
throw error
})
})
await engageGpuFallbackAfterCrashBurst(ENGAGEMENT, handlers)
expect(handlers.persistMarker).toHaveBeenCalledTimes(1)
expect(handlers.clearMarker).not.toHaveBeenCalled()
expect(handlers.onPromptFailed).toHaveBeenCalledWith(error)
})
it('does not relaunch when a quit began while the prompt was open', async () => {
const { handlers } = createHandlers({ isQuitting: () => true })
await engageGpuFallbackAfterCrashBurst(ENGAGEMENT, handlers)
expect(handlers.restartIntoSafeGraphics).not.toHaveBeenCalled()
// The marker stays: a quit is not the user declining safe graphics.
expect(handlers.clearMarker).not.toHaveBeenCalled()
})
it('does not claim a restart when the marker could not be written', async () => {
const { handlers } = createHandlers({ persistMarker: vi.fn(() => false) })
await engageGpuFallbackAfterCrashBurst(ENGAGEMENT, handlers)
expect(handlers.restartIntoSafeGraphics).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,63 @@
import type { GpuFallbackRestartDecision } from './gpu-fallback-restart-prompt'
/** The GPU-crash burst that tripped the software-rendering fallback. */
export type GpuFallbackEngagement = {
reason: string
exitCode: number | null
crashesInWindow: number
/** Wall-clock ms stamped into the persisted marker. */
engagedAt: number
}
export type GpuFallbackEngagementHandlers = {
isQuitting: () => boolean
/** Writes the build-scoped safe-graphics marker; false when it could not be persisted. */
persistMarker: (engagement: GpuFallbackEngagement) => boolean
clearMarker: () => void
promptForRestart: () => Promise<GpuFallbackRestartDecision>
onPromptFailed: (error: unknown) => void
onEngaged: (engagement: GpuFallbackEngagement) => void
onRestartDeferred: (engagement: GpuFallbackEngagement) => void
restartIntoSafeGraphics: (engagement: GpuFallbackEngagement) => void
}
/**
* Drives the safe-graphics handover once a GPU crash burst trips the threshold.
*
* Why the marker is written before the prompt: measured on Windows 11 26200 /
* Electron 43.1.0, Chromium's own GPU ladder aborts the whole browser process
* (`FATAL gpu_data_manager_impl_private.cc: GPU process isn't usable. Goodbye.`)
* on the 6th GPU crash — 1.285s after the 3rd, which is the crash that trips
* this threshold. Persisting only after the user answers a modal means the app
* dies first, no marker survives, and the next launch retries hardware
* acceleration: the crash loop never ends. An explicit "Keep Running" undoes it.
*/
export async function engageGpuFallbackAfterCrashBurst(
engagement: GpuFallbackEngagement,
handlers: GpuFallbackEngagementHandlers
): Promise<void> {
handlers.onEngaged(engagement)
const persisted = handlers.persistMarker(engagement)
let decision: GpuFallbackRestartDecision
try {
decision = await handlers.promptForRestart()
} catch (error) {
// Marker stays: the next launch is safe even though the user was never asked.
handlers.onPromptFailed(error)
return
}
if (handlers.isQuitting()) {
return
}
if (decision !== 'restart') {
if (persisted) {
handlers.clearMarker()
}
handlers.onRestartDeferred(engagement)
return
}
if (!persisted) {
return
}
handlers.restartIntoSafeGraphics(engagement)
}
+51 -46
View File
@@ -153,6 +153,7 @@ import {
configureDevUserDataPath,
configureOrcaUserDataPathEnv,
disableUnsupportedChromiumFeatures,
optOutOfHiddenPageWakeUpThrottling,
enableMainProcessGpuFeatures,
installDevParentDisconnectQuit,
installDevParentSignalQuit,
@@ -169,6 +170,7 @@ import { enableRendererHeapHeadroom } from './startup/renderer-heap-headroom'
import { argvRequestsServeMode, normalizeServeModeArgv } from './startup/serve-mode-argv'
import { ensureVirtualDisplayForHeadlessServe } from './startup/ensure-virtual-display'
import {
clearGpuFallbackMarker,
readActiveGpuFallbackMarker,
writeGpuFallbackMarker,
type GpuFallbackEnvironment,
@@ -181,10 +183,8 @@ import {
GpuCrashFallbackTracker,
isGpuFallbackCrashCandidate
} from './crash-reporting/gpu-crash-fallback-decision'
import {
promptForGpuFallbackRestart,
type GpuFallbackRestartDecision
} from './crash-reporting/gpu-fallback-restart-prompt'
import { promptForGpuFallbackRestart } from './crash-reporting/gpu-fallback-restart-prompt'
import { engageGpuFallbackAfterCrashBurst } from './crash-reporting/gpu-fallback-engagement'
import {
shouldSuppressDevEducation,
suppressDevEducationForStore
@@ -980,6 +980,8 @@ if (hasSingleInstanceLock) {
...getMainProcessLifecycleIdentity()
})
disableUnsupportedChromiumFeatures()
// Why: unconditional — a GPU-fallback launch skips enableMainProcessGpuFeatures() below.
optOutOfHiddenPageWakeUpThrottling()
configureElectronNetworkCompatibility()
enableRendererHeapHeadroom()
maybeApplyGpuFallbackForThisLaunch()
@@ -1882,54 +1884,57 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro
if (!result.shouldEngageFallback) {
return
}
recordCrashBreadcrumb('gpu_fallback_engaged', {
reason,
exitCode,
crashesInWindow: result.crashesInWindow
})
const engagedAt = Date.now()
const window = mainWindow && !mainWindow.isDestroyed() ? mainWindow : undefined
let restartDecision: GpuFallbackRestartDecision
try {
restartDecision = await promptForGpuFallbackRestart(window)
} catch (error) {
console.warn('[gpu-fallback] failed to show restart prompt:', error)
return
}
const fallbackData = {
processReason: reason,
exitCode,
crashesInWindow: result.crashesInWindow
}
if (isQuitting) {
return
}
if (restartDecision !== 'restart') {
recordDurableCrashBreadcrumb('gpu_fallback_restart_deferred', fallbackData)
return
}
const environment = getWindowsGpuFallbackEnvironment()
if (!environment) {
return
}
try {
writeGpuFallbackMarker(
app.getPath('userData'),
{
engagedAt,
crashesInWindow: result.crashesInWindow
const userDataPath = app.getPath('userData')
await engageGpuFallbackAfterCrashBurst(
{ reason, exitCode, crashesInWindow: result.crashesInWindow, engagedAt: Date.now() },
{
isQuitting: () => isQuitting,
onEngaged: (engagement) =>
recordCrashBreadcrumb('gpu_fallback_engaged', {
reason: engagement.reason,
exitCode: engagement.exitCode,
crashesInWindow: engagement.crashesInWindow
}),
persistMarker: (engagement) => {
const environment = getWindowsGpuFallbackEnvironment()
if (!environment) {
return false
}
try {
writeGpuFallbackMarker(
userDataPath,
{ engagedAt: engagement.engagedAt, crashesInWindow: engagement.crashesInWindow },
environment
)
return true
} catch (error) {
console.warn('[gpu-fallback] failed to persist marker:', error)
return false
}
},
environment
)
} catch (error) {
console.warn('[gpu-fallback] failed to persist marker:', error)
return
}
isQuitting = true
relaunchApp('gpu-fallback', fallbackData)
// Why: app.exit(0) skips before-quit, so destroy the Windows tray manually to avoid a stale icon.
destroySystemTray()
app.exit(0)
clearMarker: () => clearGpuFallbackMarker(userDataPath),
promptForRestart: () =>
promptForGpuFallbackRestart(
mainWindow && !mainWindow.isDestroyed() ? mainWindow : undefined
),
onPromptFailed: (error) =>
console.warn('[gpu-fallback] failed to show restart prompt:', error),
onRestartDeferred: () =>
recordDurableCrashBreadcrumb('gpu_fallback_restart_deferred', fallbackData),
restartIntoSafeGraphics: () => {
isQuitting = true
relaunchApp('gpu-fallback', fallbackData)
// Why: app.exit(0) skips before-quit, so destroy the Windows tray manually to avoid a stale icon.
destroySystemTray()
app.exit(0)
}
}
)
}
function recordProcessGoneCrash(
+75 -15
View File
@@ -1,4 +1,4 @@
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { homedir, tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
@@ -508,20 +508,6 @@ describe('enableMainProcessGpuFeatures', () => {
expect(app.commandLine.appendSwitch).not.toHaveBeenCalledWith('enable-unsafe-webgpu')
})
it('opts hidden pages out of intensive wake-up throttling', async () => {
const { app } = await import('electron')
const { enableMainProcessGpuFeatures } = await import('./configure-process')
delete process.env.ORCA_E2E_USER_DATA_DIR
vi.mocked(app.commandLine.appendSwitch).mockClear()
enableMainProcessGpuFeatures()
expect(app.commandLine.appendSwitch).toHaveBeenCalledWith(
'disable-features',
'IntensiveWakeUpThrottling'
)
})
it('raises the WebGL context budget above the 16-context Blink default', async () => {
const { app } = await import('electron')
const { enableMainProcessGpuFeatures } = await import('./configure-process')
@@ -754,3 +740,77 @@ describe('enableMainProcessGpuFeatures', () => {
expect(app.commandLine.appendSwitch).toHaveBeenCalledWith('enable-features', 'ExistingFeature')
})
})
describe('safe graphics mode startup switches', () => {
const originalE2EUserDataDir = process.env.ORCA_E2E_USER_DATA_DIR
afterEach(() => {
if (originalE2EUserDataDir === undefined) {
delete process.env.ORCA_E2E_USER_DATA_DIR
} else {
process.env.ORCA_E2E_USER_DATA_DIR = originalE2EUserDataDir
}
})
function disabledFeaturesFrom(appendSwitch: ReturnType<typeof vi.fn>): string[] {
return appendSwitch.mock.calls
.filter(([name]) => name === 'disable-features')
.flatMap(([, value]) => String(value ?? '').split(','))
.filter(Boolean)
}
it('opts hidden pages out of intensive wake-up throttling', async () => {
const { app } = await import('electron')
const { optOutOfHiddenPageWakeUpThrottling } = await import('./configure-process')
vi.mocked(app.commandLine.appendSwitch).mockClear()
optOutOfHiddenPageWakeUpThrottling()
expect(app.commandLine.appendSwitch).toHaveBeenCalledWith(
'disable-features',
'IntensiveWakeUpThrottling'
)
})
// Why: the defect was the call site, not the switch — a win32 safe-graphics launch runs
// `if (!gpuFallbackActiveThisLaunch) enableMainProcessGpuFeatures()` and skips everything
// parked inside it, so only an unconditional call site reaches the users a GPU crash already hit.
it('calls the throttling opt-out outside the GPU-fallback gate in index.ts', () => {
const mainSource = readFileSync(join(__dirname, '..', 'index.ts'), 'utf8')
const gateStart = mainSource.indexOf('if (!gpuFallbackActiveThisLaunch) {')
expect(gateStart).toBeGreaterThanOrEqual(0)
const gateEnd = mainSource.indexOf('\n }', gateStart)
expect(gateEnd).toBeGreaterThan(gateStart)
expect(mainSource.match(/\boptOutOfHiddenPageWakeUpThrottling\(\)/g)).toHaveLength(1)
expect(mainSource.slice(gateStart, gateEnd)).not.toContain('optOutOfHiddenPageWakeUpThrottling')
})
// Why: Chromium consumes the command line at ready, so this must stay in the pre-ready
// top-level block and never move into the whenReady callback, where appendSwitch is a silent
// no-op — the same invisible failure as parking it behind the GPU gate.
it('appends the throttling opt-out before app ready in index.ts', () => {
const mainSource = readFileSync(join(__dirname, '..', 'index.ts'), 'utf8')
const readyStart = mainSource.indexOf('void app.whenReady()')
expect(readyStart).toBeGreaterThan(0)
const callIndex = mainSource.indexOf('optOutOfHiddenPageWakeUpThrottling()')
expect(callIndex).toBeGreaterThan(0)
expect(callIndex).toBeLessThan(readyStart)
})
// Why: Chromium enables IntensiveWakeUpThrottling on every desktop platform, so the opt-out
// must never become reachable only through the GPU-feature path again.
it('does not couple the throttling opt-out to GPU feature setup', async () => {
const { app } = await import('electron')
const { enableMainProcessGpuFeatures } = await import('./configure-process')
delete process.env.ORCA_E2E_USER_DATA_DIR
vi.mocked(app.commandLine.appendSwitch).mockClear()
enableMainProcessGpuFeatures()
expect(disabledFeaturesFrom(vi.mocked(app.commandLine.appendSwitch))).not.toContain(
'IntensiveWakeUpThrottling'
)
})
})
+6 -4
View File
@@ -71,6 +71,12 @@ export function disableUnsupportedChromiumFeatures(): void {
appendDisabledChromiumFeatures([...DISABLED_CHROMIUM_FEATURES])
}
// Why: Chromium clamps hidden-page timers to 1/min after 5min on every desktop platform,
// delaying agent-done/bell notifications ~60s. Call site is unconditional (see index.ts).
export function optOutOfHiddenPageWakeUpThrottling(): void {
appendDisabledChromiumFeatures(['IntensiveWakeUpThrottling'])
}
function appendDisabledChromiumFeatures(features: string[]): void {
const existingFeatures = app.commandLine
.getSwitchValue('disable-features')
@@ -314,8 +320,4 @@ export function enableMainProcessGpuFeatures(): void {
if (features) {
app.commandLine.appendSwitch('enable-features', features)
}
// Why: IntensiveWakeUpThrottling clamps hidden-page timers to 1/min after 5min, delaying agent-done/bell notifications ~60s.
// This opt-out is skipped under GPU fallback (win32-only today); if throttling ever reaches Windows it must move out of this path.
appendDisabledChromiumFeatures(['IntensiveWakeUpThrottling'])
}