fix(gpu): engage GPU software-rendering fallback across launches

The GPU crash tracker lived only in memory, so a GPU fault that took the
whole app down reset it on every relaunch and the 3-in-30s burst rule could
never fire. Persist a build-scoped ring of wall-clock crashes next to the
fallback marker and engage when separate launches crash inside the window or
when consecutive recent launches each die on the GPU.

Both cross-launch rules count launches, never raw crashes, so recovered churn
inside one session cannot be cashed in by a later benign crash, and the streak
is bounded to 72h so launches weeks apart are not treated as one broken driver.
A renderer death trailing a suppressed GPU crash is named gpu_cascade in the
crash trail but is not recorded as a second crash, keeping the burst threshold
at 3 real faults. Declining the restart starts a 24h cooldown so "Keep Running"
is not re-asked on the first GPU crash of every later launch.

Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
Neil
2026-08-03 23:49:14 -07:00
co-authored by Orca
parent dbbeca0916
commit aa1c91d511
9 changed files with 961 additions and 8 deletions
@@ -0,0 +1,62 @@
import { describe, expect, it } from 'vitest'
import { GpuCrashCascadeAttributor } from './gpu-crash-cascade-attribution'
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 14)
describe('GpuCrashCascadeAttributor', () => {
it('claims a renderer crash that trails a suppressed GPU crash with the same exit code', () => {
const attributor = new GpuCrashCascadeAttributor()
attributor.noteSuppressedGpuCrash({ at: startedAt, exitCode: 3_000 })
expect(
attributor.claimRendererCascade({ reason: 'crashed', exitCode: 3_000, at: startedAt + 400 })
).toBe(true)
})
it('claims at most one cascade per GPU crash', () => {
const attributor = new GpuCrashCascadeAttributor()
attributor.noteSuppressedGpuCrash({ at: startedAt, exitCode: null })
expect(
attributor.claimRendererCascade({ reason: 'crashed', exitCode: null, at: startedAt + 100 })
).toBe(true)
expect(
attributor.claimRendererCascade({ reason: 'crashed', exitCode: null, at: startedAt + 200 })
).toBe(false)
})
it('ignores unrelated renderer deaths', () => {
const attributor = new GpuCrashCascadeAttributor()
attributor.noteSuppressedGpuCrash({ at: startedAt, exitCode: 3_000 })
// Different exit code: a separate fault, not the GPU cascade.
expect(
attributor.claimRendererCascade({ reason: 'crashed', exitCode: 5, at: startedAt + 100 })
).toBe(false)
// OOM/killed renderers are their own failure mode.
expect(
attributor.claimRendererCascade({
reason: 'oom',
exitCode: 3_000,
at: startedAt + 100
})
).toBe(false)
// Too late to be the same fault.
expect(
attributor.claimRendererCascade({ reason: 'crashed', exitCode: 3_000, at: startedAt + 5_000 })
).toBe(false)
})
it('never claims without an armed GPU crash', () => {
const attributor = new GpuCrashCascadeAttributor()
expect(
attributor.claimRendererCascade({ reason: 'crashed', exitCode: 3_000, at: startedAt })
).toBe(false)
})
it('ignores a renderer crash stamped before the GPU crash', () => {
const attributor = new GpuCrashCascadeAttributor()
attributor.noteSuppressedGpuCrash({ at: startedAt, exitCode: 3_000 })
// Why pinned: a backwards clock step must not open an unbounded match window.
expect(
attributor.claimRendererCascade({ reason: 'crashed', exitCode: 3_000, at: startedAt - 10_000 })
).toBe(false)
})
})
@@ -0,0 +1,49 @@
/** Renderer deaths this close behind a GPU child crash are the same fault. */
export const DEFAULT_GPU_CASCADE_WINDOW_MS = 2_000
export type GpuCascadeCrash = { at: number; exitCode: number | null }
/**
* Links a renderer crash back to the GPU child crash that caused it.
*
* Why: a GPU fault usually surfaces twice — the suppressed child-process-gone
* and, moments later, the renderer going down with the same exit code. Counting
* the tail as its own GPU crash is what lets a "one crash per launch" loop reach
* the fallback threshold, and the paired event names the real cause in the
* crash trail instead of leaving an unexplained renderer death.
*
* TODO: fold into the shared recent-process-gone ring in process-gone-recorder
* once it lands.
*/
export class GpuCrashCascadeAttributor {
private readonly windowMs: number
private pending: GpuCascadeCrash | null = null
constructor(options: { windowMs?: number } = {}) {
this.windowMs = options.windowMs ?? DEFAULT_GPU_CASCADE_WINDOW_MS
}
/** Arms attribution after a GPU child crash that did not engage fallback. */
noteSuppressedGpuCrash(crash: GpuCascadeCrash): void {
this.pending = crash
}
/**
* True when this renderer death is the tail of the armed GPU crash. Claiming
* disarms, so one GPU crash can only ever produce one cascade event.
*/
claimRendererCascade(crash: { reason: string; exitCode: number | null; at: number }): boolean {
const pending = this.pending
if (
!pending ||
crash.reason !== 'crashed' ||
crash.exitCode !== pending.exitCode ||
crash.at < pending.at ||
crash.at - pending.at > this.windowMs
) {
return false
}
this.pending = null
return true
}
}
@@ -0,0 +1,250 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import { openGpuCrashHistoryLaunch } from '../startup/gpu-crash-history-store'
import { GpuCrashFallbackTracker } from './gpu-crash-fallback-decision'
import type { GpuCrashHistoryEntry, GpuCrashHistoryLaunch } from './gpu-crash-fallback-decision'
/** In-memory stand-in for the on-disk ring, shared across simulated relaunches. */
function createHistoryFake(): {
openLaunch: () => GpuCrashHistoryLaunch
entries: GpuCrashHistoryEntry[]
} {
const entries: GpuCrashHistoryEntry[] = []
let launchSeq = 0
let declinedAt: number | null = null
return {
entries,
openLaunch: () => {
launchSeq += 1
const currentLaunchSeq = launchSeq
return {
launchSeq: currentLaunchSeq,
get declinedAt() {
return declinedAt
},
append: (crash) => {
entries.push({ ...crash, launchSeq: currentLaunchSeq })
return entries
},
noteRestartDeclined: (at) => {
declinedAt = at
}
}
}
}
}
describe('GPU fallback across relaunches', () => {
it('engages after repeated launches that each die on the first GPU crash', () => {
// F0BNM0R87SL: the GPU crash takes the app down, so every launch starts with
// an empty tracker and the in-launch burst rule can never reach 3.
const history = createHistoryFake()
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 0)
const results: boolean[] = []
for (let launch = 0; launch < 5; launch += 1) {
const tracker = new GpuCrashFallbackTracker({
windowMs: 30_000,
threshold: 3,
history: history.openLaunch()
})
const engaged = tracker.recordGpuCrash(3_000, {
at: startedAt + launch * 90_000,
exitCode: 3_000
}).shouldEngageFallback
results.push(engaged)
if (engaged) {
// The app relaunches into software rendering; later launches never happen.
break
}
}
expect(results).toEqual([false, false, true])
expect(history.entries.map((entry) => entry.launchSeq)).toEqual([1, 2, 3])
expect(history.entries.every((entry) => entry.ts >= startedAt)).toBe(true)
})
it('engages on crashes clustered in wall-clock time even when launches are not consecutive', () => {
const history = createHistoryFake()
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 0)
// Launches 1 and 3 crash, 2 does not: no streak, but three crashes in 6 min.
const crashingLaunch = (at: number): boolean => {
const tracker = new GpuCrashFallbackTracker({
windowMs: 30_000,
threshold: 3,
history: history.openLaunch()
})
return tracker.recordGpuCrash(1_000, { at, exitCode: null }).shouldEngageFallback
}
expect(crashingLaunch(startedAt)).toBe(false)
history.openLaunch()
expect(crashingLaunch(startedAt + 120_000)).toBe(false)
history.openLaunch()
expect(crashingLaunch(startedAt + 360_000)).toBe(true)
})
it('ignores crashes that aged out of the cross-launch window', () => {
const history = createHistoryFake()
const startedAt = Date.UTC(2026, 7, 3, 10, 0, 0)
const crashingLaunch = (at: number): boolean => {
const tracker = new GpuCrashFallbackTracker({
windowMs: 30_000,
threshold: 3,
crossLaunchWindowMs: 30 * 60_000,
history: history.openLaunch()
})
return tracker.recordGpuCrash(1_000, { at, exitCode: null }).shouldEngageFallback
}
// One crash a day apart is a flaky app-start race, not a broken driver — and
// the crash-free launches in between keep the streak from ever forming.
expect(crashingLaunch(startedAt)).toBe(false)
history.openLaunch()
expect(crashingLaunch(startedAt + 86_400_000)).toBe(false)
history.openLaunch()
expect(crashingLaunch(startedAt + 2 * 86_400_000)).toBe(false)
})
it('leaves the closest real-world non-burst alone once crashes are persisted', () => {
const history = createHistoryFake()
const tracker = new GpuCrashFallbackTracker({
windowMs: 30_000,
threshold: 3,
history: history.openLaunch()
})
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 0)
// Field telemetry's tightest 4-crash launch that is *not* a broken driver.
// The in-launch rule already ruled it out; the persisted rules must not
// re-litigate a single launch behind its back.
for (const at of [0, 29_531, 55_136, 74_178]) {
expect(
tracker.recordGpuCrash(at, { at: startedAt + at, exitCode: 3_000 }).shouldEngageFallback
).toBe(false)
}
})
it('does not let one recovered launch bankroll a later benign crash', () => {
const userDataPath = mkdtempSync(join(tmpdir(), 'gpu-cross-launch-churn-'))
const environment = {
appVersion: '1.4.167',
electronVersion: '38.2.0',
platform: 'win32' as const
}
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 0)
try {
// F0BGRN5912M through the real ring: 4 recovered crashes in one launch that
// the burst rule spares. Counting raw crashes would let a single ordinary
// crash in the next launch cash them in.
const first = new GpuCrashFallbackTracker({
windowMs: 30_000,
threshold: 3,
history: openGpuCrashHistoryLaunch(userDataPath, environment)
})
for (const at of [0, 29_531, 55_136, 74_178]) {
expect(
first.recordGpuCrash(at, { at: startedAt + at, exitCode: 34 }).shouldEngageFallback
).toBe(false)
}
const second = new GpuCrashFallbackTracker({
windowMs: 30_000,
threshold: 3,
history: openGpuCrashHistoryLaunch(userDataPath, environment)
})
expect(
second.recordGpuCrash(1_000, { at: startedAt + 374_178, exitCode: 34 }).shouldEngageFallback
).toBe(false)
} finally {
rmSync(userDataPath, { recursive: true, force: true })
}
})
it('ignores a crashing-launch streak spread over weeks', () => {
const history = createHistoryFake()
const day = 86_400_000
const startedAt = Date.UTC(2026, 7, 3, 10, 0, 0)
const crashingLaunch = (at: number): boolean =>
new GpuCrashFallbackTracker({
windowMs: 30_000,
threshold: 3,
history: history.openLaunch()
}).recordGpuCrash(1_000, { at, exitCode: 34 }).shouldEngageFallback
// One lone GPU crash per session is ordinary Chromium churn (13 of the 14
// GPU-crash reports in the corpus). Only a *recent* run of them is a driver.
expect(crashingLaunch(startedAt)).toBe(false)
expect(crashingLaunch(startedAt + 7 * day)).toBe(false)
expect(crashingLaunch(startedAt + 14 * day)).toBe(false)
})
it('still engages for a once-a-day user whose every launch dies on the GPU', () => {
const history = createHistoryFake()
const day = 86_400_000
const startedAt = Date.UTC(2026, 7, 3, 10, 0, 0)
const crashingLaunch = (at: number): boolean =>
new GpuCrashFallbackTracker({
windowMs: 30_000,
threshold: 3,
history: history.openLaunch()
}).recordGpuCrash(1_000, { at, exitCode: 34 }).shouldEngageFallback
expect(crashingLaunch(startedAt)).toBe(false)
expect(crashingLaunch(startedAt + day)).toBe(false)
expect(crashingLaunch(startedAt + 2 * day)).toBe(true)
})
it('stops re-prompting after the user keeps running, then asks again a day later', () => {
const history = createHistoryFake()
const startedAt = Date.UTC(2026, 7, 3, 10, 0, 0)
const crashingLaunch = (at: number): boolean => {
const launch = history.openLaunch()
const engaged = new GpuCrashFallbackTracker({
windowMs: 30_000,
threshold: 3,
history: launch
}).recordGpuCrash(1_000, { at, exitCode: 34 }).shouldEngageFallback
if (engaged) {
// The user picks "Keep Running" every time.
launch.noteRestartDeclined(at)
}
return engaged
}
expect(crashingLaunch(startedAt)).toBe(false)
expect(crashingLaunch(startedAt + 600_000)).toBe(false)
expect(crashingLaunch(startedAt + 1_200_000)).toBe(true)
const declinedAt = startedAt + 1_200_000
expect(crashingLaunch(declinedAt + 600_000)).toBe(false)
expect(crashingLaunch(declinedAt + 12 * 3_600_000)).toBe(false)
// A day on, the driver is still broken: it is fair to ask once more.
expect(crashingLaunch(declinedAt + 25 * 3_600_000)).toBe(true)
})
it('survives real relaunches through the on-disk ring', () => {
const userDataPath = mkdtempSync(join(tmpdir(), 'gpu-cross-launch-'))
const environment = {
appVersion: '1.4.167',
electronVersion: '38.2.0',
platform: 'win32' as const
}
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 14)
try {
const relaunch = (at: number): boolean =>
new GpuCrashFallbackTracker({
windowMs: 30_000,
threshold: 3,
history: openGpuCrashHistoryLaunch(userDataPath, environment)
}).recordGpuCrash(2_000, { at, exitCode: 3_000 }).shouldEngageFallback
expect(relaunch(startedAt)).toBe(false)
expect(relaunch(startedAt + 88_000)).toBe(false)
expect(relaunch(startedAt + 190_000)).toBe(true)
} finally {
rmSync(userDataPath, { recursive: true, force: true })
}
})
it('keeps the in-launch burst rule intact when nothing is persisted', () => {
const tracker = new GpuCrashFallbackTracker({ windowMs: 30_000, threshold: 3 })
expect(tracker.recordGpuCrash(500).shouldEngageFallback).toBe(false)
expect(tracker.recordGpuCrash(8_000).shouldEngageFallback).toBe(false)
expect(tracker.recordGpuCrash(16_000)).toEqual({
shouldEngageFallback: true,
crashesInWindow: 3
})
})
})
@@ -0,0 +1,77 @@
import { describe, expect, it } from 'vitest'
import { GpuCrashFallbackCoordinator } from './gpu-crash-fallback-coordinator'
import { GpuCrashFallbackTracker } from './gpu-crash-fallback-decision'
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 14)
/** One physical GPU fault as the app sees it: child death, then the renderer. */
function feedGpuFault(
coordinator: GpuCrashFallbackCoordinator,
msSinceLaunch: number,
options: { rendererDelayMs?: number; exitCode?: number | null } = {}
): { engaged: boolean; crashesInWindow: number; cascade: boolean } {
const exitCode = options.exitCode ?? 34
const at = startedAt + msSinceLaunch
const verdict = coordinator.recordGpuChildCrash({ msSinceLaunch, at, exitCode })
const cascade = coordinator.claimRendererCascade({
reason: 'crashed',
exitCode,
at: at + (options.rendererDelayMs ?? 83)
})
return {
engaged: verdict.shouldEngageFallback,
crashesInWindow: verdict.crashesInWindow,
cascade
}
}
function createCoordinator(): GpuCrashFallbackCoordinator {
return new GpuCrashFallbackCoordinator({
tracker: new GpuCrashFallbackTracker({ windowMs: 30_000, threshold: 3 })
})
}
describe('GpuCrashFallbackCoordinator', () => {
it('counts one GPU fault once even when the renderer dies behind it', () => {
// F0BNM0R87SL shape: the renderer goes down ~83ms after the GPU child with
// the same exit code. Counting that tail too would halve the tuned threshold.
const coordinator = createCoordinator()
expect(feedGpuFault(coordinator, 1_000)).toEqual({
engaged: false,
crashesInWindow: 1,
cascade: true
})
expect(feedGpuFault(coordinator, 21_000)).toEqual({
engaged: false,
crashesInWindow: 2,
cascade: true
})
})
it('still engages on the third real fault inside the window', () => {
const coordinator = createCoordinator()
feedGpuFault(coordinator, 0)
feedGpuFault(coordinator, 10_000)
expect(feedGpuFault(coordinator, 20_000)).toEqual({
engaged: true,
crashesInWindow: 3,
cascade: false
})
})
it('does not arm attribution for the crash that engaged fallback', () => {
const coordinator = createCoordinator()
feedGpuFault(coordinator, 0)
feedGpuFault(coordinator, 10_000)
// The engaging crash relaunches the app; its renderer tail is teardown noise.
expect(feedGpuFault(coordinator, 20_000).cascade).toBe(false)
})
it('leaves unrelated renderer deaths unclaimed', () => {
const coordinator = createCoordinator()
coordinator.recordGpuChildCrash({ msSinceLaunch: 1_000, at: startedAt, exitCode: 34 })
expect(
coordinator.claimRendererCascade({ reason: 'oom', exitCode: 34, at: startedAt + 100 })
).toBe(false)
})
})
@@ -0,0 +1,53 @@
import { GpuCrashCascadeAttributor } from './gpu-crash-cascade-attribution'
import type { GpuCrashFallbackTracker } from './gpu-crash-fallback-decision'
export type GpuCrashFallbackVerdict = {
shouldEngageFallback: boolean
crashesInWindow: number
}
/**
* Routes GPU faults into the fallback tracker exactly once each.
*
* Why: one physical GPU fault surfaces twice — the suppressed GPU child death
* and, ~100ms later, the renderer going down with the same exit code. The tail
* names the cause in the crash trail, but recording it as a second crash would
* halve the tuned burst threshold (3 in 30s would fire after 2 real faults), so
* it is attribution only.
*/
export class GpuCrashFallbackCoordinator {
private readonly tracker: GpuCrashFallbackTracker
private readonly attributor: GpuCrashCascadeAttributor
constructor(options: {
tracker: GpuCrashFallbackTracker
attributor?: GpuCrashCascadeAttributor
}) {
this.tracker = options.tracker
this.attributor = options.attributor ?? new GpuCrashCascadeAttributor()
}
/** Counts a GPU child crash and arms cascade attribution for its renderer tail. */
recordGpuChildCrash(crash: {
msSinceLaunch: number
at: number
exitCode: number | null
}): GpuCrashFallbackVerdict {
const verdict = this.tracker.recordGpuCrash(crash.msSinceLaunch, {
at: crash.at,
exitCode: crash.exitCode
})
if (!verdict.shouldEngageFallback) {
this.attributor.noteSuppressedGpuCrash({ at: crash.at, exitCode: crash.exitCode })
}
return verdict
}
/**
* True when this renderer death is the tail of a GPU fault already counted by
* `recordGpuChildCrash` — for the crash trail, never for the fallback rules.
*/
claimRendererCascade(crash: { reason: string; exitCode: number | null; at: number }): boolean {
return this.attributor.claimRendererCascade(crash)
}
}
@@ -1,8 +1,39 @@
export type GpuCrashHistoryEntry = {
/** Wall clock (Date.now); performance.now() restarts at 0 on every relaunch. */
ts: number
exitCode: number | null
/** Launch counter that produced the crash, incremented once per launch. */
launchSeq: number
}
/** Per-launch handle on the persisted crash ring (startup/gpu-crash-history-store). */
export type GpuCrashHistoryLaunch = {
readonly launchSeq: number
/** Wall clock of the last declined fallback restart on this build, if any. */
readonly declinedAt: number | null
/** Appends this launch's crash and returns the retained ring, oldest first. */
append(crash: { ts: number; exitCode: number | null }): readonly GpuCrashHistoryEntry[]
/** Starts the re-prompt cooldown after the user chose to keep running. */
noteRestartDeclined(at: number): void
}
export type GpuCrashFallbackOptions = {
/** Rolling span over which clustered GPU crashes indicate a broken driver. */
windowMs: number
/** GPU child crashes within the window that trigger software-rendering fallback. */
threshold: number
/** Persisted, build-scoped crash ring; omitted the tracker is in-launch only. */
history?: GpuCrashHistoryLaunch | null
/** Wall-clock span over which crashes from earlier launches still count. */
crossLaunchWindowMs?: number
/** Distinct crashing launches inside `crossLaunchWindowMs` that trigger fallback. */
crossLaunchThreshold?: number
/** Back-to-back launches that each saw a GPU crash and trigger fallback. */
crashingLaunchStreak?: number
/** Wall-clock span the whole streak must fit inside to still describe this machine. */
crashingLaunchStreakWindowMs?: number
/** Quiet period after the user declines the restart before prompting again. */
declineCooldownMs?: number
}
const GPU_FALLBACK_CRASH_REASONS = new Set(['abnormal-exit', 'crashed', 'launch-failed'])
@@ -15,6 +46,76 @@ const GPU_FALLBACK_CRASH_REASONS = new Set(['abnormal-exit', 'crashed', 'launch-
export const DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS = 30_000
export const DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD = 3
// Why: the in-launch burst rule can never fire when the GPU fault takes the whole
// app down (F0BNM0R87SL crashes once per launch, forever), so the same evidence is
// re-evaluated across launches from the persisted ring.
export const DEFAULT_GPU_CROSS_LAUNCH_WINDOW_MS = 30 * 60_000
export const DEFAULT_GPU_CROSS_LAUNCH_THRESHOLD = 3
export const DEFAULT_GPU_CRASHING_LAUNCH_STREAK = 3
// Why: three crashing launches only describe *this* machine's driver if they
// happened around the same time; a user who opens Orca once a day fits, one who
// opens it weekly is looking at three unrelated bits of Chromium churn.
export const DEFAULT_GPU_CRASHING_LAUNCH_STREAK_WINDOW_MS = 72 * 60 * 60_000
// Why: "Keep Running" has to mean something — re-asking on the first GPU crash
// of every later launch is worse than the churn it warns about.
export const DEFAULT_GPU_FALLBACK_DECLINE_COOLDOWN_MS = 24 * 60 * 60_000
/**
* Cross-launch verdict over the persisted ring: either enough separate launches
* crashed inside the window, or enough back-to-back launches each died on the GPU.
*
* Both rules count launches, never raw crashes. Churn inside a single launch is
* the burst rule's call, and letting extra crashes from one launch fill a looser
* window would relaunch sessions the burst rule deliberately spares (field
* session F0BGRN5912M: 4 recovered crashes in 74s, plus one benign crash later).
*
* Both rules are also wall-clock bounded. `launchSeq` gaps break the streak, but
* a crashing launch is only evidence while it is recent — see the streak window.
*/
export function evaluateCrossLaunchGpuCrashes(
entries: readonly GpuCrashHistoryEntry[],
options: {
now: number
launchSeq: number
windowMs: number
threshold: number
crashingLaunchStreak: number
crashingLaunchStreakWindowMs: number
}
): {
crashesInWindow: number
crashingLaunchesInWindow: number
launchStreak: number
shouldEngage: boolean
} {
const cutoff = options.now - options.windowMs
const inWindow = entries.filter((entry) => entry.ts >= cutoff && entry.ts <= options.now)
const launchesInWindow = new Set(inWindow.map((entry) => entry.launchSeq))
const streakCutoff = options.now - options.crashingLaunchStreakWindowMs
const newestCrashPerLaunch = new Map<number, number>()
for (const entry of entries) {
newestCrashPerLaunch.set(
entry.launchSeq,
Math.max(entry.ts, newestCrashPerLaunch.get(entry.launchSeq) ?? entry.ts)
)
}
let launchStreak = 0
for (;;) {
const newestCrash = newestCrashPerLaunch.get(options.launchSeq - launchStreak)
if (newestCrash === undefined || newestCrash < streakCutoff) {
break
}
launchStreak += 1
}
return {
crashesInWindow: inWindow.length,
crashingLaunchesInWindow: launchesInWindow.size,
launchStreak,
shouldEngage:
launchesInWindow.size >= options.threshold || launchStreak >= options.crashingLaunchStreak
}
}
/**
* Tracks GPU child-process crashes and decides when to fall back to software
* rendering on the next launch. Pure and deterministic: callers pass `now`
@@ -32,11 +133,24 @@ export class GpuCrashFallbackTracker {
private readonly threshold: number
// Newest-last crash times (ms since launch), pruned to the rolling window.
private readonly recentCrashes: number[] = []
private readonly history: GpuCrashHistoryLaunch | null
private readonly crossLaunchWindowMs: number
private readonly crossLaunchThreshold: number
private readonly crashingLaunchStreak: number
private readonly crashingLaunchStreakWindowMs: number
private readonly declineCooldownMs: number
private engaged = false
constructor(options: GpuCrashFallbackOptions) {
this.windowMs = options.windowMs
this.threshold = options.threshold
this.history = options.history ?? null
this.crossLaunchWindowMs = options.crossLaunchWindowMs ?? DEFAULT_GPU_CROSS_LAUNCH_WINDOW_MS
this.crossLaunchThreshold = options.crossLaunchThreshold ?? DEFAULT_GPU_CROSS_LAUNCH_THRESHOLD
this.crashingLaunchStreak = options.crashingLaunchStreak ?? DEFAULT_GPU_CRASHING_LAUNCH_STREAK
this.crashingLaunchStreakWindowMs =
options.crashingLaunchStreakWindowMs ?? DEFAULT_GPU_CRASHING_LAUNCH_STREAK_WINDOW_MS
this.declineCooldownMs = options.declineCooldownMs ?? DEFAULT_GPU_FALLBACK_DECLINE_COOLDOWN_MS
}
/**
@@ -45,7 +159,10 @@ export class GpuCrashFallbackTracker {
* Returns false after fallback already engaged, so the caller relaunches at
* most once.
*/
recordGpuCrash(msSinceLaunch: number): {
recordGpuCrash(
msSinceLaunch: number,
crash: { at?: number; exitCode?: number | null } = {}
): {
shouldEngageFallback: boolean
crashesInWindow: number
} {
@@ -62,10 +179,30 @@ export class GpuCrashFallbackTracker {
stale += 1
}
this.recentCrashes.splice(0, stale)
const wallClock = crash.at ?? Date.now()
// Evidence keeps accruing through the cooldown so the next prompt sees the truth.
const persisted = this.history?.append({ ts: wallClock, exitCode: crash.exitCode ?? null })
if (this.isWithinDeclineCooldown(wallClock)) {
return { shouldEngageFallback: false, crashesInWindow: this.recentCrashes.length }
}
if (this.recentCrashes.length >= this.threshold) {
this.engaged = true
return { shouldEngageFallback: true, crashesInWindow: this.recentCrashes.length }
}
if (persisted && this.history) {
const crossLaunch = evaluateCrossLaunchGpuCrashes(persisted, {
now: wallClock,
launchSeq: this.history.launchSeq,
windowMs: this.crossLaunchWindowMs,
threshold: this.crossLaunchThreshold,
crashingLaunchStreak: this.crashingLaunchStreak,
crashingLaunchStreakWindowMs: this.crashingLaunchStreakWindowMs
})
if (crossLaunch.shouldEngage) {
this.engaged = true
return { shouldEngageFallback: true, crashesInWindow: crossLaunch.crashesInWindow }
}
}
return { shouldEngageFallback: false, crashesInWindow: this.recentCrashes.length }
}
@@ -73,6 +210,13 @@ export class GpuCrashFallbackTracker {
return this.engaged
}
// Why |Δ|: a clock stepped backwards past the decline must not pin the prompt
// off forever — asking once too often beats never asking again.
private isWithinDeclineCooldown(wallClock: number): boolean {
const declinedAt = this.history?.declinedAt ?? null
return declinedAt !== null && Math.abs(wallClock - declinedAt) < this.declineCooldownMs
}
/** Crash times currently inside the window. Exposed to assert the pruning invariant. */
windowSnapshot(): readonly number[] {
return [...this.recentCrashes]
+45 -7
View File
@@ -135,12 +135,15 @@ import {
type WindowsGpuFallbackEnvironment
} from './startup/gpu-fallback-marker'
import { applyGpuFallbackCommandLineSwitches } from './startup/gpu-fallback-switches'
import { clearGpuCrashHistory, openGpuCrashHistoryLaunch } from './startup/gpu-crash-history-store'
import {
DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD,
DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS,
GpuCrashFallbackTracker,
isGpuFallbackCrashCandidate
isGpuFallbackCrashCandidate,
type GpuCrashHistoryLaunch
} from './crash-reporting/gpu-crash-fallback-decision'
import { GpuCrashFallbackCoordinator } from './crash-reporting/gpu-crash-fallback-coordinator'
import {
promptForGpuFallbackRestart,
type GpuFallbackRestartDecision
@@ -362,10 +365,8 @@ let managedWslCliReconciliationReady: Promise<void> = Promise.resolve()
let managedWslCliStartupBarrierReady: Promise<void> = Promise.resolve()
// Why: the serve barrier fails open, so this state tells headless clients a WSL PTY launch may still race an un-migrated registration ('settled' = off-Windows no-op).
let managedWslCliReconciliationStatus: 'pending' | 'settled' | 'failed' = 'settled'
const gpuCrashFallbackTracker = new GpuCrashFallbackTracker({
windowMs: DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS,
threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD
})
let gpuCrashHistoryLaunch: GpuCrashHistoryLaunch | null = null
let gpuCrashFallbackCoordinator: GpuCrashFallbackCoordinator | null = null
let gpuFallbackActiveThisLaunch = false
let gpuFeatureStatus: Electron.GPUFeatureStatus | null = null
let localPtyStartupReady: Promise<void> = Promise.resolve()
@@ -1221,6 +1222,7 @@ function openMainWindow(): BrowserWindow {
},
webContentsId
)
attributeGpuCascadeCrash(details.reason, details.exitCode ?? null)
},
shouldRecoverRenderer: (details, webContentsId) =>
shouldRecoverRendererAfterProcessGone({
@@ -1556,7 +1558,11 @@ function maybeApplyGpuFallbackForThisLaunch(): void {
if (isServeMode || process.platform !== 'win32') {
return
}
const marker = readActiveGpuFallbackMarker(app.getPath('userData'), getGpuFallbackEnvironment())
const environment = getGpuFallbackEnvironment()
// Why: claim this launch's sequence number even when nothing crashes — a
// crash-free launch is the only evidence that a crashing streak ended.
gpuCrashHistoryLaunch = openGpuCrashHistoryLaunch(app.getPath('userData'), environment)
const marker = readActiveGpuFallbackMarker(app.getPath('userData'), environment)
if (!marker) {
return
}
@@ -1571,13 +1577,39 @@ function maybeApplyGpuFallbackForThisLaunch(): void {
})
}
function getGpuCrashFallbackCoordinator(): GpuCrashFallbackCoordinator {
gpuCrashFallbackCoordinator ??= new GpuCrashFallbackCoordinator({
tracker: new GpuCrashFallbackTracker({
windowMs: DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS,
threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD,
history: gpuCrashHistoryLaunch
})
})
return gpuCrashFallbackCoordinator
}
// Why: names the GPU fault behind an otherwise unexplained renderer death; the
// GPU child crash it trails was already counted, so nothing is recorded here.
function attributeGpuCascadeCrash(reason: string, exitCode: number | null): void {
if (
!getGpuCrashFallbackCoordinator().claimRendererCascade({ reason, exitCode, at: Date.now() })
) {
return
}
recordCrashBreadcrumb('gpu_cascade', { reason, exitCode })
}
// Why: a burst of GPU child crashes means HW acceleration is unusable — persist a build-scoped marker and offer software rendering.
async function handleGpuChildCrash(reason: string, exitCode: number | null): Promise<void> {
// Software rendering already active or shutting down: nothing more to do.
if (gpuFallbackActiveThisLaunch || isQuitting || isServeMode) {
return
}
const result = gpuCrashFallbackTracker.recordGpuCrash(performance.now())
const result = getGpuCrashFallbackCoordinator().recordGpuChildCrash({
msSinceLaunch: performance.now(),
at: Date.now(),
exitCode
})
if (!result.shouldEngageFallback) {
return
}
@@ -1605,6 +1637,9 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro
}
if (restartDecision !== 'restart') {
recordDurableCrashBreadcrumb('gpu_fallback_restart_deferred', fallbackData)
// Why: the persisted evidence still satisfies the rules, so without a
// cooldown "Keep Running" re-prompts on the first GPU crash of every launch.
gpuCrashHistoryLaunch?.noteRestartDeclined(Date.now())
return
}
const environment = getWindowsGpuFallbackEnvironment()
@@ -1624,6 +1659,9 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro
console.warn('[gpu-fallback] failed to persist marker:', error)
return
}
// Why: the marker now carries the verdict; stale crashes must not re-engage
// the moment the user turns hardware acceleration back on.
clearGpuCrashHistory(app.getPath('userData'))
isQuitting = true
relaunchApp('gpu-fallback', fallbackData)
// Why: app.exit(0) skips before-quit, so destroy the Windows tray manually to avoid a stale icon.
@@ -0,0 +1,129 @@
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
import {
clearGpuCrashHistory,
GPU_CRASH_HISTORY_FILE,
GPU_CRASH_HISTORY_MAX_ENTRIES,
openGpuCrashHistoryLaunch,
readGpuCrashHistory
} from './gpu-crash-history-store'
import type { GpuFallbackEnvironment } from './gpu-fallback-marker'
const environment: GpuFallbackEnvironment = {
appVersion: '1.4.167',
electronVersion: '38.2.0',
platform: 'win32'
}
let userDataPath: string
beforeEach(() => {
userDataPath = mkdtempSync(join(tmpdir(), 'gpu-crash-history-'))
})
afterEach(() => {
rmSync(userDataPath, { recursive: true, force: true })
})
describe('openGpuCrashHistoryLaunch', () => {
it('bumps the launch counter on every launch, crash or not', () => {
expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(1)
expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(2)
expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(3)
})
it('persists crashes across launches with wall-clock timestamps', () => {
const first = openGpuCrashHistoryLaunch(userDataPath, environment)
first.append({ ts: 1_770_000_000_000, exitCode: 3_000 })
const second = openGpuCrashHistoryLaunch(userDataPath, environment)
expect(second.append({ ts: 1_770_000_090_000, exitCode: 3_000 })).toEqual([
{ ts: 1_770_000_000_000, exitCode: 3_000, launchSeq: 1 },
{ ts: 1_770_000_090_000, exitCode: 3_000, launchSeq: 2 }
])
})
it('starts over after an app update so a fixed build gets a fresh hardware attempt', () => {
const before = openGpuCrashHistoryLaunch(userDataPath, environment)
before.append({ ts: 1_770_000_000_000, exitCode: 3_000 })
const upgraded = openGpuCrashHistoryLaunch(userDataPath, {
...environment,
appVersion: '1.4.168'
})
expect(upgraded.launchSeq).toBe(1)
expect(upgraded.append({ ts: 1_770_000_100_000, exitCode: null })).toEqual([
{ ts: 1_770_000_100_000, exitCode: null, launchSeq: 1 }
])
})
it('bounds the ring', () => {
const launch = openGpuCrashHistoryLaunch(userDataPath, environment)
let entries: readonly { ts: number }[] = []
for (let index = 0; index < GPU_CRASH_HISTORY_MAX_ENTRIES + 5; index += 1) {
entries = launch.append({ ts: 1_770_000_000_000 + index, exitCode: 3_000 })
}
expect(entries).toHaveLength(GPU_CRASH_HISTORY_MAX_ENTRIES)
expect(entries[0]?.ts).toBe(1_770_000_000_000 + 5)
})
it('survives a corrupt file', () => {
writeFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), '{not json')
const launch = openGpuCrashHistoryLaunch(userDataPath, environment)
expect(launch.launchSeq).toBe(1)
expect(readGpuCrashHistory(userDataPath, environment)?.entries).toEqual([])
})
it('drops malformed entries but keeps valid ones', () => {
writeFileSync(
join(userDataPath, GPU_CRASH_HISTORY_FILE),
JSON.stringify({
schemeVersion: 1,
appVersion: environment.appVersion,
electronVersion: environment.electronVersion,
platform: environment.platform,
launchSeq: 4,
entries: [
{ ts: 'nope', exitCode: 1, launchSeq: 1 },
{ ts: 1_770_000_000_000, exitCode: null, launchSeq: 4 }
]
})
)
expect(readGpuCrashHistory(userDataPath, environment)?.entries).toEqual([
{ ts: 1_770_000_000_000, exitCode: null, launchSeq: 4 }
])
expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(5)
})
it('remembers a declined restart across launches without losing crashes', () => {
const first = openGpuCrashHistoryLaunch(userDataPath, environment)
first.append({ ts: 1_770_000_000_000, exitCode: 3_000 })
first.noteRestartDeclined(1_770_000_000_500)
const second = openGpuCrashHistoryLaunch(userDataPath, environment)
expect(second.declinedAt).toBe(1_770_000_000_500)
expect(second.append({ ts: 1_770_000_090_000, exitCode: 3_000 })).toHaveLength(2)
expect(openGpuCrashHistoryLaunch(userDataPath, environment).declinedAt).toBe(1_770_000_000_500)
})
it('reports no decline for a fresh or upgraded build', () => {
expect(openGpuCrashHistoryLaunch(userDataPath, environment).declinedAt).toBeNull()
openGpuCrashHistoryLaunch(userDataPath, environment).noteRestartDeclined(1_770_000_000_500)
expect(
openGpuCrashHistoryLaunch(userDataPath, { ...environment, appVersion: '1.4.168' }).declinedAt
).toBeNull()
})
it('clears the ring when fallback engages', () => {
openGpuCrashHistoryLaunch(userDataPath, environment).append({ ts: 1, exitCode: 3_000 })
clearGpuCrashHistory(userDataPath)
expect(readGpuCrashHistory(userDataPath, environment)).toBeNull()
expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(1)
})
it('writes the ring where the marker lives', () => {
openGpuCrashHistoryLaunch(userDataPath, environment)
expect(
JSON.parse(readFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), 'utf-8')).launchSeq
).toBe(1)
})
})
+151
View File
@@ -0,0 +1,151 @@
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { dirname, join } from 'node:path'
import type {
GpuCrashHistoryEntry,
GpuCrashHistoryLaunch
} from '../crash-reporting/gpu-crash-fallback-decision'
import type { GpuFallbackEnvironment } from './gpu-fallback-marker'
/**
* Build-scoped ring of GPU child crashes, stored next to the GPU fallback marker.
*
* Why on disk: a GPU fault that kills the app leaves the in-memory tracker empty
* on the next launch, so a "crash once per launch, forever" loop never reaches
* the in-launch burst threshold. Timestamps are wall clock — performance.now()
* restarts at 0 every launch and cannot be compared across them.
*/
export const GPU_CRASH_HISTORY_FILE = 'gpu-crash-history.json'
export const GPU_CRASH_HISTORY_SCHEME_VERSION = 1
/** Deep enough for the streak + window rules, small enough to write synchronously. */
export const GPU_CRASH_HISTORY_MAX_ENTRIES = 16
type GpuCrashHistoryFile = {
schemeVersion: number
appVersion: string
electronVersion: string
platform: NodeJS.Platform
launchSeq: number
entries: GpuCrashHistoryEntry[]
/** Wall clock of the last "Keep Running" answer; null when never declined. */
declinedAt: number | null
}
function historyPath(userDataPath: string): string {
return join(userDataPath, GPU_CRASH_HISTORY_FILE)
}
function isValidEntry(value: unknown): value is GpuCrashHistoryEntry {
const entry = value as Partial<GpuCrashHistoryEntry> | null
return (
typeof entry === 'object' &&
entry !== null &&
typeof entry.ts === 'number' &&
Number.isFinite(entry.ts) &&
(entry.exitCode === null || typeof entry.exitCode === 'number') &&
typeof entry.launchSeq === 'number' &&
Number.isFinite(entry.launchSeq)
)
}
export function readGpuCrashHistory(
userDataPath: string,
environment: GpuFallbackEnvironment
): GpuCrashHistoryFile | null {
try {
const parsed = JSON.parse(readFileSync(historyPath(userDataPath), 'utf-8')) as Partial<
Record<keyof GpuCrashHistoryFile, unknown>
>
if (
parsed.schemeVersion !== GPU_CRASH_HISTORY_SCHEME_VERSION ||
parsed.appVersion !== environment.appVersion ||
parsed.electronVersion !== environment.electronVersion ||
parsed.platform !== environment.platform ||
typeof parsed.launchSeq !== 'number' ||
!Number.isFinite(parsed.launchSeq) ||
!Array.isArray(parsed.entries)
) {
// Why: a different build (or a corrupt file) gets a clean slate — the
// evidence describes a driver/binary pairing that no longer exists.
return null
}
return {
schemeVersion: GPU_CRASH_HISTORY_SCHEME_VERSION,
appVersion: environment.appVersion,
electronVersion: environment.electronVersion,
platform: environment.platform,
launchSeq: parsed.launchSeq,
entries: parsed.entries.filter(isValidEntry).slice(-GPU_CRASH_HISTORY_MAX_ENTRIES),
declinedAt:
typeof parsed.declinedAt === 'number' && Number.isFinite(parsed.declinedAt)
? parsed.declinedAt
: null
}
} catch {
// missing or unreadable means no history
}
return null
}
function writeGpuCrashHistory(userDataPath: string, file: GpuCrashHistoryFile): void {
try {
const target = historyPath(userDataPath)
mkdirSync(dirname(target), { recursive: true })
writeFileSync(target, JSON.stringify(file))
} catch {
// best effort: losing history only costs a slower fallback decision
}
}
export function clearGpuCrashHistory(userDataPath: string): void {
try {
rmSync(historyPath(userDataPath), { force: true })
} catch {
// best effort; the next launch revalidates the file anyway
}
}
/**
* Claims the next launch sequence number and hands back an append port.
*
* Called once per launch even when no GPU crash follows: a crash-free launch has
* to bump the counter, otherwise a gap in the sequence — the only evidence that
* the streak broke — would never be recorded.
*/
export function openGpuCrashHistoryLaunch(
userDataPath: string,
environment: GpuFallbackEnvironment
): GpuCrashHistoryLaunch {
const existing = readGpuCrashHistory(userDataPath, environment)
const launchSeq = (existing?.launchSeq ?? 0) + 1
let entries = existing?.entries ?? []
let declinedAt = existing?.declinedAt ?? null
const file: GpuCrashHistoryFile = {
schemeVersion: GPU_CRASH_HISTORY_SCHEME_VERSION,
appVersion: environment.appVersion,
electronVersion: environment.electronVersion,
platform: environment.platform,
launchSeq,
entries,
declinedAt
}
writeGpuCrashHistory(userDataPath, file)
return {
launchSeq,
// Getter, not a snapshot: a decline taken mid-launch must be visible at once.
get declinedAt() {
return declinedAt
},
append: (crash) => {
entries = [...entries, { ts: crash.ts, exitCode: crash.exitCode, launchSeq }].slice(
-GPU_CRASH_HISTORY_MAX_ENTRIES
)
writeGpuCrashHistory(userDataPath, { ...file, entries, declinedAt })
return entries
},
noteRestartDeclined: (at) => {
declinedAt = at
writeGpuCrashHistory(userDataPath, { ...file, entries, declinedAt })
}
}
}