mirror of
https://github.com/stablyai/orca.git
synced 2026-10-20 16:03:20 +00:00
fix(gpu): engage GPU software-rendering fallback across launches
The GPU crash tracker lived only in memory, so a GPU fault that took the whole app down reset it on every relaunch and the 3-in-30s burst rule could never fire. Persist a build-scoped ring of wall-clock crashes next to the fallback marker and engage when separate launches crash inside the window or when consecutive recent launches each die on the GPU. Both cross-launch rules count launches, never raw crashes, so recovered churn inside one session cannot be cashed in by a later benign crash, and the streak is bounded to 72h so launches weeks apart are not treated as one broken driver. A renderer death trailing a suppressed GPU crash is named gpu_cascade in the crash trail but is not recorded as a second crash, keeping the burst threshold at 3 real faults. Declining the restart starts a 24h cooldown so "Keep Running" is not re-asked on the first GPU crash of every later launch. Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { GpuCrashCascadeAttributor } from './gpu-crash-cascade-attribution'
|
||||
|
||||
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 14)
|
||||
|
||||
describe('GpuCrashCascadeAttributor', () => {
|
||||
it('claims a renderer crash that trails a suppressed GPU crash with the same exit code', () => {
|
||||
const attributor = new GpuCrashCascadeAttributor()
|
||||
attributor.noteSuppressedGpuCrash({ at: startedAt, exitCode: 3_000 })
|
||||
expect(
|
||||
attributor.claimRendererCascade({ reason: 'crashed', exitCode: 3_000, at: startedAt + 400 })
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('claims at most one cascade per GPU crash', () => {
|
||||
const attributor = new GpuCrashCascadeAttributor()
|
||||
attributor.noteSuppressedGpuCrash({ at: startedAt, exitCode: null })
|
||||
expect(
|
||||
attributor.claimRendererCascade({ reason: 'crashed', exitCode: null, at: startedAt + 100 })
|
||||
).toBe(true)
|
||||
expect(
|
||||
attributor.claimRendererCascade({ reason: 'crashed', exitCode: null, at: startedAt + 200 })
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('ignores unrelated renderer deaths', () => {
|
||||
const attributor = new GpuCrashCascadeAttributor()
|
||||
attributor.noteSuppressedGpuCrash({ at: startedAt, exitCode: 3_000 })
|
||||
// Different exit code: a separate fault, not the GPU cascade.
|
||||
expect(
|
||||
attributor.claimRendererCascade({ reason: 'crashed', exitCode: 5, at: startedAt + 100 })
|
||||
).toBe(false)
|
||||
// OOM/killed renderers are their own failure mode.
|
||||
expect(
|
||||
attributor.claimRendererCascade({
|
||||
reason: 'oom',
|
||||
exitCode: 3_000,
|
||||
at: startedAt + 100
|
||||
})
|
||||
).toBe(false)
|
||||
// Too late to be the same fault.
|
||||
expect(
|
||||
attributor.claimRendererCascade({ reason: 'crashed', exitCode: 3_000, at: startedAt + 5_000 })
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('never claims without an armed GPU crash', () => {
|
||||
const attributor = new GpuCrashCascadeAttributor()
|
||||
expect(
|
||||
attributor.claimRendererCascade({ reason: 'crashed', exitCode: 3_000, at: startedAt })
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('ignores a renderer crash stamped before the GPU crash', () => {
|
||||
const attributor = new GpuCrashCascadeAttributor()
|
||||
attributor.noteSuppressedGpuCrash({ at: startedAt, exitCode: 3_000 })
|
||||
// Why pinned: a backwards clock step must not open an unbounded match window.
|
||||
expect(
|
||||
attributor.claimRendererCascade({ reason: 'crashed', exitCode: 3_000, at: startedAt - 10_000 })
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,49 @@
|
||||
/** Renderer deaths this close behind a GPU child crash are the same fault. */
|
||||
export const DEFAULT_GPU_CASCADE_WINDOW_MS = 2_000
|
||||
|
||||
export type GpuCascadeCrash = { at: number; exitCode: number | null }
|
||||
|
||||
/**
|
||||
* Links a renderer crash back to the GPU child crash that caused it.
|
||||
*
|
||||
* Why: a GPU fault usually surfaces twice — the suppressed child-process-gone
|
||||
* and, moments later, the renderer going down with the same exit code. Counting
|
||||
* the tail as its own GPU crash is what lets a "one crash per launch" loop reach
|
||||
* the fallback threshold, and the paired event names the real cause in the
|
||||
* crash trail instead of leaving an unexplained renderer death.
|
||||
*
|
||||
* TODO: fold into the shared recent-process-gone ring in process-gone-recorder
|
||||
* once it lands.
|
||||
*/
|
||||
export class GpuCrashCascadeAttributor {
|
||||
private readonly windowMs: number
|
||||
private pending: GpuCascadeCrash | null = null
|
||||
|
||||
constructor(options: { windowMs?: number } = {}) {
|
||||
this.windowMs = options.windowMs ?? DEFAULT_GPU_CASCADE_WINDOW_MS
|
||||
}
|
||||
|
||||
/** Arms attribution after a GPU child crash that did not engage fallback. */
|
||||
noteSuppressedGpuCrash(crash: GpuCascadeCrash): void {
|
||||
this.pending = crash
|
||||
}
|
||||
|
||||
/**
|
||||
* True when this renderer death is the tail of the armed GPU crash. Claiming
|
||||
* disarms, so one GPU crash can only ever produce one cascade event.
|
||||
*/
|
||||
claimRendererCascade(crash: { reason: string; exitCode: number | null; at: number }): boolean {
|
||||
const pending = this.pending
|
||||
if (
|
||||
!pending ||
|
||||
crash.reason !== 'crashed' ||
|
||||
crash.exitCode !== pending.exitCode ||
|
||||
crash.at < pending.at ||
|
||||
crash.at - pending.at > this.windowMs
|
||||
) {
|
||||
return false
|
||||
}
|
||||
this.pending = null
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { openGpuCrashHistoryLaunch } from '../startup/gpu-crash-history-store'
|
||||
import { GpuCrashFallbackTracker } from './gpu-crash-fallback-decision'
|
||||
import type { GpuCrashHistoryEntry, GpuCrashHistoryLaunch } from './gpu-crash-fallback-decision'
|
||||
|
||||
/** In-memory stand-in for the on-disk ring, shared across simulated relaunches. */
|
||||
function createHistoryFake(): {
|
||||
openLaunch: () => GpuCrashHistoryLaunch
|
||||
entries: GpuCrashHistoryEntry[]
|
||||
} {
|
||||
const entries: GpuCrashHistoryEntry[] = []
|
||||
let launchSeq = 0
|
||||
let declinedAt: number | null = null
|
||||
return {
|
||||
entries,
|
||||
openLaunch: () => {
|
||||
launchSeq += 1
|
||||
const currentLaunchSeq = launchSeq
|
||||
return {
|
||||
launchSeq: currentLaunchSeq,
|
||||
get declinedAt() {
|
||||
return declinedAt
|
||||
},
|
||||
append: (crash) => {
|
||||
entries.push({ ...crash, launchSeq: currentLaunchSeq })
|
||||
return entries
|
||||
},
|
||||
noteRestartDeclined: (at) => {
|
||||
declinedAt = at
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('GPU fallback across relaunches', () => {
|
||||
it('engages after repeated launches that each die on the first GPU crash', () => {
|
||||
// F0BNM0R87SL: the GPU crash takes the app down, so every launch starts with
|
||||
// an empty tracker and the in-launch burst rule can never reach 3.
|
||||
const history = createHistoryFake()
|
||||
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 0)
|
||||
const results: boolean[] = []
|
||||
for (let launch = 0; launch < 5; launch += 1) {
|
||||
const tracker = new GpuCrashFallbackTracker({
|
||||
windowMs: 30_000,
|
||||
threshold: 3,
|
||||
history: history.openLaunch()
|
||||
})
|
||||
const engaged = tracker.recordGpuCrash(3_000, {
|
||||
at: startedAt + launch * 90_000,
|
||||
exitCode: 3_000
|
||||
}).shouldEngageFallback
|
||||
results.push(engaged)
|
||||
if (engaged) {
|
||||
// The app relaunches into software rendering; later launches never happen.
|
||||
break
|
||||
}
|
||||
}
|
||||
expect(results).toEqual([false, false, true])
|
||||
expect(history.entries.map((entry) => entry.launchSeq)).toEqual([1, 2, 3])
|
||||
expect(history.entries.every((entry) => entry.ts >= startedAt)).toBe(true)
|
||||
})
|
||||
|
||||
it('engages on crashes clustered in wall-clock time even when launches are not consecutive', () => {
|
||||
const history = createHistoryFake()
|
||||
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 0)
|
||||
// Launches 1 and 3 crash, 2 does not: no streak, but three crashes in 6 min.
|
||||
const crashingLaunch = (at: number): boolean => {
|
||||
const tracker = new GpuCrashFallbackTracker({
|
||||
windowMs: 30_000,
|
||||
threshold: 3,
|
||||
history: history.openLaunch()
|
||||
})
|
||||
return tracker.recordGpuCrash(1_000, { at, exitCode: null }).shouldEngageFallback
|
||||
}
|
||||
expect(crashingLaunch(startedAt)).toBe(false)
|
||||
history.openLaunch()
|
||||
expect(crashingLaunch(startedAt + 120_000)).toBe(false)
|
||||
history.openLaunch()
|
||||
expect(crashingLaunch(startedAt + 360_000)).toBe(true)
|
||||
})
|
||||
|
||||
it('ignores crashes that aged out of the cross-launch window', () => {
|
||||
const history = createHistoryFake()
|
||||
const startedAt = Date.UTC(2026, 7, 3, 10, 0, 0)
|
||||
const crashingLaunch = (at: number): boolean => {
|
||||
const tracker = new GpuCrashFallbackTracker({
|
||||
windowMs: 30_000,
|
||||
threshold: 3,
|
||||
crossLaunchWindowMs: 30 * 60_000,
|
||||
history: history.openLaunch()
|
||||
})
|
||||
return tracker.recordGpuCrash(1_000, { at, exitCode: null }).shouldEngageFallback
|
||||
}
|
||||
// One crash a day apart is a flaky app-start race, not a broken driver — and
|
||||
// the crash-free launches in between keep the streak from ever forming.
|
||||
expect(crashingLaunch(startedAt)).toBe(false)
|
||||
history.openLaunch()
|
||||
expect(crashingLaunch(startedAt + 86_400_000)).toBe(false)
|
||||
history.openLaunch()
|
||||
expect(crashingLaunch(startedAt + 2 * 86_400_000)).toBe(false)
|
||||
})
|
||||
|
||||
it('leaves the closest real-world non-burst alone once crashes are persisted', () => {
|
||||
const history = createHistoryFake()
|
||||
const tracker = new GpuCrashFallbackTracker({
|
||||
windowMs: 30_000,
|
||||
threshold: 3,
|
||||
history: history.openLaunch()
|
||||
})
|
||||
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 0)
|
||||
// Field telemetry's tightest 4-crash launch that is *not* a broken driver.
|
||||
// The in-launch rule already ruled it out; the persisted rules must not
|
||||
// re-litigate a single launch behind its back.
|
||||
for (const at of [0, 29_531, 55_136, 74_178]) {
|
||||
expect(
|
||||
tracker.recordGpuCrash(at, { at: startedAt + at, exitCode: 3_000 }).shouldEngageFallback
|
||||
).toBe(false)
|
||||
}
|
||||
})
|
||||
|
||||
it('does not let one recovered launch bankroll a later benign crash', () => {
|
||||
const userDataPath = mkdtempSync(join(tmpdir(), 'gpu-cross-launch-churn-'))
|
||||
const environment = {
|
||||
appVersion: '1.4.167',
|
||||
electronVersion: '38.2.0',
|
||||
platform: 'win32' as const
|
||||
}
|
||||
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 0)
|
||||
try {
|
||||
// F0BGRN5912M through the real ring: 4 recovered crashes in one launch that
|
||||
// the burst rule spares. Counting raw crashes would let a single ordinary
|
||||
// crash in the next launch cash them in.
|
||||
const first = new GpuCrashFallbackTracker({
|
||||
windowMs: 30_000,
|
||||
threshold: 3,
|
||||
history: openGpuCrashHistoryLaunch(userDataPath, environment)
|
||||
})
|
||||
for (const at of [0, 29_531, 55_136, 74_178]) {
|
||||
expect(
|
||||
first.recordGpuCrash(at, { at: startedAt + at, exitCode: 34 }).shouldEngageFallback
|
||||
).toBe(false)
|
||||
}
|
||||
const second = new GpuCrashFallbackTracker({
|
||||
windowMs: 30_000,
|
||||
threshold: 3,
|
||||
history: openGpuCrashHistoryLaunch(userDataPath, environment)
|
||||
})
|
||||
expect(
|
||||
second.recordGpuCrash(1_000, { at: startedAt + 374_178, exitCode: 34 }).shouldEngageFallback
|
||||
).toBe(false)
|
||||
} finally {
|
||||
rmSync(userDataPath, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('ignores a crashing-launch streak spread over weeks', () => {
|
||||
const history = createHistoryFake()
|
||||
const day = 86_400_000
|
||||
const startedAt = Date.UTC(2026, 7, 3, 10, 0, 0)
|
||||
const crashingLaunch = (at: number): boolean =>
|
||||
new GpuCrashFallbackTracker({
|
||||
windowMs: 30_000,
|
||||
threshold: 3,
|
||||
history: history.openLaunch()
|
||||
}).recordGpuCrash(1_000, { at, exitCode: 34 }).shouldEngageFallback
|
||||
// One lone GPU crash per session is ordinary Chromium churn (13 of the 14
|
||||
// GPU-crash reports in the corpus). Only a *recent* run of them is a driver.
|
||||
expect(crashingLaunch(startedAt)).toBe(false)
|
||||
expect(crashingLaunch(startedAt + 7 * day)).toBe(false)
|
||||
expect(crashingLaunch(startedAt + 14 * day)).toBe(false)
|
||||
})
|
||||
|
||||
it('still engages for a once-a-day user whose every launch dies on the GPU', () => {
|
||||
const history = createHistoryFake()
|
||||
const day = 86_400_000
|
||||
const startedAt = Date.UTC(2026, 7, 3, 10, 0, 0)
|
||||
const crashingLaunch = (at: number): boolean =>
|
||||
new GpuCrashFallbackTracker({
|
||||
windowMs: 30_000,
|
||||
threshold: 3,
|
||||
history: history.openLaunch()
|
||||
}).recordGpuCrash(1_000, { at, exitCode: 34 }).shouldEngageFallback
|
||||
expect(crashingLaunch(startedAt)).toBe(false)
|
||||
expect(crashingLaunch(startedAt + day)).toBe(false)
|
||||
expect(crashingLaunch(startedAt + 2 * day)).toBe(true)
|
||||
})
|
||||
|
||||
it('stops re-prompting after the user keeps running, then asks again a day later', () => {
|
||||
const history = createHistoryFake()
|
||||
const startedAt = Date.UTC(2026, 7, 3, 10, 0, 0)
|
||||
const crashingLaunch = (at: number): boolean => {
|
||||
const launch = history.openLaunch()
|
||||
const engaged = new GpuCrashFallbackTracker({
|
||||
windowMs: 30_000,
|
||||
threshold: 3,
|
||||
history: launch
|
||||
}).recordGpuCrash(1_000, { at, exitCode: 34 }).shouldEngageFallback
|
||||
if (engaged) {
|
||||
// The user picks "Keep Running" every time.
|
||||
launch.noteRestartDeclined(at)
|
||||
}
|
||||
return engaged
|
||||
}
|
||||
expect(crashingLaunch(startedAt)).toBe(false)
|
||||
expect(crashingLaunch(startedAt + 600_000)).toBe(false)
|
||||
expect(crashingLaunch(startedAt + 1_200_000)).toBe(true)
|
||||
const declinedAt = startedAt + 1_200_000
|
||||
expect(crashingLaunch(declinedAt + 600_000)).toBe(false)
|
||||
expect(crashingLaunch(declinedAt + 12 * 3_600_000)).toBe(false)
|
||||
// A day on, the driver is still broken: it is fair to ask once more.
|
||||
expect(crashingLaunch(declinedAt + 25 * 3_600_000)).toBe(true)
|
||||
})
|
||||
|
||||
it('survives real relaunches through the on-disk ring', () => {
|
||||
const userDataPath = mkdtempSync(join(tmpdir(), 'gpu-cross-launch-'))
|
||||
const environment = {
|
||||
appVersion: '1.4.167',
|
||||
electronVersion: '38.2.0',
|
||||
platform: 'win32' as const
|
||||
}
|
||||
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 14)
|
||||
try {
|
||||
const relaunch = (at: number): boolean =>
|
||||
new GpuCrashFallbackTracker({
|
||||
windowMs: 30_000,
|
||||
threshold: 3,
|
||||
history: openGpuCrashHistoryLaunch(userDataPath, environment)
|
||||
}).recordGpuCrash(2_000, { at, exitCode: 3_000 }).shouldEngageFallback
|
||||
expect(relaunch(startedAt)).toBe(false)
|
||||
expect(relaunch(startedAt + 88_000)).toBe(false)
|
||||
expect(relaunch(startedAt + 190_000)).toBe(true)
|
||||
} finally {
|
||||
rmSync(userDataPath, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps the in-launch burst rule intact when nothing is persisted', () => {
|
||||
const tracker = new GpuCrashFallbackTracker({ windowMs: 30_000, threshold: 3 })
|
||||
expect(tracker.recordGpuCrash(500).shouldEngageFallback).toBe(false)
|
||||
expect(tracker.recordGpuCrash(8_000).shouldEngageFallback).toBe(false)
|
||||
expect(tracker.recordGpuCrash(16_000)).toEqual({
|
||||
shouldEngageFallback: true,
|
||||
crashesInWindow: 3
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { GpuCrashFallbackCoordinator } from './gpu-crash-fallback-coordinator'
|
||||
import { GpuCrashFallbackTracker } from './gpu-crash-fallback-decision'
|
||||
|
||||
const startedAt = Date.UTC(2026, 7, 3, 22, 40, 14)
|
||||
|
||||
/** One physical GPU fault as the app sees it: child death, then the renderer. */
|
||||
function feedGpuFault(
|
||||
coordinator: GpuCrashFallbackCoordinator,
|
||||
msSinceLaunch: number,
|
||||
options: { rendererDelayMs?: number; exitCode?: number | null } = {}
|
||||
): { engaged: boolean; crashesInWindow: number; cascade: boolean } {
|
||||
const exitCode = options.exitCode ?? 34
|
||||
const at = startedAt + msSinceLaunch
|
||||
const verdict = coordinator.recordGpuChildCrash({ msSinceLaunch, at, exitCode })
|
||||
const cascade = coordinator.claimRendererCascade({
|
||||
reason: 'crashed',
|
||||
exitCode,
|
||||
at: at + (options.rendererDelayMs ?? 83)
|
||||
})
|
||||
return {
|
||||
engaged: verdict.shouldEngageFallback,
|
||||
crashesInWindow: verdict.crashesInWindow,
|
||||
cascade
|
||||
}
|
||||
}
|
||||
|
||||
function createCoordinator(): GpuCrashFallbackCoordinator {
|
||||
return new GpuCrashFallbackCoordinator({
|
||||
tracker: new GpuCrashFallbackTracker({ windowMs: 30_000, threshold: 3 })
|
||||
})
|
||||
}
|
||||
|
||||
describe('GpuCrashFallbackCoordinator', () => {
|
||||
it('counts one GPU fault once even when the renderer dies behind it', () => {
|
||||
// F0BNM0R87SL shape: the renderer goes down ~83ms after the GPU child with
|
||||
// the same exit code. Counting that tail too would halve the tuned threshold.
|
||||
const coordinator = createCoordinator()
|
||||
expect(feedGpuFault(coordinator, 1_000)).toEqual({
|
||||
engaged: false,
|
||||
crashesInWindow: 1,
|
||||
cascade: true
|
||||
})
|
||||
expect(feedGpuFault(coordinator, 21_000)).toEqual({
|
||||
engaged: false,
|
||||
crashesInWindow: 2,
|
||||
cascade: true
|
||||
})
|
||||
})
|
||||
|
||||
it('still engages on the third real fault inside the window', () => {
|
||||
const coordinator = createCoordinator()
|
||||
feedGpuFault(coordinator, 0)
|
||||
feedGpuFault(coordinator, 10_000)
|
||||
expect(feedGpuFault(coordinator, 20_000)).toEqual({
|
||||
engaged: true,
|
||||
crashesInWindow: 3,
|
||||
cascade: false
|
||||
})
|
||||
})
|
||||
|
||||
it('does not arm attribution for the crash that engaged fallback', () => {
|
||||
const coordinator = createCoordinator()
|
||||
feedGpuFault(coordinator, 0)
|
||||
feedGpuFault(coordinator, 10_000)
|
||||
// The engaging crash relaunches the app; its renderer tail is teardown noise.
|
||||
expect(feedGpuFault(coordinator, 20_000).cascade).toBe(false)
|
||||
})
|
||||
|
||||
it('leaves unrelated renderer deaths unclaimed', () => {
|
||||
const coordinator = createCoordinator()
|
||||
coordinator.recordGpuChildCrash({ msSinceLaunch: 1_000, at: startedAt, exitCode: 34 })
|
||||
expect(
|
||||
coordinator.claimRendererCascade({ reason: 'oom', exitCode: 34, at: startedAt + 100 })
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,53 @@
|
||||
import { GpuCrashCascadeAttributor } from './gpu-crash-cascade-attribution'
|
||||
import type { GpuCrashFallbackTracker } from './gpu-crash-fallback-decision'
|
||||
|
||||
export type GpuCrashFallbackVerdict = {
|
||||
shouldEngageFallback: boolean
|
||||
crashesInWindow: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Routes GPU faults into the fallback tracker exactly once each.
|
||||
*
|
||||
* Why: one physical GPU fault surfaces twice — the suppressed GPU child death
|
||||
* and, ~100ms later, the renderer going down with the same exit code. The tail
|
||||
* names the cause in the crash trail, but recording it as a second crash would
|
||||
* halve the tuned burst threshold (3 in 30s would fire after 2 real faults), so
|
||||
* it is attribution only.
|
||||
*/
|
||||
export class GpuCrashFallbackCoordinator {
|
||||
private readonly tracker: GpuCrashFallbackTracker
|
||||
private readonly attributor: GpuCrashCascadeAttributor
|
||||
|
||||
constructor(options: {
|
||||
tracker: GpuCrashFallbackTracker
|
||||
attributor?: GpuCrashCascadeAttributor
|
||||
}) {
|
||||
this.tracker = options.tracker
|
||||
this.attributor = options.attributor ?? new GpuCrashCascadeAttributor()
|
||||
}
|
||||
|
||||
/** Counts a GPU child crash and arms cascade attribution for its renderer tail. */
|
||||
recordGpuChildCrash(crash: {
|
||||
msSinceLaunch: number
|
||||
at: number
|
||||
exitCode: number | null
|
||||
}): GpuCrashFallbackVerdict {
|
||||
const verdict = this.tracker.recordGpuCrash(crash.msSinceLaunch, {
|
||||
at: crash.at,
|
||||
exitCode: crash.exitCode
|
||||
})
|
||||
if (!verdict.shouldEngageFallback) {
|
||||
this.attributor.noteSuppressedGpuCrash({ at: crash.at, exitCode: crash.exitCode })
|
||||
}
|
||||
return verdict
|
||||
}
|
||||
|
||||
/**
|
||||
* True when this renderer death is the tail of a GPU fault already counted by
|
||||
* `recordGpuChildCrash` — for the crash trail, never for the fallback rules.
|
||||
*/
|
||||
claimRendererCascade(crash: { reason: string; exitCode: number | null; at: number }): boolean {
|
||||
return this.attributor.claimRendererCascade(crash)
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,39 @@
|
||||
export type GpuCrashHistoryEntry = {
|
||||
/** Wall clock (Date.now); performance.now() restarts at 0 on every relaunch. */
|
||||
ts: number
|
||||
exitCode: number | null
|
||||
/** Launch counter that produced the crash, incremented once per launch. */
|
||||
launchSeq: number
|
||||
}
|
||||
|
||||
/** Per-launch handle on the persisted crash ring (startup/gpu-crash-history-store). */
|
||||
export type GpuCrashHistoryLaunch = {
|
||||
readonly launchSeq: number
|
||||
/** Wall clock of the last declined fallback restart on this build, if any. */
|
||||
readonly declinedAt: number | null
|
||||
/** Appends this launch's crash and returns the retained ring, oldest first. */
|
||||
append(crash: { ts: number; exitCode: number | null }): readonly GpuCrashHistoryEntry[]
|
||||
/** Starts the re-prompt cooldown after the user chose to keep running. */
|
||||
noteRestartDeclined(at: number): void
|
||||
}
|
||||
|
||||
export type GpuCrashFallbackOptions = {
|
||||
/** Rolling span over which clustered GPU crashes indicate a broken driver. */
|
||||
windowMs: number
|
||||
/** GPU child crashes within the window that trigger software-rendering fallback. */
|
||||
threshold: number
|
||||
/** Persisted, build-scoped crash ring; omitted the tracker is in-launch only. */
|
||||
history?: GpuCrashHistoryLaunch | null
|
||||
/** Wall-clock span over which crashes from earlier launches still count. */
|
||||
crossLaunchWindowMs?: number
|
||||
/** Distinct crashing launches inside `crossLaunchWindowMs` that trigger fallback. */
|
||||
crossLaunchThreshold?: number
|
||||
/** Back-to-back launches that each saw a GPU crash and trigger fallback. */
|
||||
crashingLaunchStreak?: number
|
||||
/** Wall-clock span the whole streak must fit inside to still describe this machine. */
|
||||
crashingLaunchStreakWindowMs?: number
|
||||
/** Quiet period after the user declines the restart before prompting again. */
|
||||
declineCooldownMs?: number
|
||||
}
|
||||
|
||||
const GPU_FALLBACK_CRASH_REASONS = new Set(['abnormal-exit', 'crashed', 'launch-failed'])
|
||||
@@ -15,6 +46,76 @@ const GPU_FALLBACK_CRASH_REASONS = new Set(['abnormal-exit', 'crashed', 'launch-
|
||||
export const DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS = 30_000
|
||||
export const DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD = 3
|
||||
|
||||
// Why: the in-launch burst rule can never fire when the GPU fault takes the whole
|
||||
// app down (F0BNM0R87SL crashes once per launch, forever), so the same evidence is
|
||||
// re-evaluated across launches from the persisted ring.
|
||||
export const DEFAULT_GPU_CROSS_LAUNCH_WINDOW_MS = 30 * 60_000
|
||||
export const DEFAULT_GPU_CROSS_LAUNCH_THRESHOLD = 3
|
||||
export const DEFAULT_GPU_CRASHING_LAUNCH_STREAK = 3
|
||||
// Why: three crashing launches only describe *this* machine's driver if they
|
||||
// happened around the same time; a user who opens Orca once a day fits, one who
|
||||
// opens it weekly is looking at three unrelated bits of Chromium churn.
|
||||
export const DEFAULT_GPU_CRASHING_LAUNCH_STREAK_WINDOW_MS = 72 * 60 * 60_000
|
||||
// Why: "Keep Running" has to mean something — re-asking on the first GPU crash
|
||||
// of every later launch is worse than the churn it warns about.
|
||||
export const DEFAULT_GPU_FALLBACK_DECLINE_COOLDOWN_MS = 24 * 60 * 60_000
|
||||
|
||||
/**
|
||||
* Cross-launch verdict over the persisted ring: either enough separate launches
|
||||
* crashed inside the window, or enough back-to-back launches each died on the GPU.
|
||||
*
|
||||
* Both rules count launches, never raw crashes. Churn inside a single launch is
|
||||
* the burst rule's call, and letting extra crashes from one launch fill a looser
|
||||
* window would relaunch sessions the burst rule deliberately spares (field
|
||||
* session F0BGRN5912M: 4 recovered crashes in 74s, plus one benign crash later).
|
||||
*
|
||||
* Both rules are also wall-clock bounded. `launchSeq` gaps break the streak, but
|
||||
* a crashing launch is only evidence while it is recent — see the streak window.
|
||||
*/
|
||||
export function evaluateCrossLaunchGpuCrashes(
|
||||
entries: readonly GpuCrashHistoryEntry[],
|
||||
options: {
|
||||
now: number
|
||||
launchSeq: number
|
||||
windowMs: number
|
||||
threshold: number
|
||||
crashingLaunchStreak: number
|
||||
crashingLaunchStreakWindowMs: number
|
||||
}
|
||||
): {
|
||||
crashesInWindow: number
|
||||
crashingLaunchesInWindow: number
|
||||
launchStreak: number
|
||||
shouldEngage: boolean
|
||||
} {
|
||||
const cutoff = options.now - options.windowMs
|
||||
const inWindow = entries.filter((entry) => entry.ts >= cutoff && entry.ts <= options.now)
|
||||
const launchesInWindow = new Set(inWindow.map((entry) => entry.launchSeq))
|
||||
const streakCutoff = options.now - options.crashingLaunchStreakWindowMs
|
||||
const newestCrashPerLaunch = new Map<number, number>()
|
||||
for (const entry of entries) {
|
||||
newestCrashPerLaunch.set(
|
||||
entry.launchSeq,
|
||||
Math.max(entry.ts, newestCrashPerLaunch.get(entry.launchSeq) ?? entry.ts)
|
||||
)
|
||||
}
|
||||
let launchStreak = 0
|
||||
for (;;) {
|
||||
const newestCrash = newestCrashPerLaunch.get(options.launchSeq - launchStreak)
|
||||
if (newestCrash === undefined || newestCrash < streakCutoff) {
|
||||
break
|
||||
}
|
||||
launchStreak += 1
|
||||
}
|
||||
return {
|
||||
crashesInWindow: inWindow.length,
|
||||
crashingLaunchesInWindow: launchesInWindow.size,
|
||||
launchStreak,
|
||||
shouldEngage:
|
||||
launchesInWindow.size >= options.threshold || launchStreak >= options.crashingLaunchStreak
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tracks GPU child-process crashes and decides when to fall back to software
|
||||
* rendering on the next launch. Pure and deterministic: callers pass `now`
|
||||
@@ -32,11 +133,24 @@ export class GpuCrashFallbackTracker {
|
||||
private readonly threshold: number
|
||||
// Newest-last crash times (ms since launch), pruned to the rolling window.
|
||||
private readonly recentCrashes: number[] = []
|
||||
private readonly history: GpuCrashHistoryLaunch | null
|
||||
private readonly crossLaunchWindowMs: number
|
||||
private readonly crossLaunchThreshold: number
|
||||
private readonly crashingLaunchStreak: number
|
||||
private readonly crashingLaunchStreakWindowMs: number
|
||||
private readonly declineCooldownMs: number
|
||||
private engaged = false
|
||||
|
||||
constructor(options: GpuCrashFallbackOptions) {
|
||||
this.windowMs = options.windowMs
|
||||
this.threshold = options.threshold
|
||||
this.history = options.history ?? null
|
||||
this.crossLaunchWindowMs = options.crossLaunchWindowMs ?? DEFAULT_GPU_CROSS_LAUNCH_WINDOW_MS
|
||||
this.crossLaunchThreshold = options.crossLaunchThreshold ?? DEFAULT_GPU_CROSS_LAUNCH_THRESHOLD
|
||||
this.crashingLaunchStreak = options.crashingLaunchStreak ?? DEFAULT_GPU_CRASHING_LAUNCH_STREAK
|
||||
this.crashingLaunchStreakWindowMs =
|
||||
options.crashingLaunchStreakWindowMs ?? DEFAULT_GPU_CRASHING_LAUNCH_STREAK_WINDOW_MS
|
||||
this.declineCooldownMs = options.declineCooldownMs ?? DEFAULT_GPU_FALLBACK_DECLINE_COOLDOWN_MS
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -45,7 +159,10 @@ export class GpuCrashFallbackTracker {
|
||||
* Returns false after fallback already engaged, so the caller relaunches at
|
||||
* most once.
|
||||
*/
|
||||
recordGpuCrash(msSinceLaunch: number): {
|
||||
recordGpuCrash(
|
||||
msSinceLaunch: number,
|
||||
crash: { at?: number; exitCode?: number | null } = {}
|
||||
): {
|
||||
shouldEngageFallback: boolean
|
||||
crashesInWindow: number
|
||||
} {
|
||||
@@ -62,10 +179,30 @@ export class GpuCrashFallbackTracker {
|
||||
stale += 1
|
||||
}
|
||||
this.recentCrashes.splice(0, stale)
|
||||
const wallClock = crash.at ?? Date.now()
|
||||
// Evidence keeps accruing through the cooldown so the next prompt sees the truth.
|
||||
const persisted = this.history?.append({ ts: wallClock, exitCode: crash.exitCode ?? null })
|
||||
if (this.isWithinDeclineCooldown(wallClock)) {
|
||||
return { shouldEngageFallback: false, crashesInWindow: this.recentCrashes.length }
|
||||
}
|
||||
if (this.recentCrashes.length >= this.threshold) {
|
||||
this.engaged = true
|
||||
return { shouldEngageFallback: true, crashesInWindow: this.recentCrashes.length }
|
||||
}
|
||||
if (persisted && this.history) {
|
||||
const crossLaunch = evaluateCrossLaunchGpuCrashes(persisted, {
|
||||
now: wallClock,
|
||||
launchSeq: this.history.launchSeq,
|
||||
windowMs: this.crossLaunchWindowMs,
|
||||
threshold: this.crossLaunchThreshold,
|
||||
crashingLaunchStreak: this.crashingLaunchStreak,
|
||||
crashingLaunchStreakWindowMs: this.crashingLaunchStreakWindowMs
|
||||
})
|
||||
if (crossLaunch.shouldEngage) {
|
||||
this.engaged = true
|
||||
return { shouldEngageFallback: true, crashesInWindow: crossLaunch.crashesInWindow }
|
||||
}
|
||||
}
|
||||
return { shouldEngageFallback: false, crashesInWindow: this.recentCrashes.length }
|
||||
}
|
||||
|
||||
@@ -73,6 +210,13 @@ export class GpuCrashFallbackTracker {
|
||||
return this.engaged
|
||||
}
|
||||
|
||||
// Why |Δ|: a clock stepped backwards past the decline must not pin the prompt
|
||||
// off forever — asking once too often beats never asking again.
|
||||
private isWithinDeclineCooldown(wallClock: number): boolean {
|
||||
const declinedAt = this.history?.declinedAt ?? null
|
||||
return declinedAt !== null && Math.abs(wallClock - declinedAt) < this.declineCooldownMs
|
||||
}
|
||||
|
||||
/** Crash times currently inside the window. Exposed to assert the pruning invariant. */
|
||||
windowSnapshot(): readonly number[] {
|
||||
return [...this.recentCrashes]
|
||||
|
||||
+45
-7
@@ -135,12 +135,15 @@ import {
|
||||
type WindowsGpuFallbackEnvironment
|
||||
} from './startup/gpu-fallback-marker'
|
||||
import { applyGpuFallbackCommandLineSwitches } from './startup/gpu-fallback-switches'
|
||||
import { clearGpuCrashHistory, openGpuCrashHistoryLaunch } from './startup/gpu-crash-history-store'
|
||||
import {
|
||||
DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD,
|
||||
DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS,
|
||||
GpuCrashFallbackTracker,
|
||||
isGpuFallbackCrashCandidate
|
||||
isGpuFallbackCrashCandidate,
|
||||
type GpuCrashHistoryLaunch
|
||||
} from './crash-reporting/gpu-crash-fallback-decision'
|
||||
import { GpuCrashFallbackCoordinator } from './crash-reporting/gpu-crash-fallback-coordinator'
|
||||
import {
|
||||
promptForGpuFallbackRestart,
|
||||
type GpuFallbackRestartDecision
|
||||
@@ -362,10 +365,8 @@ let managedWslCliReconciliationReady: Promise<void> = Promise.resolve()
|
||||
let managedWslCliStartupBarrierReady: Promise<void> = Promise.resolve()
|
||||
// Why: the serve barrier fails open, so this state tells headless clients a WSL PTY launch may still race an un-migrated registration ('settled' = off-Windows no-op).
|
||||
let managedWslCliReconciliationStatus: 'pending' | 'settled' | 'failed' = 'settled'
|
||||
const gpuCrashFallbackTracker = new GpuCrashFallbackTracker({
|
||||
windowMs: DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS,
|
||||
threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD
|
||||
})
|
||||
let gpuCrashHistoryLaunch: GpuCrashHistoryLaunch | null = null
|
||||
let gpuCrashFallbackCoordinator: GpuCrashFallbackCoordinator | null = null
|
||||
let gpuFallbackActiveThisLaunch = false
|
||||
let gpuFeatureStatus: Electron.GPUFeatureStatus | null = null
|
||||
let localPtyStartupReady: Promise<void> = Promise.resolve()
|
||||
@@ -1221,6 +1222,7 @@ function openMainWindow(): BrowserWindow {
|
||||
},
|
||||
webContentsId
|
||||
)
|
||||
attributeGpuCascadeCrash(details.reason, details.exitCode ?? null)
|
||||
},
|
||||
shouldRecoverRenderer: (details, webContentsId) =>
|
||||
shouldRecoverRendererAfterProcessGone({
|
||||
@@ -1556,7 +1558,11 @@ function maybeApplyGpuFallbackForThisLaunch(): void {
|
||||
if (isServeMode || process.platform !== 'win32') {
|
||||
return
|
||||
}
|
||||
const marker = readActiveGpuFallbackMarker(app.getPath('userData'), getGpuFallbackEnvironment())
|
||||
const environment = getGpuFallbackEnvironment()
|
||||
// Why: claim this launch's sequence number even when nothing crashes — a
|
||||
// crash-free launch is the only evidence that a crashing streak ended.
|
||||
gpuCrashHistoryLaunch = openGpuCrashHistoryLaunch(app.getPath('userData'), environment)
|
||||
const marker = readActiveGpuFallbackMarker(app.getPath('userData'), environment)
|
||||
if (!marker) {
|
||||
return
|
||||
}
|
||||
@@ -1571,13 +1577,39 @@ function maybeApplyGpuFallbackForThisLaunch(): void {
|
||||
})
|
||||
}
|
||||
|
||||
function getGpuCrashFallbackCoordinator(): GpuCrashFallbackCoordinator {
|
||||
gpuCrashFallbackCoordinator ??= new GpuCrashFallbackCoordinator({
|
||||
tracker: new GpuCrashFallbackTracker({
|
||||
windowMs: DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS,
|
||||
threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD,
|
||||
history: gpuCrashHistoryLaunch
|
||||
})
|
||||
})
|
||||
return gpuCrashFallbackCoordinator
|
||||
}
|
||||
|
||||
// Why: names the GPU fault behind an otherwise unexplained renderer death; the
|
||||
// GPU child crash it trails was already counted, so nothing is recorded here.
|
||||
function attributeGpuCascadeCrash(reason: string, exitCode: number | null): void {
|
||||
if (
|
||||
!getGpuCrashFallbackCoordinator().claimRendererCascade({ reason, exitCode, at: Date.now() })
|
||||
) {
|
||||
return
|
||||
}
|
||||
recordCrashBreadcrumb('gpu_cascade', { reason, exitCode })
|
||||
}
|
||||
|
||||
// Why: a burst of GPU child crashes means HW acceleration is unusable — persist a build-scoped marker and offer software rendering.
|
||||
async function handleGpuChildCrash(reason: string, exitCode: number | null): Promise<void> {
|
||||
// Software rendering already active or shutting down: nothing more to do.
|
||||
if (gpuFallbackActiveThisLaunch || isQuitting || isServeMode) {
|
||||
return
|
||||
}
|
||||
const result = gpuCrashFallbackTracker.recordGpuCrash(performance.now())
|
||||
const result = getGpuCrashFallbackCoordinator().recordGpuChildCrash({
|
||||
msSinceLaunch: performance.now(),
|
||||
at: Date.now(),
|
||||
exitCode
|
||||
})
|
||||
if (!result.shouldEngageFallback) {
|
||||
return
|
||||
}
|
||||
@@ -1605,6 +1637,9 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro
|
||||
}
|
||||
if (restartDecision !== 'restart') {
|
||||
recordDurableCrashBreadcrumb('gpu_fallback_restart_deferred', fallbackData)
|
||||
// Why: the persisted evidence still satisfies the rules, so without a
|
||||
// cooldown "Keep Running" re-prompts on the first GPU crash of every launch.
|
||||
gpuCrashHistoryLaunch?.noteRestartDeclined(Date.now())
|
||||
return
|
||||
}
|
||||
const environment = getWindowsGpuFallbackEnvironment()
|
||||
@@ -1624,6 +1659,9 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro
|
||||
console.warn('[gpu-fallback] failed to persist marker:', error)
|
||||
return
|
||||
}
|
||||
// Why: the marker now carries the verdict; stale crashes must not re-engage
|
||||
// the moment the user turns hardware acceleration back on.
|
||||
clearGpuCrashHistory(app.getPath('userData'))
|
||||
isQuitting = true
|
||||
relaunchApp('gpu-fallback', fallbackData)
|
||||
// Why: app.exit(0) skips before-quit, so destroy the Windows tray manually to avoid a stale icon.
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
import {
|
||||
clearGpuCrashHistory,
|
||||
GPU_CRASH_HISTORY_FILE,
|
||||
GPU_CRASH_HISTORY_MAX_ENTRIES,
|
||||
openGpuCrashHistoryLaunch,
|
||||
readGpuCrashHistory
|
||||
} from './gpu-crash-history-store'
|
||||
import type { GpuFallbackEnvironment } from './gpu-fallback-marker'
|
||||
|
||||
const environment: GpuFallbackEnvironment = {
|
||||
appVersion: '1.4.167',
|
||||
electronVersion: '38.2.0',
|
||||
platform: 'win32'
|
||||
}
|
||||
|
||||
let userDataPath: string
|
||||
|
||||
beforeEach(() => {
|
||||
userDataPath = mkdtempSync(join(tmpdir(), 'gpu-crash-history-'))
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(userDataPath, { recursive: true, force: true })
|
||||
})
|
||||
|
||||
describe('openGpuCrashHistoryLaunch', () => {
|
||||
it('bumps the launch counter on every launch, crash or not', () => {
|
||||
expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(1)
|
||||
expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(2)
|
||||
expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(3)
|
||||
})
|
||||
|
||||
it('persists crashes across launches with wall-clock timestamps', () => {
|
||||
const first = openGpuCrashHistoryLaunch(userDataPath, environment)
|
||||
first.append({ ts: 1_770_000_000_000, exitCode: 3_000 })
|
||||
const second = openGpuCrashHistoryLaunch(userDataPath, environment)
|
||||
expect(second.append({ ts: 1_770_000_090_000, exitCode: 3_000 })).toEqual([
|
||||
{ ts: 1_770_000_000_000, exitCode: 3_000, launchSeq: 1 },
|
||||
{ ts: 1_770_000_090_000, exitCode: 3_000, launchSeq: 2 }
|
||||
])
|
||||
})
|
||||
|
||||
it('starts over after an app update so a fixed build gets a fresh hardware attempt', () => {
|
||||
const before = openGpuCrashHistoryLaunch(userDataPath, environment)
|
||||
before.append({ ts: 1_770_000_000_000, exitCode: 3_000 })
|
||||
const upgraded = openGpuCrashHistoryLaunch(userDataPath, {
|
||||
...environment,
|
||||
appVersion: '1.4.168'
|
||||
})
|
||||
expect(upgraded.launchSeq).toBe(1)
|
||||
expect(upgraded.append({ ts: 1_770_000_100_000, exitCode: null })).toEqual([
|
||||
{ ts: 1_770_000_100_000, exitCode: null, launchSeq: 1 }
|
||||
])
|
||||
})
|
||||
|
||||
it('bounds the ring', () => {
|
||||
const launch = openGpuCrashHistoryLaunch(userDataPath, environment)
|
||||
let entries: readonly { ts: number }[] = []
|
||||
for (let index = 0; index < GPU_CRASH_HISTORY_MAX_ENTRIES + 5; index += 1) {
|
||||
entries = launch.append({ ts: 1_770_000_000_000 + index, exitCode: 3_000 })
|
||||
}
|
||||
expect(entries).toHaveLength(GPU_CRASH_HISTORY_MAX_ENTRIES)
|
||||
expect(entries[0]?.ts).toBe(1_770_000_000_000 + 5)
|
||||
})
|
||||
|
||||
it('survives a corrupt file', () => {
|
||||
writeFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), '{not json')
|
||||
const launch = openGpuCrashHistoryLaunch(userDataPath, environment)
|
||||
expect(launch.launchSeq).toBe(1)
|
||||
expect(readGpuCrashHistory(userDataPath, environment)?.entries).toEqual([])
|
||||
})
|
||||
|
||||
it('drops malformed entries but keeps valid ones', () => {
|
||||
writeFileSync(
|
||||
join(userDataPath, GPU_CRASH_HISTORY_FILE),
|
||||
JSON.stringify({
|
||||
schemeVersion: 1,
|
||||
appVersion: environment.appVersion,
|
||||
electronVersion: environment.electronVersion,
|
||||
platform: environment.platform,
|
||||
launchSeq: 4,
|
||||
entries: [
|
||||
{ ts: 'nope', exitCode: 1, launchSeq: 1 },
|
||||
{ ts: 1_770_000_000_000, exitCode: null, launchSeq: 4 }
|
||||
]
|
||||
})
|
||||
)
|
||||
expect(readGpuCrashHistory(userDataPath, environment)?.entries).toEqual([
|
||||
{ ts: 1_770_000_000_000, exitCode: null, launchSeq: 4 }
|
||||
])
|
||||
expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(5)
|
||||
})
|
||||
|
||||
it('remembers a declined restart across launches without losing crashes', () => {
|
||||
const first = openGpuCrashHistoryLaunch(userDataPath, environment)
|
||||
first.append({ ts: 1_770_000_000_000, exitCode: 3_000 })
|
||||
first.noteRestartDeclined(1_770_000_000_500)
|
||||
const second = openGpuCrashHistoryLaunch(userDataPath, environment)
|
||||
expect(second.declinedAt).toBe(1_770_000_000_500)
|
||||
expect(second.append({ ts: 1_770_000_090_000, exitCode: 3_000 })).toHaveLength(2)
|
||||
expect(openGpuCrashHistoryLaunch(userDataPath, environment).declinedAt).toBe(1_770_000_000_500)
|
||||
})
|
||||
|
||||
it('reports no decline for a fresh or upgraded build', () => {
|
||||
expect(openGpuCrashHistoryLaunch(userDataPath, environment).declinedAt).toBeNull()
|
||||
openGpuCrashHistoryLaunch(userDataPath, environment).noteRestartDeclined(1_770_000_000_500)
|
||||
expect(
|
||||
openGpuCrashHistoryLaunch(userDataPath, { ...environment, appVersion: '1.4.168' }).declinedAt
|
||||
).toBeNull()
|
||||
})
|
||||
|
||||
it('clears the ring when fallback engages', () => {
|
||||
openGpuCrashHistoryLaunch(userDataPath, environment).append({ ts: 1, exitCode: 3_000 })
|
||||
clearGpuCrashHistory(userDataPath)
|
||||
expect(readGpuCrashHistory(userDataPath, environment)).toBeNull()
|
||||
expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(1)
|
||||
})
|
||||
|
||||
it('writes the ring where the marker lives', () => {
|
||||
openGpuCrashHistoryLaunch(userDataPath, environment)
|
||||
expect(
|
||||
JSON.parse(readFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), 'utf-8')).launchSeq
|
||||
).toBe(1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,151 @@
|
||||
import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { dirname, join } from 'node:path'
|
||||
import type {
|
||||
GpuCrashHistoryEntry,
|
||||
GpuCrashHistoryLaunch
|
||||
} from '../crash-reporting/gpu-crash-fallback-decision'
|
||||
import type { GpuFallbackEnvironment } from './gpu-fallback-marker'
|
||||
|
||||
/**
|
||||
* Build-scoped ring of GPU child crashes, stored next to the GPU fallback marker.
|
||||
*
|
||||
* Why on disk: a GPU fault that kills the app leaves the in-memory tracker empty
|
||||
* on the next launch, so a "crash once per launch, forever" loop never reaches
|
||||
* the in-launch burst threshold. Timestamps are wall clock — performance.now()
|
||||
* restarts at 0 every launch and cannot be compared across them.
|
||||
*/
|
||||
|
||||
export const GPU_CRASH_HISTORY_FILE = 'gpu-crash-history.json'
|
||||
export const GPU_CRASH_HISTORY_SCHEME_VERSION = 1
|
||||
/** Deep enough for the streak + window rules, small enough to write synchronously. */
|
||||
export const GPU_CRASH_HISTORY_MAX_ENTRIES = 16
|
||||
|
||||
type GpuCrashHistoryFile = {
|
||||
schemeVersion: number
|
||||
appVersion: string
|
||||
electronVersion: string
|
||||
platform: NodeJS.Platform
|
||||
launchSeq: number
|
||||
entries: GpuCrashHistoryEntry[]
|
||||
/** Wall clock of the last "Keep Running" answer; null when never declined. */
|
||||
declinedAt: number | null
|
||||
}
|
||||
|
||||
function historyPath(userDataPath: string): string {
|
||||
return join(userDataPath, GPU_CRASH_HISTORY_FILE)
|
||||
}
|
||||
|
||||
function isValidEntry(value: unknown): value is GpuCrashHistoryEntry {
|
||||
const entry = value as Partial<GpuCrashHistoryEntry> | null
|
||||
return (
|
||||
typeof entry === 'object' &&
|
||||
entry !== null &&
|
||||
typeof entry.ts === 'number' &&
|
||||
Number.isFinite(entry.ts) &&
|
||||
(entry.exitCode === null || typeof entry.exitCode === 'number') &&
|
||||
typeof entry.launchSeq === 'number' &&
|
||||
Number.isFinite(entry.launchSeq)
|
||||
)
|
||||
}
|
||||
|
||||
export function readGpuCrashHistory(
|
||||
userDataPath: string,
|
||||
environment: GpuFallbackEnvironment
|
||||
): GpuCrashHistoryFile | null {
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(historyPath(userDataPath), 'utf-8')) as Partial<
|
||||
Record<keyof GpuCrashHistoryFile, unknown>
|
||||
>
|
||||
if (
|
||||
parsed.schemeVersion !== GPU_CRASH_HISTORY_SCHEME_VERSION ||
|
||||
parsed.appVersion !== environment.appVersion ||
|
||||
parsed.electronVersion !== environment.electronVersion ||
|
||||
parsed.platform !== environment.platform ||
|
||||
typeof parsed.launchSeq !== 'number' ||
|
||||
!Number.isFinite(parsed.launchSeq) ||
|
||||
!Array.isArray(parsed.entries)
|
||||
) {
|
||||
// Why: a different build (or a corrupt file) gets a clean slate — the
|
||||
// evidence describes a driver/binary pairing that no longer exists.
|
||||
return null
|
||||
}
|
||||
return {
|
||||
schemeVersion: GPU_CRASH_HISTORY_SCHEME_VERSION,
|
||||
appVersion: environment.appVersion,
|
||||
electronVersion: environment.electronVersion,
|
||||
platform: environment.platform,
|
||||
launchSeq: parsed.launchSeq,
|
||||
entries: parsed.entries.filter(isValidEntry).slice(-GPU_CRASH_HISTORY_MAX_ENTRIES),
|
||||
declinedAt:
|
||||
typeof parsed.declinedAt === 'number' && Number.isFinite(parsed.declinedAt)
|
||||
? parsed.declinedAt
|
||||
: null
|
||||
}
|
||||
} catch {
|
||||
// missing or unreadable means no history
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function writeGpuCrashHistory(userDataPath: string, file: GpuCrashHistoryFile): void {
|
||||
try {
|
||||
const target = historyPath(userDataPath)
|
||||
mkdirSync(dirname(target), { recursive: true })
|
||||
writeFileSync(target, JSON.stringify(file))
|
||||
} catch {
|
||||
// best effort: losing history only costs a slower fallback decision
|
||||
}
|
||||
}
|
||||
|
||||
export function clearGpuCrashHistory(userDataPath: string): void {
|
||||
try {
|
||||
rmSync(historyPath(userDataPath), { force: true })
|
||||
} catch {
|
||||
// best effort; the next launch revalidates the file anyway
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Claims the next launch sequence number and hands back an append port.
|
||||
*
|
||||
* Called once per launch even when no GPU crash follows: a crash-free launch has
|
||||
* to bump the counter, otherwise a gap in the sequence — the only evidence that
|
||||
* the streak broke — would never be recorded.
|
||||
*/
|
||||
export function openGpuCrashHistoryLaunch(
|
||||
userDataPath: string,
|
||||
environment: GpuFallbackEnvironment
|
||||
): GpuCrashHistoryLaunch {
|
||||
const existing = readGpuCrashHistory(userDataPath, environment)
|
||||
const launchSeq = (existing?.launchSeq ?? 0) + 1
|
||||
let entries = existing?.entries ?? []
|
||||
let declinedAt = existing?.declinedAt ?? null
|
||||
const file: GpuCrashHistoryFile = {
|
||||
schemeVersion: GPU_CRASH_HISTORY_SCHEME_VERSION,
|
||||
appVersion: environment.appVersion,
|
||||
electronVersion: environment.electronVersion,
|
||||
platform: environment.platform,
|
||||
launchSeq,
|
||||
entries,
|
||||
declinedAt
|
||||
}
|
||||
writeGpuCrashHistory(userDataPath, file)
|
||||
return {
|
||||
launchSeq,
|
||||
// Getter, not a snapshot: a decline taken mid-launch must be visible at once.
|
||||
get declinedAt() {
|
||||
return declinedAt
|
||||
},
|
||||
append: (crash) => {
|
||||
entries = [...entries, { ts: crash.ts, exitCode: crash.exitCode, launchSeq }].slice(
|
||||
-GPU_CRASH_HISTORY_MAX_ENTRIES
|
||||
)
|
||||
writeGpuCrashHistory(userDataPath, { ...file, entries, declinedAt })
|
||||
return entries
|
||||
},
|
||||
noteRestartDeclined: (at) => {
|
||||
declinedAt = at
|
||||
writeGpuCrashHistory(userDataPath, { ...file, entries, declinedAt })
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user