From aa1c91d511b32a7f0aa3664e439fddf375ee2eec Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Mon, 3 Aug 2026 23:20:08 -0700 Subject: [PATCH] fix(gpu): engage GPU software-rendering fallback across launches The GPU crash tracker lived only in memory, so a GPU fault that took the whole app down reset it on every relaunch and the 3-in-30s burst rule could never fire. Persist a build-scoped ring of wall-clock crashes next to the fallback marker and engage when separate launches crash inside the window or when consecutive recent launches each die on the GPU. Both cross-launch rules count launches, never raw crashes, so recovered churn inside one session cannot be cashed in by a later benign crash, and the streak is bounded to 72h so launches weeks apart are not treated as one broken driver. A renderer death trailing a suppressed GPU crash is named gpu_cascade in the crash trail but is not recorded as a second crash, keeping the burst threshold at 3 real faults. Declining the restart starts a 24h cooldown so "Keep Running" is not re-asked on the first GPU crash of every later launch. Co-authored-by: Orca --- .../gpu-crash-cascade-attribution.test.ts | 62 +++++ .../gpu-crash-cascade-attribution.ts | 49 ++++ .../gpu-crash-cross-launch-fallback.test.ts | 250 ++++++++++++++++++ .../gpu-crash-fallback-coordinator.test.ts | 77 ++++++ .../gpu-crash-fallback-coordinator.ts | 53 ++++ .../gpu-crash-fallback-decision.ts | 146 +++++++++- src/main/index.ts | 52 +++- .../startup/gpu-crash-history-store.test.ts | 129 +++++++++ src/main/startup/gpu-crash-history-store.ts | 151 +++++++++++ 9 files changed, 961 insertions(+), 8 deletions(-) create mode 100644 src/main/crash-reporting/gpu-crash-cascade-attribution.test.ts create mode 100644 src/main/crash-reporting/gpu-crash-cascade-attribution.ts create mode 100644 src/main/crash-reporting/gpu-crash-cross-launch-fallback.test.ts create mode 100644 src/main/crash-reporting/gpu-crash-fallback-coordinator.test.ts create mode 100644 src/main/crash-reporting/gpu-crash-fallback-coordinator.ts create mode 100644 src/main/startup/gpu-crash-history-store.test.ts create mode 100644 src/main/startup/gpu-crash-history-store.ts diff --git a/src/main/crash-reporting/gpu-crash-cascade-attribution.test.ts b/src/main/crash-reporting/gpu-crash-cascade-attribution.test.ts new file mode 100644 index 00000000000..2653f61e375 --- /dev/null +++ b/src/main/crash-reporting/gpu-crash-cascade-attribution.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from 'vitest' +import { GpuCrashCascadeAttributor } from './gpu-crash-cascade-attribution' + +const startedAt = Date.UTC(2026, 7, 3, 22, 40, 14) + +describe('GpuCrashCascadeAttributor', () => { + it('claims a renderer crash that trails a suppressed GPU crash with the same exit code', () => { + const attributor = new GpuCrashCascadeAttributor() + attributor.noteSuppressedGpuCrash({ at: startedAt, exitCode: 3_000 }) + expect( + attributor.claimRendererCascade({ reason: 'crashed', exitCode: 3_000, at: startedAt + 400 }) + ).toBe(true) + }) + + it('claims at most one cascade per GPU crash', () => { + const attributor = new GpuCrashCascadeAttributor() + attributor.noteSuppressedGpuCrash({ at: startedAt, exitCode: null }) + expect( + attributor.claimRendererCascade({ reason: 'crashed', exitCode: null, at: startedAt + 100 }) + ).toBe(true) + expect( + attributor.claimRendererCascade({ reason: 'crashed', exitCode: null, at: startedAt + 200 }) + ).toBe(false) + }) + + it('ignores unrelated renderer deaths', () => { + const attributor = new GpuCrashCascadeAttributor() + attributor.noteSuppressedGpuCrash({ at: startedAt, exitCode: 3_000 }) + // Different exit code: a separate fault, not the GPU cascade. + expect( + attributor.claimRendererCascade({ reason: 'crashed', exitCode: 5, at: startedAt + 100 }) + ).toBe(false) + // OOM/killed renderers are their own failure mode. + expect( + attributor.claimRendererCascade({ + reason: 'oom', + exitCode: 3_000, + at: startedAt + 100 + }) + ).toBe(false) + // Too late to be the same fault. + expect( + attributor.claimRendererCascade({ reason: 'crashed', exitCode: 3_000, at: startedAt + 5_000 }) + ).toBe(false) + }) + + it('never claims without an armed GPU crash', () => { + const attributor = new GpuCrashCascadeAttributor() + expect( + attributor.claimRendererCascade({ reason: 'crashed', exitCode: 3_000, at: startedAt }) + ).toBe(false) + }) + + it('ignores a renderer crash stamped before the GPU crash', () => { + const attributor = new GpuCrashCascadeAttributor() + attributor.noteSuppressedGpuCrash({ at: startedAt, exitCode: 3_000 }) + // Why pinned: a backwards clock step must not open an unbounded match window. + expect( + attributor.claimRendererCascade({ reason: 'crashed', exitCode: 3_000, at: startedAt - 10_000 }) + ).toBe(false) + }) +}) diff --git a/src/main/crash-reporting/gpu-crash-cascade-attribution.ts b/src/main/crash-reporting/gpu-crash-cascade-attribution.ts new file mode 100644 index 00000000000..a9402b75366 --- /dev/null +++ b/src/main/crash-reporting/gpu-crash-cascade-attribution.ts @@ -0,0 +1,49 @@ +/** Renderer deaths this close behind a GPU child crash are the same fault. */ +export const DEFAULT_GPU_CASCADE_WINDOW_MS = 2_000 + +export type GpuCascadeCrash = { at: number; exitCode: number | null } + +/** + * Links a renderer crash back to the GPU child crash that caused it. + * + * Why: a GPU fault usually surfaces twice — the suppressed child-process-gone + * and, moments later, the renderer going down with the same exit code. Counting + * the tail as its own GPU crash is what lets a "one crash per launch" loop reach + * the fallback threshold, and the paired event names the real cause in the + * crash trail instead of leaving an unexplained renderer death. + * + * TODO: fold into the shared recent-process-gone ring in process-gone-recorder + * once it lands. + */ +export class GpuCrashCascadeAttributor { + private readonly windowMs: number + private pending: GpuCascadeCrash | null = null + + constructor(options: { windowMs?: number } = {}) { + this.windowMs = options.windowMs ?? DEFAULT_GPU_CASCADE_WINDOW_MS + } + + /** Arms attribution after a GPU child crash that did not engage fallback. */ + noteSuppressedGpuCrash(crash: GpuCascadeCrash): void { + this.pending = crash + } + + /** + * True when this renderer death is the tail of the armed GPU crash. Claiming + * disarms, so one GPU crash can only ever produce one cascade event. + */ + claimRendererCascade(crash: { reason: string; exitCode: number | null; at: number }): boolean { + const pending = this.pending + if ( + !pending || + crash.reason !== 'crashed' || + crash.exitCode !== pending.exitCode || + crash.at < pending.at || + crash.at - pending.at > this.windowMs + ) { + return false + } + this.pending = null + return true + } +} diff --git a/src/main/crash-reporting/gpu-crash-cross-launch-fallback.test.ts b/src/main/crash-reporting/gpu-crash-cross-launch-fallback.test.ts new file mode 100644 index 00000000000..28fe02336a7 --- /dev/null +++ b/src/main/crash-reporting/gpu-crash-cross-launch-fallback.test.ts @@ -0,0 +1,250 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { openGpuCrashHistoryLaunch } from '../startup/gpu-crash-history-store' +import { GpuCrashFallbackTracker } from './gpu-crash-fallback-decision' +import type { GpuCrashHistoryEntry, GpuCrashHistoryLaunch } from './gpu-crash-fallback-decision' + +/** In-memory stand-in for the on-disk ring, shared across simulated relaunches. */ +function createHistoryFake(): { + openLaunch: () => GpuCrashHistoryLaunch + entries: GpuCrashHistoryEntry[] +} { + const entries: GpuCrashHistoryEntry[] = [] + let launchSeq = 0 + let declinedAt: number | null = null + return { + entries, + openLaunch: () => { + launchSeq += 1 + const currentLaunchSeq = launchSeq + return { + launchSeq: currentLaunchSeq, + get declinedAt() { + return declinedAt + }, + append: (crash) => { + entries.push({ ...crash, launchSeq: currentLaunchSeq }) + return entries + }, + noteRestartDeclined: (at) => { + declinedAt = at + } + } + } + } +} + +describe('GPU fallback across relaunches', () => { + it('engages after repeated launches that each die on the first GPU crash', () => { + // F0BNM0R87SL: the GPU crash takes the app down, so every launch starts with + // an empty tracker and the in-launch burst rule can never reach 3. + const history = createHistoryFake() + const startedAt = Date.UTC(2026, 7, 3, 22, 40, 0) + const results: boolean[] = [] + for (let launch = 0; launch < 5; launch += 1) { + const tracker = new GpuCrashFallbackTracker({ + windowMs: 30_000, + threshold: 3, + history: history.openLaunch() + }) + const engaged = tracker.recordGpuCrash(3_000, { + at: startedAt + launch * 90_000, + exitCode: 3_000 + }).shouldEngageFallback + results.push(engaged) + if (engaged) { + // The app relaunches into software rendering; later launches never happen. + break + } + } + expect(results).toEqual([false, false, true]) + expect(history.entries.map((entry) => entry.launchSeq)).toEqual([1, 2, 3]) + expect(history.entries.every((entry) => entry.ts >= startedAt)).toBe(true) + }) + + it('engages on crashes clustered in wall-clock time even when launches are not consecutive', () => { + const history = createHistoryFake() + const startedAt = Date.UTC(2026, 7, 3, 22, 40, 0) + // Launches 1 and 3 crash, 2 does not: no streak, but three crashes in 6 min. + const crashingLaunch = (at: number): boolean => { + const tracker = new GpuCrashFallbackTracker({ + windowMs: 30_000, + threshold: 3, + history: history.openLaunch() + }) + return tracker.recordGpuCrash(1_000, { at, exitCode: null }).shouldEngageFallback + } + expect(crashingLaunch(startedAt)).toBe(false) + history.openLaunch() + expect(crashingLaunch(startedAt + 120_000)).toBe(false) + history.openLaunch() + expect(crashingLaunch(startedAt + 360_000)).toBe(true) + }) + + it('ignores crashes that aged out of the cross-launch window', () => { + const history = createHistoryFake() + const startedAt = Date.UTC(2026, 7, 3, 10, 0, 0) + const crashingLaunch = (at: number): boolean => { + const tracker = new GpuCrashFallbackTracker({ + windowMs: 30_000, + threshold: 3, + crossLaunchWindowMs: 30 * 60_000, + history: history.openLaunch() + }) + return tracker.recordGpuCrash(1_000, { at, exitCode: null }).shouldEngageFallback + } + // One crash a day apart is a flaky app-start race, not a broken driver — and + // the crash-free launches in between keep the streak from ever forming. + expect(crashingLaunch(startedAt)).toBe(false) + history.openLaunch() + expect(crashingLaunch(startedAt + 86_400_000)).toBe(false) + history.openLaunch() + expect(crashingLaunch(startedAt + 2 * 86_400_000)).toBe(false) + }) + + it('leaves the closest real-world non-burst alone once crashes are persisted', () => { + const history = createHistoryFake() + const tracker = new GpuCrashFallbackTracker({ + windowMs: 30_000, + threshold: 3, + history: history.openLaunch() + }) + const startedAt = Date.UTC(2026, 7, 3, 22, 40, 0) + // Field telemetry's tightest 4-crash launch that is *not* a broken driver. + // The in-launch rule already ruled it out; the persisted rules must not + // re-litigate a single launch behind its back. + for (const at of [0, 29_531, 55_136, 74_178]) { + expect( + tracker.recordGpuCrash(at, { at: startedAt + at, exitCode: 3_000 }).shouldEngageFallback + ).toBe(false) + } + }) + + it('does not let one recovered launch bankroll a later benign crash', () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'gpu-cross-launch-churn-')) + const environment = { + appVersion: '1.4.167', + electronVersion: '38.2.0', + platform: 'win32' as const + } + const startedAt = Date.UTC(2026, 7, 3, 22, 40, 0) + try { + // F0BGRN5912M through the real ring: 4 recovered crashes in one launch that + // the burst rule spares. Counting raw crashes would let a single ordinary + // crash in the next launch cash them in. + const first = new GpuCrashFallbackTracker({ + windowMs: 30_000, + threshold: 3, + history: openGpuCrashHistoryLaunch(userDataPath, environment) + }) + for (const at of [0, 29_531, 55_136, 74_178]) { + expect( + first.recordGpuCrash(at, { at: startedAt + at, exitCode: 34 }).shouldEngageFallback + ).toBe(false) + } + const second = new GpuCrashFallbackTracker({ + windowMs: 30_000, + threshold: 3, + history: openGpuCrashHistoryLaunch(userDataPath, environment) + }) + expect( + second.recordGpuCrash(1_000, { at: startedAt + 374_178, exitCode: 34 }).shouldEngageFallback + ).toBe(false) + } finally { + rmSync(userDataPath, { recursive: true, force: true }) + } + }) + + it('ignores a crashing-launch streak spread over weeks', () => { + const history = createHistoryFake() + const day = 86_400_000 + const startedAt = Date.UTC(2026, 7, 3, 10, 0, 0) + const crashingLaunch = (at: number): boolean => + new GpuCrashFallbackTracker({ + windowMs: 30_000, + threshold: 3, + history: history.openLaunch() + }).recordGpuCrash(1_000, { at, exitCode: 34 }).shouldEngageFallback + // One lone GPU crash per session is ordinary Chromium churn (13 of the 14 + // GPU-crash reports in the corpus). Only a *recent* run of them is a driver. + expect(crashingLaunch(startedAt)).toBe(false) + expect(crashingLaunch(startedAt + 7 * day)).toBe(false) + expect(crashingLaunch(startedAt + 14 * day)).toBe(false) + }) + + it('still engages for a once-a-day user whose every launch dies on the GPU', () => { + const history = createHistoryFake() + const day = 86_400_000 + const startedAt = Date.UTC(2026, 7, 3, 10, 0, 0) + const crashingLaunch = (at: number): boolean => + new GpuCrashFallbackTracker({ + windowMs: 30_000, + threshold: 3, + history: history.openLaunch() + }).recordGpuCrash(1_000, { at, exitCode: 34 }).shouldEngageFallback + expect(crashingLaunch(startedAt)).toBe(false) + expect(crashingLaunch(startedAt + day)).toBe(false) + expect(crashingLaunch(startedAt + 2 * day)).toBe(true) + }) + + it('stops re-prompting after the user keeps running, then asks again a day later', () => { + const history = createHistoryFake() + const startedAt = Date.UTC(2026, 7, 3, 10, 0, 0) + const crashingLaunch = (at: number): boolean => { + const launch = history.openLaunch() + const engaged = new GpuCrashFallbackTracker({ + windowMs: 30_000, + threshold: 3, + history: launch + }).recordGpuCrash(1_000, { at, exitCode: 34 }).shouldEngageFallback + if (engaged) { + // The user picks "Keep Running" every time. + launch.noteRestartDeclined(at) + } + return engaged + } + expect(crashingLaunch(startedAt)).toBe(false) + expect(crashingLaunch(startedAt + 600_000)).toBe(false) + expect(crashingLaunch(startedAt + 1_200_000)).toBe(true) + const declinedAt = startedAt + 1_200_000 + expect(crashingLaunch(declinedAt + 600_000)).toBe(false) + expect(crashingLaunch(declinedAt + 12 * 3_600_000)).toBe(false) + // A day on, the driver is still broken: it is fair to ask once more. + expect(crashingLaunch(declinedAt + 25 * 3_600_000)).toBe(true) + }) + + it('survives real relaunches through the on-disk ring', () => { + const userDataPath = mkdtempSync(join(tmpdir(), 'gpu-cross-launch-')) + const environment = { + appVersion: '1.4.167', + electronVersion: '38.2.0', + platform: 'win32' as const + } + const startedAt = Date.UTC(2026, 7, 3, 22, 40, 14) + try { + const relaunch = (at: number): boolean => + new GpuCrashFallbackTracker({ + windowMs: 30_000, + threshold: 3, + history: openGpuCrashHistoryLaunch(userDataPath, environment) + }).recordGpuCrash(2_000, { at, exitCode: 3_000 }).shouldEngageFallback + expect(relaunch(startedAt)).toBe(false) + expect(relaunch(startedAt + 88_000)).toBe(false) + expect(relaunch(startedAt + 190_000)).toBe(true) + } finally { + rmSync(userDataPath, { recursive: true, force: true }) + } + }) + + it('keeps the in-launch burst rule intact when nothing is persisted', () => { + const tracker = new GpuCrashFallbackTracker({ windowMs: 30_000, threshold: 3 }) + expect(tracker.recordGpuCrash(500).shouldEngageFallback).toBe(false) + expect(tracker.recordGpuCrash(8_000).shouldEngageFallback).toBe(false) + expect(tracker.recordGpuCrash(16_000)).toEqual({ + shouldEngageFallback: true, + crashesInWindow: 3 + }) + }) +}) diff --git a/src/main/crash-reporting/gpu-crash-fallback-coordinator.test.ts b/src/main/crash-reporting/gpu-crash-fallback-coordinator.test.ts new file mode 100644 index 00000000000..0824e9c0cd4 --- /dev/null +++ b/src/main/crash-reporting/gpu-crash-fallback-coordinator.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from 'vitest' +import { GpuCrashFallbackCoordinator } from './gpu-crash-fallback-coordinator' +import { GpuCrashFallbackTracker } from './gpu-crash-fallback-decision' + +const startedAt = Date.UTC(2026, 7, 3, 22, 40, 14) + +/** One physical GPU fault as the app sees it: child death, then the renderer. */ +function feedGpuFault( + coordinator: GpuCrashFallbackCoordinator, + msSinceLaunch: number, + options: { rendererDelayMs?: number; exitCode?: number | null } = {} +): { engaged: boolean; crashesInWindow: number; cascade: boolean } { + const exitCode = options.exitCode ?? 34 + const at = startedAt + msSinceLaunch + const verdict = coordinator.recordGpuChildCrash({ msSinceLaunch, at, exitCode }) + const cascade = coordinator.claimRendererCascade({ + reason: 'crashed', + exitCode, + at: at + (options.rendererDelayMs ?? 83) + }) + return { + engaged: verdict.shouldEngageFallback, + crashesInWindow: verdict.crashesInWindow, + cascade + } +} + +function createCoordinator(): GpuCrashFallbackCoordinator { + return new GpuCrashFallbackCoordinator({ + tracker: new GpuCrashFallbackTracker({ windowMs: 30_000, threshold: 3 }) + }) +} + +describe('GpuCrashFallbackCoordinator', () => { + it('counts one GPU fault once even when the renderer dies behind it', () => { + // F0BNM0R87SL shape: the renderer goes down ~83ms after the GPU child with + // the same exit code. Counting that tail too would halve the tuned threshold. + const coordinator = createCoordinator() + expect(feedGpuFault(coordinator, 1_000)).toEqual({ + engaged: false, + crashesInWindow: 1, + cascade: true + }) + expect(feedGpuFault(coordinator, 21_000)).toEqual({ + engaged: false, + crashesInWindow: 2, + cascade: true + }) + }) + + it('still engages on the third real fault inside the window', () => { + const coordinator = createCoordinator() + feedGpuFault(coordinator, 0) + feedGpuFault(coordinator, 10_000) + expect(feedGpuFault(coordinator, 20_000)).toEqual({ + engaged: true, + crashesInWindow: 3, + cascade: false + }) + }) + + it('does not arm attribution for the crash that engaged fallback', () => { + const coordinator = createCoordinator() + feedGpuFault(coordinator, 0) + feedGpuFault(coordinator, 10_000) + // The engaging crash relaunches the app; its renderer tail is teardown noise. + expect(feedGpuFault(coordinator, 20_000).cascade).toBe(false) + }) + + it('leaves unrelated renderer deaths unclaimed', () => { + const coordinator = createCoordinator() + coordinator.recordGpuChildCrash({ msSinceLaunch: 1_000, at: startedAt, exitCode: 34 }) + expect( + coordinator.claimRendererCascade({ reason: 'oom', exitCode: 34, at: startedAt + 100 }) + ).toBe(false) + }) +}) diff --git a/src/main/crash-reporting/gpu-crash-fallback-coordinator.ts b/src/main/crash-reporting/gpu-crash-fallback-coordinator.ts new file mode 100644 index 00000000000..ed12c30dd0e --- /dev/null +++ b/src/main/crash-reporting/gpu-crash-fallback-coordinator.ts @@ -0,0 +1,53 @@ +import { GpuCrashCascadeAttributor } from './gpu-crash-cascade-attribution' +import type { GpuCrashFallbackTracker } from './gpu-crash-fallback-decision' + +export type GpuCrashFallbackVerdict = { + shouldEngageFallback: boolean + crashesInWindow: number +} + +/** + * Routes GPU faults into the fallback tracker exactly once each. + * + * Why: one physical GPU fault surfaces twice — the suppressed GPU child death + * and, ~100ms later, the renderer going down with the same exit code. The tail + * names the cause in the crash trail, but recording it as a second crash would + * halve the tuned burst threshold (3 in 30s would fire after 2 real faults), so + * it is attribution only. + */ +export class GpuCrashFallbackCoordinator { + private readonly tracker: GpuCrashFallbackTracker + private readonly attributor: GpuCrashCascadeAttributor + + constructor(options: { + tracker: GpuCrashFallbackTracker + attributor?: GpuCrashCascadeAttributor + }) { + this.tracker = options.tracker + this.attributor = options.attributor ?? new GpuCrashCascadeAttributor() + } + + /** Counts a GPU child crash and arms cascade attribution for its renderer tail. */ + recordGpuChildCrash(crash: { + msSinceLaunch: number + at: number + exitCode: number | null + }): GpuCrashFallbackVerdict { + const verdict = this.tracker.recordGpuCrash(crash.msSinceLaunch, { + at: crash.at, + exitCode: crash.exitCode + }) + if (!verdict.shouldEngageFallback) { + this.attributor.noteSuppressedGpuCrash({ at: crash.at, exitCode: crash.exitCode }) + } + return verdict + } + + /** + * True when this renderer death is the tail of a GPU fault already counted by + * `recordGpuChildCrash` — for the crash trail, never for the fallback rules. + */ + claimRendererCascade(crash: { reason: string; exitCode: number | null; at: number }): boolean { + return this.attributor.claimRendererCascade(crash) + } +} diff --git a/src/main/crash-reporting/gpu-crash-fallback-decision.ts b/src/main/crash-reporting/gpu-crash-fallback-decision.ts index e962952c695..fd98dd4a1c2 100644 --- a/src/main/crash-reporting/gpu-crash-fallback-decision.ts +++ b/src/main/crash-reporting/gpu-crash-fallback-decision.ts @@ -1,8 +1,39 @@ +export type GpuCrashHistoryEntry = { + /** Wall clock (Date.now); performance.now() restarts at 0 on every relaunch. */ + ts: number + exitCode: number | null + /** Launch counter that produced the crash, incremented once per launch. */ + launchSeq: number +} + +/** Per-launch handle on the persisted crash ring (startup/gpu-crash-history-store). */ +export type GpuCrashHistoryLaunch = { + readonly launchSeq: number + /** Wall clock of the last declined fallback restart on this build, if any. */ + readonly declinedAt: number | null + /** Appends this launch's crash and returns the retained ring, oldest first. */ + append(crash: { ts: number; exitCode: number | null }): readonly GpuCrashHistoryEntry[] + /** Starts the re-prompt cooldown after the user chose to keep running. */ + noteRestartDeclined(at: number): void +} + export type GpuCrashFallbackOptions = { /** Rolling span over which clustered GPU crashes indicate a broken driver. */ windowMs: number /** GPU child crashes within the window that trigger software-rendering fallback. */ threshold: number + /** Persisted, build-scoped crash ring; omitted the tracker is in-launch only. */ + history?: GpuCrashHistoryLaunch | null + /** Wall-clock span over which crashes from earlier launches still count. */ + crossLaunchWindowMs?: number + /** Distinct crashing launches inside `crossLaunchWindowMs` that trigger fallback. */ + crossLaunchThreshold?: number + /** Back-to-back launches that each saw a GPU crash and trigger fallback. */ + crashingLaunchStreak?: number + /** Wall-clock span the whole streak must fit inside to still describe this machine. */ + crashingLaunchStreakWindowMs?: number + /** Quiet period after the user declines the restart before prompting again. */ + declineCooldownMs?: number } const GPU_FALLBACK_CRASH_REASONS = new Set(['abnormal-exit', 'crashed', 'launch-failed']) @@ -15,6 +46,76 @@ const GPU_FALLBACK_CRASH_REASONS = new Set(['abnormal-exit', 'crashed', 'launch- export const DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS = 30_000 export const DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD = 3 +// Why: the in-launch burst rule can never fire when the GPU fault takes the whole +// app down (F0BNM0R87SL crashes once per launch, forever), so the same evidence is +// re-evaluated across launches from the persisted ring. +export const DEFAULT_GPU_CROSS_LAUNCH_WINDOW_MS = 30 * 60_000 +export const DEFAULT_GPU_CROSS_LAUNCH_THRESHOLD = 3 +export const DEFAULT_GPU_CRASHING_LAUNCH_STREAK = 3 +// Why: three crashing launches only describe *this* machine's driver if they +// happened around the same time; a user who opens Orca once a day fits, one who +// opens it weekly is looking at three unrelated bits of Chromium churn. +export const DEFAULT_GPU_CRASHING_LAUNCH_STREAK_WINDOW_MS = 72 * 60 * 60_000 +// Why: "Keep Running" has to mean something — re-asking on the first GPU crash +// of every later launch is worse than the churn it warns about. +export const DEFAULT_GPU_FALLBACK_DECLINE_COOLDOWN_MS = 24 * 60 * 60_000 + +/** + * Cross-launch verdict over the persisted ring: either enough separate launches + * crashed inside the window, or enough back-to-back launches each died on the GPU. + * + * Both rules count launches, never raw crashes. Churn inside a single launch is + * the burst rule's call, and letting extra crashes from one launch fill a looser + * window would relaunch sessions the burst rule deliberately spares (field + * session F0BGRN5912M: 4 recovered crashes in 74s, plus one benign crash later). + * + * Both rules are also wall-clock bounded. `launchSeq` gaps break the streak, but + * a crashing launch is only evidence while it is recent — see the streak window. + */ +export function evaluateCrossLaunchGpuCrashes( + entries: readonly GpuCrashHistoryEntry[], + options: { + now: number + launchSeq: number + windowMs: number + threshold: number + crashingLaunchStreak: number + crashingLaunchStreakWindowMs: number + } +): { + crashesInWindow: number + crashingLaunchesInWindow: number + launchStreak: number + shouldEngage: boolean +} { + const cutoff = options.now - options.windowMs + const inWindow = entries.filter((entry) => entry.ts >= cutoff && entry.ts <= options.now) + const launchesInWindow = new Set(inWindow.map((entry) => entry.launchSeq)) + const streakCutoff = options.now - options.crashingLaunchStreakWindowMs + const newestCrashPerLaunch = new Map() + for (const entry of entries) { + newestCrashPerLaunch.set( + entry.launchSeq, + Math.max(entry.ts, newestCrashPerLaunch.get(entry.launchSeq) ?? entry.ts) + ) + } + let launchStreak = 0 + for (;;) { + const newestCrash = newestCrashPerLaunch.get(options.launchSeq - launchStreak) + if (newestCrash === undefined || newestCrash < streakCutoff) { + break + } + launchStreak += 1 + } + return { + crashesInWindow: inWindow.length, + crashingLaunchesInWindow: launchesInWindow.size, + launchStreak, + shouldEngage: + launchesInWindow.size >= options.threshold || launchStreak >= options.crashingLaunchStreak + } +} + /** * Tracks GPU child-process crashes and decides when to fall back to software * rendering on the next launch. Pure and deterministic: callers pass `now` @@ -32,11 +133,24 @@ export class GpuCrashFallbackTracker { private readonly threshold: number // Newest-last crash times (ms since launch), pruned to the rolling window. private readonly recentCrashes: number[] = [] + private readonly history: GpuCrashHistoryLaunch | null + private readonly crossLaunchWindowMs: number + private readonly crossLaunchThreshold: number + private readonly crashingLaunchStreak: number + private readonly crashingLaunchStreakWindowMs: number + private readonly declineCooldownMs: number private engaged = false constructor(options: GpuCrashFallbackOptions) { this.windowMs = options.windowMs this.threshold = options.threshold + this.history = options.history ?? null + this.crossLaunchWindowMs = options.crossLaunchWindowMs ?? DEFAULT_GPU_CROSS_LAUNCH_WINDOW_MS + this.crossLaunchThreshold = options.crossLaunchThreshold ?? DEFAULT_GPU_CROSS_LAUNCH_THRESHOLD + this.crashingLaunchStreak = options.crashingLaunchStreak ?? DEFAULT_GPU_CRASHING_LAUNCH_STREAK + this.crashingLaunchStreakWindowMs = + options.crashingLaunchStreakWindowMs ?? DEFAULT_GPU_CRASHING_LAUNCH_STREAK_WINDOW_MS + this.declineCooldownMs = options.declineCooldownMs ?? DEFAULT_GPU_FALLBACK_DECLINE_COOLDOWN_MS } /** @@ -45,7 +159,10 @@ export class GpuCrashFallbackTracker { * Returns false after fallback already engaged, so the caller relaunches at * most once. */ - recordGpuCrash(msSinceLaunch: number): { + recordGpuCrash( + msSinceLaunch: number, + crash: { at?: number; exitCode?: number | null } = {} + ): { shouldEngageFallback: boolean crashesInWindow: number } { @@ -62,10 +179,30 @@ export class GpuCrashFallbackTracker { stale += 1 } this.recentCrashes.splice(0, stale) + const wallClock = crash.at ?? Date.now() + // Evidence keeps accruing through the cooldown so the next prompt sees the truth. + const persisted = this.history?.append({ ts: wallClock, exitCode: crash.exitCode ?? null }) + if (this.isWithinDeclineCooldown(wallClock)) { + return { shouldEngageFallback: false, crashesInWindow: this.recentCrashes.length } + } if (this.recentCrashes.length >= this.threshold) { this.engaged = true return { shouldEngageFallback: true, crashesInWindow: this.recentCrashes.length } } + if (persisted && this.history) { + const crossLaunch = evaluateCrossLaunchGpuCrashes(persisted, { + now: wallClock, + launchSeq: this.history.launchSeq, + windowMs: this.crossLaunchWindowMs, + threshold: this.crossLaunchThreshold, + crashingLaunchStreak: this.crashingLaunchStreak, + crashingLaunchStreakWindowMs: this.crashingLaunchStreakWindowMs + }) + if (crossLaunch.shouldEngage) { + this.engaged = true + return { shouldEngageFallback: true, crashesInWindow: crossLaunch.crashesInWindow } + } + } return { shouldEngageFallback: false, crashesInWindow: this.recentCrashes.length } } @@ -73,6 +210,13 @@ export class GpuCrashFallbackTracker { return this.engaged } + // Why |Δ|: a clock stepped backwards past the decline must not pin the prompt + // off forever — asking once too often beats never asking again. + private isWithinDeclineCooldown(wallClock: number): boolean { + const declinedAt = this.history?.declinedAt ?? null + return declinedAt !== null && Math.abs(wallClock - declinedAt) < this.declineCooldownMs + } + /** Crash times currently inside the window. Exposed to assert the pruning invariant. */ windowSnapshot(): readonly number[] { return [...this.recentCrashes] diff --git a/src/main/index.ts b/src/main/index.ts index 3521070b55c..7b5897b6c9e 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -135,12 +135,15 @@ import { type WindowsGpuFallbackEnvironment } from './startup/gpu-fallback-marker' import { applyGpuFallbackCommandLineSwitches } from './startup/gpu-fallback-switches' +import { clearGpuCrashHistory, openGpuCrashHistoryLaunch } from './startup/gpu-crash-history-store' import { DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD, DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS, GpuCrashFallbackTracker, - isGpuFallbackCrashCandidate + isGpuFallbackCrashCandidate, + type GpuCrashHistoryLaunch } from './crash-reporting/gpu-crash-fallback-decision' +import { GpuCrashFallbackCoordinator } from './crash-reporting/gpu-crash-fallback-coordinator' import { promptForGpuFallbackRestart, type GpuFallbackRestartDecision @@ -362,10 +365,8 @@ let managedWslCliReconciliationReady: Promise = Promise.resolve() let managedWslCliStartupBarrierReady: Promise = Promise.resolve() // Why: the serve barrier fails open, so this state tells headless clients a WSL PTY launch may still race an un-migrated registration ('settled' = off-Windows no-op). let managedWslCliReconciliationStatus: 'pending' | 'settled' | 'failed' = 'settled' -const gpuCrashFallbackTracker = new GpuCrashFallbackTracker({ - windowMs: DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS, - threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD -}) +let gpuCrashHistoryLaunch: GpuCrashHistoryLaunch | null = null +let gpuCrashFallbackCoordinator: GpuCrashFallbackCoordinator | null = null let gpuFallbackActiveThisLaunch = false let gpuFeatureStatus: Electron.GPUFeatureStatus | null = null let localPtyStartupReady: Promise = Promise.resolve() @@ -1221,6 +1222,7 @@ function openMainWindow(): BrowserWindow { }, webContentsId ) + attributeGpuCascadeCrash(details.reason, details.exitCode ?? null) }, shouldRecoverRenderer: (details, webContentsId) => shouldRecoverRendererAfterProcessGone({ @@ -1556,7 +1558,11 @@ function maybeApplyGpuFallbackForThisLaunch(): void { if (isServeMode || process.platform !== 'win32') { return } - const marker = readActiveGpuFallbackMarker(app.getPath('userData'), getGpuFallbackEnvironment()) + const environment = getGpuFallbackEnvironment() + // Why: claim this launch's sequence number even when nothing crashes — a + // crash-free launch is the only evidence that a crashing streak ended. + gpuCrashHistoryLaunch = openGpuCrashHistoryLaunch(app.getPath('userData'), environment) + const marker = readActiveGpuFallbackMarker(app.getPath('userData'), environment) if (!marker) { return } @@ -1571,13 +1577,39 @@ function maybeApplyGpuFallbackForThisLaunch(): void { }) } +function getGpuCrashFallbackCoordinator(): GpuCrashFallbackCoordinator { + gpuCrashFallbackCoordinator ??= new GpuCrashFallbackCoordinator({ + tracker: new GpuCrashFallbackTracker({ + windowMs: DEFAULT_GPU_CRASH_FALLBACK_WINDOW_MS, + threshold: DEFAULT_GPU_CRASH_FALLBACK_THRESHOLD, + history: gpuCrashHistoryLaunch + }) + }) + return gpuCrashFallbackCoordinator +} + +// Why: names the GPU fault behind an otherwise unexplained renderer death; the +// GPU child crash it trails was already counted, so nothing is recorded here. +function attributeGpuCascadeCrash(reason: string, exitCode: number | null): void { + if ( + !getGpuCrashFallbackCoordinator().claimRendererCascade({ reason, exitCode, at: Date.now() }) + ) { + return + } + recordCrashBreadcrumb('gpu_cascade', { reason, exitCode }) +} + // Why: a burst of GPU child crashes means HW acceleration is unusable — persist a build-scoped marker and offer software rendering. async function handleGpuChildCrash(reason: string, exitCode: number | null): Promise { // Software rendering already active or shutting down: nothing more to do. if (gpuFallbackActiveThisLaunch || isQuitting || isServeMode) { return } - const result = gpuCrashFallbackTracker.recordGpuCrash(performance.now()) + const result = getGpuCrashFallbackCoordinator().recordGpuChildCrash({ + msSinceLaunch: performance.now(), + at: Date.now(), + exitCode + }) if (!result.shouldEngageFallback) { return } @@ -1605,6 +1637,9 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro } if (restartDecision !== 'restart') { recordDurableCrashBreadcrumb('gpu_fallback_restart_deferred', fallbackData) + // Why: the persisted evidence still satisfies the rules, so without a + // cooldown "Keep Running" re-prompts on the first GPU crash of every launch. + gpuCrashHistoryLaunch?.noteRestartDeclined(Date.now()) return } const environment = getWindowsGpuFallbackEnvironment() @@ -1624,6 +1659,9 @@ async function handleGpuChildCrash(reason: string, exitCode: number | null): Pro console.warn('[gpu-fallback] failed to persist marker:', error) return } + // Why: the marker now carries the verdict; stale crashes must not re-engage + // the moment the user turns hardware acceleration back on. + clearGpuCrashHistory(app.getPath('userData')) isQuitting = true relaunchApp('gpu-fallback', fallbackData) // Why: app.exit(0) skips before-quit, so destroy the Windows tray manually to avoid a stale icon. diff --git a/src/main/startup/gpu-crash-history-store.test.ts b/src/main/startup/gpu-crash-history-store.test.ts new file mode 100644 index 00000000000..eb5a61c1f0b --- /dev/null +++ b/src/main/startup/gpu-crash-history-store.test.ts @@ -0,0 +1,129 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + clearGpuCrashHistory, + GPU_CRASH_HISTORY_FILE, + GPU_CRASH_HISTORY_MAX_ENTRIES, + openGpuCrashHistoryLaunch, + readGpuCrashHistory +} from './gpu-crash-history-store' +import type { GpuFallbackEnvironment } from './gpu-fallback-marker' + +const environment: GpuFallbackEnvironment = { + appVersion: '1.4.167', + electronVersion: '38.2.0', + platform: 'win32' +} + +let userDataPath: string + +beforeEach(() => { + userDataPath = mkdtempSync(join(tmpdir(), 'gpu-crash-history-')) +}) + +afterEach(() => { + rmSync(userDataPath, { recursive: true, force: true }) +}) + +describe('openGpuCrashHistoryLaunch', () => { + it('bumps the launch counter on every launch, crash or not', () => { + expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(1) + expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(2) + expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(3) + }) + + it('persists crashes across launches with wall-clock timestamps', () => { + const first = openGpuCrashHistoryLaunch(userDataPath, environment) + first.append({ ts: 1_770_000_000_000, exitCode: 3_000 }) + const second = openGpuCrashHistoryLaunch(userDataPath, environment) + expect(second.append({ ts: 1_770_000_090_000, exitCode: 3_000 })).toEqual([ + { ts: 1_770_000_000_000, exitCode: 3_000, launchSeq: 1 }, + { ts: 1_770_000_090_000, exitCode: 3_000, launchSeq: 2 } + ]) + }) + + it('starts over after an app update so a fixed build gets a fresh hardware attempt', () => { + const before = openGpuCrashHistoryLaunch(userDataPath, environment) + before.append({ ts: 1_770_000_000_000, exitCode: 3_000 }) + const upgraded = openGpuCrashHistoryLaunch(userDataPath, { + ...environment, + appVersion: '1.4.168' + }) + expect(upgraded.launchSeq).toBe(1) + expect(upgraded.append({ ts: 1_770_000_100_000, exitCode: null })).toEqual([ + { ts: 1_770_000_100_000, exitCode: null, launchSeq: 1 } + ]) + }) + + it('bounds the ring', () => { + const launch = openGpuCrashHistoryLaunch(userDataPath, environment) + let entries: readonly { ts: number }[] = [] + for (let index = 0; index < GPU_CRASH_HISTORY_MAX_ENTRIES + 5; index += 1) { + entries = launch.append({ ts: 1_770_000_000_000 + index, exitCode: 3_000 }) + } + expect(entries).toHaveLength(GPU_CRASH_HISTORY_MAX_ENTRIES) + expect(entries[0]?.ts).toBe(1_770_000_000_000 + 5) + }) + + it('survives a corrupt file', () => { + writeFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), '{not json') + const launch = openGpuCrashHistoryLaunch(userDataPath, environment) + expect(launch.launchSeq).toBe(1) + expect(readGpuCrashHistory(userDataPath, environment)?.entries).toEqual([]) + }) + + it('drops malformed entries but keeps valid ones', () => { + writeFileSync( + join(userDataPath, GPU_CRASH_HISTORY_FILE), + JSON.stringify({ + schemeVersion: 1, + appVersion: environment.appVersion, + electronVersion: environment.electronVersion, + platform: environment.platform, + launchSeq: 4, + entries: [ + { ts: 'nope', exitCode: 1, launchSeq: 1 }, + { ts: 1_770_000_000_000, exitCode: null, launchSeq: 4 } + ] + }) + ) + expect(readGpuCrashHistory(userDataPath, environment)?.entries).toEqual([ + { ts: 1_770_000_000_000, exitCode: null, launchSeq: 4 } + ]) + expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(5) + }) + + it('remembers a declined restart across launches without losing crashes', () => { + const first = openGpuCrashHistoryLaunch(userDataPath, environment) + first.append({ ts: 1_770_000_000_000, exitCode: 3_000 }) + first.noteRestartDeclined(1_770_000_000_500) + const second = openGpuCrashHistoryLaunch(userDataPath, environment) + expect(second.declinedAt).toBe(1_770_000_000_500) + expect(second.append({ ts: 1_770_000_090_000, exitCode: 3_000 })).toHaveLength(2) + expect(openGpuCrashHistoryLaunch(userDataPath, environment).declinedAt).toBe(1_770_000_000_500) + }) + + it('reports no decline for a fresh or upgraded build', () => { + expect(openGpuCrashHistoryLaunch(userDataPath, environment).declinedAt).toBeNull() + openGpuCrashHistoryLaunch(userDataPath, environment).noteRestartDeclined(1_770_000_000_500) + expect( + openGpuCrashHistoryLaunch(userDataPath, { ...environment, appVersion: '1.4.168' }).declinedAt + ).toBeNull() + }) + + it('clears the ring when fallback engages', () => { + openGpuCrashHistoryLaunch(userDataPath, environment).append({ ts: 1, exitCode: 3_000 }) + clearGpuCrashHistory(userDataPath) + expect(readGpuCrashHistory(userDataPath, environment)).toBeNull() + expect(openGpuCrashHistoryLaunch(userDataPath, environment).launchSeq).toBe(1) + }) + + it('writes the ring where the marker lives', () => { + openGpuCrashHistoryLaunch(userDataPath, environment) + expect( + JSON.parse(readFileSync(join(userDataPath, GPU_CRASH_HISTORY_FILE), 'utf-8')).launchSeq + ).toBe(1) + }) +}) diff --git a/src/main/startup/gpu-crash-history-store.ts b/src/main/startup/gpu-crash-history-store.ts new file mode 100644 index 00000000000..e6ecc02a275 --- /dev/null +++ b/src/main/startup/gpu-crash-history-store.ts @@ -0,0 +1,151 @@ +import { mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import type { + GpuCrashHistoryEntry, + GpuCrashHistoryLaunch +} from '../crash-reporting/gpu-crash-fallback-decision' +import type { GpuFallbackEnvironment } from './gpu-fallback-marker' + +/** + * Build-scoped ring of GPU child crashes, stored next to the GPU fallback marker. + * + * Why on disk: a GPU fault that kills the app leaves the in-memory tracker empty + * on the next launch, so a "crash once per launch, forever" loop never reaches + * the in-launch burst threshold. Timestamps are wall clock — performance.now() + * restarts at 0 every launch and cannot be compared across them. + */ + +export const GPU_CRASH_HISTORY_FILE = 'gpu-crash-history.json' +export const GPU_CRASH_HISTORY_SCHEME_VERSION = 1 +/** Deep enough for the streak + window rules, small enough to write synchronously. */ +export const GPU_CRASH_HISTORY_MAX_ENTRIES = 16 + +type GpuCrashHistoryFile = { + schemeVersion: number + appVersion: string + electronVersion: string + platform: NodeJS.Platform + launchSeq: number + entries: GpuCrashHistoryEntry[] + /** Wall clock of the last "Keep Running" answer; null when never declined. */ + declinedAt: number | null +} + +function historyPath(userDataPath: string): string { + return join(userDataPath, GPU_CRASH_HISTORY_FILE) +} + +function isValidEntry(value: unknown): value is GpuCrashHistoryEntry { + const entry = value as Partial | null + return ( + typeof entry === 'object' && + entry !== null && + typeof entry.ts === 'number' && + Number.isFinite(entry.ts) && + (entry.exitCode === null || typeof entry.exitCode === 'number') && + typeof entry.launchSeq === 'number' && + Number.isFinite(entry.launchSeq) + ) +} + +export function readGpuCrashHistory( + userDataPath: string, + environment: GpuFallbackEnvironment +): GpuCrashHistoryFile | null { + try { + const parsed = JSON.parse(readFileSync(historyPath(userDataPath), 'utf-8')) as Partial< + Record + > + if ( + parsed.schemeVersion !== GPU_CRASH_HISTORY_SCHEME_VERSION || + parsed.appVersion !== environment.appVersion || + parsed.electronVersion !== environment.electronVersion || + parsed.platform !== environment.platform || + typeof parsed.launchSeq !== 'number' || + !Number.isFinite(parsed.launchSeq) || + !Array.isArray(parsed.entries) + ) { + // Why: a different build (or a corrupt file) gets a clean slate — the + // evidence describes a driver/binary pairing that no longer exists. + return null + } + return { + schemeVersion: GPU_CRASH_HISTORY_SCHEME_VERSION, + appVersion: environment.appVersion, + electronVersion: environment.electronVersion, + platform: environment.platform, + launchSeq: parsed.launchSeq, + entries: parsed.entries.filter(isValidEntry).slice(-GPU_CRASH_HISTORY_MAX_ENTRIES), + declinedAt: + typeof parsed.declinedAt === 'number' && Number.isFinite(parsed.declinedAt) + ? parsed.declinedAt + : null + } + } catch { + // missing or unreadable means no history + } + return null +} + +function writeGpuCrashHistory(userDataPath: string, file: GpuCrashHistoryFile): void { + try { + const target = historyPath(userDataPath) + mkdirSync(dirname(target), { recursive: true }) + writeFileSync(target, JSON.stringify(file)) + } catch { + // best effort: losing history only costs a slower fallback decision + } +} + +export function clearGpuCrashHistory(userDataPath: string): void { + try { + rmSync(historyPath(userDataPath), { force: true }) + } catch { + // best effort; the next launch revalidates the file anyway + } +} + +/** + * Claims the next launch sequence number and hands back an append port. + * + * Called once per launch even when no GPU crash follows: a crash-free launch has + * to bump the counter, otherwise a gap in the sequence — the only evidence that + * the streak broke — would never be recorded. + */ +export function openGpuCrashHistoryLaunch( + userDataPath: string, + environment: GpuFallbackEnvironment +): GpuCrashHistoryLaunch { + const existing = readGpuCrashHistory(userDataPath, environment) + const launchSeq = (existing?.launchSeq ?? 0) + 1 + let entries = existing?.entries ?? [] + let declinedAt = existing?.declinedAt ?? null + const file: GpuCrashHistoryFile = { + schemeVersion: GPU_CRASH_HISTORY_SCHEME_VERSION, + appVersion: environment.appVersion, + electronVersion: environment.electronVersion, + platform: environment.platform, + launchSeq, + entries, + declinedAt + } + writeGpuCrashHistory(userDataPath, file) + return { + launchSeq, + // Getter, not a snapshot: a decline taken mid-launch must be visible at once. + get declinedAt() { + return declinedAt + }, + append: (crash) => { + entries = [...entries, { ts: crash.ts, exitCode: crash.exitCode, launchSeq }].slice( + -GPU_CRASH_HISTORY_MAX_ENTRIES + ) + writeGpuCrashHistory(userDataPath, { ...file, entries, declinedAt }) + return entries + }, + noteRestartDeclined: (at) => { + declinedAt = at + writeGpuCrashHistory(userDataPath, { ...file, entries, declinedAt }) + } + } +}